[pypy-commit] pypy arm-backend-2: merge default
bivab
noreply at buildbot.pypy.org
Tue Oct 25 11:07:41 CEST 2011
Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r48409:f534b61e5913
Date: 2011-10-10 09:24 +0200
http://bitbucket.org/pypy/pypy/changeset/f534b61e5913/
Log: merge default
diff too long, truncating to 10000 out of 17472 lines
diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -1,2 +1,3 @@
b590cf6de4190623aad9aa698694c22e614d67b9 release-1.5
b48df0bf4e75b81d98f19ce89d4a7dc3e1dab5e5 benchmarked
+d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6
diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py
--- a/dotviewer/graphparse.py
+++ b/dotviewer/graphparse.py
@@ -36,48 +36,45 @@
print >> sys.stderr, "Warning: could not guess file type, using 'dot'"
return 'unknown'
-def dot2plain(content, contenttype, use_codespeak=False):
- if contenttype == 'plain':
- # already a .plain file
- return content
+def dot2plain_graphviz(content, contenttype, use_codespeak=False):
+ if contenttype != 'neato':
+ cmdline = 'dot -Tplain'
+ else:
+ cmdline = 'neato -Tplain'
+ #print >> sys.stderr, '* running:', cmdline
+ close_fds = sys.platform != 'win32'
+ p = subprocess.Popen(cmdline, shell=True, close_fds=close_fds,
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+ (child_in, child_out) = (p.stdin, p.stdout)
+ try:
+ import thread
+ except ImportError:
+ bkgndwrite(child_in, content)
+ else:
+ thread.start_new_thread(bkgndwrite, (child_in, content))
+ plaincontent = child_out.read()
+ child_out.close()
+ if not plaincontent: # 'dot' is likely not installed
+ raise PlainParseError("no result from running 'dot'")
+ return plaincontent
- if not use_codespeak:
- if contenttype != 'neato':
- cmdline = 'dot -Tplain'
- else:
- cmdline = 'neato -Tplain'
- #print >> sys.stderr, '* running:', cmdline
- close_fds = sys.platform != 'win32'
- p = subprocess.Popen(cmdline, shell=True, close_fds=close_fds,
- stdin=subprocess.PIPE, stdout=subprocess.PIPE)
- (child_in, child_out) = (p.stdin, p.stdout)
- try:
- import thread
- except ImportError:
- bkgndwrite(child_in, content)
- else:
- thread.start_new_thread(bkgndwrite, (child_in, content))
- plaincontent = child_out.read()
- child_out.close()
- if not plaincontent: # 'dot' is likely not installed
- raise PlainParseError("no result from running 'dot'")
- else:
- import urllib
- request = urllib.urlencode({'dot': content})
- url = 'http://codespeak.net/pypy/convertdot.cgi'
- print >> sys.stderr, '* posting:', url
- g = urllib.urlopen(url, data=request)
- result = []
- while True:
- data = g.read(16384)
- if not data:
- break
- result.append(data)
- g.close()
- plaincontent = ''.join(result)
- # very simple-minded way to give a somewhat better error message
- if plaincontent.startswith('<body'):
- raise Exception("the dot on codespeak has very likely crashed")
+def dot2plain_codespeak(content, contenttype):
+ import urllib
+ request = urllib.urlencode({'dot': content})
+ url = 'http://codespeak.net/pypy/convertdot.cgi'
+ print >> sys.stderr, '* posting:', url
+ g = urllib.urlopen(url, data=request)
+ result = []
+ while True:
+ data = g.read(16384)
+ if not data:
+ break
+ result.append(data)
+ g.close()
+ plaincontent = ''.join(result)
+ # very simple-minded way to give a somewhat better error message
+ if plaincontent.startswith('<body'):
+ raise Exception("the dot on codespeak has very likely crashed")
return plaincontent
def bkgndwrite(f, data):
@@ -148,10 +145,13 @@
def parse_dot(graph_id, content, links={}, fixedfont=False):
contenttype = guess_type(content)
- try:
- plaincontent = dot2plain(content, contenttype, use_codespeak=False)
- return list(parse_plain(graph_id, plaincontent, links, fixedfont))
- except PlainParseError:
- # failed, retry via codespeak
- plaincontent = dot2plain(content, contenttype, use_codespeak=True)
- return list(parse_plain(graph_id, plaincontent, links, fixedfont))
+ if contenttype == 'plain':
+ plaincontent = content
+ else:
+ try:
+ plaincontent = dot2plain_graphviz(content, contenttype)
+ except PlainParseError, e:
+ print e
+ # failed, retry via codespeak
+ plaincontent = dot2plain_codespeak(content, contenttype)
+ return list(parse_plain(graph_id, plaincontent, links, fixedfont))
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -317,7 +317,7 @@
RegrTest('test_multibytecodec.py', usemodules='_multibytecodec'),
RegrTest('test_multibytecodec_support.py', skip="not a test"),
RegrTest('test_multifile.py'),
- RegrTest('test_multiprocessing.py', skip='FIXME leaves subprocesses'),
+ RegrTest('test_multiprocessing.py', skip="FIXME leaves subprocesses"),
RegrTest('test_mutants.py', core="possibly"),
RegrTest('test_mutex.py'),
RegrTest('test_netrc.py'),
diff --git a/lib-python/modified-2.7/gzip.py b/lib-python/modified-2.7/gzip.py
deleted file mode 100644
--- a/lib-python/modified-2.7/gzip.py
+++ /dev/null
@@ -1,514 +0,0 @@
-"""Functions that read and write gzipped files.
-
-The user of the file doesn't have to worry about the compression,
-but random access is not allowed."""
-
-# based on Andrew Kuchling's minigzip.py distributed with the zlib module
-
-import struct, sys, time, os
-import zlib
-import io
-import __builtin__
-
-__all__ = ["GzipFile","open"]
-
-FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16
-
-READ, WRITE = 1, 2
-
-def write32u(output, value):
- # The L format writes the bit pattern correctly whether signed
- # or unsigned.
- output.write(struct.pack("<L", value))
-
-def read32(input):
- return struct.unpack("<I", input.read(4))[0]
-
-def open(filename, mode="rb", compresslevel=9):
- """Shorthand for GzipFile(filename, mode, compresslevel).
-
- The filename argument is required; mode defaults to 'rb'
- and compresslevel defaults to 9.
-
- """
- return GzipFile(filename, mode, compresslevel)
-
-class GzipFile(io.BufferedIOBase):
- """The GzipFile class simulates most of the methods of a file object with
- the exception of the readinto() and truncate() methods.
-
- """
-
- myfileobj = None
- max_read_chunk = 10 * 1024 * 1024 # 10Mb
-
- def __init__(self, filename=None, mode=None,
- compresslevel=9, fileobj=None, mtime=None):
- """Constructor for the GzipFile class.
-
- At least one of fileobj and filename must be given a
- non-trivial value.
-
- The new class instance is based on fileobj, which can be a regular
- file, a StringIO object, or any other object which simulates a file.
- It defaults to None, in which case filename is opened to provide
- a file object.
-
- When fileobj is not None, the filename argument is only used to be
- included in the gzip file header, which may includes the original
- filename of the uncompressed file. It defaults to the filename of
- fileobj, if discernible; otherwise, it defaults to the empty string,
- and in this case the original filename is not included in the header.
-
- The mode argument can be any of 'r', 'rb', 'a', 'ab', 'w', or 'wb',
- depending on whether the file will be read or written. The default
- is the mode of fileobj if discernible; otherwise, the default is 'rb'.
- Be aware that only the 'rb', 'ab', and 'wb' values should be used
- for cross-platform portability.
-
- The compresslevel argument is an integer from 1 to 9 controlling the
- level of compression; 1 is fastest and produces the least compression,
- and 9 is slowest and produces the most compression. The default is 9.
-
- The mtime argument is an optional numeric timestamp to be written
- to the stream when compressing. All gzip compressed streams
- are required to contain a timestamp. If omitted or None, the
- current time is used. This module ignores the timestamp when
- decompressing; however, some programs, such as gunzip, make use
- of it. The format of the timestamp is the same as that of the
- return value of time.time() and of the st_mtime member of the
- object returned by os.stat().
-
- """
-
- # guarantee the file is opened in binary mode on platforms
- # that care about that sort of thing
- if mode and 'b' not in mode:
- mode += 'b'
- if fileobj is None:
- fileobj = self.myfileobj = __builtin__.open(filename, mode or 'rb')
- if filename is None:
- if hasattr(fileobj, 'name'): filename = fileobj.name
- else: filename = ''
- if mode is None:
- if hasattr(fileobj, 'mode'): mode = fileobj.mode
- else: mode = 'rb'
-
- if mode[0:1] == 'r':
- self.mode = READ
- # Set flag indicating start of a new member
- self._new_member = True
- # Buffer data read from gzip file. extrastart is offset in
- # stream where buffer starts. extrasize is number of
- # bytes remaining in buffer from current stream position.
- self.extrabuf = ""
- self.extrasize = 0
- self.extrastart = 0
- self.name = filename
- # Starts small, scales exponentially
- self.min_readsize = 100
-
- elif mode[0:1] == 'w' or mode[0:1] == 'a':
- self.mode = WRITE
- self._init_write(filename)
- self.compress = zlib.compressobj(compresslevel,
- zlib.DEFLATED,
- -zlib.MAX_WBITS,
- zlib.DEF_MEM_LEVEL,
- 0)
- else:
- raise IOError, "Mode " + mode + " not supported"
-
- self.fileobj = fileobj
- self.offset = 0
- self.mtime = mtime
-
- if self.mode == WRITE:
- self._write_gzip_header()
-
- @property
- def filename(self):
- import warnings
- warnings.warn("use the name attribute", DeprecationWarning, 2)
- if self.mode == WRITE and self.name[-3:] != ".gz":
- return self.name + ".gz"
- return self.name
-
- def __repr__(self):
- s = repr(self.fileobj)
- return '<gzip ' + s[1:-1] + ' ' + hex(id(self)) + '>'
-
- def _check_closed(self):
- """Raises a ValueError if the underlying file object has been closed.
-
- """
- if self.closed:
- raise ValueError('I/O operation on closed file.')
-
- def _init_write(self, filename):
- self.name = filename
- self.crc = zlib.crc32("") & 0xffffffffL
- self.size = 0
- self.writebuf = []
- self.bufsize = 0
-
- def _write_gzip_header(self):
- self.fileobj.write('\037\213') # magic header
- self.fileobj.write('\010') # compression method
- fname = os.path.basename(self.name)
- if fname.endswith(".gz"):
- fname = fname[:-3]
- flags = 0
- if fname:
- flags = FNAME
- self.fileobj.write(chr(flags))
- mtime = self.mtime
- if mtime is None:
- mtime = time.time()
- write32u(self.fileobj, long(mtime))
- self.fileobj.write('\002')
- self.fileobj.write('\377')
- if fname:
- self.fileobj.write(fname + '\000')
-
- def _init_read(self):
- self.crc = zlib.crc32("") & 0xffffffffL
- self.size = 0
-
- def _read_gzip_header(self):
- magic = self.fileobj.read(2)
- if magic != '\037\213':
- raise IOError, 'Not a gzipped file'
- method = ord( self.fileobj.read(1) )
- if method != 8:
- raise IOError, 'Unknown compression method'
- flag = ord( self.fileobj.read(1) )
- self.mtime = read32(self.fileobj)
- # extraflag = self.fileobj.read(1)
- # os = self.fileobj.read(1)
- self.fileobj.read(2)
-
- if flag & FEXTRA:
- # Read & discard the extra field, if present
- xlen = ord(self.fileobj.read(1))
- xlen = xlen + 256*ord(self.fileobj.read(1))
- self.fileobj.read(xlen)
- if flag & FNAME:
- # Read and discard a null-terminated string containing the filename
- while True:
- s = self.fileobj.read(1)
- if not s or s=='\000':
- break
- if flag & FCOMMENT:
- # Read and discard a null-terminated string containing a comment
- while True:
- s = self.fileobj.read(1)
- if not s or s=='\000':
- break
- if flag & FHCRC:
- self.fileobj.read(2) # Read & discard the 16-bit header CRC
-
- def write(self,data):
- self._check_closed()
- if self.mode != WRITE:
- import errno
- raise IOError(errno.EBADF, "write() on read-only GzipFile object")
-
- if self.fileobj is None:
- raise ValueError, "write() on closed GzipFile object"
-
- # Convert data type if called by io.BufferedWriter.
- if isinstance(data, memoryview):
- data = data.tobytes()
-
- if len(data) > 0:
- self.size = self.size + len(data)
- self.crc = zlib.crc32(data, self.crc) & 0xffffffffL
- self.fileobj.write( self.compress.compress(data) )
- self.offset += len(data)
-
- return len(data)
-
- def read(self, size=-1):
- self._check_closed()
- if self.mode != READ:
- import errno
- raise IOError(errno.EBADF, "read() on write-only GzipFile object")
-
- if self.extrasize <= 0 and self.fileobj is None:
- return ''
-
- readsize = 1024
- if size < 0: # get the whole thing
- try:
- while True:
- self._read(readsize)
- readsize = min(self.max_read_chunk, readsize * 2)
- except EOFError:
- size = self.extrasize
- elif size == 0:
- return ""
- else: # just get some more of it
- try:
- while size > self.extrasize:
- self._read(readsize)
- readsize = min(self.max_read_chunk, readsize * 2)
- except EOFError:
- if size > self.extrasize:
- size = self.extrasize
-
- offset = self.offset - self.extrastart
- chunk = self.extrabuf[offset: offset + size]
- self.extrasize = self.extrasize - size
-
- self.offset += size
- return chunk
-
- def _unread(self, buf):
- self.extrasize = len(buf) + self.extrasize
- self.offset -= len(buf)
-
- def _read(self, size=1024):
- if self.fileobj is None:
- raise EOFError, "Reached EOF"
-
- if self._new_member:
- # If the _new_member flag is set, we have to
- # jump to the next member, if there is one.
- #
- # First, check if we're at the end of the file;
- # if so, it's time to stop; no more members to read.
- pos = self.fileobj.tell() # Save current position
- self.fileobj.seek(0, 2) # Seek to end of file
- if pos == self.fileobj.tell():
- raise EOFError, "Reached EOF"
- else:
- self.fileobj.seek( pos ) # Return to original position
-
- self._init_read()
- self._read_gzip_header()
- self.decompress = zlib.decompressobj(-zlib.MAX_WBITS)
- self._new_member = False
-
- # Read a chunk of data from the file
- buf = self.fileobj.read(size)
-
- # If the EOF has been reached, flush the decompression object
- # and mark this object as finished.
-
- if buf == "":
- uncompress = self.decompress.flush()
- self._read_eof()
- self._add_read_data( uncompress )
- raise EOFError, 'Reached EOF'
-
- uncompress = self.decompress.decompress(buf)
- self._add_read_data( uncompress )
-
- if self.decompress.unused_data != "":
- # Ending case: we've come to the end of a member in the file,
- # so seek back to the start of the unused data, finish up
- # this member, and read a new gzip header.
- # (The number of bytes to seek back is the length of the unused
- # data, minus 8 because _read_eof() will rewind a further 8 bytes)
- self.fileobj.seek( -len(self.decompress.unused_data)+8, 1)
-
- # Check the CRC and file size, and set the flag so we read
- # a new member on the next call
- self._read_eof()
- self._new_member = True
-
- def _add_read_data(self, data):
- self.crc = zlib.crc32(data, self.crc) & 0xffffffffL
- offset = self.offset - self.extrastart
- self.extrabuf = self.extrabuf[offset:] + data
- self.extrasize = self.extrasize + len(data)
- self.extrastart = self.offset
- self.size = self.size + len(data)
-
- def _read_eof(self):
- # We've read to the end of the file, so we have to rewind in order
- # to reread the 8 bytes containing the CRC and the file size.
- # We check the that the computed CRC and size of the
- # uncompressed data matches the stored values. Note that the size
- # stored is the true file size mod 2**32.
- self.fileobj.seek(-8, 1)
- crc32 = read32(self.fileobj)
- isize = read32(self.fileobj) # may exceed 2GB
- if crc32 != self.crc:
- raise IOError("CRC check failed %s != %s" % (hex(crc32),
- hex(self.crc)))
- elif isize != (self.size & 0xffffffffL):
- raise IOError, "Incorrect length of data produced"
-
- # Gzip files can be padded with zeroes and still have archives.
- # Consume all zero bytes and set the file position to the first
- # non-zero byte. See http://www.gzip.org/#faq8
- c = "\x00"
- while c == "\x00":
- c = self.fileobj.read(1)
- if c:
- self.fileobj.seek(-1, 1)
-
- @property
- def closed(self):
- return self.fileobj is None
-
- def close(self):
- if self.fileobj is None:
- return
- if self.mode == WRITE:
- self.fileobj.write(self.compress.flush())
- write32u(self.fileobj, self.crc)
- # self.size may exceed 2GB, or even 4GB
- write32u(self.fileobj, self.size & 0xffffffffL)
- self.fileobj = None
- elif self.mode == READ:
- self.fileobj = None
- if self.myfileobj:
- self.myfileobj.close()
- self.myfileobj = None
-
- def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH):
- self._check_closed()
- if self.mode == WRITE:
- # Ensure the compressor's buffer is flushed
- self.fileobj.write(self.compress.flush(zlib_mode))
- self.fileobj.flush()
-
- def fileno(self):
- """Invoke the underlying file object's fileno() method.
-
- This will raise AttributeError if the underlying file object
- doesn't support fileno().
- """
- return self.fileobj.fileno()
-
- def rewind(self):
- '''Return the uncompressed stream file position indicator to the
- beginning of the file'''
- if self.mode != READ:
- raise IOError("Can't rewind in write mode")
- self.fileobj.seek(0)
- self._new_member = True
- self.extrabuf = ""
- self.extrasize = 0
- self.extrastart = 0
- self.offset = 0
-
- def readable(self):
- return self.mode == READ
-
- def writable(self):
- return self.mode == WRITE
-
- def seekable(self):
- return True
-
- def seek(self, offset, whence=0):
- if whence:
- if whence == 1:
- offset = self.offset + offset
- else:
- raise ValueError('Seek from end not supported')
- if self.mode == WRITE:
- if offset < self.offset:
- raise IOError('Negative seek in write mode')
- count = offset - self.offset
- for i in range(count // 1024):
- self.write(1024 * '\0')
- self.write((count % 1024) * '\0')
- elif self.mode == READ:
- if offset == self.offset:
- self.read(0) # to make sure that this file is open
- return self.offset
- if offset < self.offset:
- # for negative seek, rewind and do positive seek
- self.rewind()
- count = offset - self.offset
- for i in range(count // 1024):
- self.read(1024)
- self.read(count % 1024)
-
- return self.offset
-
- def readline(self, size=-1):
- if size < 0:
- # Shortcut common case - newline found in buffer.
- offset = self.offset - self.extrastart
- i = self.extrabuf.find('\n', offset) + 1
- if i > 0:
- self.extrasize -= i - offset
- self.offset += i - offset
- return self.extrabuf[offset: i]
-
- size = sys.maxint
- readsize = self.min_readsize
- else:
- readsize = size
- bufs = []
- while size != 0:
- c = self.read(readsize)
- i = c.find('\n')
-
- # We set i=size to break out of the loop under two
- # conditions: 1) there's no newline, and the chunk is
- # larger than size, or 2) there is a newline, but the
- # resulting line would be longer than 'size'.
- if (size <= i) or (i == -1 and len(c) > size):
- i = size - 1
-
- if i >= 0 or c == '':
- bufs.append(c[:i + 1]) # Add portion of last chunk
- self._unread(c[i + 1:]) # Push back rest of chunk
- break
-
- # Append chunk to list, decrease 'size',
- bufs.append(c)
- size = size - len(c)
- readsize = min(size, readsize * 2)
- if readsize > self.min_readsize:
- self.min_readsize = min(readsize, self.min_readsize * 2, 512)
- return ''.join(bufs) # Return resulting line
-
-
-def _test():
- # Act like gzip; with -d, act like gunzip.
- # The input file is not deleted, however, nor are any other gzip
- # options or features supported.
- args = sys.argv[1:]
- decompress = args and args[0] == "-d"
- if decompress:
- args = args[1:]
- if not args:
- args = ["-"]
- for arg in args:
- if decompress:
- if arg == "-":
- f = GzipFile(filename="", mode="rb", fileobj=sys.stdin)
- g = sys.stdout
- else:
- if arg[-3:] != ".gz":
- print "filename doesn't end in .gz:", repr(arg)
- continue
- f = open(arg, "rb")
- g = __builtin__.open(arg[:-3], "wb")
- else:
- if arg == "-":
- f = sys.stdin
- g = GzipFile(filename="", mode="wb", fileobj=sys.stdout)
- else:
- f = __builtin__.open(arg, "rb")
- g = open(arg + ".gz", "wb")
- while True:
- chunk = f.read(1024)
- if not chunk:
- break
- g.write(chunk)
- if g is not sys.stdout:
- g.close()
- if f is not sys.stdin:
- f.close()
-
-if __name__ == '__main__':
- _test()
diff --git a/lib-python/modified-2.7/httplib.py b/lib-python/modified-2.7/httplib.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/httplib.py
@@ -0,0 +1,1377 @@
+"""HTTP/1.1 client library
+
+<intro stuff goes here>
+<other stuff, too>
+
+HTTPConnection goes through a number of "states", which define when a client
+may legally make another request or fetch the response for a particular
+request. This diagram details these state transitions:
+
+ (null)
+ |
+ | HTTPConnection()
+ v
+ Idle
+ |
+ | putrequest()
+ v
+ Request-started
+ |
+ | ( putheader() )* endheaders()
+ v
+ Request-sent
+ |
+ | response = getresponse()
+ v
+ Unread-response [Response-headers-read]
+ |\____________________
+ | |
+ | response.read() | putrequest()
+ v v
+ Idle Req-started-unread-response
+ ______/|
+ / |
+ response.read() | | ( putheader() )* endheaders()
+ v v
+ Request-started Req-sent-unread-response
+ |
+ | response.read()
+ v
+ Request-sent
+
+This diagram presents the following rules:
+ -- a second request may not be started until {response-headers-read}
+ -- a response [object] cannot be retrieved until {request-sent}
+ -- there is no differentiation between an unread response body and a
+ partially read response body
+
+Note: this enforcement is applied by the HTTPConnection class. The
+ HTTPResponse class does not enforce this state machine, which
+ implies sophisticated clients may accelerate the request/response
+ pipeline. Caution should be taken, though: accelerating the states
+ beyond the above pattern may imply knowledge of the server's
+ connection-close behavior for certain requests. For example, it
+ is impossible to tell whether the server will close the connection
+ UNTIL the response headers have been read; this means that further
+ requests cannot be placed into the pipeline until it is known that
+ the server will NOT be closing the connection.
+
+Logical State __state __response
+------------- ------- ----------
+Idle _CS_IDLE None
+Request-started _CS_REQ_STARTED None
+Request-sent _CS_REQ_SENT None
+Unread-response _CS_IDLE <response_class>
+Req-started-unread-response _CS_REQ_STARTED <response_class>
+Req-sent-unread-response _CS_REQ_SENT <response_class>
+"""
+
+from array import array
+import os
+import socket
+from sys import py3kwarning
+from urlparse import urlsplit
+import warnings
+with warnings.catch_warnings():
+ if py3kwarning:
+ warnings.filterwarnings("ignore", ".*mimetools has been removed",
+ DeprecationWarning)
+ import mimetools
+
+try:
+ from cStringIO import StringIO
+except ImportError:
+ from StringIO import StringIO
+
+__all__ = ["HTTP", "HTTPResponse", "HTTPConnection",
+ "HTTPException", "NotConnected", "UnknownProtocol",
+ "UnknownTransferEncoding", "UnimplementedFileMode",
+ "IncompleteRead", "InvalidURL", "ImproperConnectionState",
+ "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
+ "BadStatusLine", "error", "responses"]
+
+HTTP_PORT = 80
+HTTPS_PORT = 443
+
+_UNKNOWN = 'UNKNOWN'
+
+# connection states
+_CS_IDLE = 'Idle'
+_CS_REQ_STARTED = 'Request-started'
+_CS_REQ_SENT = 'Request-sent'
+
+# status codes
+# informational
+CONTINUE = 100
+SWITCHING_PROTOCOLS = 101
+PROCESSING = 102
+
+# successful
+OK = 200
+CREATED = 201
+ACCEPTED = 202
+NON_AUTHORITATIVE_INFORMATION = 203
+NO_CONTENT = 204
+RESET_CONTENT = 205
+PARTIAL_CONTENT = 206
+MULTI_STATUS = 207
+IM_USED = 226
+
+# redirection
+MULTIPLE_CHOICES = 300
+MOVED_PERMANENTLY = 301
+FOUND = 302
+SEE_OTHER = 303
+NOT_MODIFIED = 304
+USE_PROXY = 305
+TEMPORARY_REDIRECT = 307
+
+# client error
+BAD_REQUEST = 400
+UNAUTHORIZED = 401
+PAYMENT_REQUIRED = 402
+FORBIDDEN = 403
+NOT_FOUND = 404
+METHOD_NOT_ALLOWED = 405
+NOT_ACCEPTABLE = 406
+PROXY_AUTHENTICATION_REQUIRED = 407
+REQUEST_TIMEOUT = 408
+CONFLICT = 409
+GONE = 410
+LENGTH_REQUIRED = 411
+PRECONDITION_FAILED = 412
+REQUEST_ENTITY_TOO_LARGE = 413
+REQUEST_URI_TOO_LONG = 414
+UNSUPPORTED_MEDIA_TYPE = 415
+REQUESTED_RANGE_NOT_SATISFIABLE = 416
+EXPECTATION_FAILED = 417
+UNPROCESSABLE_ENTITY = 422
+LOCKED = 423
+FAILED_DEPENDENCY = 424
+UPGRADE_REQUIRED = 426
+
+# server error
+INTERNAL_SERVER_ERROR = 500
+NOT_IMPLEMENTED = 501
+BAD_GATEWAY = 502
+SERVICE_UNAVAILABLE = 503
+GATEWAY_TIMEOUT = 504
+HTTP_VERSION_NOT_SUPPORTED = 505
+INSUFFICIENT_STORAGE = 507
+NOT_EXTENDED = 510
+
+# Mapping status codes to official W3C names
+responses = {
+ 100: 'Continue',
+ 101: 'Switching Protocols',
+
+ 200: 'OK',
+ 201: 'Created',
+ 202: 'Accepted',
+ 203: 'Non-Authoritative Information',
+ 204: 'No Content',
+ 205: 'Reset Content',
+ 206: 'Partial Content',
+
+ 300: 'Multiple Choices',
+ 301: 'Moved Permanently',
+ 302: 'Found',
+ 303: 'See Other',
+ 304: 'Not Modified',
+ 305: 'Use Proxy',
+ 306: '(Unused)',
+ 307: 'Temporary Redirect',
+
+ 400: 'Bad Request',
+ 401: 'Unauthorized',
+ 402: 'Payment Required',
+ 403: 'Forbidden',
+ 404: 'Not Found',
+ 405: 'Method Not Allowed',
+ 406: 'Not Acceptable',
+ 407: 'Proxy Authentication Required',
+ 408: 'Request Timeout',
+ 409: 'Conflict',
+ 410: 'Gone',
+ 411: 'Length Required',
+ 412: 'Precondition Failed',
+ 413: 'Request Entity Too Large',
+ 414: 'Request-URI Too Long',
+ 415: 'Unsupported Media Type',
+ 416: 'Requested Range Not Satisfiable',
+ 417: 'Expectation Failed',
+
+ 500: 'Internal Server Error',
+ 501: 'Not Implemented',
+ 502: 'Bad Gateway',
+ 503: 'Service Unavailable',
+ 504: 'Gateway Timeout',
+ 505: 'HTTP Version Not Supported',
+}
+
+# maximal amount of data to read at one time in _safe_read
+MAXAMOUNT = 1048576
+
+class HTTPMessage(mimetools.Message):
+
+ def addheader(self, key, value):
+ """Add header for field key handling repeats."""
+ prev = self.dict.get(key)
+ if prev is None:
+ self.dict[key] = value
+ else:
+ combined = ", ".join((prev, value))
+ self.dict[key] = combined
+
+ def addcontinue(self, key, more):
+ """Add more field data from a continuation line."""
+ prev = self.dict[key]
+ self.dict[key] = prev + "\n " + more
+
+ def readheaders(self):
+ """Read header lines.
+
+ Read header lines up to the entirely blank line that terminates them.
+ The (normally blank) line that ends the headers is skipped, but not
+ included in the returned list. If a non-header line ends the headers,
+ (which is an error), an attempt is made to backspace over it; it is
+ never included in the returned list.
+
+ The variable self.status is set to the empty string if all went well,
+ otherwise it is an error message. The variable self.headers is a
+ completely uninterpreted list of lines contained in the header (so
+ printing them will reproduce the header exactly as it appears in the
+ file).
+
+ If multiple header fields with the same name occur, they are combined
+ according to the rules in RFC 2616 sec 4.2:
+
+ Appending each subsequent field-value to the first, each separated
+ by a comma. The order in which header fields with the same field-name
+ are received is significant to the interpretation of the combined
+ field value.
+ """
+ # XXX The implementation overrides the readheaders() method of
+ # rfc822.Message. The base class design isn't amenable to
+ # customized behavior here so the method here is a copy of the
+ # base class code with a few small changes.
+
+ self.dict = {}
+ self.unixfrom = ''
+ self.headers = hlist = []
+ self.status = ''
+ headerseen = ""
+ firstline = 1
+ startofline = unread = tell = None
+ if hasattr(self.fp, 'unread'):
+ unread = self.fp.unread
+ elif self.seekable:
+ tell = self.fp.tell
+ while True:
+ if tell:
+ try:
+ startofline = tell()
+ except IOError:
+ startofline = tell = None
+ self.seekable = 0
+ line = self.fp.readline()
+ if not line:
+ self.status = 'EOF in headers'
+ break
+ # Skip unix From name time lines
+ if firstline and line.startswith('From '):
+ self.unixfrom = self.unixfrom + line
+ continue
+ firstline = 0
+ if headerseen and line[0] in ' \t':
+ # XXX Not sure if continuation lines are handled properly
+ # for http and/or for repeating headers
+ # It's a continuation line.
+ hlist.append(line)
+ self.addcontinue(headerseen, line.strip())
+ continue
+ elif self.iscomment(line):
+ # It's a comment. Ignore it.
+ continue
+ elif self.islast(line):
+ # Note! No pushback here! The delimiter line gets eaten.
+ break
+ headerseen = self.isheader(line)
+ if headerseen:
+ # It's a legal header line, save it.
+ hlist.append(line)
+ self.addheader(headerseen, line[len(headerseen)+1:].strip())
+ continue
+ else:
+ # It's not a header line; throw it back and stop here.
+ if not self.dict:
+ self.status = 'No headers'
+ else:
+ self.status = 'Non-header line where header expected'
+ # Try to undo the read.
+ if unread:
+ unread(line)
+ elif tell:
+ self.fp.seek(startofline)
+ else:
+ self.status = self.status + '; bad seek'
+ break
+
+class HTTPResponse:
+
+ # strict: If true, raise BadStatusLine if the status line can't be
+ # parsed as a valid HTTP/1.0 or 1.1 status line. By default it is
+ # false because it prevents clients from talking to HTTP/0.9
+ # servers. Note that a response with a sufficiently corrupted
+ # status line will look like an HTTP/0.9 response.
+
+ # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
+
+ def __init__(self, sock, debuglevel=0, strict=0, method=None, buffering=False):
+ if buffering:
+ # The caller won't be using any sock.recv() calls, so buffering
+ # is fine and recommended for performance.
+ self.fp = sock.makefile('rb')
+ else:
+ # The buffer size is specified as zero, because the headers of
+ # the response are read with readline(). If the reads were
+ # buffered the readline() calls could consume some of the
+ # response, which make be read via a recv() on the underlying
+ # socket.
+ self.fp = sock.makefile('rb', 0)
+ self.debuglevel = debuglevel
+ self.strict = strict
+ self._method = method
+
+ self.msg = None
+
+ # from the Status-Line of the response
+ self.version = _UNKNOWN # HTTP-Version
+ self.status = _UNKNOWN # Status-Code
+ self.reason = _UNKNOWN # Reason-Phrase
+
+ self.chunked = _UNKNOWN # is "chunked" being used?
+ self.chunk_left = _UNKNOWN # bytes left to read in current chunk
+ self.length = _UNKNOWN # number of bytes left in response
+ self.will_close = _UNKNOWN # conn will close at end of response
+
+ def _read_status(self):
+ # Initialize with Simple-Response defaults
+ line = self.fp.readline()
+ if self.debuglevel > 0:
+ print "reply:", repr(line)
+ if not line:
+ # Presumably, the server closed the connection before
+ # sending a valid response.
+ raise BadStatusLine(line)
+ try:
+ [version, status, reason] = line.split(None, 2)
+ except ValueError:
+ try:
+ [version, status] = line.split(None, 1)
+ reason = ""
+ except ValueError:
+ # empty version will cause next test to fail and status
+ # will be treated as 0.9 response.
+ version = ""
+ if not version.startswith('HTTP/'):
+ if self.strict:
+ self.close()
+ raise BadStatusLine(line)
+ else:
+ # assume it's a Simple-Response from an 0.9 server
+ self.fp = LineAndFileWrapper(line, self.fp)
+ return "HTTP/0.9", 200, ""
+
+ # The status code is a three-digit number
+ try:
+ status = int(status)
+ if status < 100 or status > 999:
+ raise BadStatusLine(line)
+ except ValueError:
+ raise BadStatusLine(line)
+ return version, status, reason
+
+ def begin(self):
+ if self.msg is not None:
+ # we've already started reading the response
+ return
+
+ # read until we get a non-100 response
+ while True:
+ version, status, reason = self._read_status()
+ if status != CONTINUE:
+ break
+ # skip the header from the 100 response
+ while True:
+ skip = self.fp.readline().strip()
+ if not skip:
+ break
+ if self.debuglevel > 0:
+ print "header:", skip
+
+ self.status = status
+ self.reason = reason.strip()
+ if version == 'HTTP/1.0':
+ self.version = 10
+ elif version.startswith('HTTP/1.'):
+ self.version = 11 # use HTTP/1.1 code for HTTP/1.x where x>=1
+ elif version == 'HTTP/0.9':
+ self.version = 9
+ else:
+ raise UnknownProtocol(version)
+
+ if self.version == 9:
+ self.length = None
+ self.chunked = 0
+ self.will_close = 1
+ self.msg = HTTPMessage(StringIO())
+ return
+
+ self.msg = HTTPMessage(self.fp, 0)
+ if self.debuglevel > 0:
+ for hdr in self.msg.headers:
+ print "header:", hdr,
+
+ # don't let the msg keep an fp
+ self.msg.fp = None
+
+ # are we using the chunked-style of transfer encoding?
+ tr_enc = self.msg.getheader('transfer-encoding')
+ if tr_enc and tr_enc.lower() == "chunked":
+ self.chunked = 1
+ self.chunk_left = None
+ else:
+ self.chunked = 0
+
+ # will the connection close at the end of the response?
+ self.will_close = self._check_close()
+
+ # do we have a Content-Length?
+ # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
+ length = self.msg.getheader('content-length')
+ if length and not self.chunked:
+ try:
+ self.length = int(length)
+ except ValueError:
+ self.length = None
+ else:
+ if self.length < 0: # ignore nonsensical negative lengths
+ self.length = None
+ else:
+ self.length = None
+
+ # does the body have a fixed length? (of zero)
+ if (status == NO_CONTENT or status == NOT_MODIFIED or
+ 100 <= status < 200 or # 1xx codes
+ self._method == 'HEAD'):
+ self.length = 0
+
+ # if the connection remains open, and we aren't using chunked, and
+ # a content-length was not provided, then assume that the connection
+ # WILL close.
+ if not self.will_close and \
+ not self.chunked and \
+ self.length is None:
+ self.will_close = 1
+
+ def _check_close(self):
+ conn = self.msg.getheader('connection')
+ if self.version == 11:
+ # An HTTP/1.1 proxy is assumed to stay open unless
+ # explicitly closed.
+ conn = self.msg.getheader('connection')
+ if conn and "close" in conn.lower():
+ return True
+ return False
+
+ # Some HTTP/1.0 implementations have support for persistent
+ # connections, using rules different than HTTP/1.1.
+
+ # For older HTTP, Keep-Alive indicates persistent connection.
+ if self.msg.getheader('keep-alive'):
+ return False
+
+ # At least Akamai returns a "Connection: Keep-Alive" header,
+ # which was supposed to be sent by the client.
+ if conn and "keep-alive" in conn.lower():
+ return False
+
+ # Proxy-Connection is a netscape hack.
+ pconn = self.msg.getheader('proxy-connection')
+ if pconn and "keep-alive" in pconn.lower():
+ return False
+
+ # otherwise, assume it will close
+ return True
+
+ def close(self):
+ if self.fp:
+ self.fp.close()
+ self.fp = None
+
+ def isclosed(self):
+ # NOTE: it is possible that we will not ever call self.close(). This
+ # case occurs when will_close is TRUE, length is None, and we
+ # read up to the last byte, but NOT past it.
+ #
+ # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
+ # called, meaning self.isclosed() is meaningful.
+ return self.fp is None
+
+ # XXX It would be nice to have readline and __iter__ for this, too.
+
+ def read(self, amt=None):
+ if self.fp is None:
+ return ''
+
+ if self._method == 'HEAD':
+ self.close()
+ return ''
+
+ if self.chunked:
+ return self._read_chunked(amt)
+
+ if amt is None:
+ # unbounded read
+ if self.length is None:
+ s = self.fp.read()
+ else:
+ s = self._safe_read(self.length)
+ self.length = 0
+ self.close() # we read everything
+ return s
+
+ if self.length is not None:
+ if amt > self.length:
+ # clip the read to the "end of response"
+ amt = self.length
+
+ # we do not use _safe_read() here because this may be a .will_close
+ # connection, and the user is reading more bytes than will be provided
+ # (for example, reading in 1k chunks)
+ s = self.fp.read(amt)
+ if self.length is not None:
+ self.length -= len(s)
+ if not self.length:
+ self.close()
+ return s
+
+ def _read_chunked(self, amt):
+ assert self.chunked != _UNKNOWN
+ chunk_left = self.chunk_left
+ value = []
+ while True:
+ if chunk_left is None:
+ line = self.fp.readline()
+ i = line.find(';')
+ if i >= 0:
+ line = line[:i] # strip chunk-extensions
+ try:
+ chunk_left = int(line, 16)
+ except ValueError:
+ # close the connection as protocol synchronisation is
+ # probably lost
+ self.close()
+ raise IncompleteRead(''.join(value))
+ if chunk_left == 0:
+ break
+ if amt is None:
+ value.append(self._safe_read(chunk_left))
+ elif amt < chunk_left:
+ value.append(self._safe_read(amt))
+ self.chunk_left = chunk_left - amt
+ return ''.join(value)
+ elif amt == chunk_left:
+ value.append(self._safe_read(amt))
+ self._safe_read(2) # toss the CRLF at the end of the chunk
+ self.chunk_left = None
+ return ''.join(value)
+ else:
+ value.append(self._safe_read(chunk_left))
+ amt -= chunk_left
+
+ # we read the whole chunk, get another
+ self._safe_read(2) # toss the CRLF at the end of the chunk
+ chunk_left = None
+
+ # read and discard trailer up to the CRLF terminator
+ ### note: we shouldn't have any trailers!
+ while True:
+ line = self.fp.readline()
+ if not line:
+ # a vanishingly small number of sites EOF without
+ # sending the trailer
+ break
+ if line == '\r\n':
+ break
+
+ # we read everything; close the "file"
+ self.close()
+
+ return ''.join(value)
+
+ def _safe_read(self, amt):
+ """Read the number of bytes requested, compensating for partial reads.
+
+ Normally, we have a blocking socket, but a read() can be interrupted
+ by a signal (resulting in a partial read).
+
+ Note that we cannot distinguish between EOF and an interrupt when zero
+ bytes have been read. IncompleteRead() will be raised in this
+ situation.
+
+ This function should be used when <amt> bytes "should" be present for
+ reading. If the bytes are truly not available (due to EOF), then the
+ IncompleteRead exception can be used to detect the problem.
+ """
+ # NOTE(gps): As of svn r74426 socket._fileobject.read(x) will never
+ # return less than x bytes unless EOF is encountered. It now handles
+ # signal interruptions (socket.error EINTR) internally. This code
+ # never caught that exception anyways. It seems largely pointless.
+ # self.fp.read(amt) will work fine.
+ s = []
+ while amt > 0:
+ chunk = self.fp.read(min(amt, MAXAMOUNT))
+ if not chunk:
+ raise IncompleteRead(''.join(s), amt)
+ s.append(chunk)
+ amt -= len(chunk)
+ return ''.join(s)
+
+ def fileno(self):
+ return self.fp.fileno()
+
+ def getheader(self, name, default=None):
+ if self.msg is None:
+ raise ResponseNotReady()
+ return self.msg.getheader(name, default)
+
+ def getheaders(self):
+ """Return list of (header, value) tuples."""
+ if self.msg is None:
+ raise ResponseNotReady()
+ return self.msg.items()
+
+
+class HTTPConnection:
+
+ _http_vsn = 11
+ _http_vsn_str = 'HTTP/1.1'
+
+ response_class = HTTPResponse
+ default_port = HTTP_PORT
+ auto_open = 1
+ debuglevel = 0
+ strict = 0
+
+ def __init__(self, host, port=None, strict=None,
+ timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None):
+ self.timeout = timeout
+ self.source_address = source_address
+ self.sock = None
+ self._buffer = []
+ self.__response = None
+ self.__state = _CS_IDLE
+ self._method = None
+ self._tunnel_host = None
+ self._tunnel_port = None
+ self._tunnel_headers = {}
+
+ self._set_hostport(host, port)
+ if strict is not None:
+ self.strict = strict
+
+ def set_tunnel(self, host, port=None, headers=None):
+ """ Sets up the host and the port for the HTTP CONNECT Tunnelling.
+
+ The headers argument should be a mapping of extra HTTP headers
+ to send with the CONNECT request.
+ """
+ self._tunnel_host = host
+ self._tunnel_port = port
+ if headers:
+ self._tunnel_headers = headers
+ else:
+ self._tunnel_headers.clear()
+
+ def _set_hostport(self, host, port):
+ if port is None:
+ i = host.rfind(':')
+ j = host.rfind(']') # ipv6 addresses have [...]
+ if i > j:
+ try:
+ port = int(host[i+1:])
+ except ValueError:
+ raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
+ host = host[:i]
+ else:
+ port = self.default_port
+ if host and host[0] == '[' and host[-1] == ']':
+ host = host[1:-1]
+ self.host = host
+ self.port = port
+
+ def set_debuglevel(self, level):
+ self.debuglevel = level
+
+ def _tunnel(self):
+ self._set_hostport(self._tunnel_host, self._tunnel_port)
+ self.send("CONNECT %s:%d HTTP/1.0\r\n" % (self.host, self.port))
+ for header, value in self._tunnel_headers.iteritems():
+ self.send("%s: %s\r\n" % (header, value))
+ self.send("\r\n")
+ response = self.response_class(self.sock, strict = self.strict,
+ method = self._method)
+ (version, code, message) = response._read_status()
+
+ if code != 200:
+ self.close()
+ raise socket.error("Tunnel connection failed: %d %s" % (code,
+ message.strip()))
+ while True:
+ line = response.fp.readline()
+ if line == '\r\n': break
+
+
+ def connect(self):
+ """Connect to the host and port specified in __init__."""
+ self.sock = socket.create_connection((self.host,self.port),
+ self.timeout, self.source_address)
+
+ if self._tunnel_host:
+ self._tunnel()
+
+ def close(self):
+ """Close the connection to the HTTP server."""
+ if self.sock:
+ self.sock.close() # close it manually... there may be other refs
+ self.sock = None
+ if self.__response:
+ self.__response.close()
+ self.__response = None
+ self.__state = _CS_IDLE
+
+ def send(self, data):
+ """Send `data' to the server."""
+ if self.sock is None:
+ if self.auto_open:
+ self.connect()
+ else:
+ raise NotConnected()
+
+ if self.debuglevel > 0:
+ print "send:", repr(data)
+ blocksize = 8192
+ if hasattr(data,'read') and not isinstance(data, array):
+ if self.debuglevel > 0: print "sendIng a read()able"
+ datablock = data.read(blocksize)
+ while datablock:
+ self.sock.sendall(datablock)
+ datablock = data.read(blocksize)
+ else:
+ self.sock.sendall(data)
+
+ def _output(self, s):
+ """Add a line of output to the current request buffer.
+
+ Assumes that the line does *not* end with \\r\\n.
+ """
+ self._buffer.append(s)
+
+ def _send_output(self, message_body=None):
+ """Send the currently buffered request and clear the buffer.
+
+ Appends an extra \\r\\n to the buffer.
+ A message_body may be specified, to be appended to the request.
+ """
+ self._buffer.extend(("", ""))
+ msg = "\r\n".join(self._buffer)
+ del self._buffer[:]
+ # If msg and message_body are sent in a single send() call,
+ # it will avoid performance problems caused by the interaction
+ # between delayed ack and the Nagle algorithim.
+ if isinstance(message_body, str):
+ msg += message_body
+ message_body = None
+ self.send(msg)
+ if message_body is not None:
+ #message_body was not a string (i.e. it is a file) and
+ #we must run the risk of Nagle
+ self.send(message_body)
+
+ def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):
+ """Send a request to the server.
+
+ `method' specifies an HTTP request method, e.g. 'GET'.
+ `url' specifies the object being requested, e.g. '/index.html'.
+ `skip_host' if True does not add automatically a 'Host:' header
+ `skip_accept_encoding' if True does not add automatically an
+ 'Accept-Encoding:' header
+ """
+
+ # if a prior response has been completed, then forget about it.
+ if self.__response and self.__response.isclosed():
+ self.__response = None
+
+
+ # in certain cases, we cannot issue another request on this connection.
+ # this occurs when:
+ # 1) we are in the process of sending a request. (_CS_REQ_STARTED)
+ # 2) a response to a previous request has signalled that it is going
+ # to close the connection upon completion.
+ # 3) the headers for the previous response have not been read, thus
+ # we cannot determine whether point (2) is true. (_CS_REQ_SENT)
+ #
+ # if there is no prior response, then we can request at will.
+ #
+ # if point (2) is true, then we will have passed the socket to the
+ # response (effectively meaning, "there is no prior response"), and
+ # will open a new one when a new request is made.
+ #
+ # Note: if a prior response exists, then we *can* start a new request.
+ # We are not allowed to begin fetching the response to this new
+ # request, however, until that prior response is complete.
+ #
+ if self.__state == _CS_IDLE:
+ self.__state = _CS_REQ_STARTED
+ else:
+ raise CannotSendRequest()
+
+ # Save the method we use, we need it later in the response phase
+ self._method = method
+ if not url:
+ url = '/'
+ hdr = '%s %s %s' % (method, url, self._http_vsn_str)
+
+ self._output(hdr)
+
+ if self._http_vsn == 11:
+ # Issue some standard headers for better HTTP/1.1 compliance
+
+ if not skip_host:
+ # this header is issued *only* for HTTP/1.1
+ # connections. more specifically, this means it is
+ # only issued when the client uses the new
+ # HTTPConnection() class. backwards-compat clients
+ # will be using HTTP/1.0 and those clients may be
+ # issuing this header themselves. we should NOT issue
+ # it twice; some web servers (such as Apache) barf
+ # when they see two Host: headers
+
+ # If we need a non-standard port,include it in the
+ # header. If the request is going through a proxy,
+ # but the host of the actual URL, not the host of the
+ # proxy.
+
+ netloc = ''
+ if url.startswith('http'):
+ nil, netloc, nil, nil, nil = urlsplit(url)
+
+ if netloc:
+ try:
+ netloc_enc = netloc.encode("ascii")
+ except UnicodeEncodeError:
+ netloc_enc = netloc.encode("idna")
+ self.putheader('Host', netloc_enc)
+ else:
+ try:
+ host_enc = self.host.encode("ascii")
+ except UnicodeEncodeError:
+ host_enc = self.host.encode("idna")
+ # Wrap the IPv6 Host Header with [] (RFC 2732)
+ if host_enc.find(':') >= 0:
+ host_enc = "[" + host_enc + "]"
+ if self.port == self.default_port:
+ self.putheader('Host', host_enc)
+ else:
+ self.putheader('Host', "%s:%s" % (host_enc, self.port))
+
+ # note: we are assuming that clients will not attempt to set these
+ # headers since *this* library must deal with the
+ # consequences. this also means that when the supporting
+ # libraries are updated to recognize other forms, then this
+ # code should be changed (removed or updated).
+
+ # we only want a Content-Encoding of "identity" since we don't
+ # support encodings such as x-gzip or x-deflate.
+ if not skip_accept_encoding:
+ self.putheader('Accept-Encoding', 'identity')
+
+ # we can accept "chunked" Transfer-Encodings, but no others
+ # NOTE: no TE header implies *only* "chunked"
+ #self.putheader('TE', 'chunked')
+
+ # if TE is supplied in the header, then it must appear in a
+ # Connection header.
+ #self.putheader('Connection', 'TE')
+
+ else:
+ # For HTTP/1.0, the server will assume "not chunked"
+ pass
+
+ def putheader(self, header, *values):
+ """Send a request header line to the server.
+
+ For example: h.putheader('Accept', 'text/html')
+ """
+ if self.__state != _CS_REQ_STARTED:
+ raise CannotSendHeader()
+
+ hdr = '%s: %s' % (header, '\r\n\t'.join([str(v) for v in values]))
+ self._output(hdr)
+
+ def endheaders(self, message_body=None):
+ """Indicate that the last header line has been sent to the server.
+
+ This method sends the request to the server. The optional
+ message_body argument can be used to pass message body
+ associated with the request. The message body will be sent in
+ the same packet as the message headers if possible. The
+ message_body should be a string.
+ """
+ if self.__state == _CS_REQ_STARTED:
+ self.__state = _CS_REQ_SENT
+ else:
+ raise CannotSendHeader()
+ self._send_output(message_body)
+
+ def request(self, method, url, body=None, headers={}):
+ """Send a complete request to the server."""
+ self._send_request(method, url, body, headers)
+
+ def _set_content_length(self, body):
+ # Set the content-length based on the body.
+ thelen = None
+ try:
+ thelen = str(len(body))
+ except TypeError, te:
+ # If this is a file-like object, try to
+ # fstat its file descriptor
+ try:
+ thelen = str(os.fstat(body.fileno()).st_size)
+ except (AttributeError, OSError):
+ # Don't send a length if this failed
+ if self.debuglevel > 0: print "Cannot stat!!"
+
+ if thelen is not None:
+ self.putheader('Content-Length', thelen)
+
+ def _send_request(self, method, url, body, headers):
+ # Honor explicitly requested Host: and Accept-Encoding: headers.
+ header_names = dict.fromkeys([k.lower() for k in headers])
+ skips = {}
+ if 'host' in header_names:
+ skips['skip_host'] = 1
+ if 'accept-encoding' in header_names:
+ skips['skip_accept_encoding'] = 1
+
+ self.putrequest(method, url, **skips)
+
+ if body and ('content-length' not in header_names):
+ self._set_content_length(body)
+ for hdr, value in headers.iteritems():
+ self.putheader(hdr, value)
+ self.endheaders(body)
+
+ def getresponse(self, buffering=False):
+ "Get the response from the server."
+
+ # if a prior response has been completed, then forget about it.
+ if self.__response and self.__response.isclosed():
+ self.__response = None
+
+ #
+ # if a prior response exists, then it must be completed (otherwise, we
+ # cannot read this response's header to determine the connection-close
+ # behavior)
+ #
+ # note: if a prior response existed, but was connection-close, then the
+ # socket and response were made independent of this HTTPConnection
+ # object since a new request requires that we open a whole new
+ # connection
+ #
+ # this means the prior response had one of two states:
+ # 1) will_close: this connection was reset and the prior socket and
+ # response operate independently
+ # 2) persistent: the response was retained and we await its
+ # isclosed() status to become true.
+ #
+ if self.__state != _CS_REQ_SENT or self.__response:
+ raise ResponseNotReady()
+
+ args = (self.sock,)
+ kwds = {"strict":self.strict, "method":self._method}
+ if self.debuglevel > 0:
+ args += (self.debuglevel,)
+ if buffering:
+ #only add this keyword if non-default, for compatibility with
+ #other response_classes.
+ kwds["buffering"] = True;
+ response = self.response_class(*args, **kwds)
+
+ try:
+ response.begin()
+ except:
+ response.close()
+ raise
+ assert response.will_close != _UNKNOWN
+ self.__state = _CS_IDLE
+
+ if response.will_close:
+ # this effectively passes the connection to the response
+ self.close()
+ else:
+ # remember this, so we can tell when it is complete
+ self.__response = response
+
+ return response
+
+
+class HTTP:
+ "Compatibility class with httplib.py from 1.5."
+
+ _http_vsn = 10
+ _http_vsn_str = 'HTTP/1.0'
+
+ debuglevel = 0
+
+ _connection_class = HTTPConnection
+
+ def __init__(self, host='', port=None, strict=None):
+ "Provide a default host, since the superclass requires one."
+
+ # some joker passed 0 explicitly, meaning default port
+ if port == 0:
+ port = None
+
+ # Note that we may pass an empty string as the host; this will throw
+ # an error when we attempt to connect. Presumably, the client code
+ # will call connect before then, with a proper host.
+ self._setup(self._connection_class(host, port, strict))
+
+ def _setup(self, conn):
+ self._conn = conn
+
+ # set up delegation to flesh out interface
+ self.send = conn.send
+ self.putrequest = conn.putrequest
+ self.putheader = conn.putheader
+ self.endheaders = conn.endheaders
+ self.set_debuglevel = conn.set_debuglevel
+
+ conn._http_vsn = self._http_vsn
+ conn._http_vsn_str = self._http_vsn_str
+
+ self.file = None
+
+ def connect(self, host=None, port=None):
+ "Accept arguments to set the host/port, since the superclass doesn't."
+
+ if host is not None:
+ self._conn._set_hostport(host, port)
+ self._conn.connect()
+
+ def getfile(self):
+ "Provide a getfile, since the superclass' does not use this concept."
+ return self.file
+
+ def getreply(self, buffering=False):
+ """Compat definition since superclass does not define it.
+
+ Returns a tuple consisting of:
+ - server status code (e.g. '200' if all goes well)
+ - server "reason" corresponding to status code
+ - any RFC822 headers in the response from the server
+ """
+ try:
+ if not buffering:
+ response = self._conn.getresponse()
+ else:
+ #only add this keyword if non-default for compatibility
+ #with other connection classes
+ response = self._conn.getresponse(buffering)
+ except BadStatusLine, e:
+ ### hmm. if getresponse() ever closes the socket on a bad request,
+ ### then we are going to have problems with self.sock
+
+ ### should we keep this behavior? do people use it?
+ # keep the socket open (as a file), and return it
+ self.file = self._conn.sock.makefile('rb', 0)
+
+ # close our socket -- we want to restart after any protocol error
+ self.close()
+
+ self.headers = None
+ return -1, e.line, None
+
+ self.headers = response.msg
+ self.file = response.fp
+ return response.status, response.reason, response.msg
+
+ def close(self):
+ self._conn.close()
+
+ # note that self.file == response.fp, which gets closed by the
+ # superclass. just clear the object ref here.
+ ### hmm. messy. if status==-1, then self.file is owned by us.
+ ### well... we aren't explicitly closing, but losing this ref will
+ ### do it
+ self.file = None
+
+try:
+ import ssl
+except ImportError:
+ pass
+else:
+ class HTTPSConnection(HTTPConnection):
+ "This class allows communication via SSL."
+
+ default_port = HTTPS_PORT
+
+ def __init__(self, host, port=None, key_file=None, cert_file=None,
+ strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+ source_address=None):
+ HTTPConnection.__init__(self, host, port, strict, timeout,
+ source_address)
+ self.key_file = key_file
+ self.cert_file = cert_file
+
+ def connect(self):
+ "Connect to a host on a given (SSL) port."
+
+ sock = socket.create_connection((self.host, self.port),
+ self.timeout, self.source_address)
+ if self._tunnel_host:
+ self.sock = sock
+ self._tunnel()
+ self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
+
+ __all__.append("HTTPSConnection")
+
+ class HTTPS(HTTP):
+ """Compatibility with 1.5 httplib interface
+
+ Python 1.5.2 did not have an HTTPS class, but it defined an
+ interface for sending http requests that is also useful for
+ https.
+ """
+
+ _connection_class = HTTPSConnection
+
+ def __init__(self, host='', port=None, key_file=None, cert_file=None,
+ strict=None):
+ # provide a default host, pass the X509 cert info
+
+ # urf. compensate for bad input.
+ if port == 0:
+ port = None
+ self._setup(self._connection_class(host, port, key_file,
+ cert_file, strict))
+
+ # we never actually use these for anything, but we keep them
+ # here for compatibility with post-1.5.2 CVS.
+ self.key_file = key_file
+ self.cert_file = cert_file
+
+
+ def FakeSocket (sock, sslobj):
+ warnings.warn("FakeSocket is deprecated, and won't be in 3.x. " +
+ "Use the result of ssl.wrap_socket() directly instead.",
+ DeprecationWarning, stacklevel=2)
+ return sslobj
+
+
+class HTTPException(Exception):
+ # Subclasses that define an __init__ must call Exception.__init__
+ # or define self.args. Otherwise, str() will fail.
+ pass
+
+class NotConnected(HTTPException):
+ pass
+
+class InvalidURL(HTTPException):
+ pass
+
+class UnknownProtocol(HTTPException):
+ def __init__(self, version):
+ self.args = version,
+ self.version = version
+
+class UnknownTransferEncoding(HTTPException):
+ pass
+
+class UnimplementedFileMode(HTTPException):
+ pass
+
+class IncompleteRead(HTTPException):
+ def __init__(self, partial, expected=None):
+ self.args = partial,
+ self.partial = partial
+ self.expected = expected
+ def __repr__(self):
+ if self.expected is not None:
+ e = ', %i more expected' % self.expected
+ else:
+ e = ''
+ return 'IncompleteRead(%i bytes read%s)' % (len(self.partial), e)
+ def __str__(self):
+ return repr(self)
+
+class ImproperConnectionState(HTTPException):
+ pass
+
+class CannotSendRequest(ImproperConnectionState):
+ pass
+
+class CannotSendHeader(ImproperConnectionState):
+ pass
+
+class ResponseNotReady(ImproperConnectionState):
+ pass
+
+class BadStatusLine(HTTPException):
+ def __init__(self, line):
+ if not line:
+ line = repr(line)
+ self.args = line,
+ self.line = line
+
+# for backwards compatibility
+error = HTTPException
+
+class LineAndFileWrapper:
+ """A limited file-like object for HTTP/0.9 responses."""
+
+ # The status-line parsing code calls readline(), which normally
+ # get the HTTP status line. For a 0.9 response, however, this is
+ # actually the first line of the body! Clients need to get a
+ # readable file object that contains that line.
+
+ def __init__(self, line, file):
+ self._line = line
+ self._file = file
+ self._line_consumed = 0
+ self._line_offset = 0
+ self._line_left = len(line)
+
+ def __getattr__(self, attr):
+ return getattr(self._file, attr)
+
+ def _done(self):
+ # called when the last byte is read from the line. After the
+ # call, all read methods are delegated to the underlying file
+ # object.
+ self._line_consumed = 1
+ self.read = self._file.read
+ self.readline = self._file.readline
+ self.readlines = self._file.readlines
+
+ def read(self, amt=None):
+ if self._line_consumed:
+ return self._file.read(amt)
+ assert self._line_left
+ if amt is None or amt > self._line_left:
+ s = self._line[self._line_offset:]
+ self._done()
+ if amt is None:
+ return s + self._file.read()
+ else:
+ return s + self._file.read(amt - len(s))
+ else:
+ assert amt <= self._line_left
+ i = self._line_offset
+ j = i + amt
+ s = self._line[i:j]
+ self._line_offset = j
+ self._line_left -= amt
+ if self._line_left == 0:
+ self._done()
+ return s
+
+ def readline(self):
+ if self._line_consumed:
+ return self._file.readline()
+ assert self._line_left
+ s = self._line[self._line_offset:]
+ self._done()
+ return s
+
+ def readlines(self, size=None):
+ if self._line_consumed:
+ return self._file.readlines(size)
+ assert self._line_left
+ L = [self._line[self._line_offset:]]
+ self._done()
+ if size is None:
+ return L + self._file.readlines()
+ else:
+ return L + self._file.readlines(size)
+
+def test():
+ """Test this module.
+
+ A hodge podge of tests collected here, because they have too many
+ external dependencies for the regular test suite.
+ """
+
+ import sys
+ import getopt
+ opts, args = getopt.getopt(sys.argv[1:], 'd')
+ dl = 0
+ for o, a in opts:
+ if o == '-d': dl = dl + 1
+ host = 'www.python.org'
+ selector = '/'
+ if args[0:]: host = args[0]
+ if args[1:]: selector = args[1]
+ h = HTTP()
+ h.set_debuglevel(dl)
+ h.connect(host)
+ h.putrequest('GET', selector)
+ h.endheaders()
+ status, reason, headers = h.getreply()
+ print 'status =', status
+ print 'reason =', reason
+ print "read", len(h.getfile().read())
+ print
+ if headers:
+ for header in headers.headers: print header.strip()
+ print
+
+ # minimal test that code to extract host from url works
+ class HTTP11(HTTP):
+ _http_vsn = 11
+ _http_vsn_str = 'HTTP/1.1'
+
+ h = HTTP11('www.python.org')
+ h.putrequest('GET', 'http://www.python.org/~jeremy/')
+ h.endheaders()
+ h.getreply()
+ h.close()
+
+ try:
+ import ssl
+ except ImportError:
+ pass
+ else:
+
+ for host, selector in (('sourceforge.net', '/projects/python'),
+ ):
+ print "https://%s%s" % (host, selector)
+ hs = HTTPS()
+ hs.set_debuglevel(dl)
+ hs.connect(host)
+ hs.putrequest('GET', selector)
+ hs.endheaders()
+ status, reason, headers = hs.getreply()
+ print 'status =', status
+ print 'reason =', reason
+ print "read", len(hs.getfile().read())
+ print
+ if headers:
+ for header in headers.headers: print header.strip()
+ print
+
+if __name__ == '__main__':
+ test()
diff --git a/lib-python/modified-2.7/tarfile.py b/lib-python/modified-2.7/tarfile.py
--- a/lib-python/modified-2.7/tarfile.py
+++ b/lib-python/modified-2.7/tarfile.py
@@ -252,8 +252,8 @@
the high bit set. So we calculate two checksums, unsigned and
signed.
"""
- unsigned_chksum = 256 + sum(struct.unpack("148B8x356B", buf[:512]))
- signed_chksum = 256 + sum(struct.unpack("148b8x356b", buf[:512]))
+ unsigned_chksum = 256 + sum(struct.unpack("148B", buf[:148]) + struct.unpack("356B", buf[156:512]))
+ signed_chksum = 256 + sum(struct.unpack("148b", buf[:148]) + struct.unpack("356b", buf[156:512]))
return unsigned_chksum, signed_chksum
def copyfileobj(src, dst, length=None):
@@ -265,6 +265,7 @@
if length is None:
shutil.copyfileobj(src, dst)
return
+
BUFSIZE = 16 * 1024
blocks, remainder = divmod(length, BUFSIZE)
for b in xrange(blocks):
@@ -801,19 +802,19 @@
if self.closed:
raise ValueError("I/O operation on closed file")
+ buf = ""
if self.buffer:
if size is None:
- buf = self.buffer + self.fileobj.read()
+ buf = self.buffer
self.buffer = ""
else:
buf = self.buffer[:size]
self.buffer = self.buffer[size:]
- buf += self.fileobj.read(size - len(buf))
+
+ if size is None:
+ buf += self.fileobj.read()
else:
- if size is None:
- buf = self.fileobj.read()
- else:
- buf = self.fileobj.read(size)
+ buf += self.fileobj.read(size - len(buf))
self.position += len(buf)
return buf
diff --git a/lib-python/modified-2.7/test/test_multiprocessing.py b/lib-python/modified-2.7/test/test_multiprocessing.py
--- a/lib-python/modified-2.7/test/test_multiprocessing.py
+++ b/lib-python/modified-2.7/test/test_multiprocessing.py
@@ -510,7 +510,6 @@
p.join()
- @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
def test_qsize(self):
q = self.Queue()
try:
@@ -532,7 +531,6 @@
time.sleep(DELTA)
q.task_done()
- @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
def test_task_done(self):
queue = self.JoinableQueue()
@@ -1091,7 +1089,6 @@
class _TestPoolWorkerLifetime(BaseTestCase):
ALLOWED_TYPES = ('processes', )
- @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
def test_pool_worker_lifetime(self):
p = multiprocessing.Pool(3, maxtasksperchild=10)
self.assertEqual(3, len(p._pool))
@@ -1280,7 +1277,6 @@
queue = manager.get_queue()
queue.put('hello world')
- @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
def test_rapid_restart(self):
authkey = os.urandom(32)
manager = QueueManager(
@@ -1297,6 +1293,7 @@
queue = manager.get_queue()
self.assertEqual(queue.get(), 'hello world')
del queue
+ test_support.gc_collect()
manager.shutdown()
manager = QueueManager(
address=addr, authkey=authkey, serializer=SERIALIZER)
@@ -1573,7 +1570,6 @@
ALLOWED_TYPES = ('processes',)
- @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
def test_heap(self):
iterations = 5000
maxblocks = 50
diff --git a/lib-python/modified-2.7/test/test_sys_settrace.py b/lib-python/modified-2.7/test/test_sys_settrace.py
--- a/lib-python/modified-2.7/test/test_sys_settrace.py
+++ b/lib-python/modified-2.7/test/test_sys_settrace.py
@@ -286,11 +286,11 @@
self.compare_events(func.func_code.co_firstlineno,
tracer.events, func.events)
- def set_and_retrieve_none(self):
+ def test_set_and_retrieve_none(self):
sys.settrace(None)
assert sys.gettrace() is None
- def set_and_retrieve_func(self):
+ def test_set_and_retrieve_func(self):
def fn(*args):
pass
diff --git a/lib-python/modified-2.7/test/test_urllib2.py b/lib-python/modified-2.7/test/test_urllib2.py
--- a/lib-python/modified-2.7/test/test_urllib2.py
+++ b/lib-python/modified-2.7/test/test_urllib2.py
@@ -307,6 +307,9 @@
def getresponse(self):
return MockHTTPResponse(MockFile(), {}, 200, "OK")
+ def close(self):
+ pass
+
class MockHandler:
# useful for testing handler machinery
# see add_ordered_mock_handlers() docstring
diff --git a/lib-python/modified-2.7/urllib2.py b/lib-python/modified-2.7/urllib2.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/urllib2.py
@@ -0,0 +1,1440 @@
+"""An extensible library for opening URLs using a variety of protocols
+
+The simplest way to use this module is to call the urlopen function,
+which accepts a string containing a URL or a Request object (described
+below). It opens the URL and returns the results as file-like
+object; the returned object has some extra methods described below.
+
+The OpenerDirector manages a collection of Handler objects that do
+all the actual work. Each Handler implements a particular protocol or
+option. The OpenerDirector is a composite object that invokes the
+Handlers needed to open the requested URL. For example, the
+HTTPHandler performs HTTP GET and POST requests and deals with
+non-error returns. The HTTPRedirectHandler automatically deals with
+HTTP 301, 302, 303 and 307 redirect errors, and the HTTPDigestAuthHandler
+deals with digest authentication.
+
+urlopen(url, data=None) -- Basic usage is the same as original
+urllib. pass the url and optionally data to post to an HTTP URL, and
+get a file-like object back. One difference is that you can also pass
+a Request instance instead of URL. Raises a URLError (subclass of
+IOError); for HTTP errors, raises an HTTPError, which can also be
+treated as a valid response.
+
+build_opener -- Function that creates a new OpenerDirector instance.
+Will install the default handlers. Accepts one or more Handlers as
+arguments, either instances or Handler classes that it will
+instantiate. If one of the argument is a subclass of the default
+handler, the argument will be installed instead of the default.
+
+install_opener -- Installs a new opener as the default opener.
+
+objects of interest:
+
+OpenerDirector -- Sets up the User Agent as the Python-urllib client and manages
+the Handler classes, while dealing with requests and responses.
+
+Request -- An object that encapsulates the state of a request. The
+state can be as simple as the URL. It can also include extra HTTP
+headers, e.g. a User-Agent.
+
+BaseHandler --
+
+exceptions:
+URLError -- A subclass of IOError, individual protocols have their own
+specific subclass.
+
+HTTPError -- Also a valid HTTP response, so you can treat an HTTP error
+as an exceptional event or valid response.
+
+internals:
+BaseHandler and parent
+_call_chain conventions
+
+Example usage:
+
+import urllib2
+
+# set up authentication info
+authinfo = urllib2.HTTPBasicAuthHandler()
+authinfo.add_password(realm='PDQ Application',
+ uri='https://mahler:8092/site-updates.py',
+ user='klem',
+ passwd='geheim$parole')
+
+proxy_support = urllib2.ProxyHandler({"http" : "http://ahad-haam:3128"})
+
+# build a new opener that adds authentication and caching FTP handlers
+opener = urllib2.build_opener(proxy_support, authinfo, urllib2.CacheFTPHandler)
+
+# install it
+urllib2.install_opener(opener)
+
+f = urllib2.urlopen('http://www.python.org/')
+
+
+"""
+
+# XXX issues:
+# If an authentication error handler that tries to perform
+# authentication for some reason but fails, how should the error be
+# signalled? The client needs to know the HTTP error code. But if
+# the handler knows that the problem was, e.g., that it didn't know
+# that hash algo that requested in the challenge, it would be good to
+# pass that information along to the client, too.
+# ftp errors aren't handled cleanly
+# check digest against correct (i.e. non-apache) implementation
+
+# Possible extensions:
+# complex proxies XXX not sure what exactly was meant by this
+# abstract factory for opener
+
+import base64
+import hashlib
+import httplib
+import mimetools
+import os
+import posixpath
+import random
+import re
+import socket
+import sys
+import time
+import urlparse
+import bisect
+
+try:
+ from cStringIO import StringIO
+except ImportError:
+ from StringIO import StringIO
+
+from urllib import (unwrap, unquote, splittype, splithost, quote,
+ addinfourl, splitport, splittag,
+ splitattr, ftpwrapper, splituser, splitpasswd, splitvalue)
+
+# support for FileHandler, proxies via environment variables
+from urllib import localhost, url2pathname, getproxies, proxy_bypass
+
+# used in User-Agent header sent
+__version__ = sys.version[:3]
+
+_opener = None
+def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+ global _opener
+ if _opener is None:
+ _opener = build_opener()
+ return _opener.open(url, data, timeout)
+
+def install_opener(opener):
+ global _opener
+ _opener = opener
+
+# do these error classes make sense?
+# make sure all of the IOError stuff is overridden. we just want to be
+# subtypes.
+
+class URLError(IOError):
+ # URLError is a sub-type of IOError, but it doesn't share any of
+ # the implementation. need to override __init__ and __str__.
+ # It sets self.args for compatibility with other EnvironmentError
+ # subclasses, but args doesn't have the typical format with errno in
+ # slot 0 and strerror in slot 1. This may be better than nothing.
+ def __init__(self, reason):
+ self.args = reason,
+ self.reason = reason
+
+ def __str__(self):
+ return '<urlopen error %s>' % self.reason
+
+class HTTPError(URLError, addinfourl):
+ """Raised when HTTP error occurs, but also acts like non-error return"""
+ __super_init = addinfourl.__init__
+
+ def __init__(self, url, code, msg, hdrs, fp):
+ self.code = code
+ self.msg = msg
+ self.hdrs = hdrs
+ self.fp = fp
+ self.filename = url
+ # The addinfourl classes depend on fp being a valid file
+ # object. In some cases, the HTTPError may not have a valid
+ # file object. If this happens, the simplest workaround is to
+ # not initialize the base classes.
+ if fp is not None:
+ self.__super_init(fp, hdrs, url, code)
+
+ def __str__(self):
+ return 'HTTP Error %s: %s' % (self.code, self.msg)
+
+# copied from cookielib.py
+_cut_port_re = re.compile(r":\d+$")
+def request_host(request):
+ """Return request-host, as defined by RFC 2965.
+
+ Variation from RFC: returned value is lowercased, for convenient
+ comparison.
+
+ """
+ url = request.get_full_url()
+ host = urlparse.urlparse(url)[1]
+ if host == "":
+ host = request.get_header("Host", "")
+
+ # remove port, if present
+ host = _cut_port_re.sub("", host, 1)
+ return host.lower()
+
+class Request:
+
+ def __init__(self, url, data=None, headers={},
+ origin_req_host=None, unverifiable=False):
+ # unwrap('<URL:type://host/path>') --> 'type://host/path'
+ self.__original = unwrap(url)
+ self.__original, fragment = splittag(self.__original)
+ self.type = None
+ # self.__r_type is what's left after doing the splittype
+ self.host = None
+ self.port = None
+ self._tunnel_host = None
+ self.data = data
+ self.headers = {}
+ for key, value in headers.items():
+ self.add_header(key, value)
+ self.unredirected_hdrs = {}
+ if origin_req_host is None:
+ origin_req_host = request_host(self)
+ self.origin_req_host = origin_req_host
+ self.unverifiable = unverifiable
+
+ def __getattr__(self, attr):
+ # XXX this is a fallback mechanism to guard against these
+ # methods getting called in a non-standard order. this may be
+ # too complicated and/or unnecessary.
+ # XXX should the __r_XXX attributes be public?
+ if attr[:12] == '_Request__r_':
+ name = attr[12:]
+ if hasattr(Request, 'get_' + name):
+ getattr(self, 'get_' + name)()
+ return getattr(self, attr)
+ raise AttributeError, attr
+
+ def get_method(self):
+ if self.has_data():
+ return "POST"
+ else:
+ return "GET"
+
+ # XXX these helper methods are lame
+
+ def add_data(self, data):
+ self.data = data
+
+ def has_data(self):
+ return self.data is not None
+
+ def get_data(self):
+ return self.data
+
+ def get_full_url(self):
+ return self.__original
+
+ def get_type(self):
+ if self.type is None:
+ self.type, self.__r_type = splittype(self.__original)
+ if self.type is None:
+ raise ValueError, "unknown url type: %s" % self.__original
+ return self.type
+
+ def get_host(self):
+ if self.host is None:
+ self.host, self.__r_host = splithost(self.__r_type)
+ if self.host:
+ self.host = unquote(self.host)
+ return self.host
+
+ def get_selector(self):
+ return self.__r_host
+
+ def set_proxy(self, host, type):
+ if self.type == 'https' and not self._tunnel_host:
+ self._tunnel_host = self.host
+ else:
+ self.type = type
+ self.__r_host = self.__original
+
+ self.host = host
+
+ def has_proxy(self):
+ return self.__r_host == self.__original
+
+ def get_origin_req_host(self):
+ return self.origin_req_host
+
+ def is_unverifiable(self):
+ return self.unverifiable
+
+ def add_header(self, key, val):
+ # useful for something like authentication
+ self.headers[key.capitalize()] = val
+
+ def add_unredirected_header(self, key, val):
+ # will not be added to a redirected request
+ self.unredirected_hdrs[key.capitalize()] = val
+
+ def has_header(self, header_name):
+ return (header_name in self.headers or
+ header_name in self.unredirected_hdrs)
+
+ def get_header(self, header_name, default=None):
+ return self.headers.get(
+ header_name,
+ self.unredirected_hdrs.get(header_name, default))
+
+ def header_items(self):
+ hdrs = self.unredirected_hdrs.copy()
+ hdrs.update(self.headers)
+ return hdrs.items()
+
+class OpenerDirector:
+ def __init__(self):
+ client_version = "Python-urllib/%s" % __version__
+ self.addheaders = [('User-agent', client_version)]
+ # manage the individual handlers
+ self.handlers = []
+ self.handle_open = {}
+ self.handle_error = {}
+ self.process_response = {}
+ self.process_request = {}
+
+ def add_handler(self, handler):
+ if not hasattr(handler, "add_parent"):
+ raise TypeError("expected BaseHandler instance, got %r" %
+ type(handler))
+
+ added = False
+ for meth in dir(handler):
+ if meth in ["redirect_request", "do_open", "proxy_open"]:
+ # oops, coincidental match
+ continue
+
+ i = meth.find("_")
+ protocol = meth[:i]
+ condition = meth[i+1:]
+
+ if condition.startswith("error"):
+ j = condition.find("_") + i + 1
+ kind = meth[j+1:]
+ try:
+ kind = int(kind)
+ except ValueError:
+ pass
+ lookup = self.handle_error.get(protocol, {})
+ self.handle_error[protocol] = lookup
+ elif condition == "open":
+ kind = protocol
+ lookup = self.handle_open
+ elif condition == "response":
+ kind = protocol
+ lookup = self.process_response
+ elif condition == "request":
+ kind = protocol
+ lookup = self.process_request
+ else:
+ continue
+
+ handlers = lookup.setdefault(kind, [])
+ if handlers:
+ bisect.insort(handlers, handler)
+ else:
+ handlers.append(handler)
+ added = True
+
+ if added:
+ # the handlers must work in an specific order, the order
+ # is specified in a Handler attribute
+ bisect.insort(self.handlers, handler)
+ handler.add_parent(self)
+
+ def close(self):
+ # Only exists for backwards compatibility.
+ pass
+
+ def _call_chain(self, chain, kind, meth_name, *args):
+ # Handlers raise an exception if no one else should try to handle
+ # the request, or return None if they can't but another handler
+ # could. Otherwise, they return the response.
+ handlers = chain.get(kind, ())
+ for handler in handlers:
+ func = getattr(handler, meth_name)
+
+ result = func(*args)
+ if result is not None:
+ return result
+
+ def open(self, fullurl, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+ # accept a URL or a Request object
+ if isinstance(fullurl, basestring):
+ req = Request(fullurl, data)
+ else:
+ req = fullurl
+ if data is not None:
+ req.add_data(data)
+
+ req.timeout = timeout
+ protocol = req.get_type()
+
+ # pre-process request
+ meth_name = protocol+"_request"
+ for processor in self.process_request.get(protocol, []):
+ meth = getattr(processor, meth_name)
+ req = meth(req)
+
+ response = self._open(req, data)
+
+ # post-process response
+ meth_name = protocol+"_response"
+ for processor in self.process_response.get(protocol, []):
+ meth = getattr(processor, meth_name)
+ try:
+ response = meth(req, response)
+ except:
+ response.close()
+ raise
+
+ return response
+
+ def _open(self, req, data=None):
+ result = self._call_chain(self.handle_open, 'default',
+ 'default_open', req)
+ if result:
+ return result
+
+ protocol = req.get_type()
+ result = self._call_chain(self.handle_open, protocol, protocol +
+ '_open', req)
+ if result:
+ return result
+
+ return self._call_chain(self.handle_open, 'unknown',
+ 'unknown_open', req)
+
+ def error(self, proto, *args):
+ if proto in ('http', 'https'):
+ # XXX http[s] protocols are special-cased
+ dict = self.handle_error['http'] # https is not different than http
+ proto = args[2] # YUCK!
+ meth_name = 'http_error_%s' % proto
+ http_err = 1
+ orig_args = args
+ else:
+ dict = self.handle_error
+ meth_name = proto + '_error'
+ http_err = 0
+ args = (dict, proto, meth_name) + args
+ result = self._call_chain(*args)
+ if result:
+ return result
+
+ if http_err:
+ args = (dict, 'default', 'http_error_default') + orig_args
+ return self._call_chain(*args)
+
+# XXX probably also want an abstract factory that knows when it makes
+# sense to skip a superclass in favor of a subclass and when it might
+# make sense to include both
+
+def build_opener(*handlers):
+ """Create an opener object from a list of handlers.
+
+ The opener will use several default handlers, including support
+ for HTTP, FTP and when applicable, HTTPS.
+
+ If any of the handlers passed as arguments are subclasses of the
+ default handlers, the default handlers will not be used.
+ """
+ import types
+ def isclass(obj):
+ return isinstance(obj, (types.ClassType, type))
+
+ opener = OpenerDirector()
+ default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
+ HTTPDefaultErrorHandler, HTTPRedirectHandler,
+ FTPHandler, FileHandler, HTTPErrorProcessor]
+ if hasattr(httplib, 'HTTPS'):
+ default_classes.append(HTTPSHandler)
+ skip = set()
+ for klass in default_classes:
+ for check in handlers:
+ if isclass(check):
+ if issubclass(check, klass):
+ skip.add(klass)
+ elif isinstance(check, klass):
+ skip.add(klass)
+ for klass in skip:
+ default_classes.remove(klass)
+
+ for klass in default_classes:
+ opener.add_handler(klass())
+
+ for h in handlers:
+ if isclass(h):
+ h = h()
+ opener.add_handler(h)
+ return opener
+
+class BaseHandler:
+ handler_order = 500
+
+ def add_parent(self, parent):
+ self.parent = parent
+
+ def close(self):
+ # Only exists for backwards compatibility
+ pass
+
+ def __lt__(self, other):
+ if not hasattr(other, "handler_order"):
+ # Try to preserve the old behavior of having custom classes
+ # inserted after default ones (works only for custom user
+ # classes which are not aware of handler_order).
+ return True
+ return self.handler_order < other.handler_order
+
+
+class HTTPErrorProcessor(BaseHandler):
+ """Process HTTP error responses."""
+ handler_order = 1000 # after all other processing
+
+ def http_response(self, request, response):
+ code, msg, hdrs = response.code, response.msg, response.info()
+
+ # According to RFC 2616, "2xx" code indicates that the client's
+ # request was successfully received, understood, and accepted.
+ if not (200 <= code < 300):
+ response = self.parent.error(
+ 'http', request, response, code, msg, hdrs)
+
+ return response
+
+ https_response = http_response
+
+class HTTPDefaultErrorHandler(BaseHandler):
+ def http_error_default(self, req, fp, code, msg, hdrs):
+ raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
+
+class HTTPRedirectHandler(BaseHandler):
+ # maximum number of redirections to any single URL
+ # this is needed because of the state that cookies introduce
+ max_repeats = 4
+ # maximum total number of redirections (regardless of URL) before
+ # assuming we're in a loop
+ max_redirections = 10
+
+ def redirect_request(self, req, fp, code, msg, headers, newurl):
+ """Return a Request or None in response to a redirect.
+
+ This is called by the http_error_30x methods when a
+ redirection response is received. If a redirection should
+ take place, return a new Request to allow http_error_30x to
+ perform the redirect. Otherwise, raise HTTPError if no-one
+ else should try to handle this url. Return None if you can't
+ but another Handler might.
+ """
+ m = req.get_method()
+ if (code in (301, 302, 303, 307) and m in ("GET", "HEAD")
+ or code in (301, 302, 303) and m == "POST"):
+ # Strictly (according to RFC 2616), 301 or 302 in response
+ # to a POST MUST NOT cause a redirection without confirmation
+ # from the user (of urllib2, in this case). In practice,
+ # essentially all clients do redirect in this case, so we
+ # do the same.
+ # be conciliant with URIs containing a space
+ newurl = newurl.replace(' ', '%20')
+ newheaders = dict((k,v) for k,v in req.headers.items()
+ if k.lower() not in ("content-length", "content-type")
+ )
+ return Request(newurl,
+ headers=newheaders,
+ origin_req_host=req.get_origin_req_host(),
+ unverifiable=True)
+ else:
+ raise HTTPError(req.get_full_url(), code, msg, headers, fp)
+
+ # Implementation note: To avoid the server sending us into an
+ # infinite loop, the request object needs to track what URLs we
+ # have already seen. Do this by adding a handler-specific
+ # attribute to the Request object.
+ def http_error_302(self, req, fp, code, msg, headers):
+ # Some servers (incorrectly) return multiple Location headers
+ # (so probably same goes for URI). Use first header.
+ if 'location' in headers:
+ newurl = headers.getheaders('location')[0]
+ elif 'uri' in headers:
+ newurl = headers.getheaders('uri')[0]
+ else:
+ return
+
+ # fix a possible malformed URL
+ urlparts = urlparse.urlparse(newurl)
+ if not urlparts.path:
+ urlparts = list(urlparts)
+ urlparts[2] = "/"
+ newurl = urlparse.urlunparse(urlparts)
+
+ newurl = urlparse.urljoin(req.get_full_url(), newurl)
+
+ # XXX Probably want to forget about the state of the current
+ # request, although that might interact poorly with other
+ # handlers that also use handler-specific request attributes
+ new = self.redirect_request(req, fp, code, msg, headers, newurl)
+ if new is None:
+ return
+
+ # loop detection
+ # .redirect_dict has a key url if url was previously visited.
+ if hasattr(req, 'redirect_dict'):
+ visited = new.redirect_dict = req.redirect_dict
+ if (visited.get(newurl, 0) >= self.max_repeats or
+ len(visited) >= self.max_redirections):
+ raise HTTPError(req.get_full_url(), code,
+ self.inf_msg + msg, headers, fp)
+ else:
+ visited = new.redirect_dict = req.redirect_dict = {}
+ visited[newurl] = visited.get(newurl, 0) + 1
+
+ # Don't close the fp until we are sure that we won't use it
+ # with HTTPError.
+ fp.read()
+ fp.close()
+
+ return self.parent.open(new, timeout=req.timeout)
+
+ http_error_301 = http_error_303 = http_error_307 = http_error_302
+
+ inf_msg = "The HTTP server returned a redirect error that would " \
+ "lead to an infinite loop.\n" \
+ "The last 30x error message was:\n"
+
+
+def _parse_proxy(proxy):
+ """Return (scheme, user, password, host/port) given a URL or an authority.
+
+ If a URL is supplied, it must have an authority (host:port) component.
+ According to RFC 3986, having an authority component means the URL must
+ have two slashes after the scheme:
+
+ >>> _parse_proxy('file:/ftp.example.com/')
+ Traceback (most recent call last):
+ ValueError: proxy URL with no authority: 'file:/ftp.example.com/'
+
+ The first three items of the returned tuple may be None.
+
+ Examples of authority parsing:
+
+ >>> _parse_proxy('proxy.example.com')
+ (None, None, None, 'proxy.example.com')
+ >>> _parse_proxy('proxy.example.com:3128')
+ (None, None, None, 'proxy.example.com:3128')
+
+ The authority component may optionally include userinfo (assumed to be
+ username:password):
+
+ >>> _parse_proxy('joe:password at proxy.example.com')
+ (None, 'joe', 'password', 'proxy.example.com')
+ >>> _parse_proxy('joe:password at proxy.example.com:3128')
+ (None, 'joe', 'password', 'proxy.example.com:3128')
+
+ Same examples, but with URLs instead:
+
+ >>> _parse_proxy('http://proxy.example.com/')
+ ('http', None, None, 'proxy.example.com')
+ >>> _parse_proxy('http://proxy.example.com:3128/')
+ ('http', None, None, 'proxy.example.com:3128')
+ >>> _parse_proxy('http://joe:password@proxy.example.com/')
+ ('http', 'joe', 'password', 'proxy.example.com')
+ >>> _parse_proxy('http://joe:password@proxy.example.com:3128')
+ ('http', 'joe', 'password', 'proxy.example.com:3128')
+
+ Everything after the authority is ignored:
+
+ >>> _parse_proxy('ftp://joe:password@proxy.example.com/rubbish:3128')
+ ('ftp', 'joe', 'password', 'proxy.example.com')
+
+ Test for no trailing '/' case:
+
+ >>> _parse_proxy('http://joe:password@proxy.example.com')
+ ('http', 'joe', 'password', 'proxy.example.com')
+
+ """
+ scheme, r_scheme = splittype(proxy)
+ if not r_scheme.startswith("/"):
+ # authority
+ scheme = None
+ authority = proxy
+ else:
+ # URL
+ if not r_scheme.startswith("//"):
+ raise ValueError("proxy URL with no authority: %r" % proxy)
+ # We have an authority, so for RFC 3986-compliant URLs (by ss 3.
+ # and 3.3.), path is empty or starts with '/'
+ end = r_scheme.find("/", 2)
+ if end == -1:
+ end = None
+ authority = r_scheme[2:end]
+ userinfo, hostport = splituser(authority)
+ if userinfo is not None:
+ user, password = splitpasswd(userinfo)
+ else:
+ user = password = None
+ return scheme, user, password, hostport
+
+class ProxyHandler(BaseHandler):
+ # Proxies must be in front
+ handler_order = 100
+
+ def __init__(self, proxies=None):
+ if proxies is None:
+ proxies = getproxies()
+ assert hasattr(proxies, 'has_key'), "proxies must be a mapping"
+ self.proxies = proxies
+ for type, url in proxies.items():
+ setattr(self, '%s_open' % type,
+ lambda r, proxy=url, type=type, meth=self.proxy_open: \
+ meth(r, proxy, type))
+
+ def proxy_open(self, req, proxy, type):
+ orig_type = req.get_type()
+ proxy_type, user, password, hostport = _parse_proxy(proxy)
+
+ if proxy_type is None:
+ proxy_type = orig_type
+
+ if req.host and proxy_bypass(req.host):
+ return None
+
+ if user and password:
+ user_pass = '%s:%s' % (unquote(user), unquote(password))
+ creds = base64.b64encode(user_pass).strip()
+ req.add_header('Proxy-authorization', 'Basic ' + creds)
+ hostport = unquote(hostport)
+ req.set_proxy(hostport, proxy_type)
+
+ if orig_type == proxy_type or orig_type == 'https':
+ # let other handlers take care of it
+ return None
+ else:
+ # need to start over, because the other handlers don't
+ # grok the proxy's URL type
+ # e.g. if we have a constructor arg proxies like so:
+ # {'http': 'ftp://proxy.example.com'}, we may end up turning
+ # a request for http://acme.example.com/a into one for
+ # ftp://proxy.example.com/a
+ return self.parent.open(req, timeout=req.timeout)
+
+class HTTPPasswordMgr:
+
+ def __init__(self):
+ self.passwd = {}
+
+ def add_password(self, realm, uri, user, passwd):
+ # uri could be a single URI or a sequence
+ if isinstance(uri, basestring):
+ uri = [uri]
+ if not realm in self.passwd:
+ self.passwd[realm] = {}
+ for default_port in True, False:
+ reduced_uri = tuple(
+ [self.reduce_uri(u, default_port) for u in uri])
+ self.passwd[realm][reduced_uri] = (user, passwd)
+
+ def find_user_password(self, realm, authuri):
+ domains = self.passwd.get(realm, {})
+ for default_port in True, False:
+ reduced_authuri = self.reduce_uri(authuri, default_port)
+ for uris, authinfo in domains.iteritems():
+ for uri in uris:
+ if self.is_suburi(uri, reduced_authuri):
+ return authinfo
+ return None, None
+
+ def reduce_uri(self, uri, default_port=True):
+ """Accept authority or URI and extract only the authority and path."""
+ # note HTTP URLs do not have a userinfo component
+ parts = urlparse.urlsplit(uri)
+ if parts[1]:
+ # URI
+ scheme = parts[0]
+ authority = parts[1]
+ path = parts[2] or '/'
+ else:
+ # host or host:port
+ scheme = None
+ authority = uri
+ path = '/'
+ host, port = splitport(authority)
+ if default_port and port is None and scheme is not None:
+ dport = {"http": 80,
+ "https": 443,
+ }.get(scheme)
+ if dport is not None:
+ authority = "%s:%d" % (host, dport)
+ return authority, path
+
+ def is_suburi(self, base, test):
+ """Check if test is below base in a URI tree
+
+ Both args must be URIs in reduced form.
+ """
+ if base == test:
+ return True
+ if base[0] != test[0]:
+ return False
+ common = posixpath.commonprefix((base[1], test[1]))
+ if len(common) == len(base[1]):
+ return True
+ return False
+
+
+class HTTPPasswordMgrWithDefaultRealm(HTTPPasswordMgr):
+
+ def find_user_password(self, realm, authuri):
+ user, password = HTTPPasswordMgr.find_user_password(self, realm,
+ authuri)
+ if user is not None:
+ return user, password
+ return HTTPPasswordMgr.find_user_password(self, None, authuri)
+
+
+class AbstractBasicAuthHandler:
+
+ # XXX this allows for multiple auth-schemes, but will stupidly pick
+ # the last one with a realm specified.
+
+ # allow for double- and single-quoted realm values
+ # (single quotes are a violation of the RFC, but appear in the wild)
+ rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+'
+ 'realm=(["\'])(.*?)\\2', re.I)
+
+ # XXX could pre-emptively send auth info already accepted (RFC 2617,
+ # end of section 2, and section 1.2 immediately after "credentials"
+ # production).
+
+ def __init__(self, password_mgr=None):
+ if password_mgr is None:
+ password_mgr = HTTPPasswordMgr()
+ self.passwd = password_mgr
+ self.add_password = self.passwd.add_password
+ self.retried = 0
+
+ def reset_retry_count(self):
+ self.retried = 0
+
+ def http_error_auth_reqed(self, authreq, host, req, headers):
+ # host may be an authority (without userinfo) or a URL with an
+ # authority
+ # XXX could be multiple headers
+ authreq = headers.get(authreq, None)
+
+ if self.retried > 5:
+ # retry sending the username:password 5 times before failing.
+ raise HTTPError(req.get_full_url(), 401, "basic auth failed",
+ headers, None)
+ else:
+ self.retried += 1
+
+ if authreq:
+ mo = AbstractBasicAuthHandler.rx.search(authreq)
+ if mo:
+ scheme, quote, realm = mo.groups()
+ if scheme.lower() == 'basic':
+ response = self.retry_http_basic_auth(host, req, realm)
+ if response and response.code != 401:
+ self.retried = 0
+ return response
+
+ def retry_http_basic_auth(self, host, req, realm):
+ user, pw = self.passwd.find_user_password(realm, host)
+ if pw is not None:
+ raw = "%s:%s" % (user, pw)
+ auth = 'Basic %s' % base64.b64encode(raw).strip()
+ if req.headers.get(self.auth_header, None) == auth:
+ return None
+ req.add_unredirected_header(self.auth_header, auth)
+ return self.parent.open(req, timeout=req.timeout)
+ else:
+ return None
+
+
+class HTTPBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+ auth_header = 'Authorization'
+
+ def http_error_401(self, req, fp, code, msg, headers):
+ url = req.get_full_url()
+ response = self.http_error_auth_reqed('www-authenticate',
+ url, req, headers)
+ self.reset_retry_count()
+ return response
+
+
+class ProxyBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+ auth_header = 'Proxy-authorization'
+
+ def http_error_407(self, req, fp, code, msg, headers):
+ # http_error_auth_reqed requires that there is no userinfo component in
+ # authority. Assume there isn't one, since urllib2 does not (and
+ # should not, RFC 3986 s. 3.2.1) support requests for URLs containing
+ # userinfo.
+ authority = req.get_host()
+ response = self.http_error_auth_reqed('proxy-authenticate',
+ authority, req, headers)
+ self.reset_retry_count()
+ return response
+
+
+def randombytes(n):
+ """Return n random bytes."""
+ # Use /dev/urandom if it is available. Fall back to random module
+ # if not. It might be worthwhile to extend this function to use
+ # other platform-specific mechanisms for getting random bytes.
+ if os.path.exists("/dev/urandom"):
+ f = open("/dev/urandom")
+ s = f.read(n)
+ f.close()
+ return s
+ else:
+ L = [chr(random.randrange(0, 256)) for i in range(n)]
+ return "".join(L)
+
+class AbstractDigestAuthHandler:
+ # Digest authentication is specified in RFC 2617.
+
+ # XXX The client does not inspect the Authentication-Info header
+ # in a successful response.
+
+ # XXX It should be possible to test this implementation against
+ # a mock server that just generates a static set of challenges.
+
+ # XXX qop="auth-int" supports is shaky
+
+ def __init__(self, passwd=None):
+ if passwd is None:
+ passwd = HTTPPasswordMgr()
+ self.passwd = passwd
+ self.add_password = self.passwd.add_password
+ self.retried = 0
+ self.nonce_count = 0
+ self.last_nonce = None
+
+ def reset_retry_count(self):
+ self.retried = 0
+
+ def http_error_auth_reqed(self, auth_header, host, req, headers):
+ authreq = headers.get(auth_header, None)
+ if self.retried > 5:
+ # Don't fail endlessly - if we failed once, we'll probably
+ # fail a second time. Hm. Unless the Password Manager is
+ # prompting for the information. Crap. This isn't great
+ # but it's better than the current 'repeat until recursion
+ # depth exceeded' approach <wink>
+ raise HTTPError(req.get_full_url(), 401, "digest auth failed",
+ headers, None)
+ else:
+ self.retried += 1
+ if authreq:
+ scheme = authreq.split()[0]
+ if scheme.lower() == 'digest':
+ return self.retry_http_digest_auth(req, authreq)
+
+ def retry_http_digest_auth(self, req, auth):
+ token, challenge = auth.split(' ', 1)
+ chal = parse_keqv_list(parse_http_list(challenge))
+ auth = self.get_authorization(req, chal)
+ if auth:
+ auth_val = 'Digest %s' % auth
+ if req.headers.get(self.auth_header, None) == auth_val:
+ return None
+ req.add_unredirected_header(self.auth_header, auth_val)
+ resp = self.parent.open(req, timeout=req.timeout)
+ return resp
+
+ def get_cnonce(self, nonce):
+ # The cnonce-value is an opaque
+ # quoted string value provided by the client and used by both client
+ # and server to avoid chosen plaintext attacks, to provide mutual
+ # authentication, and to provide some message integrity protection.
+ # This isn't a fabulous effort, but it's probably Good Enough.
+ dig = hashlib.sha1("%s:%s:%s:%s" % (self.nonce_count, nonce, time.ctime(),
+ randombytes(8))).hexdigest()
+ return dig[:16]
+
+ def get_authorization(self, req, chal):
+ try:
+ realm = chal['realm']
+ nonce = chal['nonce']
+ qop = chal.get('qop')
+ algorithm = chal.get('algorithm', 'MD5')
+ # mod_digest doesn't send an opaque, even though it isn't
+ # supposed to be optional
+ opaque = chal.get('opaque', None)
+ except KeyError:
+ return None
+
+ H, KD = self.get_algorithm_impls(algorithm)
+ if H is None:
+ return None
+
+ user, pw = self.passwd.find_user_password(realm, req.get_full_url())
+ if user is None:
+ return None
+
+ # XXX not implemented yet
+ if req.has_data():
+ entdig = self.get_entity_digest(req.get_data(), chal)
+ else:
+ entdig = None
+
+ A1 = "%s:%s:%s" % (user, realm, pw)
+ A2 = "%s:%s" % (req.get_method(),
+ # XXX selector: what about proxies and full urls
+ req.get_selector())
+ if qop == 'auth':
+ if nonce == self.last_nonce:
+ self.nonce_count += 1
+ else:
+ self.nonce_count = 1
+ self.last_nonce = nonce
+
+ ncvalue = '%08x' % self.nonce_count
+ cnonce = self.get_cnonce(nonce)
+ noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2))
+ respdig = KD(H(A1), noncebit)
+ elif qop is None:
+ respdig = KD(H(A1), "%s:%s" % (nonce, H(A2)))
+ else:
+ # XXX handle auth-int.
+ raise URLError("qop '%s' is not supported." % qop)
+
+ # XXX should the partial digests be encoded too?
+
+ base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
+ 'response="%s"' % (user, realm, nonce, req.get_selector(),
+ respdig)
+ if opaque:
+ base += ', opaque="%s"' % opaque
+ if entdig:
+ base += ', digest="%s"' % entdig
+ base += ', algorithm="%s"' % algorithm
+ if qop:
+ base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)
+ return base
+
+ def get_algorithm_impls(self, algorithm):
+ # algorithm should be case-insensitive according to RFC2617
+ algorithm = algorithm.upper()
+ # lambdas assume digest modules are imported at the top level
+ if algorithm == 'MD5':
+ H = lambda x: hashlib.md5(x).hexdigest()
+ elif algorithm == 'SHA':
+ H = lambda x: hashlib.sha1(x).hexdigest()
+ # XXX MD5-sess
+ KD = lambda s, d: H("%s:%s" % (s, d))
+ return H, KD
+
+ def get_entity_digest(self, data, chal):
+ # XXX not implemented yet
+ return None
+
+
+class HTTPDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+ """An authentication protocol defined by RFC 2069
+
+ Digest authentication improves on basic authentication because it
+ does not transmit passwords in the clear.
+ """
+
+ auth_header = 'Authorization'
+ handler_order = 490 # before Basic auth
+
+ def http_error_401(self, req, fp, code, msg, headers):
+ host = urlparse.urlparse(req.get_full_url())[1]
+ retry = self.http_error_auth_reqed('www-authenticate',
+ host, req, headers)
+ self.reset_retry_count()
+ return retry
+
+
+class ProxyDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+
+ auth_header = 'Proxy-Authorization'
+ handler_order = 490 # before Basic auth
+
+ def http_error_407(self, req, fp, code, msg, headers):
+ host = req.get_host()
+ retry = self.http_error_auth_reqed('proxy-authenticate',
+ host, req, headers)
+ self.reset_retry_count()
+ return retry
+
+class AbstractHTTPHandler(BaseHandler):
+
+ def __init__(self, debuglevel=0):
+ self._debuglevel = debuglevel
+
+ def set_http_debuglevel(self, level):
+ self._debuglevel = level
+
+ def do_request_(self, request):
+ host = request.get_host()
+ if not host:
+ raise URLError('no host given')
+
+ if request.has_data(): # POST
+ data = request.get_data()
+ if not request.has_header('Content-type'):
+ request.add_unredirected_header(
+ 'Content-type',
+ 'application/x-www-form-urlencoded')
+ if not request.has_header('Content-length'):
+ request.add_unredirected_header(
+ 'Content-length', '%d' % len(data))
+
+ sel_host = host
+ if request.has_proxy():
+ scheme, sel = splittype(request.get_selector())
+ sel_host, sel_path = splithost(sel)
+
+ if not request.has_header('Host'):
+ request.add_unredirected_header('Host', sel_host)
+ for name, value in self.parent.addheaders:
+ name = name.capitalize()
+ if not request.has_header(name):
+ request.add_unredirected_header(name, value)
+
+ return request
+
+ def do_open(self, http_class, req):
+ """Return an addinfourl object for the request, using http_class.
+
+ http_class must implement the HTTPConnection API from httplib.
+ The addinfourl return value is a file-like object. It also
+ has methods and attributes including:
+ - info(): return a mimetools.Message object for the headers
+ - geturl(): return the original request URL
+ - code: HTTP status code
+ """
+ host = req.get_host()
+ if not host:
+ raise URLError('no host given')
+
+ h = http_class(host, timeout=req.timeout) # will parse host:port
+ h.set_debuglevel(self._debuglevel)
+
+ headers = dict(req.unredirected_hdrs)
+ headers.update(dict((k, v) for k, v in req.headers.items()
+ if k not in headers))
+
+ # We want to make an HTTP/1.1 request, but the addinfourl
+ # class isn't prepared to deal with a persistent connection.
+ # It will try to read all remaining data from the socket,
+ # which will block while the server waits for the next request.
+ # So make sure the connection gets closed after the (only)
+ # request.
+ headers["Connection"] = "close"
+ headers = dict(
+ (name.title(), val) for name, val in headers.items())
+
+ if req._tunnel_host:
+ tunnel_headers = {}
+ proxy_auth_hdr = "Proxy-Authorization"
+ if proxy_auth_hdr in headers:
+ tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr]
+ # Proxy-Authorization should not be sent to origin
+ # server.
+ del headers[proxy_auth_hdr]
+ h.set_tunnel(req._tunnel_host, headers=tunnel_headers)
+
+ try:
+ h.request(req.get_method(), req.get_selector(), req.data, headers)
+ try:
+ r = h.getresponse(buffering=True)
+ except TypeError: #buffering kw not supported
+ r = h.getresponse()
+ except socket.error, err: # XXX what error?
+ h.close()
+ raise URLError(err)
+
+ # Pick apart the HTTPResponse object to get the addinfourl
+ # object initialized properly.
+
+ # Wrap the HTTPResponse object in socket's file object adapter
+ # for Windows. That adapter calls recv(), so delegate recv()
+ # to read(). This weird wrapping allows the returned object to
+ # have readline() and readlines() methods.
+
+ # XXX It might be better to extract the read buffering code
+ # out of socket._fileobject() and into a base class.
+
+ r.recv = r.read
+ fp = socket._fileobject(r, close=True)
+
+ resp = addinfourl(fp, r.msg, req.get_full_url())
+ resp.code = r.status
+ resp.msg = r.reason
+ return resp
+
+
+class HTTPHandler(AbstractHTTPHandler):
+
+ def http_open(self, req):
+ return self.do_open(httplib.HTTPConnection, req)
+
+ http_request = AbstractHTTPHandler.do_request_
+
+if hasattr(httplib, 'HTTPS'):
+ class HTTPSHandler(AbstractHTTPHandler):
+
+ def https_open(self, req):
+ return self.do_open(httplib.HTTPSConnection, req)
+
+ https_request = AbstractHTTPHandler.do_request_
+
+class HTTPCookieProcessor(BaseHandler):
+ def __init__(self, cookiejar=None):
+ import cookielib
+ if cookiejar is None:
+ cookiejar = cookielib.CookieJar()
+ self.cookiejar = cookiejar
+
+ def http_request(self, request):
+ self.cookiejar.add_cookie_header(request)
+ return request
+
+ def http_response(self, request, response):
+ self.cookiejar.extract_cookies(response, request)
+ return response
+
+ https_request = http_request
+ https_response = http_response
+
+class UnknownHandler(BaseHandler):
+ def unknown_open(self, req):
+ type = req.get_type()
+ raise URLError('unknown url type: %s' % type)
+
+def parse_keqv_list(l):
+ """Parse list of key=value strings where keys are not duplicated."""
+ parsed = {}
+ for elt in l:
+ k, v = elt.split('=', 1)
+ if v[0] == '"' and v[-1] == '"':
+ v = v[1:-1]
+ parsed[k] = v
+ return parsed
+
+def parse_http_list(s):
+ """Parse lists as described by RFC 2068 Section 2.
+
+ In particular, parse comma-separated lists where the elements of
+ the list may include quoted-strings. A quoted-string could
+ contain a comma. A non-quoted string could have quotes in the
+ middle. Neither commas nor quotes count if they are escaped.
+ Only double-quotes count, not single-quotes.
+ """
+ res = []
+ part = ''
+
+ escape = quote = False
+ for cur in s:
+ if escape:
+ part += cur
+ escape = False
+ continue
+ if quote:
+ if cur == '\\':
+ escape = True
+ continue
+ elif cur == '"':
+ quote = False
+ part += cur
+ continue
+
+ if cur == ',':
+ res.append(part)
+ part = ''
+ continue
+
+ if cur == '"':
+ quote = True
+
+ part += cur
+
+ # append last part
+ if part:
+ res.append(part)
+
+ return [part.strip() for part in res]
+
+def _safe_gethostbyname(host):
+ try:
+ return socket.gethostbyname(host)
+ except socket.gaierror:
+ return None
+
+class FileHandler(BaseHandler):
+ # Use local file or FTP depending on form of URL
+ def file_open(self, req):
+ url = req.get_selector()
+ if url[:2] == '//' and url[2:3] != '/' and (req.host and
+ req.host != 'localhost'):
+ req.type = 'ftp'
+ return self.parent.open(req)
+ else:
+ return self.open_local_file(req)
+
+ # names for the localhost
+ names = None
+ def get_names(self):
+ if FileHandler.names is None:
+ try:
+ FileHandler.names = tuple(
+ socket.gethostbyname_ex('localhost')[2] +
+ socket.gethostbyname_ex(socket.gethostname())[2])
+ except socket.gaierror:
+ FileHandler.names = (socket.gethostbyname('localhost'),)
+ return FileHandler.names
+
+ # not entirely sure what the rules are here
+ def open_local_file(self, req):
+ import email.utils
+ import mimetypes
+ host = req.get_host()
+ filename = req.get_selector()
+ localfile = url2pathname(filename)
+ try:
+ stats = os.stat(localfile)
+ size = stats.st_size
+ modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
+ mtype = mimetypes.guess_type(filename)[0]
+ headers = mimetools.Message(StringIO(
+ 'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
+ (mtype or 'text/plain', size, modified)))
+ if host:
+ host, port = splitport(host)
+ if not host or \
+ (not port and _safe_gethostbyname(host) in self.get_names()):
+ if host:
+ origurl = 'file://' + host + filename
+ else:
+ origurl = 'file://' + filename
+ return addinfourl(open(localfile, 'rb'), headers, origurl)
+ except OSError, msg:
+ # urllib2 users shouldn't expect OSErrors coming from urlopen()
+ raise URLError(msg)
+ raise URLError('file not on local host')
+
+class FTPHandler(BaseHandler):
+ def ftp_open(self, req):
+ import ftplib
+ import mimetypes
+ host = req.get_host()
+ if not host:
+ raise URLError('ftp error: no host given')
+ host, port = splitport(host)
+ if port is None:
+ port = ftplib.FTP_PORT
+ else:
+ port = int(port)
+
+ # username/password handling
+ user, host = splituser(host)
+ if user:
+ user, passwd = splitpasswd(user)
+ else:
+ passwd = None
+ host = unquote(host)
+ user = user or ''
+ passwd = passwd or ''
+
+ try:
+ host = socket.gethostbyname(host)
+ except socket.error, msg:
+ raise URLError(msg)
+ path, attrs = splitattr(req.get_selector())
+ dirs = path.split('/')
+ dirs = map(unquote, dirs)
+ dirs, file = dirs[:-1], dirs[-1]
+ if dirs and not dirs[0]:
+ dirs = dirs[1:]
+ try:
+ fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
+ type = file and 'I' or 'D'
+ for attr in attrs:
+ attr, value = splitvalue(attr)
+ if attr.lower() == 'type' and \
+ value in ('a', 'A', 'i', 'I', 'd', 'D'):
+ type = value.upper()
+ fp, retrlen = fw.retrfile(file, type)
+ headers = ""
+ mtype = mimetypes.guess_type(req.get_full_url())[0]
+ if mtype:
+ headers += "Content-type: %s\n" % mtype
+ if retrlen is not None and retrlen >= 0:
+ headers += "Content-length: %d\n" % retrlen
+ sf = StringIO(headers)
+ headers = mimetools.Message(sf)
+ return addinfourl(fp, headers, req.get_full_url())
+ except ftplib.all_errors, msg:
+ raise URLError, ('ftp error: %s' % msg), sys.exc_info()[2]
+
+ def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+ fw = ftpwrapper(user, passwd, host, port, dirs, timeout)
+## fw.ftp.set_debuglevel(1)
+ return fw
+
+class CacheFTPHandler(FTPHandler):
+ # XXX would be nice to have pluggable cache strategies
+ # XXX this stuff is definitely not thread safe
+ def __init__(self):
+ self.cache = {}
+ self.timeout = {}
+ self.soonest = 0
+ self.delay = 60
+ self.max_conns = 16
+
+ def setTimeout(self, t):
+ self.delay = t
+
+ def setMaxConns(self, m):
+ self.max_conns = m
+
+ def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+ key = user, host, port, '/'.join(dirs), timeout
+ if key in self.cache:
+ self.timeout[key] = time.time() + self.delay
+ else:
+ self.cache[key] = ftpwrapper(user, passwd, host, port, dirs, timeout)
+ self.timeout[key] = time.time() + self.delay
+ self.check_cache()
+ return self.cache[key]
+
+ def check_cache(self):
+ # first check for old ones
+ t = time.time()
+ if self.soonest <= t:
+ for k, v in self.timeout.items():
+ if v < t:
+ self.cache[k].close()
+ del self.cache[k]
+ del self.timeout[k]
+ self.soonest = min(self.timeout.values())
+
+ # then check the size
+ if len(self.cache) == self.max_conns:
+ for k, v in self.timeout.items():
+ if v == self.soonest:
+ del self.cache[k]
+ del self.timeout[k]
+ break
+ self.soonest = min(self.timeout.values())
diff --git a/lib_pypy/_functools.py b/lib_pypy/_functools.py
--- a/lib_pypy/_functools.py
+++ b/lib_pypy/_functools.py
@@ -14,10 +14,9 @@
raise TypeError("the first argument must be callable")
self.func = func
self.args = args
- self.keywords = keywords
+ self.keywords = keywords or None
def __call__(self, *fargs, **fkeywords):
- newkeywords = self.keywords.copy()
- newkeywords.update(fkeywords)
- return self.func(*(self.args + fargs), **newkeywords)
-
+ if self.keywords is not None:
+ fkeywords = dict(self.keywords, **fkeywords)
+ return self.func(*(self.args + fargs), **fkeywords)
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -96,7 +96,16 @@
@property
def gr_frame(self):
- raise NotImplementedError("attribute 'gr_frame' of greenlet objects")
+ # xxx this doesn't work when called on either the current or
+ # the main greenlet of another thread
+ if self is getcurrent():
+ return None
+ if self.__main:
+ self = getcurrent()
+ f = _continulet.__reduce__(self)[2][0]
+ if not f:
+ return None
+ return f.f_back.f_back.f_back # go past start(), __switch(), switch()
# ____________________________________________________________
# Internal stuff
diff --git a/lib_pypy/pypy_test/test_stackless_pickling.py b/lib_pypy/pypy_test/test_stackless_pickling.py
--- a/lib_pypy/pypy_test/test_stackless_pickling.py
+++ b/lib_pypy/pypy_test/test_stackless_pickling.py
@@ -1,7 +1,3 @@
-"""
-this test should probably not run from CPython or py.py.
-I'm not entirely sure, how to do that.
-"""
from __future__ import absolute_import
from py.test import skip
try:
@@ -16,11 +12,15 @@
class Test_StacklessPickling:
+ def test_pickle_main_coroutine(self):
+ import stackless, pickle
+ s = pickle.dumps(stackless.coroutine.getcurrent())
+ print s
+ c = pickle.loads(s)
+ assert c is stackless.coroutine.getcurrent()
+
def test_basic_tasklet_pickling(self):
- try:
- import stackless
- except ImportError:
- skip("can't load stackless and don't know why!!!")
+ import stackless
from stackless import run, schedule, tasklet
import pickle
diff --git a/lib_pypy/pyrepl/completing_reader.py b/lib_pypy/pyrepl/completing_reader.py
--- a/lib_pypy/pyrepl/completing_reader.py
+++ b/lib_pypy/pyrepl/completing_reader.py
@@ -229,7 +229,8 @@
def after_command(self, cmd):
super(CompletingReader, self).after_command(cmd)
- if not isinstance(cmd, complete) and not isinstance(cmd, self_insert):
+ if not isinstance(cmd, self.commands['complete']) \
+ and not isinstance(cmd, self.commands['self_insert']):
self.cmpltn_reset()
def calc_screen(self):
diff --git a/lib_pypy/stackless.py b/lib_pypy/stackless.py
--- a/lib_pypy/stackless.py
+++ b/lib_pypy/stackless.py
@@ -5,7 +5,6 @@
"""
-import traceback
import _continuation
class TaskletExit(Exception):
@@ -14,33 +13,46 @@
CoroutineExit = TaskletExit
+def _coroutine_getcurrent():
+ "Returns the current coroutine (i.e. the one which called this function)."
+ try:
+ return _tls.current_coroutine
+ except AttributeError:
+ # first call in this thread: current == main
+ return _coroutine_getmain()
+
+def _coroutine_getmain():
+ try:
+ return _tls.main_coroutine
+ except AttributeError:
+ # create the main coroutine for this thread
+ continulet = _continuation.continulet
+ main = coroutine()
+ main._frame = continulet.__new__(continulet)
+ main._is_started = -1
+ _tls.current_coroutine = _tls.main_coroutine = main
+ return _tls.main_coroutine
+
+
class coroutine(object):
- "we can't have continulet as a base, because continulets can't be rebound"
+ _is_started = 0 # 0=no, 1=yes, -1=main
def __init__(self):
self._frame = None
- self.is_zombie = False
-
- def __getattr__(self, attr):
- return getattr(self._frame, attr)
-
- def __del__(self):
- self.is_zombie = True
- del self._frame
- self._frame = None
def bind(self, func, *argl, **argd):
"""coro.bind(f, *argl, **argd) -> None.
binds function f to coro. f will be called with
arguments *argl, **argd
"""
- if self._frame is None or not self._frame.is_pending():
- def run(c):
- _tls.current_coroutine = self
- return func(*argl, **argd)
- self._frame = frame = _continuation.continulet(run)
- else:
+ if self.is_alive:
raise ValueError("cannot bind a bound coroutine")
+ def run(c):
+ _tls.current_coroutine = self
+ self._is_started = 1
+ return func(*argl, **argd)
+ self._is_started = 0
+ self._frame = _continuation.continulet(run)
def switch(self):
"""coro.switch() -> returnvalue
@@ -48,7 +60,7 @@
f finishes, the returnvalue is that of f, otherwise
None is returned
"""
- current = _getcurrent()
+ current = _coroutine_getcurrent()
try:
current._frame.switch(to=self._frame)
finally:
@@ -56,37 +68,30 @@
def kill(self):
"""coro.kill() : kill coroutine coro"""
- current = _getcurrent()
+ current = _coroutine_getcurrent()
try:
current._frame.throw(CoroutineExit, to=self._frame)
finally:
_tls.current_coroutine = current
- def _is_alive(self):
- if self._frame is None:
- return False
- return not self._frame.is_pending()
- is_alive = property(_is_alive)
- del _is_alive
+ @property
+ def is_alive(self):
+ return self._is_started < 0 or (
+ self._frame is not None and self._frame.is_pending())
- def getcurrent():
- """coroutine.getcurrent() -> the currently running coroutine"""
- return _getcurrent()
- getcurrent = staticmethod(getcurrent)
+ @property
+ def is_zombie(self):
+ return self._is_started > 0 and not self._frame.is_pending()
+
+ getcurrent = staticmethod(_coroutine_getcurrent)
def __reduce__(self):
- raise TypeError, 'pickling is not possible based upon continulets'
+ if self._is_started < 0:
+ return _coroutine_getmain, ()
+ else:
+ return type(self), (), self.__dict__
-def _getcurrent():
- "Returns the current coroutine (i.e. the one which called this function)."
- try:
- return _tls.current_coroutine
- except AttributeError:
- # first call in this thread: current == main
- _coroutine_create_main()
- return _tls.current_coroutine
-
try:
from thread import _local
except ImportError:
@@ -95,14 +100,8 @@
_tls = _local()
-def _coroutine_create_main():
- # create the main coroutine for this thread
- _tls.current_coroutine = None
- main_coroutine = coroutine()
- typ = _continuation.continulet
- main_coroutine._frame = typ.__new__(typ)
- _tls.main_coroutine = main_coroutine
- _tls.current_coroutine = main_coroutine
+
+# ____________________________________________________________
from collections import deque
@@ -148,10 +147,7 @@
_last_task = next
assert not next.blocked
if next is not current:
- #try:
- next.switch()
- #except CoroutineExit: --- they are the same anyway
- # raise TaskletExit
+ next.switch()
return current
def set_schedule_callback(callback):
@@ -175,34 +171,6 @@
raise self.type, self.value, self.traceback
#
-# helpers for pickling
-#
-
-_stackless_primitive_registry = {}
-
-def register_stackless_primitive(thang, retval_expr='None'):
- import types
- func = thang
- if isinstance(thang, types.MethodType):
- func = thang.im_func
- code = func.func_code
- _stackless_primitive_registry[code] = retval_expr
- # It is not too nice to attach info via the code object, but
- # I can't think of a better solution without a real transform.
-
-def rewrite_stackless_primitive(coro_state, alive, tempval):
- flags, frame, thunk, parent = coro_state
- while frame is not None:
- retval_expr = _stackless_primitive_registry.get(frame.f_code)
- if retval_expr:
- # this tasklet needs to stop pickling here and return its value.
- tempval = eval(retval_expr, globals(), frame.f_locals)
- coro_state = flags, frame, thunk, parent
- break
- frame = frame.f_back
- return coro_state, alive, tempval
-
-#
#
class channel(object):
@@ -354,8 +322,6 @@
"""
return self._channel_action(None, -1)
- register_stackless_primitive(receive, retval_expr='receiver.tempval')
-
def send_exception(self, exp_type, msg):
self.send(bomb(exp_type, exp_type(msg)))
@@ -372,9 +338,8 @@
the runnables list.
"""
return self._channel_action(msg, 1)
-
- register_stackless_primitive(send)
-
+
+
class tasklet(coroutine):
"""
A tasklet object represents a tiny task in a Python thread.
@@ -456,7 +421,7 @@
self.func = None
coroutine.bind(self, _func)
- back = _getcurrent()
+ back = _coroutine_getcurrent()
coroutine.switch(self)
self.alive = True
_scheduler_append(self)
@@ -480,39 +445,6 @@
raise RuntimeError, "The current tasklet cannot be removed."
# not sure if I will revive this " Use t=tasklet().capture()"
_scheduler_remove(self)
-
- def __reduce__(self):
- one, two, coro_state = coroutine.__reduce__(self)
- assert one is coroutine
- assert two == ()
- # we want to get rid of the parent thing.
- # for now, we just drop it
- a, frame, c, d = coro_state
-
- # Removing all frames related to stackless.py.
- # They point to stuff we don't want to be pickled.
-
- pickleframe = frame
- while frame is not None:
- if frame.f_code == schedule.func_code:
- # Removing everything including and after the
- # call to stackless.schedule()
- pickleframe = frame.f_back
- break
- frame = frame.f_back
- if d:
- assert isinstance(d, coroutine)
- coro_state = a, pickleframe, c, None
- coro_state, alive, tempval = rewrite_stackless_primitive(coro_state, self.alive, self.tempval)
- inst_dict = self.__dict__.copy()
- inst_dict.pop('tempval', None)
- return self.__class__, (), (coro_state, alive, tempval, inst_dict)
-
- def __setstate__(self, (coro_state, alive, tempval, inst_dict)):
- coroutine.__setstate__(self, coro_state)
- self.__dict__.update(inst_dict)
- self.alive = alive
- self.tempval = tempval
def getmain():
"""
@@ -601,30 +533,7 @@
global _last_task
_global_task_id = 0
_main_tasklet = coroutine.getcurrent()
- try:
- _main_tasklet.__class__ = tasklet
- except TypeError: # we are running pypy-c
- class TaskletProxy(object):
- """TaskletProxy is needed to give the _main_coroutine tasklet behaviour"""
- def __init__(self, coro):
- self._coro = coro
-
- def __getattr__(self,attr):
- return getattr(self._coro,attr)
-
- def __str__(self):
- return '<tasklet %s a:%s>' % (self._task_id, self.is_alive)
-
- def __reduce__(self):
- return getmain, ()
-
- __repr__ = __str__
-
-
- global _main_coroutine
- _main_coroutine = _main_tasklet
- _main_tasklet = TaskletProxy(_main_tasklet)
- assert _main_tasklet.is_alive and not _main_tasklet.is_zombie
+ _main_tasklet.__class__ = tasklet # XXX HAAAAAAAAAAAAAAAAAAAAACK
_last_task = _main_tasklet
tasklet._init.im_func(_main_tasklet, label='main')
_squeue = deque()
diff --git a/py/_code/source.py b/py/_code/source.py
--- a/py/_code/source.py
+++ b/py/_code/source.py
@@ -139,7 +139,7 @@
trysource = self[start:end]
if trysource.isparseable():
return start, end
- return start, end
+ return start, len(self)
def getblockend(self, lineno):
# XXX
diff --git a/pypy/annotation/annrpython.py b/pypy/annotation/annrpython.py
--- a/pypy/annotation/annrpython.py
+++ b/pypy/annotation/annrpython.py
@@ -149,7 +149,7 @@
desc = olddesc.bind_self(classdef)
args = self.bookkeeper.build_args("simple_call", args_s[:])
desc.consider_call_site(self.bookkeeper, desc.getcallfamily(), [desc],
- args, annmodel.s_ImpossibleValue)
+ args, annmodel.s_ImpossibleValue, None)
result = []
def schedule(graph, inputcells):
result.append((graph, inputcells))
diff --git a/pypy/annotation/bookkeeper.py b/pypy/annotation/bookkeeper.py
--- a/pypy/annotation/bookkeeper.py
+++ b/pypy/annotation/bookkeeper.py
@@ -209,8 +209,8 @@
self.consider_call_site(call_op)
for pbc, args_s in self.emulated_pbc_calls.itervalues():
- self.consider_call_site_for_pbc(pbc, 'simple_call',
- args_s, s_ImpossibleValue)
+ self.consider_call_site_for_pbc(pbc, 'simple_call',
+ args_s, s_ImpossibleValue, None)
self.emulated_pbc_calls = {}
finally:
self.leave()
@@ -257,18 +257,18 @@
args_s = [lltype_to_annotation(adtmeth.ll_ptrtype)] + args_s
if isinstance(s_callable, SomePBC):
s_result = binding(call_op.result, s_ImpossibleValue)
- self.consider_call_site_for_pbc(s_callable,
- call_op.opname,
- args_s, s_result)
+ self.consider_call_site_for_pbc(s_callable, call_op.opname, args_s,
+ s_result, call_op)
- def consider_call_site_for_pbc(self, s_callable, opname, args_s, s_result):
+ def consider_call_site_for_pbc(self, s_callable, opname, args_s, s_result,
+ call_op):
descs = list(s_callable.descriptions)
if not descs:
return
family = descs[0].getcallfamily()
args = self.build_args(opname, args_s)
s_callable.getKind().consider_call_site(self, family, descs, args,
- s_result)
+ s_result, call_op)
def getuniqueclassdef(self, cls):
"""Get the ClassDef associated with the given user cls.
@@ -656,6 +656,7 @@
whence = None
else:
whence = emulated # callback case
+ op = None
s_previous_result = s_ImpossibleValue
def schedule(graph, inputcells):
@@ -663,7 +664,7 @@
results = []
for desc in descs:
- results.append(desc.pycall(schedule, args, s_previous_result))
+ results.append(desc.pycall(schedule, args, s_previous_result, op))
s_result = unionof(*results)
return s_result
diff --git a/pypy/annotation/description.py b/pypy/annotation/description.py
--- a/pypy/annotation/description.py
+++ b/pypy/annotation/description.py
@@ -255,7 +255,11 @@
raise TypeError, "signature mismatch: %s" % e.getmsg(self.name)
return inputcells
- def specialize(self, inputcells):
+ def specialize(self, inputcells, op=None):
+ if (op is None and
+ getattr(self.bookkeeper, "position_key", None) is not None):
+ _, block, i = self.bookkeeper.position_key
+ op = block.operations[i]
if self.specializer is None:
# get the specializer based on the tag of the 'pyobj'
# (if any), according to the current policy
@@ -269,11 +273,14 @@
enforceargs = Sig(*enforceargs)
self.pyobj._annenforceargs_ = enforceargs
enforceargs(self, inputcells) # can modify inputcells in-place
- return self.specializer(self, inputcells)
+ if getattr(self.pyobj, '_annspecialcase_', '').endswith("call_location"):
+ return self.specializer(self, inputcells, op)
+ else:
+ return self.specializer(self, inputcells)
- def pycall(self, schedule, args, s_previous_result):
+ def pycall(self, schedule, args, s_previous_result, op=None):
inputcells = self.parse_arguments(args)
- result = self.specialize(inputcells)
+ result = self.specialize(inputcells, op)
if isinstance(result, FunctionGraph):
graph = result # common case
# if that graph has a different signature, we need to re-parse
@@ -296,17 +303,17 @@
None, # selfclassdef
name)
- def consider_call_site(bookkeeper, family, descs, args, s_result):
+ def consider_call_site(bookkeeper, family, descs, args, s_result, op):
shape = rawshape(args)
- row = FunctionDesc.row_to_consider(descs, args)
+ row = FunctionDesc.row_to_consider(descs, args, op)
family.calltable_add_row(shape, row)
consider_call_site = staticmethod(consider_call_site)
- def variant_for_call_site(bookkeeper, family, descs, args):
+ def variant_for_call_site(bookkeeper, family, descs, args, op):
shape = rawshape(args)
bookkeeper.enter(None)
try:
- row = FunctionDesc.row_to_consider(descs, args)
+ row = FunctionDesc.row_to_consider(descs, args, op)
finally:
bookkeeper.leave()
index = family.calltable_lookup_row(shape, row)
@@ -316,7 +323,7 @@
def rowkey(self):
return self
- def row_to_consider(descs, args):
+ def row_to_consider(descs, args, op):
# see comments in CallFamily
from pypy.annotation.model import s_ImpossibleValue
row = {}
@@ -324,7 +331,7 @@
def enlist(graph, ignore):
row[desc.rowkey()] = graph
return s_ImpossibleValue # meaningless
- desc.pycall(enlist, args, s_ImpossibleValue)
+ desc.pycall(enlist, args, s_ImpossibleValue, op)
return row
row_to_consider = staticmethod(row_to_consider)
@@ -521,7 +528,7 @@
"specialization" % (self.name,))
return self.getclassdef(None)
- def pycall(self, schedule, args, s_previous_result):
+ def pycall(self, schedule, args, s_previous_result, op=None):
from pypy.annotation.model import SomeInstance, SomeImpossibleValue
if self.specialize:
if self.specialize == 'specialize:ctr_location':
@@ -664,7 +671,7 @@
cdesc = cdesc.basedesc
return s_result # common case
- def consider_call_site(bookkeeper, family, descs, args, s_result):
+ def consider_call_site(bookkeeper, family, descs, args, s_result, op):
from pypy.annotation.model import SomeInstance, SomePBC, s_None
if len(descs) == 1:
# call to a single class, look at the result annotation
@@ -709,7 +716,7 @@
initdescs[0].mergecallfamilies(*initdescs[1:])
initfamily = initdescs[0].getcallfamily()
MethodDesc.consider_call_site(bookkeeper, initfamily, initdescs,
- args, s_None)
+ args, s_None, op)
consider_call_site = staticmethod(consider_call_site)
def getallbases(self):
@@ -782,13 +789,13 @@
def getuniquegraph(self):
return self.funcdesc.getuniquegraph()
- def pycall(self, schedule, args, s_previous_result):
+ def pycall(self, schedule, args, s_previous_result, op=None):
from pypy.annotation.model import SomeInstance
if self.selfclassdef is None:
raise Exception("calling %r" % (self,))
s_instance = SomeInstance(self.selfclassdef, flags = self.flags)
args = args.prepend(s_instance)
- return self.funcdesc.pycall(schedule, args, s_previous_result)
+ return self.funcdesc.pycall(schedule, args, s_previous_result, op)
def bind_under(self, classdef, name):
self.bookkeeper.warning("rebinding an already bound %r" % (self,))
@@ -801,10 +808,10 @@
self.name,
flags)
- def consider_call_site(bookkeeper, family, descs, args, s_result):
+ def consider_call_site(bookkeeper, family, descs, args, s_result, op):
shape = rawshape(args, nextra=1) # account for the extra 'self'
funcdescs = [methoddesc.funcdesc for methoddesc in descs]
- row = FunctionDesc.row_to_consider(descs, args)
+ row = FunctionDesc.row_to_consider(descs, args, op)
family.calltable_add_row(shape, row)
consider_call_site = staticmethod(consider_call_site)
@@ -956,16 +963,16 @@
return '<MethodOfFrozenDesc %r of %r>' % (self.funcdesc,
self.frozendesc)
- def pycall(self, schedule, args, s_previous_result):
+ def pycall(self, schedule, args, s_previous_result, op=None):
from pypy.annotation.model import SomePBC
s_self = SomePBC([self.frozendesc])
args = args.prepend(s_self)
- return self.funcdesc.pycall(schedule, args, s_previous_result)
+ return self.funcdesc.pycall(schedule, args, s_previous_result, op)
- def consider_call_site(bookkeeper, family, descs, args, s_result):
+ def consider_call_site(bookkeeper, family, descs, args, s_result, op):
shape = rawshape(args, nextra=1) # account for the extra 'self'
funcdescs = [mofdesc.funcdesc for mofdesc in descs]
- row = FunctionDesc.row_to_consider(descs, args)
+ row = FunctionDesc.row_to_consider(descs, args, op)
family.calltable_add_row(shape, row)
consider_call_site = staticmethod(consider_call_site)
diff --git a/pypy/annotation/policy.py b/pypy/annotation/policy.py
--- a/pypy/annotation/policy.py
+++ b/pypy/annotation/policy.py
@@ -1,7 +1,7 @@
# base annotation policy for specialization
from pypy.annotation.specialize import default_specialize as default
-from pypy.annotation.specialize import specialize_argvalue, specialize_argtype, specialize_arglistitemtype
-from pypy.annotation.specialize import memo
+from pypy.annotation.specialize import specialize_argvalue, specialize_argtype, specialize_arglistitemtype, specialize_arg_or_var
+from pypy.annotation.specialize import memo, specialize_call_location
# for some reason, model must be imported first,
# or we create a cycle.
from pypy.annotation import model as annmodel
@@ -73,8 +73,10 @@
default_specialize = staticmethod(default)
specialize__memo = staticmethod(memo)
specialize__arg = staticmethod(specialize_argvalue) # specialize:arg(N)
+ specialize__arg_or_var = staticmethod(specialize_arg_or_var)
specialize__argtype = staticmethod(specialize_argtype) # specialize:argtype(N)
specialize__arglistitemtype = staticmethod(specialize_arglistitemtype)
+ specialize__call_location = staticmethod(specialize_call_location)
def specialize__ll(pol, *args):
from pypy.rpython.annlowlevel import LowLevelAnnotatorPolicy
diff --git a/pypy/annotation/specialize.py b/pypy/annotation/specialize.py
--- a/pypy/annotation/specialize.py
+++ b/pypy/annotation/specialize.py
@@ -353,6 +353,16 @@
key = tuple(key)
return maybe_star_args(funcdesc, key, args_s)
+def specialize_arg_or_var(funcdesc, args_s, *argindices):
+ for argno in argindices:
+ if not args_s[argno].is_constant():
+ break
+ else:
+ # all constant
+ return specialize_argvalue(funcdesc, args_s, *argindices)
+ # some not constant
+ return maybe_star_args(funcdesc, None, args_s)
+
def specialize_argtype(funcdesc, args_s, *argindices):
key = tuple([args_s[i].knowntype for i in argindices])
for cls in key:
@@ -370,3 +380,7 @@
else:
key = s.listdef.listitem.s_value.knowntype
return maybe_star_args(funcdesc, key, args_s)
+
+def specialize_call_location(funcdesc, args_s, op):
+ assert op is not None
+ return maybe_star_args(funcdesc, op, args_s)
diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -1099,8 +1099,8 @@
allocdesc = a.bookkeeper.getdesc(alloc)
s_C1 = a.bookkeeper.immutablevalue(C1)
s_C2 = a.bookkeeper.immutablevalue(C2)
- graph1 = allocdesc.specialize([s_C1])
- graph2 = allocdesc.specialize([s_C2])
+ graph1 = allocdesc.specialize([s_C1], None)
+ graph2 = allocdesc.specialize([s_C2], None)
assert a.binding(graph1.getreturnvar()).classdef == C1df
assert a.binding(graph2.getreturnvar()).classdef == C2df
assert graph1 in a.translator.graphs
@@ -1135,8 +1135,8 @@
allocdesc = a.bookkeeper.getdesc(alloc)
s_C1 = a.bookkeeper.immutablevalue(C1)
s_C2 = a.bookkeeper.immutablevalue(C2)
- graph1 = allocdesc.specialize([s_C1, s_C2])
- graph2 = allocdesc.specialize([s_C2, s_C2])
+ graph1 = allocdesc.specialize([s_C1, s_C2], None)
+ graph2 = allocdesc.specialize([s_C2, s_C2], None)
assert a.binding(graph1.getreturnvar()).classdef == C1df
assert a.binding(graph2.getreturnvar()).classdef == C2df
assert graph1 in a.translator.graphs
@@ -1194,6 +1194,33 @@
assert len(executedesc._cache[(0, 'star', 2)].startblock.inputargs) == 4
assert len(executedesc._cache[(1, 'star', 3)].startblock.inputargs) == 5
+ def test_specialize_arg_or_var(self):
+ def f(a):
+ return 1
+ f._annspecialcase_ = 'specialize:arg_or_var(0)'
+
+ def fn(a):
+ return f(3) + f(a)
+
+ a = self.RPythonAnnotator()
+ a.build_types(fn, [int])
+ executedesc = a.bookkeeper.getdesc(f)
+ assert sorted(executedesc._cache.keys()) == [None, (3,)]
+ # we got two different special
+
+ def test_specialize_call_location(self):
+ def g(a):
+ return a
+ g._annspecialcase_ = "specialize:call_location"
+ def f(x):
+ return g(x)
+ f._annspecialcase_ = "specialize:argtype(0)"
+ def h(y):
+ w = f(y)
+ return int(f(str(y))) + w
+ a = self.RPythonAnnotator()
+ assert a.build_types(h, [int]) == annmodel.SomeInteger()
+
def test_assert_list_doesnt_lose_info(self):
class T(object):
pass
@@ -3177,6 +3204,8 @@
s = a.build_types(f, [])
assert isinstance(s, annmodel.SomeList)
assert not s.listdef.listitem.resized
+ assert not s.listdef.listitem.immutable
+ assert s.listdef.listitem.mutated
def test_delslice(self):
def f():
diff --git a/pypy/annotation/unaryop.py b/pypy/annotation/unaryop.py
--- a/pypy/annotation/unaryop.py
+++ b/pypy/annotation/unaryop.py
@@ -352,6 +352,7 @@
check_negative_slice(s_start, s_stop)
if not isinstance(s_iterable, SomeList):
raise Exception("list[start:stop] = x: x must be a list")
+ lst.listdef.mutate()
lst.listdef.agree(s_iterable.listdef)
# note that setslice is not allowed to resize a list in RPython
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -21,8 +21,6 @@
* `Papers`_: Academic papers, talks, and related projects
-* `Videos`_: Videos of PyPy talks and presentations
-
* `speed.pypy.org`_: Daily benchmarks of how fast PyPy is
* `potential project ideas`_: In case you want to get your feet wet...
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -53,6 +53,18 @@
this is an ideal task to get started, because it does not require any deep
knowledge of the internals.
+Optimized Unicode Representation
+--------------------------------
+
+CPython 3.3 will use an `optimized unicode representation`_ which switches between
+different ways to represent a unicode string, depending on whether the string
+fits into ASCII, has only two-byte characters or needs four-byte characters.
+
+The actual details would be rather differen in PyPy, but we would like to have
+the same optimization implemented.
+
+.. _`optimized unicode representation`: http://www.python.org/dev/peps/pep-0393/
+
Translation Toolchain
---------------------
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -125,6 +125,7 @@
### Manipulation ###
+ @jit.look_inside_iff(lambda self: not self._dont_jit)
def unpack(self): # slowish
"Return a ([w1,w2...], {'kw':w3...}) pair."
kwds_w = {}
@@ -245,6 +246,8 @@
### Parsing for function calls ###
+ # XXX: this should be @jit.look_inside_iff, but we need key word arguments,
+ # and it doesn't support them for now.
def _match_signature(self, w_firstarg, scope_w, signature, defaults_w=None,
blindargs=0):
"""Parse args and kwargs according to the signature of a code object,
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -3,12 +3,12 @@
from pypy.interpreter.executioncontext import ExecutionContext, ActionFlag
from pypy.interpreter.executioncontext import UserDelAction, FrameTraceAction
from pypy.interpreter.error import OperationError, operationerrfmt
-from pypy.interpreter.error import new_exception_class
+from pypy.interpreter.error import new_exception_class, typed_unwrap_error_msg
from pypy.interpreter.argument import Arguments
from pypy.interpreter.miscutils import ThreadLocals
from pypy.tool.cache import Cache
from pypy.tool.uid import HUGEVAL_BYTES
-from pypy.rlib.objectmodel import we_are_translated, newlist
+from pypy.rlib.objectmodel import we_are_translated, newlist, compute_unique_id
from pypy.rlib.debug import make_sure_not_resized
from pypy.rlib.timer import DummyTimer, Timer
from pypy.rlib.rarithmetic import r_uint
@@ -186,6 +186,28 @@
def _set_mapdict_storage_and_map(self, storage, map):
raise NotImplementedError
+ # -------------------------------------------------------------------
+
+ def str_w(self, space):
+ w_msg = typed_unwrap_error_msg(space, "string", self)
+ raise OperationError(space.w_TypeError, w_msg)
+
+ def unicode_w(self, space):
+ raise OperationError(space.w_TypeError,
+ typed_unwrap_error_msg(space, "unicode", self))
+
+ def int_w(self, space):
+ raise OperationError(space.w_TypeError,
+ typed_unwrap_error_msg(space, "integer", self))
+
+ def uint_w(self, space):
+ raise OperationError(space.w_TypeError,
+ typed_unwrap_error_msg(space, "integer", self))
+
+ def bigint_w(self, space):
+ raise OperationError(space.w_TypeError,
+ typed_unwrap_error_msg(space, "integer", self))
+
class Wrappable(W_Root):
"""A subclass of Wrappable is an internal, interpreter-level class
@@ -901,7 +923,7 @@
ec.c_call_trace(frame, w_func, args)
try:
w_res = self.call_args(w_func, args)
- except OperationError, e:
+ except OperationError:
ec.c_exception_trace(frame, w_func)
raise
ec.c_return_trace(frame, w_func, args)
@@ -947,6 +969,9 @@
def isinstance_w(self, w_obj, w_type):
return self.is_true(self.isinstance(w_obj, w_type))
+ def id(self, w_obj):
+ return self.wrap(compute_unique_id(w_obj))
+
# The code below only works
# for the simple case (new-style instance).
# These methods are patched with the full logic by the __builtin__
@@ -999,8 +1024,6 @@
def eval(self, expression, w_globals, w_locals, hidden_applevel=False):
"NOT_RPYTHON: For internal debugging."
- import types
- from pypy.interpreter.pycode import PyCode
if isinstance(expression, str):
compiler = self.createcompiler()
expression = compiler.compile(expression, '?', 'eval', 0,
@@ -1012,7 +1035,6 @@
def exec_(self, statement, w_globals, w_locals, hidden_applevel=False,
filename=None):
"NOT_RPYTHON: For internal debugging."
- import types
if filename is None:
filename = '?'
from pypy.interpreter.pycode import PyCode
@@ -1210,6 +1232,18 @@
return None
return self.str_w(w_obj)
+ def str_w(self, w_obj):
+ return w_obj.str_w(self)
+
+ def int_w(self, w_obj):
+ return w_obj.int_w(self)
+
+ def uint_w(self, w_obj):
+ return w_obj.uint_w(self)
+
+ def bigint_w(self, w_obj):
+ return w_obj.bigint_w(self)
+
def realstr_w(self, w_obj):
# Like str_w, but only works if w_obj is really of type 'str'.
if not self.is_true(self.isinstance(w_obj, self.w_str)):
@@ -1217,6 +1251,9 @@
self.wrap('argument must be a string'))
return self.str_w(w_obj)
+ def unicode_w(self, w_obj):
+ return w_obj.unicode_w(self)
+
def realunicode_w(self, w_obj):
# Like unicode_w, but only works if w_obj is really of type
# 'unicode'.
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -458,3 +458,7 @@
if module:
space.setattr(w_exc, space.wrap("__module__"), space.wrap(module))
return w_exc
+
+def typed_unwrap_error_msg(space, expected, w_obj):
+ type_name = space.type(w_obj).getname(space)
+ return space.wrap("expected %s, got %s object" % (expected, type_name))
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -1,5 +1,4 @@
import sys
-from pypy.interpreter.miscutils import Stack
from pypy.interpreter.error import OperationError
from pypy.rlib.rarithmetic import LONG_BIT
from pypy.rlib.unroll import unrolling_iterable
@@ -176,6 +175,9 @@
self.w_tracefunc = w_func
self.space.frame_trace_action.fire()
+ def gettrace(self):
+ return self.w_tracefunc
+
def setprofile(self, w_func):
"""Set the global trace function."""
if self.space.is_w(w_func, self.space.w_None):
@@ -308,7 +310,11 @@
self._nonperiodic_actions = []
self.has_bytecode_counter = False
self.fired_actions = None
- self.checkinterval_scaled = 100 * TICK_COUNTER_STEP
+ # the default value is not 100, unlike CPython 2.7, but a much
+ # larger value, because we use a technique that not only allows
+ # but actually *forces* another thread to run whenever the counter
+ # reaches zero.
+ self.checkinterval_scaled = 10000 * TICK_COUNTER_STEP
self._rebuild_action_dispatcher()
def fire(self, action):
@@ -347,6 +353,7 @@
elif interval > MAX:
interval = MAX
self.checkinterval_scaled = interval * TICK_COUNTER_STEP
+ self.reset_ticker(-1)
def _rebuild_action_dispatcher(self):
periodic_actions = unrolling_iterable(self._periodic_actions)
diff --git a/pypy/interpreter/miscutils.py b/pypy/interpreter/miscutils.py
--- a/pypy/interpreter/miscutils.py
+++ b/pypy/interpreter/miscutils.py
@@ -2,154 +2,6 @@
Miscellaneous utilities.
"""
-import types
-
-from pypy.rlib.rarithmetic import r_uint
-
-class RootStack:
- pass
-
-class Stack(RootStack):
- """Utility class implementing a stack."""
-
- _annspecialcase_ = "specialize:ctr_location" # polymorphic
-
- def __init__(self):
- self.items = []
-
- def clone(self):
- s = self.__class__()
- for item in self.items:
- try:
- item = item.clone()
- except AttributeError:
- pass
- s.push(item)
- return s
-
- def push(self, item):
- self.items.append(item)
-
- def pop(self):
- return self.items.pop()
-
- def drop(self, n):
- if n > 0:
- del self.items[-n:]
-
- def top(self, position=0):
- """'position' is 0 for the top of the stack, 1 for the item below,
- and so on. It must not be negative."""
- if position < 0:
- raise ValueError, 'negative stack position'
- if position >= len(self.items):
- raise IndexError, 'not enough entries in stack'
- return self.items[~position]
-
- def set_top(self, value, position=0):
- """'position' is 0 for the top of the stack, 1 for the item below,
- and so on. It must not be negative."""
- if position < 0:
- raise ValueError, 'negative stack position'
- if position >= len(self.items):
- raise IndexError, 'not enough entries in stack'
- self.items[~position] = value
-
- def depth(self):
- return len(self.items)
-
- def empty(self):
- return len(self.items) == 0
-
-
-class FixedStack(RootStack):
- _annspecialcase_ = "specialize:ctr_location" # polymorphic
-
- # unfortunately, we have to re-do everything
- def __init__(self):
- pass
-
- def setup(self, stacksize):
- self.ptr = r_uint(0) # we point after the last element
- self.items = [None] * stacksize
-
- def clone(self):
- # this is only needed if we support flow space
- s = self.__class__()
- s.setup(len(self.items))
- for item in self.items[:self.ptr]:
- try:
- item = item.clone()
- except AttributeError:
- pass
- s.push(item)
- return s
-
- def push(self, item):
- ptr = self.ptr
- self.items[ptr] = item
- self.ptr = ptr + 1
-
- def pop(self):
- ptr = self.ptr - 1
- ret = self.items[ptr] # you get OverflowError if the stack is empty
- self.items[ptr] = None
- self.ptr = ptr
- return ret
-
- def drop(self, n):
- while n > 0:
- n -= 1
- self.ptr -= 1
- self.items[self.ptr] = None
-
- def top(self, position=0):
- # for a fixed stack, we assume correct indices
- return self.items[self.ptr + ~position]
-
- def set_top(self, value, position=0):
- # for a fixed stack, we assume correct indices
- self.items[self.ptr + ~position] = value
-
- def depth(self):
- return self.ptr
-
- def empty(self):
- return not self.ptr
-
-
-class InitializedClass(type):
- """NOT_RPYTHON. A meta-class that allows a class to initialize itself (or
- its subclasses) by calling __initclass__() as a class method."""
- def __init__(self, name, bases, dict):
- super(InitializedClass, self).__init__(name, bases, dict)
- for basecls in self.__mro__:
- raw = basecls.__dict__.get('__initclass__')
- if isinstance(raw, types.FunctionType):
- raw(self) # call it as a class method
-
-
-class RwDictProxy(object):
- """NOT_RPYTHON. A dict-like class standing for 'cls.__dict__', to work
- around the fact that the latter is a read-only proxy for new-style
- classes."""
-
- def __init__(self, cls):
- self.cls = cls
-
- def __getitem__(self, attr):
- return self.cls.__dict__[attr]
-
- def __setitem__(self, attr, value):
- setattr(self.cls, attr, value)
-
- def __contains__(self, value):
- return value in self.cls.__dict__
-
- def items(self):
- return self.cls.__dict__.items()
-
-
class ThreadLocals:
"""Pseudo thread-local storage, for 'space.threadlocals'.
This is not really thread-local at all; the intention is that the PyPy
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -66,7 +66,7 @@
make_sure_not_resized(self.locals_stack_w)
check_nonneg(self.nlocals)
#
- if space.config.objspace.honor__builtins__ and w_globals is not None:
+ if space.config.objspace.honor__builtins__:
self.builtin = space.builtin.pick_builtin(w_globals)
# regular functions always have CO_OPTIMIZED and CO_NEWLOCALS.
# class bodies only have CO_NEWLOCALS.
diff --git a/pypy/interpreter/pyparser/future.py b/pypy/interpreter/pyparser/future.py
--- a/pypy/interpreter/pyparser/future.py
+++ b/pypy/interpreter/pyparser/future.py
@@ -225,14 +225,16 @@
raise DoneException
self.consume_whitespace()
- def consume_whitespace(self):
+ def consume_whitespace(self, newline_ok=False):
while 1:
c = self.getc()
if c in whitespace:
self.pos += 1
continue
- elif c == '\\':
- self.pos += 1
+ elif c == '\\' or newline_ok:
+ slash = c == '\\'
+ if slash:
+ self.pos += 1
c = self.getc()
if c == '\n':
self.pos += 1
@@ -243,8 +245,10 @@
if self.getc() == '\n':
self.pos += 1
self.atbol()
+ elif slash:
+ raise DoneException
else:
- raise DoneException
+ return
else:
return
@@ -281,7 +285,7 @@
return
else:
self.pos += 1
- self.consume_whitespace()
+ self.consume_whitespace(paren_list)
if paren_list and self.getc() == ')':
self.pos += 1
return # Handles trailing comma inside parenthesis
diff --git a/pypy/interpreter/pyparser/test/test_futureautomaton.py b/pypy/interpreter/pyparser/test/test_futureautomaton.py
--- a/pypy/interpreter/pyparser/test/test_futureautomaton.py
+++ b/pypy/interpreter/pyparser/test/test_futureautomaton.py
@@ -3,7 +3,7 @@
from pypy.tool import stdlib___future__ as fut
def run(s):
- f = future.FutureAutomaton(future.futureFlags_2_5, s)
+ f = future.FutureAutomaton(future.futureFlags_2_7, s)
try:
f.start()
except future.DoneException:
@@ -113,6 +113,14 @@
assert f.lineno == 1
assert f.col_offset == 0
+def test_paren_with_newline():
+ s = 'from __future__ import (division,\nabsolute_import)\n'
+ f = run(s)
+ assert f.pos == len(s)
+ assert f.flags == (fut.CO_FUTURE_DIVISION | fut.CO_FUTURE_ABSOLUTE_IMPORT)
+ assert f.lineno == 1
+ assert f.col_offset == 0
+
def test_multiline():
s = '"abc" #def\n #ghi\nfrom __future__ import (division as b, generators,)\nfrom __future__ import with_statement\n'
f = run(s)
diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py
--- a/pypy/interpreter/test/test_executioncontext.py
+++ b/pypy/interpreter/test/test_executioncontext.py
@@ -42,6 +42,7 @@
assert i == 9
def test_periodic_action(self):
+ from pypy.interpreter.executioncontext import ActionFlag
class DemoAction(executioncontext.PeriodicAsyncAction):
counter = 0
@@ -53,17 +54,20 @@
space = self.space
a2 = DemoAction(space)
- space.actionflag.register_periodic_action(a2, True)
try:
- for i in range(500):
- space.appexec([], """():
- n = 5
- return n + 2
- """)
- except Finished:
- pass
- checkinterval = space.actionflag.getcheckinterval()
- assert checkinterval / 10 < i < checkinterval * 1.1
+ space.actionflag.setcheckinterval(100)
+ space.actionflag.register_periodic_action(a2, True)
+ try:
+ for i in range(500):
+ space.appexec([], """():
+ n = 5
+ return n + 2
+ """)
+ except Finished:
+ pass
+ finally:
+ space.actionflag = ActionFlag() # reset to default
+ assert 10 < i < 110
def test_llprofile(self):
l = []
diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py
--- a/pypy/jit/backend/llsupport/llmodel.py
+++ b/pypy/jit/backend/llsupport/llmodel.py
@@ -498,6 +498,16 @@
u = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string)
u.chars[index] = unichr(newvalue)
+ def bh_copystrcontent(self, src, dst, srcstart, dststart, length):
+ src = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), src)
+ dst = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), dst)
+ rstr.copy_string_contents(src, dst, srcstart, dststart, length)
+
+ def bh_copyunicodecontent(self, src, dst, srcstart, dststart, length):
+ src = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), src)
+ dst = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), dst)
+ rstr.copy_unicode_contents(src, dst, srcstart, dststart, length)
+
def bh_call_i(self, func, calldescr, args_i, args_r, args_f):
assert isinstance(calldescr, BaseIntCallDescr)
if not we_are_translated():
diff --git a/pypy/jit/backend/model.py b/pypy/jit/backend/model.py
--- a/pypy/jit/backend/model.py
+++ b/pypy/jit/backend/model.py
@@ -303,6 +303,10 @@
raise NotImplementedError
def bh_unicodesetitem(self, string, index, newvalue):
raise NotImplementedError
+ def bh_copystrcontent(self, src, dst, srcstart, dststart, length):
+ raise NotImplementedError
+ def bh_copyunicodecontent(self, src, dst, srcstart, dststart, length):
+ raise NotImplementedError
def force(self, force_token):
raise NotImplementedError
diff --git a/pypy/jit/codewriter/effectinfo.py b/pypy/jit/codewriter/effectinfo.py
--- a/pypy/jit/codewriter/effectinfo.py
+++ b/pypy/jit/codewriter/effectinfo.py
@@ -74,6 +74,7 @@
OS_LLONG_UGE = 91
OS_LLONG_URSHIFT = 92
OS_LLONG_FROM_UINT = 93
+ OS_LLONG_U_TO_FLOAT = 94
#
OS_MATH_SQRT = 100
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -440,6 +440,7 @@
rewrite_op_ullong_mod_zer = _do_builtin_call
rewrite_op_gc_identityhash = _do_builtin_call
rewrite_op_gc_id = _do_builtin_call
+ rewrite_op_uint_mod = _do_builtin_call
# ----------
# getfield/setfield/mallocs etc.
@@ -883,9 +884,15 @@
v = v_arg
oplist = []
if unsigned1:
- opname = 'cast_uint_to_longlong'
+ if unsigned2:
+ opname = 'cast_uint_to_ulonglong'
+ else:
+ opname = 'cast_uint_to_longlong'
else:
- opname = 'cast_int_to_longlong'
+ if unsigned2:
+ opname = 'cast_int_to_ulonglong'
+ else:
+ opname = 'cast_int_to_longlong'
op2 = self.rewrite_operation(
SpaceOperation(opname, [v], v_result)
)
@@ -995,6 +1002,21 @@
return op2
''' % (_op, _oopspec.lower(), _oopspec, _oopspec)).compile()
+ for _op, _oopspec in [('cast_int_to_ulonglong', 'FROM_INT'),
+ ('cast_uint_to_ulonglong', 'FROM_UINT'),
+ ('cast_float_to_ulonglong', 'FROM_FLOAT'),
+ ('cast_ulonglong_to_float', 'U_TO_FLOAT'),
+ ]:
+ exec py.code.Source('''
+ def rewrite_op_%s(self, op):
+ args = op.args
+ op1 = self.prepare_builtin_call(op, "ullong_%s", args)
+ op2 = self._handle_oopspec_call(op1, args,
+ EffectInfo.OS_LLONG_%s,
+ EffectInfo.EF_ELIDABLE_CANNOT_RAISE)
+ return op2
+ ''' % (_op, _oopspec.lower(), _oopspec)).compile()
+
def _normalize(self, oplist):
if isinstance(oplist, SpaceOperation):
return [oplist]
@@ -1158,6 +1180,12 @@
return SpaceOperation('%s_assert_green' % kind, args, None)
elif oopspec_name == 'jit.current_trace_length':
return SpaceOperation('current_trace_length', [], op.result)
+ elif oopspec_name == 'jit.isconstant':
+ kind = getkind(args[0].concretetype)
+ return SpaceOperation('%s_isconstant' % kind, args, op.result)
+ elif oopspec_name == 'jit.isvirtual':
+ kind = getkind(args[0].concretetype)
+ return SpaceOperation('%s_isvirtual' % kind, args, op.result)
else:
raise AssertionError("missing support for %r" % oopspec_name)
@@ -1415,6 +1443,14 @@
else:
assert 0, "args[0].concretetype must be STR or UNICODE"
#
+ if oopspec_name == 'stroruni.copy_contents':
+ if SoU.TO == rstr.STR:
+ new_op = 'copystrcontent'
+ elif SoU.TO == rstr.UNICODE:
+ new_op = 'copyunicodecontent'
+ else:
+ assert 0
+ return SpaceOperation(new_op, args, op.result)
if oopspec_name == "stroruni.equal":
for otherindex, othername, argtypes, resulttype in [
(EffectInfo.OS_STREQ_SLICE_CHECKNULL,
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -315,18 +315,30 @@
def _ll_1_llong_from_int(x):
return r_longlong(intmask(x))
+def _ll_1_ullong_from_int(x):
+ return r_ulonglong(intmask(x))
+
def _ll_1_llong_from_uint(x):
return r_longlong(r_uint(x))
+def _ll_1_ullong_from_uint(x):
+ return r_ulonglong(r_uint(x))
+
def _ll_1_llong_to_int(xll):
return intmask(xll)
def _ll_1_llong_from_float(xf):
return r_longlong(xf)
+def _ll_1_ullong_from_float(xf):
+ return r_ulonglong(xf)
+
def _ll_1_llong_to_float(xll):
return float(rffi.cast(lltype.SignedLongLong, xll))
+def _ll_1_ullong_u_to_float(xull):
+ return float(rffi.cast(lltype.UnsignedLongLong, xull))
+
def _ll_1_llong_abs(xll):
if xll < 0:
@@ -351,20 +363,23 @@
return llop.llong_mod(lltype.SignedLongLong, xll, yll)
def _ll_2_ullong_floordiv(xll, yll):
- return llop.ullong_floordiv(lltype.SignedLongLong, xll, yll)
+ return llop.ullong_floordiv(lltype.UnsignedLongLong, xll, yll)
def _ll_2_ullong_floordiv_zer(xll, yll):
if yll == 0:
raise ZeroDivisionError
- return llop.ullong_floordiv(lltype.SignedLongLong, xll, yll)
+ return llop.ullong_floordiv(lltype.UnsignedLongLong, xll, yll)
def _ll_2_ullong_mod(xll, yll):
- return llop.ullong_mod(lltype.SignedLongLong, xll, yll)
+ return llop.ullong_mod(lltype.UnsignedLongLong, xll, yll)
def _ll_2_ullong_mod_zer(xll, yll):
if yll == 0:
raise ZeroDivisionError
- return llop.ullong_mod(lltype.SignedLongLong, xll, yll)
+ return llop.ullong_mod(lltype.UnsignedLongLong, xll, yll)
+
+def _ll_2_uint_mod(xll, yll):
+ return llop.uint_mod(lltype.Unsigned, xll, yll)
# libffi support
diff --git a/pypy/jit/codewriter/test/test_flatten.py b/pypy/jit/codewriter/test/test_flatten.py
--- a/pypy/jit/codewriter/test/test_flatten.py
+++ b/pypy/jit/codewriter/test/test_flatten.py
@@ -829,14 +829,15 @@
self.encoding_test(f, [rffi.cast(FROM, 42)], expectedstr,
transform=True)
elif TO in (rffi.LONG, rffi.ULONG):
+ if rffi.cast(FROM, -1) < 0:
+ fnname = "llong_from_int"
+ else:
+ fnname = "llong_from_uint"
if TO == rffi.LONG:
TO = rffi.LONGLONG
else:
TO = rffi.ULONGLONG
- if rffi.cast(FROM, -1) < 0:
- fnname = "llong_from_int"
- else:
- fnname = "llong_from_uint"
+ fnname = "u" + fnname
expected.pop() # remove int_return
expected.append(
"residual_call_irf_f $<* fn %s>, <Descr>, I[%s], R[], F[] -> %%f0"
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -1,5 +1,19 @@
import py
import random
+try:
+ from itertools import product
+except ImportError:
+ # Python 2.5, this is taken from the CPython docs, but simplified.
+ def product(*args):
+ # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
+ # product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111
+ pools = map(tuple, args)
+ result = [[]]
+ for pool in pools:
+ result = [x+[y] for x in result for y in pool]
+ for prod in result:
+ yield tuple(prod)
+
from pypy.objspace.flow.model import FunctionGraph, Block, Link
from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
from pypy.jit.codewriter.jtransform import Transformer
@@ -254,26 +268,35 @@
assert op1.result is None
def test_calls():
- for RESTYPE in [lltype.Signed, rclass.OBJECTPTR,
- lltype.Float, lltype.Void]:
- for with_void in [False, True]:
- for with_i in [False, True]:
- for with_r in [False, True]:
- for with_f in [False, True]:
- ARGS = []
- if with_void: ARGS += [lltype.Void, lltype.Void]
- if with_i: ARGS += [lltype.Signed, lltype.Char]
- if with_r: ARGS += [rclass.OBJECTPTR, lltype.Ptr(rstr.STR)]
- if with_f: ARGS += [lltype.Float, lltype.Float]
- random.shuffle(ARGS)
- if RESTYPE == lltype.Float: with_f = True
- if with_f: expectedkind = 'irf' # all kinds
- elif with_i: expectedkind = 'ir' # integers and references
- else: expectedkind = 'r' # only references
- yield residual_call_test, ARGS, RESTYPE, expectedkind
- yield direct_call_test, ARGS, RESTYPE, expectedkind
- yield indirect_residual_call_test, ARGS, RESTYPE, expectedkind
- yield indirect_regular_call_test, ARGS, RESTYPE, expectedkind
+ for RESTYPE, with_void, with_i, with_r, with_f in product(
+ [lltype.Signed, rclass.OBJECTPTR, lltype.Float, lltype.Void],
+ [False, True],
+ [False, True],
+ [False, True],
+ [False, True],
+ ):
+ ARGS = []
+ if with_void:
+ ARGS += [lltype.Void, lltype.Void]
+ if with_i:
+ ARGS += [lltype.Signed, lltype.Char]
+ if with_r:
+ ARGS += [rclass.OBJECTPTR, lltype.Ptr(rstr.STR)]
+ if with_f:
+ ARGS += [lltype.Float, lltype.Float]
+ random.shuffle(ARGS)
+ if RESTYPE == lltype.Float:
+ with_f = True
+ if with_f:
+ expectedkind = 'irf' # all kinds
+ elif with_i:
+ expectedkind = 'ir' # integers and references
+ else:
+ expectedkind = 'r' # only references
+ yield residual_call_test, ARGS, RESTYPE, expectedkind
+ yield direct_call_test, ARGS, RESTYPE, expectedkind
+ yield indirect_residual_call_test, ARGS, RESTYPE, expectedkind
+ yield indirect_regular_call_test, ARGS, RESTYPE, expectedkind
def get_direct_call_op(argtypes, restype):
FUNC = lltype.FuncType(argtypes, restype)
diff --git a/pypy/jit/codewriter/test/test_longlong.py b/pypy/jit/codewriter/test/test_longlong.py
--- a/pypy/jit/codewriter/test/test_longlong.py
+++ b/pypy/jit/codewriter/test/test_longlong.py
@@ -57,7 +57,8 @@
assert op1.opname == 'residual_call_irf_f'
else:
assert op1.opname == 'residual_call_irf_i'
- gotindex = getattr(EffectInfo, 'OS_' + op1.args[0].value.upper())
+ gotindex = getattr(EffectInfo,
+ 'OS_' + op1.args[0].value.upper().lstrip('U'))
assert gotindex == oopspecindex
assert op1.args[1] == 'calldescr-%d' % oopspecindex
assert list(op1.args[2]) == [v for v in vlist
@@ -192,8 +193,12 @@
[lltype.SignedLongLong], lltype.Signed)
self.do_check('cast_float_to_longlong', EffectInfo.OS_LLONG_FROM_FLOAT,
[lltype.Float], lltype.SignedLongLong)
+ self.do_check('cast_float_to_ulonglong', EffectInfo.OS_LLONG_FROM_FLOAT,
+ [lltype.Float], lltype.UnsignedLongLong)
self.do_check('cast_longlong_to_float', EffectInfo.OS_LLONG_TO_FLOAT,
[lltype.SignedLongLong], lltype.Float)
+ self.do_check('cast_ulonglong_to_float', EffectInfo.OS_LLONG_U_TO_FLOAT,
+ [lltype.UnsignedLongLong], lltype.Float)
for T1 in [lltype.SignedLongLong, lltype.UnsignedLongLong]:
for T2 in [lltype.Signed, lltype.Unsigned]:
self.do_check('cast_primitive', EffectInfo.OS_LLONG_TO_INT,
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -835,6 +835,18 @@
def bhimpl_current_trace_length():
return -1
+ @arguments("i", returns="i")
+ def bhimpl_int_isconstant(x):
+ return False
+
+ @arguments("r", returns="i")
+ def bhimpl_ref_isconstant(x):
+ return False
+
+ @arguments("r", returns="i")
+ def bhimpl_ref_isvirtual(x):
+ return False
+
# ----------
# the main hints and recursive calls
@@ -1224,6 +1236,9 @@
@arguments("cpu", "r", "i", "i")
def bhimpl_strsetitem(cpu, string, index, newchr):
cpu.bh_strsetitem(string, index, newchr)
+ @arguments("cpu", "r", "r", "i", "i", "i")
+ def bhimpl_copystrcontent(cpu, src, dst, srcstart, dststart, length):
+ cpu.bh_copystrcontent(src, dst, srcstart, dststart, length)
@arguments("cpu", "i", returns="r")
def bhimpl_newunicode(cpu, length):
@@ -1237,6 +1252,9 @@
@arguments("cpu", "r", "i", "i")
def bhimpl_unicodesetitem(cpu, unicode, index, newchr):
cpu.bh_unicodesetitem(unicode, index, newchr)
+ @arguments("cpu", "r", "r", "i", "i", "i")
+ def bhimpl_copyunicodecontent(cpu, src, dst, srcstart, dststart, length):
+ cpu.bh_copyunicodecontent(src, dst, srcstart, dststart, length)
@arguments(returns=(longlong.is_64_bit and "i" or "f"))
def bhimpl_ll_read_timestamp():
diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -11,9 +11,13 @@
self.known_class_boxes = {}
# store the boxes that contain newly allocated objects, this maps the
# boxes to a bool, the bool indicates whether or not the object has
- # escaped the trace or not, its presences in the mapping shows that it
- # was allocated inside the trace
+ # escaped the trace or not (True means the box never escaped, False
+ # means it did escape), its presences in the mapping shows that it was
+ # allocated inside the trace
self.new_boxes = {}
+ # Tracks which boxes should be marked as escaped when the key box
+ # escapes.
+ self.dependencies = {}
# contains frame boxes that are not virtualizables
self.nonstandard_virtualizables = {}
# heap cache
@@ -30,11 +34,29 @@
self.clear_caches(opnum, descr, argboxes)
def mark_escaped(self, opnum, argboxes):
- for idx, box in enumerate(argboxes):
- # setfield_gc and setarrayitem_gc don't escape their first argument
- if not (idx == 0 and opnum in [rop.SETFIELD_GC, rop.SETARRAYITEM_GC]):
- if box in self.new_boxes:
- self.new_boxes[box] = False
+ idx = 0
+ if opnum == rop.SETFIELD_GC:
+ assert len(argboxes) == 2
+ box, valuebox = argboxes
+ if self.is_unescaped(box) and self.is_unescaped(valuebox):
+ self.dependencies.setdefault(box, []).append(valuebox)
+ else:
+ self._escape(valuebox)
+ # GETFIELD_GC doesn't escape it's argument
+ elif opnum != rop.GETFIELD_GC:
+ for box in argboxes:
+ # setarrayitem_gc don't escape its first argument
+ if not (idx == 0 and opnum in [rop.SETARRAYITEM_GC]):
+ self._escape(box)
+ idx += 1
+
+ def _escape(self, box):
+ if box in self.new_boxes:
+ self.new_boxes[box] = False
+ if box in self.dependencies:
+ for dep in self.dependencies[box]:
+ self._escape(dep)
+ del self.dependencies[box]
def clear_caches(self, opnum, descr, argboxes):
if opnum == rop.SETFIELD_GC:
diff --git a/pypy/jit/metainterp/optimizeopt/__init__.py b/pypy/jit/metainterp/optimizeopt/__init__.py
--- a/pypy/jit/metainterp/optimizeopt/__init__.py
+++ b/pypy/jit/metainterp/optimizeopt/__init__.py
@@ -7,6 +7,8 @@
from pypy.jit.metainterp.optimizeopt.unroll import optimize_unroll, OptInlineShortPreamble
from pypy.jit.metainterp.optimizeopt.fficall import OptFfiCall
from pypy.jit.metainterp.optimizeopt.simplify import OptSimplify
+from pypy.jit.metainterp.optimizeopt.pure import OptPure
+from pypy.jit.metainterp.optimizeopt.earlyforce import OptEarlyForce
from pypy.rlib.jit import PARAMETERS
from pypy.rlib.unroll import unrolling_iterable
@@ -14,6 +16,8 @@
('rewrite', OptRewrite),
('virtualize', OptVirtualize),
('string', OptString),
+ ('earlyforce', OptEarlyForce),
+ ('pure', OptPure),
('heap', OptHeap),
('ffi', None),
('unroll', None)]
diff --git a/pypy/jit/metainterp/optimizeopt/earlyforce.py b/pypy/jit/metainterp/optimizeopt/earlyforce.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/optimizeopt/earlyforce.py
@@ -0,0 +1,24 @@
+from pypy.jit.metainterp.optimizeopt.optimizer import Optimization
+from pypy.jit.metainterp.optimizeopt.vstring import VAbstractStringValue
+from pypy.jit.metainterp.resoperation import rop, ResOperation
+
+class OptEarlyForce(Optimization):
+ def propagate_forward(self, op):
+ opnum = op.getopnum()
+ if (opnum != rop.SETFIELD_GC and
+ opnum != rop.SETARRAYITEM_GC and
+ opnum != rop.QUASIIMMUT_FIELD):
+
+ for arg in op.getarglist():
+ if arg in self.optimizer.values:
+ value = self.getvalue(arg)
+ value.force_box(self)
+ self.emit_operation(op)
+
+ def new(self):
+ return OptEarlyForce()
+
+ def setup(self):
+ self.optimizer.optearlyforce = self
+
+
diff --git a/pypy/jit/metainterp/optimizeopt/fficall.py b/pypy/jit/metainterp/optimizeopt/fficall.py
--- a/pypy/jit/metainterp/optimizeopt/fficall.py
+++ b/pypy/jit/metainterp/optimizeopt/fficall.py
@@ -177,10 +177,10 @@
funcinfo.descr is None):
return [op] # cannot optimize
funcsymval = self.getvalue(op.getarg(2))
- arglist = [funcsymval.force_box()]
+ arglist = [funcsymval.get_key_box()]
for push_op in funcinfo.opargs:
argval = self.getvalue(push_op.getarg(2))
- arglist.append(argval.force_box())
+ arglist.append(argval.get_key_box())
newop = ResOperation(rop.CALL_RELEASE_GIL, arglist, op.result,
descr=funcinfo.descr)
self.commit_optimization()
diff --git a/pypy/jit/metainterp/optimizeopt/heap.py b/pypy/jit/metainterp/optimizeopt/heap.py
--- a/pypy/jit/metainterp/optimizeopt/heap.py
+++ b/pypy/jit/metainterp/optimizeopt/heap.py
@@ -50,6 +50,7 @@
if not self._lazy_setfield_registered:
optheap._lazy_setfields_and_arrayitems.append(self)
self._lazy_setfield_registered = True
+
else:
# this is the case where the pending setfield ends up
# storing precisely the value that is already there,
@@ -158,12 +159,17 @@
self._lazy_setfields_and_arrayitems = []
self._remove_guard_not_invalidated = False
self._seen_guard_not_invalidated = False
+ self.posponedop = None
def force_at_end_of_preamble(self):
self.force_all_lazy_setfields_and_arrayitems()
def flush(self):
self.force_all_lazy_setfields_and_arrayitems()
+ if self.posponedop:
+ posponedop = self.posponedop
+ self.posponedop = None
+ self.next_optimization.propagate_forward(posponedop)
def new(self):
return OptHeap()
@@ -211,7 +217,15 @@
def emit_operation(self, op):
self.emitting_operation(op)
- self.next_optimization.propagate_forward(op)
+ if self.posponedop:
+ posponedop = self.posponedop
+ self.posponedop = None
+ self.next_optimization.propagate_forward(posponedop)
+ if (op.is_comparison() or op.getopnum() == rop.CALL_MAY_FORCE
+ or op.is_ovf()):
+ self.posponedop = op
+ else:
+ self.next_optimization.propagate_forward(op)
def emitting_operation(self, op):
if op.has_no_side_effect():
@@ -293,30 +307,6 @@
if indexvalue is None or indexvalue.intbound.contains(idx):
cf.force_lazy_setfield(self, can_cache)
- def fixup_guard_situation(self):
- # hackish: reverse the order of the last two operations if it makes
- # sense to avoid a situation like "int_eq/setfield_gc/guard_true",
- # which the backend (at least the x86 backend) does not handle well.
- newoperations = self.optimizer.newoperations
- if len(newoperations) < 2:
- return
- lastop = newoperations[-1]
- if (lastop.getopnum() != rop.SETFIELD_GC and
- lastop.getopnum() != rop.SETARRAYITEM_GC):
- return
- # - is_comparison() for cases like "int_eq/setfield_gc/guard_true"
- # - CALL_MAY_FORCE: "call_may_force/setfield_gc/guard_not_forced"
- # - is_ovf(): "int_add_ovf/setfield_gc/guard_no_overflow"
- prevop = newoperations[-2]
- opnum = prevop.getopnum()
- if not (prevop.is_comparison() or opnum == rop.CALL_MAY_FORCE
- or prevop.is_ovf()):
- return
- if prevop.result in lastop.getarglist():
- return
- newoperations[-2] = lastop
- newoperations[-1] = prevop
-
def _assert_valid_cf(self, cf):
# check that 'cf' is in cached_fields or cached_arrayitems
if not we_are_translated():
@@ -362,7 +352,6 @@
fieldvalue.get_key_box(), itemindex))
else:
cf.force_lazy_setfield(self)
- self.fixup_guard_situation()
return pendingfields
def optimize_GETFIELD_GC(self, op):
@@ -374,12 +363,22 @@
return
# default case: produce the operation
structvalue.ensure_nonnull()
- ###self.optimizer.optimize_default(op)
self.emit_operation(op)
# then remember the result of reading the field
fieldvalue = self.getvalue(op.result)
cf.remember_field_value(structvalue, fieldvalue, op)
+ def optimize_GETFIELD_GC_PURE(self, op):
+ structvalue = self.getvalue(op.getarg(0))
+ cf = self.field_cache(op.getdescr())
+ fieldvalue = cf.getfield_from_cache(self, structvalue)
+ if fieldvalue is not None:
+ self.make_equal_to(op.result, fieldvalue)
+ return
+ # default case: produce the operation
+ structvalue.ensure_nonnull()
+ self.emit_operation(op)
+
def optimize_SETFIELD_GC(self, op):
if self.has_pure_result(rop.GETFIELD_GC_PURE, [op.getarg(0)],
op.getdescr()):
@@ -389,6 +388,7 @@
#
cf = self.field_cache(op.getdescr())
cf.do_setfield(self, op)
+
def optimize_GETARRAYITEM_GC(self, op):
arrayvalue = self.getvalue(op.getarg(0))
@@ -413,6 +413,25 @@
fieldvalue = self.getvalue(op.result)
cf.remember_field_value(arrayvalue, fieldvalue, op)
+ def optimize_GETARRAYITEM_GC_PURE(self, op):
+ arrayvalue = self.getvalue(op.getarg(0))
+ indexvalue = self.getvalue(op.getarg(1))
+ cf = None
+ if indexvalue.is_constant():
+ arrayvalue.make_len_gt(MODE_ARRAY, op.getdescr(), indexvalue.box.getint())
+ # use the cache on (arraydescr, index), which is a constant
+ cf = self.arrayitem_cache(op.getdescr(), indexvalue.box.getint())
+ fieldvalue = cf.getfield_from_cache(self, arrayvalue)
+ if fieldvalue is not None:
+ self.make_equal_to(op.result, fieldvalue)
+ return
+ else:
+ # variable index, so make sure the lazy setarrayitems are done
+ self.force_lazy_setarrayitem(op.getdescr(), indexvalue=indexvalue)
+ # default case: produce the operation
+ arrayvalue.ensure_nonnull()
+ self.emit_operation(op)
+
def optimize_SETARRAYITEM_GC(self, op):
if self.has_pure_result(rop.GETARRAYITEM_GC_PURE, [op.getarg(0),
op.getarg(1)],
diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py
--- a/pypy/jit/metainterp/optimizeopt/optimizer.py
+++ b/pypy/jit/metainterp/optimizeopt/optimizer.py
@@ -31,8 +31,8 @@
class OptValue(object):
__metaclass__ = extendabletype
- _attrs_ = ('box', 'known_class', 'last_guard_index', 'level', 'intbound', 'lenbound')
- last_guard_index = -1
+ _attrs_ = ('box', 'known_class', 'last_guard', 'level', 'intbound', 'lenbound')
+ last_guard = None
level = LEVEL_UNKNOWN
known_class = None
@@ -71,7 +71,7 @@
guards.append(op)
elif self.level == LEVEL_KNOWNCLASS:
op = ResOperation(rop.GUARD_NONNULL, [box], None)
- guards.append(op)
+ guards.append(op)
op = ResOperation(rop.GUARD_CLASS, [box, self.known_class], None)
guards.append(op)
else:
@@ -100,7 +100,7 @@
self.make_constant(other.get_key_box())
optimizer.turned_constant(self)
elif other.level == LEVEL_KNOWNCLASS:
- self.make_constant_class(other.known_class, -1)
+ self.make_constant_class(other.known_class, None)
else:
if other.level == LEVEL_NONNULL:
self.ensure_nonnull()
@@ -112,15 +112,15 @@
self.lenbound.bound.intersect(other.lenbound.bound)
else:
self.lenbound = other.lenbound.clone()
-
- def force_box(self):
+
+ def force_box(self, optforce):
return self.box
def get_key_box(self):
return self.box
- def force_at_end_of_preamble(self, already_forced):
+ def force_at_end_of_preamble(self, already_forced, optforce):
return self
def get_args_for_fail(self, modifier):
@@ -146,7 +146,7 @@
assert isinstance(constbox, Const)
self.box = constbox
self.level = LEVEL_CONSTANT
-
+
if isinstance(constbox, ConstInt):
val = constbox.getint()
self.intbound = IntBound(val, val)
@@ -162,16 +162,16 @@
else:
return None
- def make_constant_class(self, classbox, opindex):
+ def make_constant_class(self, classbox, guardop):
assert self.level < LEVEL_KNOWNCLASS
self.known_class = classbox
self.level = LEVEL_KNOWNCLASS
- self.last_guard_index = opindex
+ self.last_guard = guardop
- def make_nonnull(self, opindex):
+ def make_nonnull(self, guardop):
assert self.level < LEVEL_NONNULL
self.level = LEVEL_NONNULL
- self.last_guard_index = opindex
+ self.last_guard = guardop
def is_nonnull(self):
level = self.level
@@ -223,6 +223,9 @@
def __init__(self, box):
self.make_constant(box)
+ def __repr__(self):
+ return 'Constant(%r)' % (self.box,)
+
CONST_0 = ConstInt(0)
CONST_1 = ConstInt(1)
CVAL_ZERO = ConstantValue(CONST_0)
@@ -237,26 +240,12 @@
def __init__(self):
pass # make rpython happy
- def propagate_begin_forward(self):
- if self.next_optimization:
- self.next_optimization.propagate_begin_forward()
-
- def propagate_end_forward(self):
- if self.next_optimization:
- self.next_optimization.propagate_end_forward()
-
def propagate_forward(self, op):
raise NotImplementedError
def emit_operation(self, op):
self.next_optimization.propagate_forward(op)
- def test_emittable(self, op):
- return self.is_emittable(op)
-
- def is_emittable(self, op):
- return self.next_optimization.test_emittable(op)
-
# FIXME: Move some of these here?
def getvalue(self, box):
return self.optimizer.getvalue(box)
@@ -286,19 +275,19 @@
return self.optimizer.new_const_item(arraydescr)
def pure(self, opnum, args, result):
- op = ResOperation(opnum, args, result)
- key = self.optimizer.make_args_key(op)
- if key not in self.optimizer.pure_operations:
- self.optimizer.pure_operations[key] = op
+ if self.optimizer.optpure:
+ self.optimizer.optpure.pure(opnum, args, result)
def has_pure_result(self, opnum, args, descr):
- op = ResOperation(opnum, args, None, descr)
- key = self.optimizer.make_args_key(op)
- op = self.optimizer.pure_operations.get(key, None)
- if op is None:
- return False
- return op.getdescr() is descr
+ if self.optimizer.optpure:
+ return self.optimizer.optpure.has_pure_result(opnum, args, descr)
+ return False
+ def get_pure_result(self, key):
+ if self.optimizer.optpure:
+ return self.optimizer.optpure.get_pure_result(key)
+ return None
+
def setup(self):
pass
@@ -319,6 +308,9 @@
def produce_potential_short_preamble_ops(self, potential_ops):
pass
+ def forget_numberings(self, box):
+ self.optimizer.forget_numberings(box)
+
class Optimizer(Optimization):
def __init__(self, metainterp_sd, loop, optimizations=None, bridge=False):
@@ -330,14 +322,16 @@
self.interned_refs = self.cpu.ts.new_ref_dict()
self.resumedata_memo = resume.ResumeDataLoopMemo(metainterp_sd)
self.bool_boxes = {}
- self.pure_operations = args_dict()
self.producer = {}
self.pendingfields = []
- self.posponedop = None
self.exception_might_have_happened = False
self.quasi_immutable_deps = None
self.opaque_pointers = {}
- self.newoperations = []
+ self.replaces_guard = {}
+ self._newoperations = []
+ self.optimizer = self
+ self.optpure = None
+ self.optearlyforce = None
if loop is not None:
self.call_pure_results = loop.call_pure_results
@@ -366,21 +360,20 @@
def flush(self):
for o in self.optimizations:
o.flush()
- assert self.posponedop is None
def new(self):
new = Optimizer(self.metainterp_sd, self.loop)
return self._new(new)
def _new(self, new):
- assert self.posponedop is None
optimizations = [o.new() for o in self.optimizations]
new.set_optimizations(optimizations)
new.quasi_immutable_deps = self.quasi_immutable_deps
return new
-
+
def produce_potential_short_preamble_ops(self, sb):
- raise NotImplementedError('This is implemented in unroll.UnrollableOptimizer')
+ for opt in self.optimizations:
+ opt.produce_potential_short_preamble_ops(sb)
def turned_constant(self, value):
for o in self.optimizations:
@@ -430,6 +423,13 @@
return constbox
return None
+ def get_newoperations(self):
+ self.flush()
+ return self._newoperations
+
+ def clear_newoperations(self):
+ self._newoperations = []
+
def make_equal_to(self, box, value, replace=False):
assert isinstance(value, OptValue)
assert replace or box not in self.values
@@ -478,15 +478,10 @@
def propagate_all_forward(self):
self.exception_might_have_happened = self.bridge
- self.newoperations = []
- self.first_optimization.propagate_begin_forward()
- self.i = 0
- while self.i < len(self.loop.operations):
- op = self.loop.operations[self.i]
+ self.clear_newoperations()
+ for op in self.loop.operations:
self.first_optimization.propagate_forward(op)
- self.i += 1
- self.first_optimization.propagate_end_forward()
- self.loop.operations = self.newoperations
+ self.loop.operations = self.get_newoperations()
self.loop.quasi_immutable_deps = self.quasi_immutable_deps
# accumulate counters
self.resumedata_memo.update_counters(self.metainterp_sd.profiler)
@@ -498,16 +493,13 @@
self.producer[op.result] = op
dispatch_opt(self, op)
- def test_emittable(self, op):
- return True
-
def emit_operation(self, op):
if op.returns_bool_result():
self.bool_boxes[self.getvalue(op.result)] = None
self._emit_operation(op)
-
+
@specialize.argtype(0)
- def _emit_operation(self, op):
+ def _emit_operation(self, op):
for i in range(op.numargs()):
arg = op.getarg(i)
try:
@@ -516,14 +508,30 @@
pass
else:
self.ensure_imported(value)
- op.setarg(i, value.force_box())
+ op.setarg(i, value.force_box(self))
self.metainterp_sd.profiler.count(jitprof.OPT_OPS)
if op.is_guard():
self.metainterp_sd.profiler.count(jitprof.OPT_GUARDS)
- op = self.store_final_boxes_in_guard(op)
+ if self.replaces_guard and op in self.replaces_guard:
+ self.replace_op(self.replaces_guard[op], op)
+ del self.replaces_guard[op]
+ return
+ else:
+ op = self.store_final_boxes_in_guard(op)
elif op.can_raise():
self.exception_might_have_happened = True
- self.newoperations.append(op)
+ self._newoperations.append(op)
+
+ def replace_op(self, old_op, new_op):
+ # XXX: Do we want to cache indexes to prevent search?
+ i = len(self._newoperations)
+ while i > 0:
+ i -= 1
+ if self._newoperations[i] is old_op:
+ self._newoperations[i] = new_op
+ break
+ else:
+ assert False
def store_final_boxes_in_guard(self, op):
descr = op.getdescr()
@@ -568,55 +576,12 @@
arg = value.get_key_box()
args[i] = arg
args[n] = ConstInt(op.getopnum())
- args[n+1] = op.getdescr()
+ args[n + 1] = op.getdescr()
return args
- @specialize.argtype(0)
def optimize_default(self, op):
- canfold = op.is_always_pure()
- if op.is_ovf():
- self.posponedop = op
- return
- if self.posponedop:
- nextop = op
- op = self.posponedop
- self.posponedop = None
- canfold = nextop.getopnum() == rop.GUARD_NO_OVERFLOW
- else:
- nextop = None
+ self.emit_operation(op)
- if canfold:
- for i in range(op.numargs()):
- if self.get_constant_box(op.getarg(i)) is None:
- break
- else:
- # all constant arguments: constant-fold away
- resbox = self.constant_fold(op)
- # note that INT_xxx_OVF is not done from here, and the
- # overflows in the INT_xxx operations are ignored
- self.make_constant(op.result, resbox)
- return
-
- # did we do the exact same operation already?
- args = self.make_args_key(op)
- oldop = self.pure_operations.get(args, None)
- if oldop is not None and oldop.getdescr() is op.getdescr():
- assert oldop.getopnum() == op.getopnum()
- self.make_equal_to(op.result, self.getvalue(oldop.result),
- True)
- return
- else:
- self.pure_operations[args] = op
- self.remember_emitting_pure(op)
-
- # otherwise, the operation remains
- self.emit_operation(op)
- if nextop:
- self.emit_operation(nextop)
-
- def remember_emitting_pure(self, op):
- pass
-
def constant_fold(self, op):
argboxes = [self.get_constant_box(op.getarg(i))
for i in range(op.numargs())]
@@ -633,11 +598,6 @@
def optimize_DEBUG_MERGE_POINT(self, op):
self.emit_operation(op)
- def optimize_CAST_OPAQUE_PTR(self, op):
- value = self.getvalue(op.getarg(0))
- self.opaque_pointers[value] = True
- self.make_equal_to(op.result, value)
-
def optimize_GETARRAYITEM_GC_PURE(self, op):
indexvalue = self.getvalue(op.getarg(1))
if indexvalue.is_constant():
@@ -658,9 +618,9 @@
arrayvalue = self.getvalue(op.getarg(0))
arrayvalue.make_len_gt(MODE_UNICODE, op.getdescr(), indexvalue.box.getint())
self.optimize_default(op)
-
-
+
+
dispatch_opt = make_dispatcher_method(Optimizer, 'optimize_',
default=Optimizer.optimize_default)
diff --git a/pypy/jit/metainterp/optimizeopt/pure.py b/pypy/jit/metainterp/optimizeopt/pure.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/optimizeopt/pure.py
@@ -0,0 +1,115 @@
+from pypy.jit.metainterp.optimizeopt.optimizer import Optimization
+from pypy.jit.metainterp.resoperation import rop, ResOperation
+from pypy.jit.metainterp.optimizeopt.util import (make_dispatcher_method,
+ args_dict)
+
+class OptPure(Optimization):
+ def __init__(self):
+ self.posponedop = None
+ self.pure_operations = args_dict()
+ self.emitted_pure_operations = {}
+
+ def propagate_forward(self, op):
+ dispatch_opt(self, op)
+
+ def optimize_default(self, op):
+ canfold = op.is_always_pure()
+ if op.is_ovf():
+ self.posponedop = op
+ return
+ if self.posponedop:
+ nextop = op
+ op = self.posponedop
+ self.posponedop = None
+ canfold = nextop.getopnum() == rop.GUARD_NO_OVERFLOW
+ else:
+ nextop = None
+
+ if canfold:
+ for i in range(op.numargs()):
+ if self.get_constant_box(op.getarg(i)) is None:
+ break
+ else:
+ # all constant arguments: constant-fold away
+ resbox = self.optimizer.constant_fold(op)
+ # note that INT_xxx_OVF is not done from here, and the
+ # overflows in the INT_xxx operations are ignored
+ self.optimizer.make_constant(op.result, resbox)
+ return
+
+ # did we do the exact same operation already?
+ args = self.optimizer.make_args_key(op)
+ oldop = self.pure_operations.get(args, None)
+ if oldop is not None and oldop.getdescr() is op.getdescr():
+ assert oldop.getopnum() == op.getopnum()
+ self.optimizer.make_equal_to(op.result, self.getvalue(oldop.result),
+ True)
+ return
+ else:
+ self.pure_operations[args] = op
+ self.remember_emitting_pure(op)
+
+ # otherwise, the operation remains
+ self.emit_operation(op)
+ if op.returns_bool_result():
+ self.optimizer.bool_boxes[self.getvalue(op.result)] = None
+ if nextop:
+ self.emit_operation(nextop)
+
+ def optimize_CALL_PURE(self, op):
+ args = self.optimizer.make_args_key(op)
+ oldop = self.pure_operations.get(args, None)
+ if oldop is not None and oldop.getdescr() is op.getdescr():
+ assert oldop.getopnum() == op.getopnum()
+ self.make_equal_to(op.result, self.getvalue(oldop.result))
+ return
+ else:
+ self.pure_operations[args] = op
+ self.remember_emitting_pure(op)
+
+ # replace CALL_PURE with just CALL
+ args = op.getarglist()
+ self.emit_operation(ResOperation(rop.CALL, args, op.result,
+ op.getdescr()))
+
+ def flush(self):
+ assert self.posponedop is None
+
+ def new(self):
+ assert self.posponedop is None
+ return OptPure()
+
+ def setup(self):
+ self.optimizer.optpure = self
+
+ def pure(self, opnum, args, result):
+ op = ResOperation(opnum, args, result)
+ key = self.optimizer.make_args_key(op)
+ if key not in self.pure_operations:
+ self.pure_operations[key] = op
+
+ def has_pure_result(self, opnum, args, descr):
+ op = ResOperation(opnum, args, None, descr)
+ key = self.optimizer.make_args_key(op)
+ op = self.pure_operations.get(key, None)
+ if op is None:
+ return False
+ return op.getdescr() is descr
+
+ def get_pure_result(self, key):
+ return self.pure_operations.get(key, None)
+
+ def remember_emitting_pure(self, op):
+ self.emitted_pure_operations[op] = True
+
+ def produce_potential_short_preamble_ops(self, sb):
+ for op in self.emitted_pure_operations:
+ if op.getopnum() == rop.GETARRAYITEM_GC_PURE or \
+ op.getopnum() == rop.STRGETITEM or \
+ op.getopnum() == rop.UNICODEGETITEM:
+ if not self.getvalue(op.getarg(1)).is_constant():
+ continue
+ sb.add_potential(op)
+
+dispatch_opt = make_dispatcher_method(OptPure, 'optimize_',
+ default=OptPure.optimize_default)
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -19,7 +19,7 @@
def new(self):
return OptRewrite()
-
+
def produce_potential_short_preamble_ops(self, sb):
for op in self.loop_invariant_producer.values():
sb.add_potential(op)
@@ -31,21 +31,8 @@
dispatch_opt(self, op)
- def test_emittable(self, op):
- opnum = op.getopnum()
- for value, cls, func in optimize_guards:
- if opnum == value:
- assert isinstance(op, cls)
- try:
- func(self, op, dryrun=True)
- return self.is_emittable(op)
- except InvalidLoop:
- return False
- return self.is_emittable(op)
-
-
def try_boolinvers(self, op, targs):
- oldop = self.optimizer.pure_operations.get(targs, None)
+ oldop = self.get_pure_result(targs)
if oldop is not None and oldop.getdescr() is op.getdescr():
value = self.getvalue(oldop.result)
if value.is_constant():
@@ -62,32 +49,35 @@
def find_rewritable_bool(self, op, args):
try:
oldopnum = opboolinvers[op.getopnum()]
+ except KeyError:
+ pass
+ else:
targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[0], args[1]],
None))
if self.try_boolinvers(op, targs):
return True
- except KeyError:
- pass
try:
oldopnum = opboolreflex[op.getopnum()] # FIXME: add INT_ADD, INT_MUL
+ except KeyError:
+ pass
+ else:
targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
None))
- oldop = self.optimizer.pure_operations.get(targs, None)
+ oldop = self.get_pure_result(targs)
if oldop is not None and oldop.getdescr() is op.getdescr():
self.make_equal_to(op.result, self.getvalue(oldop.result))
return True
- except KeyError:
- pass
try:
oldopnum = opboolinvers[opboolreflex[op.getopnum()]]
+ except KeyError:
+ pass
+ else:
targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
None))
if self.try_boolinvers(op, targs):
return True
- except KeyError:
- pass
return False
@@ -214,29 +204,7 @@
self.emit_operation(op)
self.pure(rop.FLOAT_NEG, [op.result], v1)
- def optimize_CALL_PURE(self, op):
- arg_consts = []
- for i in range(op.numargs()):
- arg = op.getarg(i)
- const = self.get_constant_box(arg)
- if const is None:
- break
- arg_consts.append(const)
- else:
- # all constant arguments: check if we already know the result
- try:
- result = self.optimizer.call_pure_results[arg_consts]
- except KeyError:
- pass
- else:
- self.make_constant(op.result, result)
- return
- # replace CALL_PURE with just CALL
- args = op.getarglist()
- self.emit_operation(ResOperation(rop.CALL, args, op.result,
- op.getdescr()))
-
- def optimize_guard(self, op, constbox, emit_operation=True, dryrun=False):
+ def optimize_guard(self, op, constbox, emit_operation=True):
value = self.getvalue(op.getarg(0))
if value.is_constant():
box = value.box
@@ -244,62 +212,57 @@
if not box.same_constant(constbox):
raise InvalidLoop
return
- if dryrun: return
if emit_operation:
self.emit_operation(op)
value.make_constant(constbox)
self.optimizer.turned_constant(value)
- def optimize_GUARD_ISNULL(self, op, dryrun=False):
+ def optimize_GUARD_ISNULL(self, op):
value = self.getvalue(op.getarg(0))
if value.is_null():
return
elif value.is_nonnull():
raise InvalidLoop
- if dryrun: return
self.emit_operation(op)
value.make_constant(self.optimizer.cpu.ts.CONST_NULL)
- def optimize_GUARD_NONNULL(self, op, dryrun=False):
+ def optimize_GUARD_NONNULL(self, op):
value = self.getvalue(op.getarg(0))
if value.is_nonnull():
return
elif value.is_null():
raise InvalidLoop
- if dryrun: return
self.emit_operation(op)
- value.make_nonnull(len(self.optimizer.newoperations) - 1)
+ value.make_nonnull(op)
- def optimize_GUARD_VALUE(self, op, dryrun=False):
+ def optimize_GUARD_VALUE(self, op):
value = self.getvalue(op.getarg(0))
- emit_operation = True
- if not dryrun and value.last_guard_index != -1:
+ if value.last_guard:
# there already has been a guard_nonnull or guard_class or
# guard_nonnull_class on this value, which is rather silly.
# replace the original guard with a guard_value
- old_guard_op = self.optimizer.newoperations[value.last_guard_index]
- new_guard_op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
- args = [old_guard_op.getarg(0), op.getarg(1)])
- self.optimizer.newoperations[value.last_guard_index] = new_guard_op
+ old_guard_op = value.last_guard
+ op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
+ args = [old_guard_op.getarg(0), op.getarg(1)])
+ self.optimizer.replaces_guard[op] = old_guard_op
# hack hack hack. Change the guard_opnum on
# new_guard_op.getdescr() so that when resuming,
# the operation is not skipped by pyjitpl.py.
- descr = new_guard_op.getdescr()
+ descr = op.getdescr()
assert isinstance(descr, compile.ResumeGuardDescr)
descr.guard_opnum = rop.GUARD_VALUE
- descr.make_a_counter_per_value(new_guard_op)
- emit_operation = False
+ descr.make_a_counter_per_value(op)
constbox = op.getarg(1)
assert isinstance(constbox, Const)
- self.optimize_guard(op, constbox, emit_operation, dryrun)
+ self.optimize_guard(op, constbox)
- def optimize_GUARD_TRUE(self, op, dryrun=False):
- self.optimize_guard(op, CONST_1, dryrun=dryrun)
+ def optimize_GUARD_TRUE(self, op):
+ self.optimize_guard(op, CONST_1)
- def optimize_GUARD_FALSE(self, op, dryrun=False):
- self.optimize_guard(op, CONST_0, dryrun=dryrun)
+ def optimize_GUARD_FALSE(self, op):
+ self.optimize_guard(op, CONST_0)
- def optimize_GUARD_CLASS(self, op, dryrun=False):
+ def optimize_GUARD_CLASS(self, op):
value = self.getvalue(op.getarg(0))
expectedclassbox = op.getarg(1)
assert isinstance(expectedclassbox, Const)
@@ -308,38 +271,32 @@
if realclassbox.same_constant(expectedclassbox):
return
raise InvalidLoop
- if dryrun: return
- emit_operation = True
- if value.last_guard_index != -1:
+ if value.last_guard:
# there already has been a guard_nonnull or guard_class or
# guard_nonnull_class on this value.
- old_guard_op = self.optimizer.newoperations[value.last_guard_index]
+ old_guard_op = value.last_guard
if old_guard_op.getopnum() == rop.GUARD_NONNULL:
# it was a guard_nonnull, which we replace with a
# guard_nonnull_class.
- new_guard_op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
+ op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
args = [old_guard_op.getarg(0), op.getarg(1)])
- self.optimizer.newoperations[value.last_guard_index] = new_guard_op
+ self.optimizer.replaces_guard[op] = old_guard_op
# hack hack hack. Change the guard_opnum on
# new_guard_op.getdescr() so that when resuming,
# the operation is not skipped by pyjitpl.py.
- descr = new_guard_op.getdescr()
+ descr = op.getdescr()
assert isinstance(descr, compile.ResumeGuardDescr)
descr.guard_opnum = rop.GUARD_NONNULL_CLASS
- emit_operation = False
- if emit_operation:
- self.emit_operation(op)
- last_guard_index = len(self.optimizer.newoperations) - 1
- else:
- last_guard_index = value.last_guard_index
- value.make_constant_class(expectedclassbox, last_guard_index)
+ self.emit_operation(op)
+ value.make_constant_class(expectedclassbox, op)
- def optimize_GUARD_NONNULL_CLASS(self, op, dryrun=False):
- self.optimize_GUARD_NONNULL(op, True)
- self.optimize_GUARD_CLASS(op, dryrun)
+ def optimize_GUARD_NONNULL_CLASS(self, op):
+ value = self.getvalue(op.getarg(0))
+ if value.is_null():
+ raise InvalidLoop
+ self.optimize_GUARD_CLASS(op)
- def optimize_GUARD_NO_EXCEPTION(self, op, dryrun=False):
- if dryrun: return
+ def optimize_GUARD_NO_EXCEPTION(self, op):
if not self.optimizer.exception_might_have_happened:
return
self.emit_operation(op)
@@ -351,7 +308,7 @@
# expects a compile-time constant
assert isinstance(arg, Const)
key = make_hashable_int(arg.getint())
-
+
resvalue = self.loop_invariant_results.get(key, None)
if resvalue is not None:
self.make_equal_to(op.result, resvalue)
@@ -459,7 +416,7 @@
newop = ResOperation(rop.SETARRAYITEM_GC,
[op.getarg(2),
ConstInt(index + dest_start),
- val.force_box()], None,
+ val.get_key_box()], None,
descr=source_value.arraydescr)
self.emit_operation(newop)
return True
@@ -467,6 +424,25 @@
return True # 0-length arraycopy
return False
+ def optimize_CALL_PURE(self, op):
+ arg_consts = []
+ for i in range(op.numargs()):
+ arg = op.getarg(i)
+ const = self.get_constant_box(arg)
+ if const is None:
+ break
+ arg_consts.append(const)
+ else:
+ # all constant arguments: check if we already know the result
+ try:
+ result = self.optimizer.call_pure_results[arg_consts]
+ except KeyError:
+ pass
+ else:
+ self.make_constant(op.result, result)
+ return
+ self.emit_operation(op)
+
def optimize_INT_FLOORDIV(self, op):
v1 = self.getvalue(op.getarg(0))
v2 = self.getvalue(op.getarg(1))
@@ -481,6 +457,10 @@
args = [op.getarg(0), ConstInt(highest_bit(val))])
self.emit_operation(op)
+ def optimize_CAST_OPAQUE_PTR(self, op):
+ value = self.getvalue(op.getarg(0))
+ self.optimizer.opaque_pointers[value] = True
+ self.make_equal_to(op.result, value)
dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_',
default=OptRewrite.emit_operation)
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -39,8 +39,9 @@
def test_sharing_field_lists_of_virtual():
class FakeOptimizer(object):
- class cpu(object):
- pass
+ class optimizer(object):
+ class cpu(object):
+ pass
opt = FakeOptimizer()
virt1 = virtualize.AbstractVirtualStructValue(opt, None)
lst1 = virt1._get_field_descr_list()
@@ -69,7 +70,7 @@
class FakeVirtualValue(virtualize.AbstractVirtualValue):
def _make_virtual(self, *args):
return FakeVInfo()
- v1 = FakeVirtualValue(None, None, None)
+ v1 = FakeVirtualValue(None, None)
vinfo1 = v1.make_virtual_info(None, [1, 2, 4])
vinfo2 = v1.make_virtual_info(None, [1, 2, 4])
assert vinfo1 is vinfo2
@@ -111,7 +112,7 @@
class BaseTestBasic(BaseTest):
- enable_opts = "intbounds:rewrite:virtualize:string:heap"
+ enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap"
def optimize_loop(self, ops, optops, call_pure_results=None):
@@ -651,8 +652,8 @@
i3 = getfield_gc(p3, descr=valuedescr)
escape(i3)
p1 = new_with_vtable(ConstClass(node_vtable))
+ p1sub = new_with_vtable(ConstClass(node_vtable2))
setfield_gc(p1, i1, descr=valuedescr)
- p1sub = new_with_vtable(ConstClass(node_vtable2))
setfield_gc(p1sub, i1, descr=valuedescr)
setfield_gc(p1, p1sub, descr=nextdescr)
jump(i1, p1, p2)
@@ -667,10 +668,10 @@
p3sub = getfield_gc(p3, descr=nextdescr)
i3 = getfield_gc(p3sub, descr=valuedescr)
escape(i3)
+ p1 = new_with_vtable(ConstClass(node_vtable))
p2sub = new_with_vtable(ConstClass(node_vtable2))
setfield_gc(p2sub, i1, descr=valuedescr)
setfield_gc(p2, p2sub, descr=nextdescr)
- p1 = new_with_vtable(ConstClass(node_vtable))
jump(i1, p1, p2)
"""
# The same as test_p123_simple, but in the end the "old" p2 contains
@@ -4711,6 +4712,35 @@
"""
self.optimize_loop(ops, expected)
+ def test_empty_copystrunicontent(self):
+ ops = """
+ [p0, p1, i0, i2, i3]
+ i4 = int_eq(i3, 0)
+ guard_true(i4) []
+ copystrcontent(p0, p1, i0, i2, i3)
+ jump(p0, p1, i0, i2, i3)
+ """
+ expected = """
+ [p0, p1, i0, i2, i3]
+ i4 = int_eq(i3, 0)
+ guard_true(i4) []
+ jump(p0, p1, i0, i2, 0)
+ """
+ self.optimize_strunicode_loop(ops, expected)
+
+ def test_empty_copystrunicontent_virtual(self):
+ ops = """
+ [p0]
+ p1 = newstr(23)
+ copystrcontent(p0, p1, 0, 0, 0)
+ jump(p0)
+ """
+ expected = """
+ [p0]
+ jump(p0)
+ """
+ self.optimize_strunicode_loop(ops, expected)
+
def test_forced_virtuals_aliasing(self):
ops = """
[i0, i1]
@@ -4738,6 +4768,27 @@
# other
self.optimize_loop(ops, expected)
+ def test_plain_virtual_string_copy_content(self):
+ ops = """
+ []
+ p0 = newstr(6)
+ copystrcontent(s"hello!", p0, 0, 0, 6)
+ p1 = call(0, p0, s"abc123", descr=strconcatdescr)
+ i0 = strgetitem(p1, 0)
+ finish(i0)
+ """
+ expected = """
+ []
+ p0 = newstr(6)
+ copystrcontent(s"hello!", p0, 0, 0, 6)
+ p1 = newstr(12)
+ copystrcontent(p0, p1, 0, 0, 6)
+ copystrcontent(s"abc123", p1, 0, 6, 6)
+ i0 = strgetitem(p1, 0)
+ finish(i0)
+ """
+ self.optimize_strunicode_loop(ops, expected)
+
class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
pass
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
@@ -36,7 +36,7 @@
class TestFfiCall(BaseTestBasic, LLtypeMixin):
- enable_opts = "intbounds:rewrite:virtualize:string:heap:ffi"
+ enable_opts = "intbounds:rewrite:virtualize:string:pure:earlyforce:heap:ffi"
class namespace:
cpu = LLtypeMixin.cpu
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -68,7 +68,7 @@
class BaseTestWithUnroll(BaseTest):
- enable_opts = "intbounds:rewrite:virtualize:string:heap:unroll"
+ enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll"
def optimize_loop(self, ops, expected, expected_preamble=None,
call_pure_results=None, expected_short=None):
@@ -102,9 +102,9 @@
print "Short Preamble:"
short = loop.preamble.token.short_preamble[0]
print short.inputargs
- print '\n'.join([str(o) for o in short.operations])
+ print '\n'.join([str(o) for o in short.operations])
print
-
+
assert expected != "crash!", "should have raised an exception"
self.assert_equal(loop, expected)
if expected_preamble:
@@ -113,7 +113,7 @@
if expected_short:
self.assert_equal(short, expected_short,
text_right='expected short preamble')
-
+
return loop
class OptimizeOptTest(BaseTestWithUnroll):
@@ -825,8 +825,8 @@
i3 = getfield_gc(p2, descr=valuedescr)
escape(i3)
p4 = new_with_vtable(ConstClass(node_vtable))
+ p1sub = new_with_vtable(ConstClass(node_vtable2))
setfield_gc(p4, i1, descr=valuedescr)
- p1sub = new_with_vtable(ConstClass(node_vtable2))
setfield_gc(p1sub, i1, descr=valuedescr)
setfield_gc(p4, p1sub, descr=nextdescr)
jump(i1, p4)
@@ -865,13 +865,7 @@
p3sub = new_with_vtable(ConstClass(node_vtable2))
setfield_gc(p3sub, i1, descr=valuedescr)
setfield_gc(p1, p3sub, descr=nextdescr)
- # XXX: We get two extra operations here because the setfield
- # above is the result of forcing p1 and thus not
- # registered with the heap optimizer. I've makred tests
- # below with VIRTUALHEAP if they suffer from this issue
- p3sub2 = getfield_gc(p1, descr=nextdescr)
- guard_nonnull_class(p3sub2, ConstClass(node_vtable2)) []
- jump(i1, p1, p3sub2)
+ jump(i1, p1, p3sub)
"""
self.optimize_loop(ops, expected, preamble)
@@ -902,9 +896,7 @@
guard_true(i2b) []
p3 = new_with_vtable(ConstClass(node_vtable))
setfield_gc(p3, i2, descr=nextdescr)
- # XXX: VIRTUALHEAP (see above)
- i3 = getfield_gc(p3, descr=nextdescr)
- jump(p3, i3)
+ jump(p3, i2)
"""
self.optimize_loop(ops, expected, preamble)
@@ -1219,7 +1211,15 @@
setfield_gc(p3, p30, descr=valuedescr)
jump(i29, p30, p3)
"""
- expected = preamble
+ expected = """
+ [i0, p1, p3]
+ i28 = int_add(i0, 1)
+ i29 = int_add(i28, 1)
+ p30 = new_with_vtable(ConstClass(node_vtable))
+ setfield_gc(p3, p30, descr=valuedescr)
+ setfield_gc(p30, i28, descr=nextdescr)
+ jump(i29, p30, p3)
+ """
self.optimize_loop(ops, expected, preamble)
def test_nonvirtual_1(self):
@@ -1411,7 +1411,7 @@
guard_isnull(p18) [p0, p8]
p31 = new(descr=ssize)
p35 = new_with_vtable(ConstClass(node_vtable))
- setfield_gc(p35, p31, descr=valuedescr)
+ setfield_gc(p35, p31, descr=valuedescr)
jump(p0, p35)
"""
expected = """
@@ -1426,7 +1426,7 @@
guard_isnull(p18) [p0, p8]
p31 = new(descr=ssize)
p35 = new_with_vtable(ConstClass(node_vtable))
- setfield_gc(p35, p31, descr=valuedescr)
+ setfield_gc(p35, p31, descr=valuedescr)
jump(p0, p35, p19, p18)
"""
expected = """
@@ -1435,7 +1435,7 @@
jump(p0, NULL)
"""
self.optimize_loop(ops, expected)
-
+
def test_varray_1(self):
ops = """
[i1]
@@ -2181,7 +2181,7 @@
jump(p1)
"""
self.optimize_loop(ops, expected)
-
+
def test_duplicate_getarrayitem_2(self):
ops = """
[p1, i0]
@@ -2199,7 +2199,7 @@
jump(p1, i7, i6)
"""
self.optimize_loop(ops, expected)
-
+
def test_duplicate_getarrayitem_after_setarrayitem_1(self):
ops = """
[p1, p2]
@@ -2408,8 +2408,8 @@
guard_class(p2, ConstClass(node_vtable)) []
p3 = getfield_gc(p1, descr=otherdescr)
guard_class(p3, ConstClass(node_vtable)) []
+ p3a = new_with_vtable(ConstClass(node_vtable))
setfield_gc(p3, p2, descr=otherdescr)
- p3a = new_with_vtable(ConstClass(node_vtable))
escape(p3a)
jump(p3a)
"""
@@ -2421,9 +2421,9 @@
# setfield_gc(p3, p2, descr=otherdescr) # p3a.other = p2a
# p1a = new_with_vtable(ConstClass(node_vtable2))
# p2a = new_with_vtable(ConstClass(node_vtable))
+ p3anew = new_with_vtable(ConstClass(node_vtable))
p2 = new_with_vtable(ConstClass(node_vtable))
setfield_gc(p3a, p2, descr=otherdescr) # p3a.other = p2a
- p3anew = new_with_vtable(ConstClass(node_vtable))
escape(p3anew)
jump(p3anew)
"""
@@ -2458,9 +2458,9 @@
p3 = getfield_gc(p1, descr=otherdescr)
guard_class(p3, ConstClass(node_vtable)) []
# p1a = new_with_vtable(ConstClass(node_vtable2))
+ p3a = new_with_vtable(ConstClass(node_vtable))
p2a = new_with_vtable(ConstClass(node_vtable))
setfield_gc(p3, p2a, descr=otherdescr)
- p3a = new_with_vtable(ConstClass(node_vtable))
escape(p3a)
# setfield_gc(p1a, p2a, descr=nextdescr)
# setfield_gc(p1a, p3a, descr=otherdescr)
@@ -2468,9 +2468,9 @@
"""
expected = """
[p2, p3]
+ p3a = new_with_vtable(ConstClass(node_vtable))
p2a = new_with_vtable(ConstClass(node_vtable))
setfield_gc(p3, p2a, descr=otherdescr)
- p3a = new_with_vtable(ConstClass(node_vtable))
escape(p3a)
jump(p2a, p3a)
"""
@@ -2790,7 +2790,6 @@
self.optimize_loop(ops, expected, preamble)
def test_remove_duplicate_pure_op_ovf_with_lazy_setfield(self):
- py.test.skip('this optimization is not yet supprted')
ops = """
[i1, p1]
i3 = int_add_ovf(i1, 1)
@@ -2812,14 +2811,14 @@
guard_no_overflow() []
i3b = int_is_true(i3)
guard_true(i3b) []
- setfield_gc(p1, i1, descr=valuedescr)
+ setfield_gc(p1, i1, descr=valuedescr)
escape(i3)
escape(i3)
jump(i1, p1, i3)
"""
expected = """
[i1, p1, i3]
- setfield_gc(p1, i1, descr=valuedescr)
+ setfield_gc(p1, i1, descr=valuedescr)
escape(i3)
escape(i3)
jump(i1, p1, i3)
@@ -2830,7 +2829,7 @@
ops = """
[p8, p11, i24]
p26 = new_with_vtable(ConstClass(node_vtable))
- setfield_gc(p26, i24, descr=adescr)
+ setfield_gc(p26, i24, descr=adescr)
i34 = getfield_gc_pure(p11, descr=valuedescr)
i35 = getfield_gc_pure(p26, descr=adescr)
i36 = int_add_ovf(i34, i35)
@@ -2839,10 +2838,10 @@
"""
expected = """
[p8, p11, i26]
- jump(p8, p11, i26)
- """
- self.optimize_loop(ops, expected)
-
+ jump(p8, p11, i26)
+ """
+ self.optimize_loop(ops, expected)
+
def test_ovf_guard_in_short_preamble2(self):
ops = """
[p8, p11, p12]
@@ -3191,13 +3190,18 @@
jump(p1, i4, i3)
'''
expected = '''
+ [p1, i4, i3, i5]
+ setfield_gc(p1, i5, descr=valuedescr)
+ jump(p1, i3, i5, i5)
+ '''
+ preamble = '''
[p1, i1, i4]
setfield_gc(p1, i1, descr=valuedescr)
i3 = call(p1, descr=plaincalldescr)
setfield_gc(p1, i3, descr=valuedescr)
- jump(p1, i4, i3)
+ jump(p1, i4, i3, i3)
'''
- self.optimize_loop(ops, expected, expected)
+ self.optimize_loop(ops, expected, preamble)
def test_call_pure_invalidates_heap_knowledge(self):
# CALL_PURE should still force the setfield_gc() to occur before it
@@ -3209,21 +3213,20 @@
jump(p1, i4, i3)
'''
expected = '''
+ [p1, i4, i3, i5]
+ setfield_gc(p1, i4, descr=valuedescr)
+ jump(p1, i3, i5, i5)
+ '''
+ preamble = '''
[p1, i1, i4]
setfield_gc(p1, i1, descr=valuedescr)
i3 = call(p1, descr=plaincalldescr)
setfield_gc(p1, i1, descr=valuedescr)
- jump(p1, i4, i3)
+ jump(p1, i4, i3, i3)
'''
- self.optimize_loop(ops, expected, expected)
+ self.optimize_loop(ops, expected, preamble)
def test_call_pure_constant_folding(self):
- # CALL_PURE is not marked as is_always_pure(), because it is wrong
- # to call the function arbitrary many times at arbitrary points in
- # time. Check that it is either constant-folded (and replaced by
- # the result of the call, recorded as the first arg), or turned into
- # a regular CALL.
- # XXX can this test be improved with unrolling?
arg_consts = [ConstInt(i) for i in (123456, 4, 5, 6)]
call_pure_results = {tuple(arg_consts): ConstInt(42)}
ops = '''
@@ -3239,14 +3242,13 @@
escape(i1)
escape(i2)
i4 = call(123456, 4, i0, 6, descr=plaincalldescr)
- jump(i0, i4)
+ jump(i0, i4, i4)
'''
expected = '''
- [i0, i2]
+ [i0, i4, i5]
escape(42)
- escape(i2)
- i4 = call(123456, 4, i0, 6, descr=plaincalldescr)
- jump(i0, i4)
+ escape(i4)
+ jump(i0, i5, i5)
'''
self.optimize_loop(ops, expected, preamble, call_pure_results)
@@ -3270,18 +3272,43 @@
escape(i2)
i4 = call(123456, 4, i0, 6, descr=plaincalldescr)
guard_no_exception() []
- jump(i0, i4)
+ jump(i0, i4, i4)
'''
expected = '''
- [i0, i2]
+ [i0, i2, i3]
escape(42)
escape(i2)
- i4 = call(123456, 4, i0, 6, descr=plaincalldescr)
- guard_no_exception() []
- jump(i0, i4)
+ jump(i0, i3, i3)
'''
self.optimize_loop(ops, expected, preamble, call_pure_results)
+ def test_call_pure_returning_virtual(self):
+ # XXX: This kind of loop invaraint call_pure will be forced
+ # both in the preamble and in the peeled loop
+ ops = '''
+ [p1, i1, i2]
+ p2 = call_pure(0, p1, i1, i2, descr=strslicedescr)
+ escape(p2)
+ jump(p1, i1, i2)
+ '''
+ preamble = '''
+ [p1, i1, i2]
+ i6 = int_sub(i2, i1)
+ p2 = newstr(i6)
+ copystrcontent(p1, p2, i1, 0, i6)
+ escape(p2)
+ jump(p1, i1, i2, i6)
+ '''
+ expected = '''
+ [p1, i1, i2, i6]
+ p2 = newstr(i6)
+ copystrcontent(p1, p2, i1, 0, i6)
+ escape(p2)
+ jump(p1, i1, i2, i6)
+ '''
+ self.optimize_loop(ops, expected, preamble)
+
+
# ----------
def test_vref_nonvirtual_nonescape(self):
@@ -5150,14 +5177,14 @@
[i0, i1, i10, i11, i2, i3, i4]
escape(i2)
escape(i3)
- escape(i4)
+ escape(i4)
i24 = int_mul_ovf(i10, i11)
guard_no_overflow() []
i23 = int_sub_ovf(i10, i11)
guard_no_overflow() []
i22 = int_add_ovf(i10, i11)
guard_no_overflow() []
- jump(i0, i1, i10, i11, i2, i3, i4)
+ jump(i0, i1, i10, i11, i2, i3, i4)
"""
self.optimize_loop(ops, expected)
@@ -5366,6 +5393,8 @@
"""
self.optimize_strunicode_loop(ops, expected, expected)
+ # XXX Should some of the call's below now be call_pure?
+
def test_str_concat_1(self):
ops = """
[p1, p2]
@@ -5699,14 +5728,14 @@
ops = """
[p0, i0]
i1 = unicodegetitem(p0, i0)
- i10 = unicodegetitem(p0, i0)
+ i10 = unicodegetitem(p0, i0)
i2 = int_lt(i1, 0)
guard_false(i2) []
jump(p0, i0)
"""
expected = """
[p0, i0]
- i1 = unicodegetitem(p0, i0)
+ i1 = unicodegetitem(p0, i0)
jump(p0, i0)
"""
self.optimize_loop(ops, expected)
@@ -5865,7 +5894,7 @@
"""
preamble = """
[p1, i1, i2, p3]
- guard_nonnull(p3) []
+ guard_nonnull(p3) []
i4 = int_sub(i2, i1)
i0 = call(0, p1, i1, i4, p3, descr=streq_slice_nonnull_descr)
escape(i0)
@@ -5931,13 +5960,18 @@
escape(i0)
jump(p1)
"""
- expected = """
+ preamble = """
[p1]
i0 = ptr_eq(p1, NULL)
escape(i0)
- jump(p1)
- """
- self.optimize_strunicode_loop_extradescrs(ops, expected, expected)
+ jump(p1, i0)
+ """
+ expected = """
+ [p1, i0]
+ escape(i0)
+ jump(p1, i0)
+ """
+ self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
def test_str_equal_none2(self):
ops = """
@@ -5946,13 +5980,18 @@
escape(i0)
jump(p1)
"""
- expected = """
+ preamble = """
[p1]
i0 = ptr_eq(p1, NULL)
escape(i0)
- jump(p1)
- """
- self.optimize_strunicode_loop_extradescrs(ops, expected, expected)
+ jump(p1, i0)
+ """
+ expected = """
+ [p1, i0]
+ escape(i0)
+ jump(p1, i0)
+ """
+ self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
def test_str_equal_nonnull1(self):
ops = """
@@ -6474,7 +6513,7 @@
setfield_gc(p3, i1, descr=adescr)
setfield_gc(p3, i2, descr=bdescr)
i5 = int_gt(ii, 42)
- guard_true(i5) []
+ guard_true(i5) []
jump(p0, p1, p3, ii2, ii, i1, i2)
"""
self.optimize_loop(ops, expected)
@@ -6500,7 +6539,7 @@
p1 = getfield_gc(p0, descr=nextdescr)
guard_nonnull_class(p1, ConstClass(node_vtable)) []
p2 = getfield_gc(p1, descr=nextdescr)
- guard_nonnull_class(p2, ConstClass(node_vtable)) []
+ guard_nonnull_class(p2, ConstClass(node_vtable)) []
jump(p0)
"""
expected = """
@@ -6514,11 +6553,11 @@
guard_class(p1, ConstClass(node_vtable)) []
p2 = getfield_gc(p1, descr=nextdescr)
guard_nonnull(p2) []
- guard_class(p2, ConstClass(node_vtable)) []
+ guard_class(p2, ConstClass(node_vtable)) []
jump(p0)
"""
self.optimize_loop(ops, expected, expected_short=short)
-
+
def test_forced_virtual_pure_getfield(self):
ops = """
[p0]
@@ -6582,7 +6621,7 @@
jump(p1, i2)
"""
self.optimize_loop(ops, expected)
-
+
def test_loopinvariant_strlen(self):
ops = """
[p9]
@@ -6715,7 +6754,7 @@
[p0, p1]
p2 = new_with_vtable(ConstClass(node_vtable))
p3 = new_with_vtable(ConstClass(node_vtable))
- setfield_gc(p2, p3, descr=nextdescr)
+ setfield_gc(p2, p3, descr=nextdescr)
jump(p2, p3)
"""
expected = """
@@ -6734,7 +6773,7 @@
jump(p2, i2)
"""
expected = """
- [p1]
+ [p1]
p2 = getarrayitem_gc(p1, 7, descr=<GcPtrArrayDescr>)
i1 = arraylen_gc(p1)
jump(p2)
@@ -6775,8 +6814,8 @@
jump(p0, p2, p1)
"""
self.optimize_loop(ops, expected, expected_short=short)
-
-
+
+
def test_loopinvariant_constant_strgetitem(self):
ops = """
[p0]
@@ -6830,11 +6869,11 @@
expected = """
[p0, i22, p1]
call(i22, descr=nonwritedescr)
- i3 = unicodelen(p1) # Should be killed by backend
+ i3 = unicodelen(p1) # Should be killed by backend
jump(p0, i22, p1)
"""
self.optimize_loop(ops, expected, expected_short=short)
-
+
def test_propagate_virtual_arryalen(self):
ops = """
[p0]
@@ -6903,7 +6942,7 @@
[p0, p1, p10, p11]
i1 = arraylen_gc(p10, descr=arraydescr)
getarrayitem_gc(p11, 1, descr=arraydescr)
- call(i1, descr=nonwritedescr)
+ call(i1, descr=nonwritedescr)
jump(p1, p0, p11, p10)
"""
self.optimize_loop(ops, expected)
@@ -6912,20 +6951,20 @@
ops = """
[p5]
i10 = getfield_gc(p5, descr=valuedescr)
- call(i10, descr=nonwritedescr)
+ call(i10, descr=nonwritedescr)
setfield_gc(p5, 1, descr=valuedescr)
jump(p5)
"""
preamble = """
[p5]
i10 = getfield_gc(p5, descr=valuedescr)
- call(i10, descr=nonwritedescr)
+ call(i10, descr=nonwritedescr)
setfield_gc(p5, 1, descr=valuedescr)
jump(p5)
"""
expected = """
[p5]
- call(1, descr=nonwritedescr)
+ call(1, descr=nonwritedescr)
jump(p5)
"""
self.optimize_loop(ops, expected, preamble)
@@ -6963,7 +7002,7 @@
[p9]
call_assembler(0, descr=asmdescr)
i18 = getfield_gc(p9, descr=valuedescr)
- guard_value(i18, 0) []
+ guard_value(i18, 0) []
jump(p9)
"""
self.optimize_loop(ops, expected)
@@ -6992,17 +7031,17 @@
i10 = getfield_gc(p5, descr=valuedescr)
i11 = getfield_gc(p6, descr=nextdescr)
call(i10, i11, descr=nonwritedescr)
- setfield_gc(p6, i10, descr=nextdescr)
+ setfield_gc(p6, i10, descr=nextdescr)
jump(p5, p6)
"""
expected = """
[p5, p6, i10, i11]
call(i10, i11, descr=nonwritedescr)
- setfield_gc(p6, i10, descr=nextdescr)
+ setfield_gc(p6, i10, descr=nextdescr)
jump(p5, p6, i10, i10)
"""
self.optimize_loop(ops, expected)
-
+
def test_cached_pure_func_of_equal_fields(self):
ops = """
[p5, p6]
@@ -7011,18 +7050,18 @@
i12 = int_add(i10, 7)
i13 = int_add(i11, 7)
call(i12, i13, descr=nonwritedescr)
- setfield_gc(p6, i10, descr=nextdescr)
+ setfield_gc(p6, i10, descr=nextdescr)
jump(p5, p6)
"""
expected = """
[p5, p6, i14, i12, i10]
i13 = int_add(i14, 7)
call(i12, i13, descr=nonwritedescr)
- setfield_gc(p6, i10, descr=nextdescr)
+ setfield_gc(p6, i10, descr=nextdescr)
jump(p5, p6, i10, i12, i10)
"""
self.optimize_loop(ops, expected)
-
+
def test_forced_counter(self):
# XXX: VIRTUALHEAP (see above)
py.test.skip("would be fixed by make heap optimizer aware of virtual setfields")
@@ -7165,7 +7204,7 @@
expected = """
[p1, p2, i2, i1]
call(i2, descr=nonwritedescr)
- setfield_gc(p2, i1, descr=nextdescr)
+ setfield_gc(p2, i1, descr=nextdescr)
jump(p1, p2, i2, i1)
"""
self.optimize_loop(ops, expected)
@@ -7185,11 +7224,65 @@
expected = """
[p1, p2, i2, i1]
call(i2, descr=nonwritedescr)
- setfield_gc(p2, i1, descr=valuedescr)
+ setfield_gc(p2, i1, descr=valuedescr)
jump(p1, p2, i2, i1)
"""
self.optimize_loop(ops, expected)
-
+
+ def test_heap_cache_forced_virtuals(self):
+ ops = """
+ [i1, i2, p0]
+ p1 = new(descr=ssize)
+ setfield_gc(p1, i1, descr=adescr)
+ setfield_gc(p1, i2, descr=bdescr)
+ call(p0, p1, descr=writeadescr)
+ i3 = getfield_gc(p1, descr=adescr)
+ i4 = getfield_gc(p1, descr=bdescr)
+ jump(i3, i4, p0)
+ """
+ expected = """
+ [i1, i2, p0]
+ p1 = new(descr=ssize)
+ setfield_gc(p1, i1, descr=adescr)
+ call(p0, p1, descr=writeadescr)
+ i3 = getfield_gc(p1, descr=adescr)
+ setfield_gc(p1, i2, descr=bdescr)
+ jump(i3, i2, p0)
+ """
+ self.optimize_loop(ops, expected)
+
+
+ def test_setarrayitem_followed_by_arraycopy(self):
+ ops = """
+ [p1, p2]
+ setarrayitem_gc(p1, 2, 10, descr=arraydescr)
+ setarrayitem_gc(p2, 3, 13, descr=arraydescr)
+ call(0, p1, p2, 0, 0, 10, descr=arraycopydescr)
+ jump(p1, p2)
+ """
+ self.optimize_loop(ops, ops)
+
+ def test_heap_cache_virtuals_forced_by_delayed_setfield(self):
+ py.test.skip('not yet supoprted')
+ ops = """
+ [i1, p0]
+ p1 = new(descr=ssize)
+ setfield_gc(p1, i1, descr=valuedescr)
+ setfield_gc(p0, p1, descr=adescr)
+ call(p0, descr=writeadescr)
+ i2 = getfield_gc(p1, descr=valuedescr)
+ jump(i2, p0)
+ """
+ expected = """
+ [i1, p0]
+ p1 = new(descr=ssize)
+ setfield_gc(p1, i1, descr=valuedescr)
+ setfield_gc(p0, p1, descr=adescr)
+ call(p0, descr=writeadescr)
+ jump(i1, p0)
+ """
+ self.optimize_loop(ops, expected)
+
class TestLLtype(OptimizeOptTest, LLtypeMixin):
pass
-
+
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_util.py b/pypy/jit/metainterp/optimizeopt/test/test_util.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_util.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_util.py
@@ -182,7 +182,8 @@
EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE,
can_invalidate=True))
arraycopydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
- EffectInfo([], [], [], [], oopspecindex=EffectInfo.OS_ARRAYCOPY))
+ EffectInfo([], [arraydescr], [], [arraydescr],
+ oopspecindex=EffectInfo.OS_ARRAYCOPY))
for _name, _os in [
('strconcatdescr', 'OS_STR_CONCAT'),
diff --git a/pypy/jit/metainterp/optimizeopt/unroll.py b/pypy/jit/metainterp/optimizeopt/unroll.py
--- a/pypy/jit/metainterp/optimizeopt/unroll.py
+++ b/pypy/jit/metainterp/optimizeopt/unroll.py
@@ -75,7 +75,6 @@
self.importable_values = {}
self.emitting_dissabled = False
self.emitted_guards = 0
- self.emitted_pure_operations = {}
def ensure_imported(self, value):
if not self.emitting_dissabled and value in self.importable_values:
@@ -96,21 +95,6 @@
new = UnrollableOptimizer(self.metainterp_sd, self.loop)
return self._new(new)
- def remember_emitting_pure(self, op):
- self.emitted_pure_operations[op] = True
-
- def produce_potential_short_preamble_ops(self, sb):
- for op in self.emitted_pure_operations:
- if op.getopnum() == rop.GETARRAYITEM_GC_PURE or \
- op.getopnum() == rop.STRGETITEM or \
- op.getopnum() == rop.UNICODEGETITEM:
- if not self.getvalue(op.getarg(1)).is_constant():
- continue
- sb.add_potential(op)
- for opt in self.optimizations:
- opt.produce_potential_short_preamble_ops(sb)
-
-
class UnrollOptimizer(Optimization):
"""Unroll the loop into two iterations. The first one will
@@ -154,7 +138,7 @@
KillHugeIntBounds(self.optimizer).apply()
- loop.preamble.operations = self.optimizer.newoperations
+ loop.preamble.operations = self.optimizer.get_newoperations()
jump_args = [self.getvalue(a).get_key_box() for a in jump_args]
start_resumedescr = loop.preamble.start_resumedescr.clone_if_mutable()
@@ -167,8 +151,9 @@
virtual_state = modifier.get_virtual_state(jump_args)
values = [self.getvalue(arg) for arg in jump_args]
- inputargs = virtual_state.make_inputargs(values)
- short_inputargs = virtual_state.make_inputargs(values, keyboxes=True)
+ inputargs = virtual_state.make_inputargs(values, self.optimizer)
+ short_inputargs = virtual_state.make_inputargs(values, self.optimizer,
+ keyboxes=True)
self.constant_inputargs = {}
for box in jump_args:
@@ -197,7 +182,7 @@
# operations needed to setup the proper state of those virtuals
# in the peeled loop
inputarg_setup_ops = []
- preamble_optimizer.newoperations = []
+ preamble_optimizer.clear_newoperations()
seen = {}
for box in inputargs:
if box in seen:
@@ -211,9 +196,8 @@
continue
seen[box] = True
value = preamble_optimizer.getvalue(box)
- value.force_box()
- preamble_optimizer.flush()
- inputarg_setup_ops += preamble_optimizer.newoperations
+ value.force_box(preamble_optimizer)
+ inputarg_setup_ops += preamble_optimizer.get_newoperations()
# Setup the state of the new optimizer by emiting the
# short preamble operations and discarding the result
@@ -226,8 +210,9 @@
if op and op.result:
preamble_value = preamble_optimizer.getvalue(op.result)
value = self.optimizer.getvalue(op.result)
- imp = ValueImporter(self, preamble_value, op)
- self.optimizer.importable_values[value] = imp
+ if not value.is_virtual():
+ imp = ValueImporter(self, preamble_value, op)
+ self.optimizer.importable_values[value] = imp
newresult = self.optimizer.getvalue(op.result).get_key_box()
if newresult is not op.result:
self.short_boxes.alias(newresult, op.result)
@@ -243,13 +228,13 @@
virtual_state)
loop.inputargs = inputargs
- args = [preamble_optimizer.getvalue(self.short_boxes.original(a)).force_box()\
+ args = [preamble_optimizer.getvalue(self.short_boxes.original(a)).force_box(preamble_optimizer)\
for a in inputargs]
jmp = ResOperation(rop.JUMP, args, None)
jmp.setdescr(loop.token)
loop.preamble.operations.append(jmp)
- loop.operations = self.optimizer.newoperations
+ loop.operations = self.optimizer.get_newoperations()
maxguards = self.optimizer.metainterp_sd.warmrunnerdesc.memory_manager.max_retrace_guards
if self.optimizer.emitted_guards > maxguards:
@@ -334,9 +319,10 @@
assert jumpop
original_jumpargs = jumpop.getarglist()[:]
values = [self.getvalue(arg) for arg in jumpop.getarglist()]
- jumpargs = virtual_state.make_inputargs(values)
+ jumpargs = virtual_state.make_inputargs(values, self.optimizer)
jumpop.initarglist(jumpargs)
- jmp_to_short_args = virtual_state.make_inputargs(values, keyboxes=True)
+ jmp_to_short_args = virtual_state.make_inputargs(values, self.optimizer,
+ keyboxes=True)
self.short_inliner = Inliner(short_inputargs, jmp_to_short_args)
for box, const in self.constant_inputargs.items():
@@ -346,11 +332,11 @@
newop = self.short_inliner.inline_op(op)
self.optimizer.send_extra_operation(newop)
- self.optimizer.flush()
+ newoperations = self.optimizer.get_newoperations()
i = j = 0
- while i < len(self.optimizer.newoperations) or j < len(jumpargs):
- if i == len(self.optimizer.newoperations):
+ while i < len(newoperations) or j < len(jumpargs):
+ if i == len(newoperations):
while j < len(jumpargs):
a = jumpargs[j]
if self.optimizer.loop.logops:
@@ -359,7 +345,7 @@
jumpargs, short_seen)
j += 1
else:
- op = self.optimizer.newoperations[i]
+ op = newoperations[i]
self.boxes_created_this_iteration[op.result] = True
args = op.getarglist()
@@ -374,6 +360,7 @@
self.import_box(a, inputargs, short, short_jumpargs,
jumpargs, short_seen)
i += 1
+ newoperations = self.optimizer.get_newoperations()
jumpop.initarglist(jumpargs)
self.optimizer.send_extra_operation(jumpop)
@@ -467,7 +454,7 @@
inputargs.append(box)
box = newresult
if box in self.optimizer.values:
- box = self.optimizer.values[box].force_box()
+ box = self.optimizer.values[box].force_box(self.optimizer)
jumpargs.append(box)
@@ -482,11 +469,6 @@
if op.getopnum() == rop.JUMP:
loop_token = op.getdescr()
assert isinstance(loop_token, LoopToken)
- # FIXME: Use a tree, similar to the tree formed by the full
- # preamble and it's bridges, instead of a list to save time and
- # memory. This should also allow better behaviour in
- # situations that the is_emittable() chain currently cant
- # handle and the inlining fails unexpectedly belwo.
short = loop_token.short_preamble
if short:
args = op.getarglist()
@@ -522,7 +504,7 @@
values = [self.getvalue(arg)
for arg in op.getarglist()]
- args = sh.virtual_state.make_inputargs(values,
+ args = sh.virtual_state.make_inputargs(values, self.optimizer,
keyboxes=True)
inliner = Inliner(sh.inputargs, args)
diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -10,13 +10,12 @@
class AbstractVirtualValue(optimizer.OptValue):
- _attrs_ = ('optimizer', 'keybox', 'source_op', '_cached_vinfo')
+ _attrs_ = ('keybox', 'source_op', '_cached_vinfo')
box = None
level = optimizer.LEVEL_NONNULL
_cached_vinfo = None
- def __init__(self, optimizer, keybox, source_op=None):
- self.optimizer = optimizer
+ def __init__(self, keybox, source_op=None):
self.keybox = keybox # only used as a key in dictionaries
self.source_op = source_op # the NEW_WITH_VTABLE/NEW_ARRAY operation
# that builds this box
@@ -29,17 +28,17 @@
return self.keybox
return self.box
- def force_box(self):
+ def force_box(self, optforce):
if self.box is None:
- self.optimizer.forget_numberings(self.keybox)
- self._really_force()
+ optforce.forget_numberings(self.keybox)
+ self._really_force(optforce)
return self.box
- def force_at_end_of_preamble(self, already_forced):
+ def force_at_end_of_preamble(self, already_forced, optforce):
value = already_forced.get(self, None)
if value:
return value
- return OptValue(self.force_box())
+ return OptValue(self.force_box(optforce))
def make_virtual_info(self, modifier, fieldnums):
if fieldnums is None:
@@ -55,7 +54,7 @@
def _make_virtual(self, modifier):
raise NotImplementedError("abstract base")
- def _really_force(self):
+ def _really_force(self, optforce):
raise NotImplementedError("abstract base")
def import_from(self, other, optimizer):
@@ -70,10 +69,11 @@
get_fielddescrlist_cache._annspecialcase_ = "specialize:memo"
class AbstractVirtualStructValue(AbstractVirtualValue):
- _attrs_ = ('_fields', '_cached_sorted_fields')
+ _attrs_ = ('_fields', 'cpu', '_cached_sorted_fields')
- def __init__(self, optimizer, keybox, source_op=None):
- AbstractVirtualValue.__init__(self, optimizer, keybox, source_op)
+ def __init__(self, cpu, keybox, source_op=None):
+ AbstractVirtualValue.__init__(self, keybox, source_op)
+ self.cpu = cpu
self._fields = {}
self._cached_sorted_fields = None
@@ -87,45 +87,44 @@
def _get_descr(self):
raise NotImplementedError
- def _is_immutable_and_filled_with_constants(self):
+ def _is_immutable_and_filled_with_constants(self, optforce):
count = self._get_descr().count_fields_if_immutable()
if count != len(self._fields): # always the case if count == -1
return False
for value in self._fields.itervalues():
- subbox = value.force_box()
+ subbox = value.force_box(optforce)
if not isinstance(subbox, Const):
return False
return True
- def force_at_end_of_preamble(self, already_forced):
+ def force_at_end_of_preamble(self, already_forced, optforce):
if self in already_forced:
return self
already_forced[self] = self
if self._fields:
for ofs in self._fields.keys():
- self._fields[ofs] = self._fields[ofs].force_at_end_of_preamble(already_forced)
+ self._fields[ofs] = self._fields[ofs].force_at_end_of_preamble(already_forced, optforce)
return self
- def _really_force(self):
+ def _really_force(self, optforce):
op = self.source_op
assert op is not None
# ^^^ This case should not occur any more (see test_bug_3).
#
if not we_are_translated():
op.name = 'FORCE ' + self.source_op.name
-
- if self._is_immutable_and_filled_with_constants():
- box = self.optimizer.constant_fold(op)
+
+ if self._is_immutable_and_filled_with_constants(optforce):
+ box = optforce.optimizer.constant_fold(op)
self.make_constant(box)
for ofs, value in self._fields.iteritems():
- subbox = value.force_box()
+ subbox = value.force_box(optforce)
assert isinstance(subbox, Const)
- execute(self.optimizer.cpu, None, rop.SETFIELD_GC,
+ execute(optforce.optimizer.cpu, None, rop.SETFIELD_GC,
ofs, box, subbox)
# keep self._fields, because it's all immutable anyway
else:
- newoperations = self.optimizer.newoperations
- newoperations.append(op)
+ optforce.emit_operation(op)
self.box = box = op.result
#
iteritems = self._fields.iteritems()
@@ -135,10 +134,11 @@
for ofs, value in iteritems:
if value.is_null():
continue
- subbox = value.force_box()
+ subbox = value.force_box(optforce)
op = ResOperation(rop.SETFIELD_GC, [box, subbox], None,
descr=ofs)
- newoperations.append(op)
+
+ optforce.emit_operation(op)
def _get_field_descr_list(self):
_cached_sorted_fields = self._cached_sorted_fields
@@ -155,7 +155,7 @@
else:
lst = self._fields.keys()
sort_descrs(lst)
- cache = get_fielddescrlist_cache(self.optimizer.cpu)
+ cache = get_fielddescrlist_cache(self.cpu)
result = cache.get(lst, None)
if result is None:
cache[lst] = lst
@@ -180,8 +180,8 @@
class VirtualValue(AbstractVirtualStructValue):
level = optimizer.LEVEL_KNOWNCLASS
- def __init__(self, optimizer, known_class, keybox, source_op=None):
- AbstractVirtualStructValue.__init__(self, optimizer, keybox, source_op)
+ def __init__(self, cpu, known_class, keybox, source_op=None):
+ AbstractVirtualStructValue.__init__(self, cpu, keybox, source_op)
assert isinstance(known_class, Const)
self.known_class = known_class
@@ -190,7 +190,7 @@
return modifier.make_virtual(self.known_class, fielddescrs)
def _get_descr(self):
- return vtable2descr(self.optimizer.cpu, self.known_class.getint())
+ return vtable2descr(self.cpu, self.known_class.getint())
def __repr__(self):
cls_name = self.known_class.value.adr.ptr._obj._TYPE._name
@@ -201,8 +201,8 @@
class VStructValue(AbstractVirtualStructValue):
- def __init__(self, optimizer, structdescr, keybox, source_op=None):
- AbstractVirtualStructValue.__init__(self, optimizer, keybox, source_op)
+ def __init__(self, cpu, structdescr, keybox, source_op=None):
+ AbstractVirtualStructValue.__init__(self, cpu, keybox, source_op)
self.structdescr = structdescr
def _make_virtual(self, modifier):
@@ -215,10 +215,10 @@
class VArrayValue(AbstractVirtualValue):
- def __init__(self, optimizer, arraydescr, size, keybox, source_op=None):
- AbstractVirtualValue.__init__(self, optimizer, keybox, source_op)
+ def __init__(self, arraydescr, constvalue, size, keybox, source_op=None):
+ AbstractVirtualValue.__init__(self, keybox, source_op)
self.arraydescr = arraydescr
- self.constvalue = optimizer.new_const_item(arraydescr)
+ self.constvalue = constvalue
self._items = [self.constvalue] * size
def getlength(self):
@@ -232,31 +232,30 @@
assert isinstance(itemvalue, optimizer.OptValue)
self._items[index] = itemvalue
- def force_at_end_of_preamble(self, already_forced):
+ def force_at_end_of_preamble(self, already_forced, optforce):
if self in already_forced:
return self
already_forced[self] = self
for index in range(len(self._items)):
- self._items[index] = self._items[index].force_at_end_of_preamble(already_forced)
+ self._items[index] = self._items[index].force_at_end_of_preamble(already_forced, optforce)
return self
- def _really_force(self):
+ def _really_force(self, optforce):
assert self.source_op is not None
if not we_are_translated():
self.source_op.name = 'FORCE ' + self.source_op.name
- newoperations = self.optimizer.newoperations
- newoperations.append(self.source_op)
+ optforce.emit_operation(self.source_op)
self.box = box = self.source_op.result
for index in range(len(self._items)):
subvalue = self._items[index]
if subvalue is not self.constvalue:
if subvalue.is_null():
continue
- subbox = subvalue.force_box()
+ subbox = subvalue.force_box(optforce)
op = ResOperation(rop.SETARRAYITEM_GC,
[box, ConstInt(index), subbox], None,
descr=self.arraydescr)
- newoperations.append(op)
+ optforce.emit_operation(op)
def get_args_for_fail(self, modifier):
if self.box is None and not modifier.already_seen_virtual(self.keybox):
@@ -279,17 +278,18 @@
return OptVirtualize()
def make_virtual(self, known_class, box, source_op=None):
- vvalue = VirtualValue(self.optimizer, known_class, box, source_op)
+ vvalue = VirtualValue(self.optimizer.cpu, known_class, box, source_op)
self.make_equal_to(box, vvalue)
return vvalue
def make_varray(self, arraydescr, size, box, source_op=None):
- vvalue = VArrayValue(self.optimizer, arraydescr, size, box, source_op)
+ constvalue = self.new_const_item(arraydescr)
+ vvalue = VArrayValue(arraydescr, constvalue, size, box, source_op)
self.make_equal_to(box, vvalue)
return vvalue
def make_vstruct(self, structdescr, box, source_op=None):
- vvalue = VStructValue(self.optimizer, structdescr, box, source_op)
+ vvalue = VStructValue(self.optimizer.cpu, structdescr, box, source_op)
self.make_equal_to(box, vvalue)
return vvalue
@@ -362,7 +362,6 @@
self.make_equal_to(op.result, fieldvalue)
else:
value.ensure_nonnull()
- ###self.heap_op_optimizer.optimize_GETFIELD_GC(op, value)
self.emit_operation(op)
# note: the following line does not mean that the two operations are
@@ -377,7 +376,6 @@
value.setfield(op.getdescr(), fieldvalue)
else:
value.ensure_nonnull()
- ###self.heap_op_optimizer.optimize_SETFIELD_GC(op, value, fieldvalue)
self.emit_operation(op)
def optimize_NEW_WITH_VTABLE(self, op):
@@ -417,7 +415,6 @@
self.make_equal_to(op.result, itemvalue)
return
value.ensure_nonnull()
- ###self.heap_op_optimizer.optimize_GETARRAYITEM_GC(op, value)
self.emit_operation(op)
# note: the following line does not mean that the two operations are
@@ -432,7 +429,6 @@
value.setitem(indexbox.getint(), self.getvalue(op.getarg(2)))
return
value.ensure_nonnull()
- ###self.heap_op_optimizer.optimize_SETARRAYITEM_GC(op, value, fieldvalue)
self.emit_operation(op)
diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -30,7 +30,7 @@
def _generate_guards(self, other, box, cpu, extra_guards):
raise InvalidLoop
- def enum_forced_boxes(self, boxes, value):
+ def enum_forced_boxes(self, boxes, value, optimizer):
raise NotImplementedError
def enum(self, virtual_state):
@@ -100,14 +100,14 @@
def _generalization_of(self, other):
raise NotImplementedError
- def enum_forced_boxes(self, boxes, value):
+ def enum_forced_boxes(self, boxes, value, optimizer):
assert isinstance(value, virtualize.AbstractVirtualStructValue)
assert value.is_virtual()
for i in range(len(self.fielddescrs)):
v = value._fields[self.fielddescrs[i]]
s = self.fieldstate[i]
if s.position > self.position:
- s.enum_forced_boxes(boxes, v)
+ s.enum_forced_boxes(boxes, v, optimizer)
def _enum(self, virtual_state):
for s in self.fieldstate:
@@ -177,14 +177,14 @@
return False
return True
- def enum_forced_boxes(self, boxes, value):
+ def enum_forced_boxes(self, boxes, value, optimizer):
assert isinstance(value, virtualize.VArrayValue)
assert value.is_virtual()
for i in range(len(self.fieldstate)):
v = value._items[i]
s = self.fieldstate[i]
if s.position > self.position:
- s.enum_forced_boxes(boxes, v)
+ s.enum_forced_boxes(boxes, v, optimizer)
def _enum(self, virtual_state):
for s in self.fieldstate:
@@ -316,11 +316,11 @@
import pdb; pdb.set_trace()
raise NotImplementedError
- def enum_forced_boxes(self, boxes, value):
+ def enum_forced_boxes(self, boxes, value, optimizer):
if self.level == LEVEL_CONSTANT:
return
assert 0 <= self.position_in_notvirtuals
- boxes[self.position_in_notvirtuals] = value.force_box()
+ boxes[self.position_in_notvirtuals] = value.force_box(optimizer)
def _enum(self, virtual_state):
if self.level == LEVEL_CONSTANT:
@@ -377,11 +377,13 @@
self.state[i].generate_guards(other.state[i], args[i],
cpu, extra_guards, renum)
- def make_inputargs(self, values, keyboxes=False):
+ def make_inputargs(self, values, optimizer, keyboxes=False):
+ if optimizer.optearlyforce:
+ optimizer = optimizer.optearlyforce
assert len(values) == len(self.state)
inputargs = [None] * len(self.notvirtuals)
for i in range(len(values)):
- self.state[i].enum_forced_boxes(inputargs, values[i])
+ self.state[i].enum_forced_boxes(inputargs, values[i], optimizer)
if keyboxes:
for i in range(len(values)):
@@ -434,7 +436,12 @@
def get_virtual_state(self, jump_args):
self.optimizer.force_at_end_of_preamble()
already_forced = {}
- values = [self.getvalue(box).force_at_end_of_preamble(already_forced)
+ if self.optimizer.optearlyforce:
+ opt = self.optimizer.optearlyforce
+ else:
+ opt = self.optimizer
+ values = [self.getvalue(box).force_at_end_of_preamble(already_forced,
+ opt)
for box in jump_args]
for value in values:
diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -43,7 +43,7 @@
class __extend__(optimizer.OptValue):
"""New methods added to the base class OptValue for this file."""
- def getstrlen(self, optimization, mode):
+ def getstrlen(self, string_optimizer, mode):
if mode is mode_string:
s = self.get_constant_string_spec(mode_string)
if s is not None:
@@ -52,12 +52,12 @@
s = self.get_constant_string_spec(mode_unicode)
if s is not None:
return ConstInt(len(s))
- if optimization is None:
+ if string_optimizer is None:
return None
self.ensure_nonnull()
- box = self.force_box()
+ box = self.force_box(string_optimizer)
lengthbox = BoxInt()
- optimization.propagate_forward(ResOperation(mode.STRLEN, [box], lengthbox))
+ string_optimizer.emit_operation(ResOperation(mode.STRLEN, [box], lengthbox))
return lengthbox
@specialize.arg(1)
@@ -68,25 +68,25 @@
else:
return None
- def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
+ def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
# Copies the pointer-to-string 'self' into the target string
# given by 'targetbox', at the specified offset. Returns the offset
# at the end of the copy.
- lengthbox = self.getstrlen(optimizer, mode)
- srcbox = self.force_box()
- return copy_str_content(optimizer, srcbox, targetbox,
+ lengthbox = self.getstrlen(string_optimizer, mode)
+ srcbox = self.force_box(string_optimizer)
+ return copy_str_content(string_optimizer, srcbox, targetbox,
CONST_0, offsetbox, lengthbox, mode)
class VAbstractStringValue(virtualize.AbstractVirtualValue):
_attrs_ = ('mode',)
- def __init__(self, optimizer, keybox, source_op, mode):
- virtualize.AbstractVirtualValue.__init__(self, optimizer, keybox,
+ def __init__(self, keybox, source_op, mode):
+ virtualize.AbstractVirtualValue.__init__(self, keybox,
source_op)
self.mode = mode
- def _really_force(self):
+ def _really_force(self, optforce):
if self.mode is mode_string:
s = self.get_constant_string_spec(mode_string)
if s is not None:
@@ -101,12 +101,12 @@
return
assert self.source_op is not None
self.box = box = self.source_op.result
- lengthbox = self.getstrlen(self.optimizer, self.mode)
+ lengthbox = self.getstrlen(optforce, self.mode)
op = ResOperation(self.mode.NEWSTR, [lengthbox], box)
if not we_are_translated():
op.name = 'FORCE'
- self.optimizer.emit_operation(op)
- self.string_copy_parts(self.optimizer, box, CONST_0, self.mode)
+ optforce.emit_operation(op)
+ self.string_copy_parts(optforce, box, CONST_0, self.mode)
class VStringPlainValue(VAbstractStringValue):
@@ -140,15 +140,20 @@
return mode.emptystr.join([mode.chr(c.box.getint())
for c in self._chars])
- def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
+ def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
+ if not self.is_virtual() and targetbox is not self.box:
+ lengthbox = self.getstrlen(string_optimizer, mode)
+ srcbox = self.force_box(string_optimizer)
+ return copy_str_content(string_optimizer, srcbox, targetbox,
+ CONST_0, offsetbox, lengthbox, mode)
for i in range(len(self._chars)):
- charbox = self._chars[i].force_box()
+ charbox = self._chars[i].force_box(string_optimizer)
if not (isinstance(charbox, Const) and charbox.same_constant(CONST_0)):
- optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
- offsetbox,
- charbox],
+ string_optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
+ offsetbox,
+ charbox],
None))
- offsetbox = _int_add(optimizer, offsetbox, CONST_1)
+ offsetbox = _int_add(string_optimizer, offsetbox, CONST_1)
return offsetbox
def get_args_for_fail(self, modifier):
@@ -182,16 +187,16 @@
self.left = left
self.right = right
- def getstrlen(self, optimizer, mode):
+ def getstrlen(self, string_optimizer, mode):
if self.lengthbox is None:
- len1box = self.left.getstrlen(optimizer, mode)
+ len1box = self.left.getstrlen(string_optimizer, mode)
if len1box is None:
return None
- len2box = self.right.getstrlen(optimizer, mode)
+ len2box = self.right.getstrlen(string_optimizer, mode)
if len2box is None:
return None
- self.lengthbox = _int_add(optimizer, len1box, len2box)
- # ^^^ may still be None, if optimizer is None
+ self.lengthbox = _int_add(string_optimizer, len1box, len2box)
+ # ^^^ may still be None, if string_optimizer is None
return self.lengthbox
@specialize.arg(1)
@@ -204,10 +209,10 @@
return None
return s1 + s2
- def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
- offsetbox = self.left.string_copy_parts(optimizer, targetbox,
+ def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
+ offsetbox = self.left.string_copy_parts(string_optimizer, targetbox,
offsetbox, mode)
- offsetbox = self.right.string_copy_parts(optimizer, targetbox,
+ offsetbox = self.right.string_copy_parts(string_optimizer, targetbox,
offsetbox, mode)
return offsetbox
@@ -246,8 +251,8 @@
self.vstart = vstart
self.vlength = vlength
- def getstrlen(self, _, mode):
- return self.vlength.force_box()
+ def getstrlen(self, optforce, mode):
+ return self.vlength.force_box(optforce)
@specialize.arg(1)
def get_constant_string_spec(self, mode):
@@ -262,11 +267,11 @@
return s1[start : start + length]
return None
- def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
- lengthbox = self.getstrlen(optimizer, mode)
- return copy_str_content(optimizer,
- self.vstr.force_box(), targetbox,
- self.vstart.force_box(), offsetbox,
+ def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
+ lengthbox = self.getstrlen(string_optimizer, mode)
+ return copy_str_content(string_optimizer,
+ self.vstr.force_box(string_optimizer), targetbox,
+ self.vstart.force_box(string_optimizer), offsetbox,
lengthbox, mode)
def get_args_for_fail(self, modifier):
@@ -295,8 +300,8 @@
return modifier.make_vstrslice(self.mode is mode_unicode)
-def copy_str_content(optimizer, srcbox, targetbox,
- srcoffsetbox, offsetbox, lengthbox, mode):
+def copy_str_content(string_optimizer, srcbox, targetbox,
+ srcoffsetbox, offsetbox, lengthbox, mode, need_next_offset=True):
if isinstance(srcbox, ConstPtr) and isinstance(srcoffsetbox, Const):
M = 5
else:
@@ -305,23 +310,26 @@
# up to M characters are done "inline", i.e. with STRGETITEM/STRSETITEM
# instead of just a COPYSTRCONTENT.
for i in range(lengthbox.value):
- charbox = _strgetitem(optimizer, srcbox, srcoffsetbox, mode)
- srcoffsetbox = _int_add(optimizer, srcoffsetbox, CONST_1)
- optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
- offsetbox,
- charbox],
+ charbox = _strgetitem(string_optimizer, srcbox, srcoffsetbox, mode)
+ srcoffsetbox = _int_add(string_optimizer, srcoffsetbox, CONST_1)
+ string_optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
+ offsetbox,
+ charbox],
None))
- offsetbox = _int_add(optimizer, offsetbox, CONST_1)
+ offsetbox = _int_add(string_optimizer, offsetbox, CONST_1)
else:
- nextoffsetbox = _int_add(optimizer, offsetbox, lengthbox)
+ if need_next_offset:
+ nextoffsetbox = _int_add(string_optimizer, offsetbox, lengthbox)
+ else:
+ nextoffsetbox = None
op = ResOperation(mode.COPYSTRCONTENT, [srcbox, targetbox,
srcoffsetbox, offsetbox,
lengthbox], None)
- optimizer.emit_operation(op)
+ string_optimizer.emit_operation(op)
offsetbox = nextoffsetbox
return offsetbox
-def _int_add(optimizer, box1, box2):
+def _int_add(string_optimizer, box1, box2):
if isinstance(box1, ConstInt):
if box1.value == 0:
return box2
@@ -329,23 +337,23 @@
return ConstInt(box1.value + box2.value)
elif isinstance(box2, ConstInt) and box2.value == 0:
return box1
- if optimizer is None:
+ if string_optimizer is None:
return None
resbox = BoxInt()
- optimizer.propagate_forward(ResOperation(rop.INT_ADD, [box1, box2], resbox))
+ string_optimizer.emit_operation(ResOperation(rop.INT_ADD, [box1, box2], resbox))
return resbox
-def _int_sub(optimizer, box1, box2):
+def _int_sub(string_optimizer, box1, box2):
if isinstance(box2, ConstInt):
if box2.value == 0:
return box1
if isinstance(box1, ConstInt):
return ConstInt(box1.value - box2.value)
resbox = BoxInt()
- optimizer.propagate_forward(ResOperation(rop.INT_SUB, [box1, box2], resbox))
+ string_optimizer.emit_operation(ResOperation(rop.INT_SUB, [box1, box2], resbox))
return resbox
-def _strgetitem(optimizer, strbox, indexbox, mode):
+def _strgetitem(string_optimizer, strbox, indexbox, mode):
if isinstance(strbox, ConstPtr) and isinstance(indexbox, ConstInt):
if mode is mode_string:
s = strbox.getref(lltype.Ptr(rstr.STR))
@@ -354,30 +362,28 @@
s = strbox.getref(lltype.Ptr(rstr.UNICODE))
return ConstInt(ord(s.chars[indexbox.getint()]))
resbox = BoxInt()
- optimizer.propagate_forward(ResOperation(mode.STRGETITEM, [strbox, indexbox],
- resbox))
+ string_optimizer.emit_operation(ResOperation(mode.STRGETITEM, [strbox, indexbox],
+ resbox))
return resbox
class OptString(optimizer.Optimization):
"Handling of strings and unicodes."
- enabled = True
-
def new(self):
return OptString()
-
+
def make_vstring_plain(self, box, source_op, mode):
- vvalue = VStringPlainValue(self.optimizer, box, source_op, mode)
+ vvalue = VStringPlainValue(box, source_op, mode)
self.make_equal_to(box, vvalue)
return vvalue
def make_vstring_concat(self, box, source_op, mode):
- vvalue = VStringConcatValue(self.optimizer, box, source_op, mode)
+ vvalue = VStringConcatValue(box, source_op, mode)
self.make_equal_to(box, vvalue)
return vvalue
def make_vstring_slice(self, box, source_op, mode):
- vvalue = VStringSliceValue(self.optimizer, box, source_op, mode)
+ vvalue = VStringSliceValue(box, source_op, mode)
self.make_equal_to(box, vvalue)
return vvalue
@@ -427,17 +433,21 @@
value.ensure_nonnull()
#
if value.is_virtual() and isinstance(value, VStringSliceValue):
- fullindexbox = _int_add(self.optimizer,
- value.vstart.force_box(),
- vindex.force_box())
+ fullindexbox = _int_add(self,
+ value.vstart.force_box(self),
+ vindex.force_box(self))
value = value.vstr
vindex = self.getvalue(fullindexbox)
#
if isinstance(value, VStringPlainValue): # even if no longer virtual
if vindex.is_constant():
- return value.getitem(vindex.box.getint())
+ res = value.getitem(vindex.box.getint())
+ # If it is uninitialized we can't return it, it was set by a
+ # COPYSTRCONTENT, not a STRSETITEM
+ if res is not optimizer.CVAL_UNINITIALIZED_ZERO:
+ return res
#
- resbox = _strgetitem(self.optimizer, value.force_box(), vindex.force_box(), mode)
+ resbox = _strgetitem(self, value.force_box(self), vindex.force_box(self), mode)
return self.getvalue(resbox)
def optimize_STRLEN(self, op):
@@ -447,9 +457,33 @@
def _optimize_STRLEN(self, op, mode):
value = self.getvalue(op.getarg(0))
- lengthbox = value.getstrlen(self.optimizer, mode)
+ lengthbox = value.getstrlen(self, mode)
self.make_equal_to(op.result, self.getvalue(lengthbox))
+ def optimize_COPYSTRCONTENT(self, op):
+ self._optimize_COPYSTRCONTENT(op, mode_string)
+ def optimize_COPYUNICODECONTENT(self, op):
+ self._optimize_COPYSTRCONTENT(op, mode_unicode)
+
+ def _optimize_COPYSTRCONTENT(self, op, mode):
+ # args: src dst srcstart dststart length
+ src = self.getvalue(op.getarg(0))
+ dst = self.getvalue(op.getarg(1))
+ srcstart = self.getvalue(op.getarg(2))
+ dststart = self.getvalue(op.getarg(3))
+ length = self.getvalue(op.getarg(4))
+
+ if length.is_constant() and length.box.getint() == 0:
+ return
+ copy_str_content(self,
+ src.force_box(self),
+ dst.force_box(self),
+ srcstart.force_box(self),
+ dststart.force_box(self),
+ length.force_box(self),
+ mode, need_next_offset=False
+ )
+
def optimize_CALL(self, op):
# dispatch based on 'oopspecindex' to a method that handles
# specifically the given oopspec call. For non-oopspec calls,
@@ -472,6 +506,8 @@
return
self.emit_operation(op)
+ optimize_CALL_PURE = optimize_CALL
+
def opt_call_str_STR2UNICODE(self, op):
# Constant-fold unicode("constant string").
# More generally, supporting non-constant but virtual cases is
@@ -511,16 +547,16 @@
return True
#
vstr.ensure_nonnull()
- lengthbox = _int_sub(self.optimizer, vstop.force_box(),
- vstart.force_box())
+ lengthbox = _int_sub(self, vstop.force_box(self),
+ vstart.force_box(self))
#
if isinstance(vstr, VStringSliceValue):
# double slicing s[i:j][k:l]
vintermediate = vstr
vstr = vintermediate.vstr
- startbox = _int_add(self.optimizer,
- vintermediate.vstart.force_box(),
- vstart.force_box())
+ startbox = _int_add(self,
+ vintermediate.vstart.force_box(self),
+ vstart.force_box(self))
vstart = self.getvalue(startbox)
#
value = self.make_vstring_slice(op.result, op, mode)
@@ -558,8 +594,8 @@
do = EffectInfo.OS_STREQ_LENGTHOK
else:
do = EffectInfo.OS_STREQ_NONNULL
- self.generate_modified_call(do, [v1.force_box(),
- v2.force_box()], op.result, mode)
+ self.generate_modified_call(do, [v1.force_box(self),
+ v2.force_box(self)], op.result, mode)
return True
return False
@@ -567,7 +603,7 @@
l2box = v2.getstrlen(None, mode)
if isinstance(l2box, ConstInt):
if l2box.value == 0:
- lengthbox = v1.getstrlen(self.optimizer, mode)
+ lengthbox = v1.getstrlen(self, mode)
seo = self.optimizer.send_extra_operation
seo(ResOperation(rop.INT_EQ, [lengthbox, CONST_0], resultbox))
return True
@@ -578,17 +614,17 @@
vchar1 = self.strgetitem(v1, optimizer.CVAL_ZERO, mode)
vchar2 = self.strgetitem(v2, optimizer.CVAL_ZERO, mode)
seo = self.optimizer.send_extra_operation
- seo(ResOperation(rop.INT_EQ, [vchar1.force_box(),
- vchar2.force_box()],
+ seo(ResOperation(rop.INT_EQ, [vchar1.force_box(self),
+ vchar2.force_box(self)],
resultbox))
return True
if isinstance(v1, VStringSliceValue):
vchar = self.strgetitem(v2, optimizer.CVAL_ZERO, mode)
do = EffectInfo.OS_STREQ_SLICE_CHAR
- self.generate_modified_call(do, [v1.vstr.force_box(),
- v1.vstart.force_box(),
- v1.vlength.force_box(),
- vchar.force_box()],
+ self.generate_modified_call(do, [v1.vstr.force_box(self),
+ v1.vstart.force_box(self),
+ v1.vlength.force_box(self),
+ vchar.force_box(self)],
resultbox, mode)
return True
#
@@ -599,10 +635,10 @@
if v1.is_null():
self.make_constant(resultbox, CONST_1)
return True
- op = ResOperation(rop.PTR_EQ, [v1.force_box(),
+ op = ResOperation(rop.PTR_EQ, [v1.force_box(self),
llhelper.CONST_NULL],
resultbox)
- self.optimizer.emit_operation(op)
+ self.emit_operation(op)
return True
#
return False
@@ -616,8 +652,8 @@
do = EffectInfo.OS_STREQ_NONNULL_CHAR
else:
do = EffectInfo.OS_STREQ_CHECKNULL_CHAR
- self.generate_modified_call(do, [v1.force_box(),
- vchar.force_box()], resultbox,
+ self.generate_modified_call(do, [v1.force_box(self),
+ vchar.force_box(self)], resultbox,
mode)
return True
#
@@ -626,10 +662,10 @@
do = EffectInfo.OS_STREQ_SLICE_NONNULL
else:
do = EffectInfo.OS_STREQ_SLICE_CHECKNULL
- self.generate_modified_call(do, [v1.vstr.force_box(),
- v1.vstart.force_box(),
- v1.vlength.force_box(),
- v2.force_box()], resultbox, mode)
+ self.generate_modified_call(do, [v1.vstr.force_box(self),
+ v1.vstart.force_box(self),
+ v1.vlength.force_box(self),
+ v2.force_box(self)], resultbox, mode)
return True
return False
@@ -639,16 +675,11 @@
calldescr, func = cic.callinfo_for_oopspec(oopspecindex)
op = ResOperation(rop.CALL, [ConstInt(func)] + args, result,
descr=calldescr)
- self.optimizer.emit_operation(op)
+ self.emit_operation(op)
def propagate_forward(self, op):
- if not self.enabled:
- self.emit_operation(op)
- return
-
dispatch_opt(self, op)
-
dispatch_opt = make_dispatcher_method(OptString, 'optimize_',
default=OptString.emit_operation)
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -210,7 +210,8 @@
self.metainterp.clear_exception()
resbox = self.execute(rop.%s, b1, b2)
self.make_result_of_lastop(resbox) # same as execute_varargs()
- self.metainterp.handle_possible_overflow_error()
+ if not isinstance(resbox, Const):
+ self.metainterp.handle_possible_overflow_error()
return resbox
''' % (_opimpl, _opimpl.upper())).compile()
@@ -401,23 +402,25 @@
self.metainterp.heapcache.new_array(resbox, lengthbox)
return resbox
- @arguments("box", "descr", "box")
- def _opimpl_getarrayitem_gc_any(self, arraybox, arraydescr, indexbox):
+ @specialize.arg(1)
+ def _do_getarrayitem_gc_any(self, op, arraybox, arraydescr, indexbox):
tobox = self.metainterp.heapcache.getarrayitem(
arraybox, arraydescr, indexbox)
if tobox:
# sanity check: see whether the current array value
# corresponds to what the cache thinks the value is
- resbox = executor.execute(self.metainterp.cpu, self.metainterp,
- rop.GETARRAYITEM_GC, arraydescr, arraybox, indexbox)
+ resbox = executor.execute(self.metainterp.cpu, self.metainterp, op,
+ arraydescr, arraybox, indexbox)
assert resbox.constbox().same_constant(tobox.constbox())
return tobox
- resbox = self.execute_with_descr(rop.GETARRAYITEM_GC,
- arraydescr, arraybox, indexbox)
+ resbox = self.execute_with_descr(op, arraydescr, arraybox, indexbox)
self.metainterp.heapcache.getarrayitem_now_known(
arraybox, arraydescr, indexbox, resbox)
return resbox
+ @arguments("box", "descr", "box")
+ def _opimpl_getarrayitem_gc_any(self, arraybox, arraydescr, indexbox):
+ return self._do_getarrayitem_gc_any(rop.GETARRAYITEM_GC, arraybox, arraydescr, indexbox)
opimpl_getarrayitem_gc_i = _opimpl_getarrayitem_gc_any
opimpl_getarrayitem_gc_r = _opimpl_getarrayitem_gc_any
@@ -433,8 +436,7 @@
@arguments("box", "descr", "box")
def _opimpl_getarrayitem_gc_pure_any(self, arraybox, arraydescr, indexbox):
- return self.execute_with_descr(rop.GETARRAYITEM_GC_PURE,
- arraydescr, arraybox, indexbox)
+ return self._do_getarrayitem_gc_any(rop.GETARRAYITEM_GC_PURE, arraybox, arraydescr, indexbox)
opimpl_getarrayitem_gc_pure_i = _opimpl_getarrayitem_gc_pure_any
opimpl_getarrayitem_gc_pure_r = _opimpl_getarrayitem_gc_pure_any
@@ -866,6 +868,14 @@
def opimpl_newunicode(self, lengthbox):
return self.execute(rop.NEWUNICODE, lengthbox)
+ @arguments("box", "box", "box", "box", "box")
+ def opimpl_copystrcontent(self, srcbox, dstbox, srcstartbox, dststartbox, lengthbox):
+ return self.execute(rop.COPYSTRCONTENT, srcbox, dstbox, srcstartbox, dststartbox, lengthbox)
+
+ @arguments("box", "box", "box", "box", "box")
+ def opimpl_copyunicodecontent(self, srcbox, dstbox, srcstartbox, dststartbox, lengthbox):
+ return self.execute(rop.COPYUNICODECONTENT, srcbox, dstbox, srcstartbox, dststartbox, lengthbox)
+
## @FixME #arguments("descr", "varargs")
## def opimpl_residual_oosend_canraise(self, methdescr, varargs):
## return self.execute_varargs(rop.OOSEND, varargs, descr=methdescr,
@@ -1060,6 +1070,18 @@
return ConstInt(trace_length)
@arguments("box")
+ def _opimpl_isconstant(self, box):
+ return ConstInt(isinstance(box, Const))
+
+ opimpl_int_isconstant = opimpl_ref_isconstant = _opimpl_isconstant
+
+ @arguments("box")
+ def _opimpl_isvirtual(self, box):
+ return ConstInt(self.metainterp.heapcache.is_unescaped(box))
+
+ opimpl_ref_isvirtual = _opimpl_isvirtual
+
+ @arguments("box")
def opimpl_virtual_ref(self, box):
# Details on the content of metainterp.virtualref_boxes:
#
@@ -1673,6 +1695,10 @@
def _record_helper_nonpure_varargs(self, opnum, resbox, descr, argboxes):
assert resbox is None or isinstance(resbox, Box)
+ if (rop._OVF_FIRST <= opnum <= rop._OVF_LAST and
+ self.last_exc_value_box is None and
+ self._all_constants_varargs(argboxes)):
+ return resbox.constbox()
# record the operation
profiler = self.staticdata.profiler
profiler.count_ops(opnum, RECORDED_OPS)
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -221,7 +221,6 @@
newop.setfailargs(self.getfailargs())
return newop
-
# ============
# arity mixins
# ============
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -1,23 +1,25 @@
+import sys
+
import py
-import sys
-from pypy.rlib.jit import JitDriver, we_are_jitted, hint, dont_look_inside
-from pypy.rlib.jit import loop_invariant, elidable, promote
-from pypy.rlib.jit import jit_debug, assert_green, AssertGreenFailed
-from pypy.rlib.jit import unroll_safe, current_trace_length
+
+from pypy import conftest
+from pypy.jit.codewriter.policy import JitPolicy, StopAtXPolicy
from pypy.jit.metainterp import pyjitpl, history
+from pypy.jit.metainterp.optimizeopt import ALL_OPTS_DICT
+from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin, noConst
+from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper
+from pypy.jit.metainterp.warmspot import get_stats
from pypy.jit.metainterp.warmstate import set_future_value
-from pypy.jit.metainterp.warmspot import get_stats
-from pypy.jit.codewriter.policy import JitPolicy, StopAtXPolicy
-from pypy import conftest
+from pypy.rlib.jit import (JitDriver, we_are_jitted, hint, dont_look_inside,
+ loop_invariant, elidable, promote, jit_debug, assert_green,
+ AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
+ isconstant, isvirtual)
from pypy.rlib.rarithmetic import ovfcheck
-from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper
from pypy.rpython.lltypesystem import lltype, llmemory, rffi
from pypy.rpython.ootypesystem import ootype
-from pypy.jit.metainterp.optimizeopt import ALL_OPTS_DICT
-from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin, noConst
+
class BasicTests:
-
def test_basic(self):
def f(x, y):
return x + y
@@ -99,14 +101,14 @@
myjitdriver.jit_merge_point(x=x, y=y, res=res)
res += x * x
x += 1
- res += x * x
+ res += x * x
y -= 1
return res
res = self.meta_interp(f, [6, 7])
assert res == 1323
self.check_loop_count(1)
self.check_loops(int_mul=1)
-
+
def test_loop_variant_mul_ovf(self):
myjitdriver = JitDriver(greens = [], reds = ['y', 'res', 'x'])
def f(x, y):
@@ -1372,7 +1374,7 @@
return x
res = self.meta_interp(f, [299], listops=True)
assert res == f(299)
- self.check_loops(guard_class=0, guard_value=3)
+ self.check_loops(guard_class=0, guard_value=3)
self.check_loops(guard_class=0, guard_value=6, everywhere=True)
def test_merge_guardnonnull_guardclass(self):
@@ -2118,7 +2120,7 @@
return sa
res = self.meta_interp(f, [32, 7])
assert res == f(32, 7)
-
+
def test_caching_setarrayitem_fixed(self):
myjitdriver = JitDriver(greens = [], reds = ['sa', 'i', 'n', 'a', 'node'])
def f(n, a):
@@ -2138,7 +2140,7 @@
return sa
res = self.meta_interp(f, [32, 7])
assert res == f(32, 7)
-
+
def test_caching_setarrayitem_var(self):
myjitdriver = JitDriver(greens = [], reds = ['sa', 'i', 'n', 'a', 'b', 'node'])
def f(n, a, b):
@@ -2668,7 +2670,7 @@
myjitdriver.set_param('threshold', 3)
myjitdriver.set_param('trace_eagerness', 1)
myjitdriver.set_param('retrace_limit', 5)
- myjitdriver.set_param('function_threshold', -1)
+ myjitdriver.set_param('function_threshold', -1)
pc = sa = i = 0
while pc < len(bytecode):
myjitdriver.jit_merge_point(pc=pc, n=n, sa=sa, i=i)
@@ -2693,12 +2695,12 @@
def g(n1, n2):
for i in range(10):
f(n1)
- for i in range(10):
+ for i in range(10):
f(n2)
nn = [10, 3]
assert self.meta_interp(g, nn) == g(*nn)
-
+
# The attempts of retracing first loop will end up retracing the
# second and thus fail 5 times, saturating the retrace_count. Instead a
# bridge back to the preamble of the first loop is produced. A guard in
@@ -2709,7 +2711,7 @@
self.check_tree_loop_count(2 + 3)
# FIXME: Add a gloabl retrace counter and test that we are not trying more than 5 times.
-
+
def g(n):
for i in range(n):
for j in range(10):
@@ -2945,15 +2947,27 @@
a = [0, 1, 2, 3, 4]
while i < n:
myjitdriver.jit_merge_point(sa=sa, n=n, a=a, i=i)
- if i < n/2:
+ if i < n / 2:
sa += a[4]
- elif i == n/2:
+ elif i == n / 2:
a.pop()
i += 1
res = self.meta_interp(f, [32])
assert res == f(32)
self.check_loops(arraylen_gc=2)
-
+
+ def test_ulonglong_mod(self):
+ myjitdriver = JitDriver(greens = [], reds = ['n', 'sa', 'i'])
+ def f(n):
+ sa = i = rffi.cast(rffi.ULONGLONG, 1)
+ while i < rffi.cast(rffi.ULONGLONG, n):
+ myjitdriver.jit_merge_point(sa=sa, n=n, i=i)
+ sa += sa % i
+ i += 1
+ res = self.meta_interp(f, [32])
+ assert res == f(32)
+
+
class TestOOtype(BasicTests, OOJitMixin):
def test_oohash(self):
@@ -3173,7 +3187,7 @@
res = self.meta_interp(f, [32])
assert res == f(32)
self.check_tree_loop_count(3)
-
+
def test_two_loopinvariant_arrays3(self):
from pypy.rpython.lltypesystem import lltype, llmemory, rffi
myjitdriver = JitDriver(greens = [], reds = ['sa', 'n', 'i', 'a'])
@@ -3197,7 +3211,7 @@
res = self.meta_interp(f, [32])
assert res == f(32)
self.check_tree_loop_count(2)
-
+
def test_two_loopinvariant_arrays_boxed(self):
class A(object):
def __init__(self, a):
@@ -3222,7 +3236,7 @@
res = self.meta_interp(f, [32])
assert res == f(32)
self.check_loops(arraylen_gc=2, everywhere=True)
-
+
def test_release_gil_flush_heap_cache(self):
if sys.platform == "win32":
py.test.skip("needs 'time'")
@@ -3298,5 +3312,183 @@
self.meta_interp(main, [10])
+ def test_look_inside_iff_const(self):
+ @look_inside_iff(lambda arg: isconstant(arg))
+ def f(arg):
+ s = 0
+ while arg > 0:
+ s += arg
+ arg -= 1
+ return s
+
+ driver = JitDriver(greens = ['code'], reds = ['n', 'arg', 's'])
+
+ def main(code, n, arg):
+ s = 0
+ while n > 0:
+ driver.jit_merge_point(code=code, n=n, arg=arg, s=s)
+ if code == 0:
+ s += f(arg)
+ else:
+ s += f(1)
+ n -= 1
+ return s
+
+ res = self.meta_interp(main, [0, 10, 2], enable_opts='')
+ assert res == main(0, 10, 2)
+ self.check_loops(call=1)
+ res = self.meta_interp(main, [1, 10, 2], enable_opts='')
+ assert res == main(1, 10, 2)
+ self.check_loops(call=0)
+
+ def test_look_inside_iff_virtual(self):
+ # There's no good reason for this to be look_inside_iff, but it's a test!
+ @look_inside_iff(lambda arg, n: isvirtual(arg))
+ def f(arg, n):
+ if n == 100:
+ for i in xrange(n):
+ n += i
+ return arg.x
+ class A(object):
+ def __init__(self, x):
+ self.x = x
+ driver = JitDriver(greens=['n'], reds=['i', 'a'])
+ def main(n):
+ i = 0
+ a = A(3)
+ while i < 20:
+ driver.jit_merge_point(i=i, n=n, a=a)
+ if n == 0:
+ i += f(a, n)
+ else:
+ i += f(A(2), n)
+ res = self.meta_interp(main, [0], enable_opts='')
+ assert res == main(0)
+ self.check_loops(call=1, getfield_gc=0)
+ res = self.meta_interp(main, [1], enable_opts='')
+ assert res == main(1)
+ self.check_loops(call=0, getfield_gc=0)
+
+ def test_reuse_elidable_result(self):
+ driver = JitDriver(reds=['n', 's'], greens = [])
+ def main(n):
+ s = 0
+ while n > 0:
+ driver.jit_merge_point(s=s, n=n)
+ s += len(str(n)) + len(str(n))
+ n -= 1
+ return s
+ res = self.meta_interp(main, [10])
+ assert res == main(10)
+ self.check_loops({
+ 'call': 1, 'guard_no_exception': 1, 'guard_true': 1, 'int_add': 2,
+ 'int_gt': 1, 'int_sub': 1, 'strlen': 1, 'jump': 1,
+ })
+
+ def test_look_inside_iff_const_getarrayitem_gc_pure(self):
+ driver = JitDriver(greens=['unroll'], reds=['s', 'n'])
+
+ class A(object):
+ _immutable_fields_ = ["x[*]"]
+ def __init__(self, x):
+ self.x = [x]
+
+ @look_inside_iff(lambda x: isconstant(x))
+ def f(x):
+ i = 0
+ for c in x:
+ i += 1
+ return i
+
+ def main(unroll, n):
+ s = 0
+ while n > 0:
+ driver.jit_merge_point(s=s, n=n, unroll=unroll)
+ if unroll:
+ x = A("xx")
+ else:
+ x = A("x" * n)
+ s += f(x.x[0])
+ n -= 1
+ return s
+
+ res = self.meta_interp(main, [0, 10])
+ assert res == main(0, 10)
+ # 2 calls, one for f() and one for char_mul
+ self.check_loops(call=2)
+ res = self.meta_interp(main, [1, 10])
+ assert res == main(1, 10)
+ self.check_loops(call=0)
+
+ def test_setarrayitem_followed_by_arraycopy(self):
+ myjitdriver = JitDriver(greens = [], reds = ['n', 'sa', 'x', 'y'])
+ def f(n):
+ sa = 0
+ x = [1,2,n]
+ y = [1,2,3]
+ while n > 0:
+ myjitdriver.jit_merge_point(sa=sa, n=n, x=x, y=y)
+ y[0] = n
+ x[0:3] = y
+ sa += x[0]
+ n -= 1
+ return sa
+ res = self.meta_interp(f, [16])
+ assert res == f(16)
+
+
+
class TestLLtype(BaseLLtypeTests, LLJitMixin):
- pass
+ def test_tagged(self):
+ py.test.skip("implement me")
+ from pypy.rlib.objectmodel import UnboxedValue
+ class Base(object):
+ __slots__ = ()
+
+ class Int(UnboxedValue, Base):
+ __slots__ = ["a"]
+
+ def is_pos(self):
+ return self.a > 0
+
+ def dec(self):
+ return Int(self.a - 1)
+
+
+ class Float(Base):
+ def __init__(self, a):
+ self.a = a
+
+ def is_pos(self):
+ return self.a > 0
+
+ def dec(self):
+ return Float(self.a - 1)
+
+ driver = JitDriver(greens=['pc', 's'], reds=['o'])
+
+ def main(fl, n, s):
+ if s:
+ s = "--j"
+ else:
+ s = "---j"
+ if fl:
+ o = Float(float(n))
+ else:
+ o = Int(n)
+ pc = 0
+ while True:
+ driver.jit_merge_point(s=s, pc=pc, o=o)
+ c = s[pc]
+ if c == "j":
+ driver.can_enter_jit(s=s, pc=pc, o=o)
+ if o.is_pos():
+ pc = 0
+ continue
+ else:
+ break
+ elif c == "-":
+ o = o.dec()
+ pc += 1
+ return pc
+ res = self.meta_interp(main, [False, 100, True], taggedpointers=True)
diff --git a/pypy/jit/metainterp/test/test_dict.py b/pypy/jit/metainterp/test/test_dict.py
--- a/pypy/jit/metainterp/test/test_dict.py
+++ b/pypy/jit/metainterp/test/test_dict.py
@@ -153,11 +153,7 @@
res = self.meta_interp(f, [100], listops=True)
assert res == f(50)
- # XXX: ideally there would be 7 calls here, but repeated CALL_PURE with
- # the same arguments are not folded, because we have conflicting
- # definitions of pure, once strhash can be appropriately folded
- # this should be decreased to seven.
- self.check_loops({"call": 8, "guard_false": 1, "guard_no_exception": 6,
+ self.check_loops({"call": 7, "guard_false": 1, "guard_no_exception": 6,
"guard_true": 1, "int_and": 1, "int_gt": 1,
"int_is_true": 1, "int_sub": 1, "jump": 1,
"new_with_vtable": 1, "setfield_gc": 1})
diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py
--- a/pypy/jit/metainterp/test/test_heapcache.py
+++ b/pypy/jit/metainterp/test/test_heapcache.py
@@ -337,6 +337,24 @@
h.invalidate_caches(rop.SETFIELD_GC, None, [box1, box2])
assert not h.is_unescaped(box2)
+ def test_unescaped_testing(self):
+ h = HeapCache()
+ h.new(box1)
+ h.new(box2)
+ assert h.is_unescaped(box1)
+ assert h.is_unescaped(box2)
+ # Putting a virtual inside of another virtual doesn't escape it.
+ h.invalidate_caches(rop.SETFIELD_GC, None, [box1, box2])
+ assert h.is_unescaped(box2)
+ # Reading a field from a virtual doesn't escape it.
+ h.invalidate_caches(rop.GETFIELD_GC, None, [box1])
+ assert h.is_unescaped(box1)
+ # Escaping a virtual transitively escapes anything inside of it.
+ assert not h.is_unescaped(box3)
+ h.invalidate_caches(rop.SETFIELD_GC, None, [box3, box1])
+ assert not h.is_unescaped(box1)
+ assert not h.is_unescaped(box2)
+
def test_unescaped_array(self):
h = HeapCache()
h.new_array(box1, lengthbox1)
diff --git a/pypy/jit/metainterp/test/test_list.py b/pypy/jit/metainterp/test/test_list.py
--- a/pypy/jit/metainterp/test/test_list.py
+++ b/pypy/jit/metainterp/test/test_list.py
@@ -34,7 +34,7 @@
l = [x + 1]
n -= 1
return l[0]
-
+
res = self.meta_interp(f, [10], listops=True)
assert res == f(10)
self.check_all_virtualized()
@@ -60,7 +60,7 @@
def test_ll_fixed_setitem_fast(self):
jitdriver = JitDriver(greens = [], reds = ['n', 'l'])
-
+
def f(n):
l = [1, 2, 3]
@@ -116,7 +116,7 @@
assert res == f(10)
py.test.skip("'[non-null] * n' gives a residual call so far")
self.check_loops(setarrayitem_gc=0, getarrayitem_gc=0, call=0)
-
+
def test_arraycopy_simpleoptimize(self):
def f():
l = [1, 2, 3, 4]
@@ -208,6 +208,26 @@
assert res == f(15)
self.check_loops(guard_exception=0)
+ def test_virtual_resize(self):
+ jitdriver = JitDriver(greens = [], reds = ['n', 's'])
+ def f(n):
+ s = 0
+ while n > 0:
+ jitdriver.jit_merge_point(n=n, s=s)
+ lst = []
+ lst += [1]
+ n -= len(lst)
+ s += lst[0]
+ lst.pop()
+ lst.append(1)
+ s /= lst.pop()
+ return s
+ res = self.meta_interp(f, [15], listops=True)
+ assert res == f(15)
+ self.check_loops({"int_add": 1, "int_sub": 1, "int_gt": 1,
+ "guard_true": 1, "jump": 1})
+
+
class TestOOtype(ListTests, OOJitMixin):
pass
@@ -236,8 +256,6 @@
return a * b
res = self.meta_interp(f, [37])
assert res == f(37)
- # There is the one actual field on a, plus 2 getfield's from the list
- # itself, 1 to get the length (which is then incremented and passed to
- # the resize func), and then a read of the items field to actually
- # perform the setarrayitem on
- self.check_loops(getfield_gc=5, everywhere=True)
+ # There is the one actual field on a, plus several fields on the list
+ # itself
+ self.check_loops(getfield_gc=10, everywhere=True)
diff --git a/pypy/jit/metainterp/test/test_longlong.py b/pypy/jit/metainterp/test/test_longlong.py
--- a/pypy/jit/metainterp/test/test_longlong.py
+++ b/pypy/jit/metainterp/test/test_longlong.py
@@ -118,6 +118,26 @@
res = self.interp_operations(f, [1000000000])
assert res == 123500000000.0
+ def test_floats_negative(self):
+ def f(i):
+ # i == 1000000000
+ f = i * -123.5
+ n = r_longlong(f)
+ compare(n, -29, 1054051584)
+ return float(n)
+ res = self.interp_operations(f, [1000000000])
+ assert res == -123500000000.0
+
+ def test_floats_ulonglong(self):
+ def f(i):
+ # i == 1000000000
+ f = i * 12350000000.0
+ n = r_ulonglong(f)
+ compare(n, -1419508847, 538116096)
+ return float(n)
+ res = self.interp_operations(f, [1000000000])
+ assert res == 12350000000000000000.0
+
def test_unsigned_compare_ops(self):
def f(n1, n2):
# n == 30002000000000
diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py
--- a/pypy/jit/metainterp/test/test_resume.py
+++ b/pypy/jit/metainterp/test/test_resume.py
@@ -576,8 +576,9 @@
class FakeOptimizer_VirtualValue(object):
- class cpu:
- pass
+ class optimizer:
+ class cpu:
+ pass
fakeoptimizer = FakeOptimizer_VirtualValue()
def ConstAddr(addr, cpu): # compatibility
@@ -1135,12 +1136,7 @@
modifier.liveboxes = {}
modifier.vfieldboxes = {}
- class FakeOptimizer(object):
- class cpu:
- pass
- def new_const_item(self, descr):
- return None
- v2 = VArrayValue(FakeOptimizer(), LLtypeMixin.arraydescr, 2, b2s)
+ v2 = VArrayValue(LLtypeMixin.arraydescr, None, 2, b2s)
v2._items = [b4s, c1s]
modifier.register_virtual_fields(b2s, [b4s, c1s])
liveboxes = []
diff --git a/pypy/jit/metainterp/test/test_slist.py b/pypy/jit/metainterp/test/test_slist.py
--- a/pypy/jit/metainterp/test/test_slist.py
+++ b/pypy/jit/metainterp/test/test_slist.py
@@ -5,7 +5,6 @@
class ListTests(object):
def test_basic_list(self):
- py.test.skip("not yet")
myjitdriver = JitDriver(greens = [], reds = ['n', 'lst'])
def f(n):
lst = []
@@ -34,7 +33,7 @@
return m
res = self.interp_operations(f, [11], listops=True)
assert res == 49
- self.check_operations_history(call=5)
+ self.check_operations_history(call=3)
def test_list_of_voids(self):
myjitdriver = JitDriver(greens = [], reds = ['n', 'lst'])
@@ -93,7 +92,7 @@
return x
res = self.meta_interp(f, [-2], listops=True)
assert res == 41
- self.check_loops(call=1, guard_value=0)
+ self.check_loops(call=0, guard_value=0)
# we don't support resizable lists on ootype
#class TestOOtype(ListTests, OOJitMixin):
diff --git a/pypy/jit/metainterp/test/test_string.py b/pypy/jit/metainterp/test/test_string.py
--- a/pypy/jit/metainterp/test/test_string.py
+++ b/pypy/jit/metainterp/test/test_string.py
@@ -1,9 +1,11 @@
import py
+
+from pypy.jit.codewriter.policy import StopAtXPolicy
+from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
+from pypy.rlib.debug import debug_print
from pypy.rlib.jit import JitDriver, dont_look_inside, we_are_jitted
-from pypy.rlib.debug import debug_print
-from pypy.jit.codewriter.policy import StopAtXPolicy
+from pypy.rlib.rstring import StringBuilder
from pypy.rpython.ootypesystem import ootype
-from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
class StringTests:
@@ -27,7 +29,7 @@
return i
res = self.meta_interp(f, [10, True, _str('h')], listops=True)
assert res == 5
- self.check_loops(**{self.CALL: 1, self.CALL_PURE: 0})
+ self.check_loops(**{self.CALL: 1, self.CALL_PURE: 0, 'everywhere': True})
def test_eq_folded(self):
_str = self._str
@@ -327,7 +329,7 @@
def test_str_slice_len_surviving(self):
_str = self._str
longstring = _str("Unrolling Trouble")
- mydriver = JitDriver(reds = ['i', 'a', 'sa'], greens = [])
+ mydriver = JitDriver(reds = ['i', 'a', 'sa'], greens = [])
def f(a):
i = sa = a
while i < len(longstring):
@@ -343,7 +345,7 @@
fillers = _str("abcdefghijklmnopqrstuvwxyz")
data = _str("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
- mydriver = JitDriver(reds = ['line', 'noise', 'res'], greens = [])
+ mydriver = JitDriver(reds = ['line', 'noise', 'res'], greens = [])
def f():
line = data
noise = fillers
@@ -370,7 +372,7 @@
def __init__(self, value):
self.value = value
mydriver = JitDriver(reds = ['ratio', 'line', 'noise', 'res'],
- greens = [])
+ greens = [])
def f():
line = Str(data)
noise = Str(fillers)
@@ -408,7 +410,7 @@
return len(sa)
assert self.meta_interp(f, [16]) == f(16)
- def test_loop_invariant_string_slize(self):
+ def test_loop_invariant_string_slice(self):
_str = self._str
mydriver = JitDriver(reds = ['i', 'n', 'sa', 's', 's1'], greens = [])
def f(n, c):
@@ -425,7 +427,7 @@
return sa
assert self.meta_interp(f, [16, 'a']) == f(16, 'a')
- def test_loop_invariant_string_slize_boxed(self):
+ def test_loop_invariant_string_slice_boxed(self):
class Str(object):
def __init__(self, value):
self.value = value
@@ -445,7 +447,7 @@
return sa
assert self.meta_interp(f, [16, 'a']) == f(16, 'a')
- def test_loop_invariant_string_slize_in_array(self):
+ def test_loop_invariant_string_slice_in_array(self):
_str = self._str
mydriver = JitDriver(reds = ['i', 'n', 'sa', 's', 's1'], greens = [])
def f(n, c):
@@ -482,6 +484,30 @@
return len(sa.val)
assert self.meta_interp(f, ['a']) == f('a')
+ def test_string_comepare_quasiimmutable(self):
+ class Sys(object):
+ _immutable_fields_ = ["defaultencoding?"]
+ def __init__(self, s):
+ self.defaultencoding = s
+ _str = self._str
+ sys = Sys(_str('ascii'))
+ mydriver = JitDriver(reds = ['n', 'sa'], greens = [])
+ def f(n):
+ sa = 0
+ sys.defaultencoding = _str('ascii')
+ while n:
+ mydriver.jit_merge_point(n=n, sa=sa)
+ if sys.defaultencoding == _str('ascii'):
+ sa += 1
+ n -= 1
+ sys.defaultencoding = _str('utf-8')
+ return sa
+ assert self.meta_interp(f, [8]) == f(8)
+ self.check_loops({'int_add': 1, 'guard_true': 1, 'int_sub': 1,
+ 'jump': 1, 'int_is_true': 1,
+ 'guard_not_invalidated': 1})
+
+
#class TestOOtype(StringTests, OOJitMixin):
# CALL = "oosend"
# CALL_PURE = "oosend_pure"
@@ -513,7 +539,7 @@
m -= 1
return 42
self.meta_interp(f, [6, 7])
- self.check_loops(call=3, # str(), _str(), escape()
+ self.check_loops(call=1, # escape()
newunicode=1, unicodegetitem=0,
unicodesetitem=1, copyunicodecontent=1)
@@ -536,3 +562,55 @@
self.check_loops(call_pure=0, call=1,
newunicode=0, unicodegetitem=0,
unicodesetitem=0, copyunicodecontent=0)
+
+ def test_join_chars(self):
+ jitdriver = JitDriver(reds=['a', 'b', 'c', 'i'], greens=[])
+ def f(a, b, c):
+ i = 0
+ while i < 10:
+ jitdriver.jit_merge_point(a=a, b=b, c=c, i=i)
+ x = []
+ if a:
+ x.append("a")
+ if b:
+ x.append("b")
+ if c:
+ x.append("c")
+ i += len("".join(x))
+ return i
+ res = self.meta_interp(f, [1, 1, 1])
+ assert res == f(True, True, True)
+ # The "".join should be unrolled, since the length of x is known since
+ # it is virtual, ensure there are no calls to ll_join_chars, or
+ # allocations.
+ self.check_loops({
+ "guard_true": 5, "int_is_true": 3, "int_lt": 2, "int_add": 2, "jump": 2,
+ }, everywhere=True)
+
+ def test_virtual_copystringcontent(self):
+ jitdriver = JitDriver(reds=['n', 'result'], greens=[])
+ def main(n):
+ result = 0
+ while n >= 0:
+ jitdriver.jit_merge_point(n=n, result=result)
+ b = StringBuilder(6)
+ b.append("Hello!")
+ result += ord(b.build()[0])
+ n -= 1
+ return result
+ res = self.meta_interp(main, [9])
+ assert res == main(9)
+
+ def test_virtual_copystringcontent2(self):
+ jitdriver = JitDriver(reds=['n', 'result'], greens=[])
+ def main(n):
+ result = 0
+ while n >= 0:
+ jitdriver.jit_merge_point(n=n, result=result)
+ b = StringBuilder(6)
+ b.append("Hello!")
+ result += ord((b.build() + "xyz")[0])
+ n -= 1
+ return result
+ res = self.meta_interp(main, [9])
+ assert res == main(9)
diff --git a/pypy/jit/metainterp/test/test_tracingopts.py b/pypy/jit/metainterp/test/test_tracingopts.py
--- a/pypy/jit/metainterp/test/test_tracingopts.py
+++ b/pypy/jit/metainterp/test/test_tracingopts.py
@@ -1,7 +1,10 @@
+import sys
+
+from pypy.jit.metainterp.test.support import LLJitMixin
+from pypy.rlib import jit
+from pypy.rlib.rarithmetic import ovfcheck
+
import py
-import sys
-from pypy.rlib import jit
-from pypy.jit.metainterp.test.support import LLJitMixin
class TestLLtype(LLJitMixin):
@@ -573,3 +576,18 @@
res = self.interp_operations(fn, [3])
assert res == 24
self.check_operations_history(getarrayitem_gc=0)
+
+ def test_fold_int_add_ovf(self):
+ def fn(n):
+ jit.promote(n)
+ try:
+ n = ovfcheck(n + 1)
+ except OverflowError:
+ return 12
+ else:
+ return n
+ res = self.interp_operations(fn, [3])
+ assert res == 4
+ self.check_operations_history(int_add_ovf=0)
+ res = self.interp_operations(fn, [sys.maxint])
+ assert res == 12
\ No newline at end of file
diff --git a/pypy/jit/metainterp/test/test_virtualstate.py b/pypy/jit/metainterp/test/test_virtualstate.py
--- a/pypy/jit/metainterp/test/test_virtualstate.py
+++ b/pypy/jit/metainterp/test/test_virtualstate.py
@@ -431,7 +431,7 @@
class BaseTestBridges(BaseTest):
- enable_opts = "intbounds:rewrite:virtualize:string:heap:unroll"
+ enable_opts = "intbounds:rewrite:virtualize:string:pure:heap:unroll"
def _do_optimize_bridge(self, bridge, call_pure_results):
from pypy.jit.metainterp.optimizeopt import optimize_bridge_1, build_opt_chain
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -53,6 +53,8 @@
extraconfigopts = {'translation.list_comprehension_operations': True}
else:
extraconfigopts = {}
+ if kwds.pop("taggedpointers", False):
+ extraconfigopts["translation.taggedpointers"] = True
interp, graph = get_interpreter(function, args,
backendopt=False, # will be done below
type_system=type_system,
diff --git a/pypy/jit/tl/pypyjit.py b/pypy/jit/tl/pypyjit.py
--- a/pypy/jit/tl/pypyjit.py
+++ b/pypy/jit/tl/pypyjit.py
@@ -40,7 +40,7 @@
config.objspace.usemodules.array = False
config.objspace.usemodules._weakref = True
config.objspace.usemodules._sre = False
-config.objspace.usemodules._lsprof = True
+config.objspace.usemodules._lsprof = False
#
config.objspace.usemodules._ffi = True
config.objspace.usemodules.micronumpy = False
@@ -77,7 +77,7 @@
def read_code():
from pypy.module.marshal.interp_marshal import dumps
-
+
filename = 'pypyjit_demo.py'
source = readfile(filename)
ec = space.getexecutioncontext()
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -3,13 +3,13 @@
"""
+from pypy.interpreter.baseobjspace import Wrappable
from pypy.interpreter.error import OperationError
-from pypy.interpreter.gateway import NoneNotWrapped
-from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.gateway import NoneNotWrapped, interp2app, unwrap_spec
from pypy.interpreter.typedef import TypeDef
-from pypy.interpreter.baseobjspace import Wrappable
+from pypy.rlib import jit
+from pypy.rlib.objectmodel import specialize
from pypy.rlib.rarithmetic import r_uint, intmask
-from pypy.rlib.objectmodel import specialize
from pypy.rlib.rbigint import rbigint
@@ -134,29 +134,15 @@
@specialize.arg(2)
+ at jit.look_inside_iff(lambda space, args, implementation_of:
+ jit.isconstant(len(args.arguments_w)) and
+ len(args.arguments_w) == 2
+)
def min_max(space, args, implementation_of):
if implementation_of == "max":
compare = space.gt
else:
compare = space.lt
-
- args_w = args.arguments_w
- if len(args_w) == 2 and not args.keywords:
- # simple case, suitable for the JIT
- w_arg0, w_arg1 = args_w
- if space.is_true(compare(w_arg0, w_arg1)):
- return w_arg0
- else:
- return w_arg1
- else:
- return min_max_loop(space, args, implementation_of)
-
- at specialize.arg(2)
-def min_max_loop(space, args, implementation_of):
- if implementation_of == "max":
- compare = space.gt
- else:
- compare = space.lt
args_w = args.arguments_w
if len(args_w) > 1:
w_sequence = space.newtuple(args_w)
@@ -207,9 +193,11 @@
return min_max(space, __args__, "max")
def min(space, __args__):
- """Return the smallest item in a sequence.
+ """min(iterable[, key=func]) -> value
+ min(a, b, c, ...[, key=func]) -> value
- If more than one argument is passed, return the minimum of them.
+ With a single iterable argument, return its smallest item.
+ With two or more arguments, return the smallest argument.
"""
return min_max(space, __args__, "min")
diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py
--- a/pypy/module/__pypy__/interp_builders.py
+++ b/pypy/module/__pypy__/interp_builders.py
@@ -16,7 +16,8 @@
def _check_done(self, space):
if self.builder is None:
- raise OperationError(space.w_ValueError, space.wrap("Can't operate on a done builder"))
+ raise OperationError(space.w_ValueError, space.wrap(
+ "Can't operate on a built builder"))
@unwrap_spec(size=int)
def descr__new__(space, w_subtype, size=-1):
@@ -31,7 +32,8 @@
def descr_append_slice(self, space, s, start, end):
self._check_done(space)
if not 0 <= start <= end <= len(s):
- raise OperationError(space.w_ValueError, space.wrap("bad start/stop"))
+ raise OperationError(space.w_ValueError, space.wrap(
+ "bad start/stop"))
self.builder.append_slice(s, start, end)
def descr_build(self, space):
@@ -40,6 +42,12 @@
self.builder = None
return w_s
+ def descr_len(self, space):
+ if self.builder is None:
+ raise OperationError(space.w_ValueError, space.wrap(
+ "no length of built builder"))
+ return space.wrap(self.builder.getlength())
+
W_Builder.__name__ = "W_%s" % name
W_Builder.typedef = TypeDef(name,
__new__ = interp2app(func_with_new_name(
@@ -48,6 +56,7 @@
append = interp2app(W_Builder.descr_append),
append_slice = interp2app(W_Builder.descr_append_slice),
build = interp2app(W_Builder.descr_build),
+ __len__ = interp2app(W_Builder.descr_len),
)
W_Builder.typedef.acceptable_as_base_class = False
return W_Builder
diff --git a/pypy/module/__pypy__/test/test_builders.py b/pypy/module/__pypy__/test/test_builders.py
--- a/pypy/module/__pypy__/test/test_builders.py
+++ b/pypy/module/__pypy__/test/test_builders.py
@@ -38,7 +38,9 @@
b = StringBuilder()
b.append("abc")
b.append("123")
+ assert len(b) == 6
b.append("you and me")
s = b.build()
+ raises(ValueError, len, b)
assert s == "abc123you and me"
- raises(ValueError, b.build)
\ No newline at end of file
+ raises(ValueError, b.build)
diff --git a/pypy/module/_continuation/__init__.py b/pypy/module/_continuation/__init__.py
--- a/pypy/module/_continuation/__init__.py
+++ b/pypy/module/_continuation/__init__.py
@@ -37,4 +37,5 @@
interpleveldefs = {
'continulet': 'interp_continuation.W_Continulet',
'permute': 'interp_continuation.permute',
+ '_p': 'interp_continuation.unpickle', # pickle support
}
diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -6,6 +6,7 @@
from pypy.interpreter.typedef import TypeDef
from pypy.interpreter.gateway import interp2app
from pypy.interpreter.pycode import PyCode
+from pypy.interpreter.pyframe import PyFrame
class W_Continulet(Wrappable):
@@ -23,24 +24,27 @@
if ec.stacklet_thread is not self.sthread:
global_state.clear()
raise geterror(self.space, "inter-thread support is missing")
- return ec
def descr_init(self, w_callable, __args__):
if self.sthread is not None:
raise geterror(self.space, "continulet already __init__ialized")
+ #
+ # hackish: build the frame "by hand", passing it the correct arguments
+ space = self.space
+ w_args, w_kwds = __args__.topacked()
+ bottomframe = space.createframe(get_entrypoint_pycode(space),
+ get_w_module_dict(space), None)
+ bottomframe.locals_stack_w[0] = space.wrap(self)
+ bottomframe.locals_stack_w[1] = w_callable
+ bottomframe.locals_stack_w[2] = w_args
+ bottomframe.locals_stack_w[3] = w_kwds
+ self.bottomframe = bottomframe
+ #
global_state.origin = self
- global_state.w_callable = w_callable
- global_state.args = __args__
- self.bottomframe = make_fresh_frame(self.space)
- self.sthread = build_sthread(self.space)
- try:
- self.h = self.sthread.new(new_stacklet_callback)
- if self.sthread.is_empty_handle(self.h): # early return
- raise MemoryError
- except MemoryError:
- self.sthread = None
- global_state.clear()
- raise getmemoryerror(self.space)
+ sthread = build_sthread(self.space)
+ self.sthread = sthread
+ h = sthread.new(new_stacklet_callback)
+ post_switch(sthread, h)
def switch(self, w_to):
sthread = self.sthread
@@ -66,7 +70,7 @@
if sthread.is_empty_handle(to.h):
global_state.clear()
raise geterror(self.space, "continulet already finished")
- ec = self.check_sthread()
+ self.check_sthread()
#
global_state.origin = self
if to is None:
@@ -76,13 +80,8 @@
# double switch: the final destination is to.h
global_state.destination = to
#
- try:
- do_switch(sthread, global_state.destination.h)
- except MemoryError:
- global_state.clear()
- raise getmemoryerror(self.space)
- #
- return get_result()
+ h = sthread.switch(global_state.destination.h)
+ return post_switch(sthread, h)
def descr_switch(self, w_value=None, w_to=None):
global_state.w_value = w_value
@@ -109,12 +108,26 @@
and not self.sthread.is_empty_handle(self.h))
return self.space.newbool(valid)
+ def descr__reduce__(self):
+ from pypy.module._continuation import interp_pickle
+ return interp_pickle.reduce(self)
+
+ def descr__setstate__(self, w_args):
+ from pypy.module._continuation import interp_pickle
+ interp_pickle.setstate(self, w_args)
+
def W_Continulet___new__(space, w_subtype, __args__):
r = space.allocate_instance(W_Continulet, w_subtype)
r.__init__(space)
return space.wrap(r)
+def unpickle(space, w_subtype):
+ """Pickle support."""
+ r = space.allocate_instance(W_Continulet, w_subtype)
+ r.__init__(space)
+ return space.wrap(r)
+
W_Continulet.typedef = TypeDef(
'continulet',
@@ -124,9 +137,10 @@
switch = interp2app(W_Continulet.descr_switch),
throw = interp2app(W_Continulet.descr_throw),
is_pending = interp2app(W_Continulet.descr_is_pending),
+ __reduce__ = interp2app(W_Continulet.descr__reduce__),
+ __setstate__= interp2app(W_Continulet.descr__setstate__),
)
-
# ____________________________________________________________
# Continulet objects maintain a dummy frame object in order to ensure
@@ -135,27 +149,40 @@
class State:
def __init__(self, space):
- from pypy.interpreter.astcompiler.consts import CO_OPTIMIZED
- self.space = space
+ self.space = space
w_module = space.getbuiltinmodule('_continuation')
self.w_error = space.getattr(w_module, space.wrap('error'))
- self.w_memoryerror = OperationError(space.w_MemoryError, space.w_None)
- self.dummy_pycode = PyCode(space, 0, 0, 0, CO_OPTIMIZED,
- '', [], [], [], '',
- '<bottom of continulet>', 0, '', [], [],
- hidden_applevel=True)
+ # the following function switches away immediately, so that
+ # continulet.__init__() doesn't immediately run func(), but it
+ # also has the hidden purpose of making sure we have a single
+ # bottomframe for the whole duration of the continulet's run.
+ # Hackish: only the func_code is used, and used in the context
+ # of w_globals == this module, so we can access the name
+ # 'continulet' directly.
+ w_code = space.appexec([], '''():
+ def start(c, func, args, kwds):
+ if continulet.switch(c) is not None:
+ raise TypeError(
+ "can\'t send non-None value to a just-started continulet")
+ return func(c, *args, **kwds)
+ return start.func_code
+ ''')
+ self.entrypoint_pycode = space.interp_w(PyCode, w_code)
+ self.entrypoint_pycode.hidden_applevel = True
+ self.w_unpickle = w_module.get('_p')
+ self.w_module_dict = w_module.getdict(space)
def geterror(space, message):
cs = space.fromcache(State)
return OperationError(cs.w_error, space.wrap(message))
-def getmemoryerror(space):
+def get_entrypoint_pycode(space):
cs = space.fromcache(State)
- return cs.w_memoryerror
+ return cs.entrypoint_pycode
-def make_fresh_frame(space):
+def get_w_module_dict(space):
cs = space.fromcache(State)
- return space.FrameClass(space, cs.dummy_pycode, None, None)
+ return cs.w_module_dict
# ____________________________________________________________
@@ -166,6 +193,9 @@
StackletThread.__init__(self, space.config)
self.space = space
self.ec = ec
+ # for unpickling
+ from pypy.rlib.rweakref import RWeakKeyDictionary
+ self.frame2continulet = RWeakKeyDictionary(PyFrame, W_Continulet)
ExecutionContext.stacklet_thread = None
@@ -176,8 +206,6 @@
def clear(self):
self.origin = None
self.destination = None
- self.w_callable = None
- self.args = None
self.w_value = None
self.propagate_exception = None
global_state = GlobalState()
@@ -185,27 +213,13 @@
def new_stacklet_callback(h, arg):
- self = global_state.origin
- w_callable = global_state.w_callable
- args = global_state.args
+ self = global_state.origin
+ self.h = h
global_state.clear()
- try:
- do_switch(self.sthread, h)
- except MemoryError:
- return h # oups! do an early return in this case
- #
space = self.space
try:
- assert self.sthread.ec.topframeref() is None
- self.sthread.ec.topframeref = jit.non_virtual_ref(self.bottomframe)
- if global_state.propagate_exception is not None:
- raise global_state.propagate_exception # just propagate it further
- if global_state.w_value is not space.w_None:
- raise OperationError(space.w_TypeError, space.wrap(
- "can't send non-None value to a just-started continulet"))
-
- args = args.prepend(self.space.wrap(self))
- w_result = space.call_args(w_callable, args)
+ frame = self.bottomframe
+ w_result = frame.execute_frame()
except Exception, e:
global_state.propagate_exception = e
else:
@@ -215,9 +229,7 @@
global_state.destination = self
return self.h
-
-def do_switch(sthread, h):
- h = sthread.switch(h)
+def post_switch(sthread, h):
origin = global_state.origin
self = global_state.destination
global_state.origin = None
@@ -228,6 +240,8 @@
sthread.ec.topframeref = self.bottomframe.f_backref
self.bottomframe.f_backref = origin.bottomframe.f_backref
origin.bottomframe.f_backref = current
+ #
+ return get_result()
def get_result():
if global_state.propagate_exception:
diff --git a/pypy/module/_continuation/interp_pickle.py b/pypy/module/_continuation/interp_pickle.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_continuation/interp_pickle.py
@@ -0,0 +1,128 @@
+from pypy.tool import stdlib_opcode as pythonopcode
+from pypy.rlib import jit
+from pypy.interpreter.error import OperationError
+from pypy.interpreter.pyframe import PyFrame
+from pypy.module._continuation.interp_continuation import State, global_state
+from pypy.module._continuation.interp_continuation import build_sthread
+from pypy.module._continuation.interp_continuation import post_switch
+from pypy.module._continuation.interp_continuation import get_result, geterror
+
+
+def getunpickle(space):
+ cs = space.fromcache(State)
+ return cs.w_unpickle
+
+
+def reduce(self):
+ # xxx this is known to be not completely correct with respect
+ # to subclasses, e.g. no __slots__ support, no looking for a
+ # __getnewargs__ or __getstate__ defined in the subclass, etc.
+ # Doing the right thing looks involved, though...
+ space = self.space
+ if self.sthread is None:
+ w_frame = space.w_False
+ elif self.sthread.is_empty_handle(self.h):
+ w_frame = space.w_None
+ else:
+ w_frame = space.wrap(self.bottomframe)
+ w_continulet_type = space.type(space.wrap(self))
+ w_dict = self.getdict(space) or space.w_None
+ args = [getunpickle(space),
+ space.newtuple([w_continulet_type]),
+ space.newtuple([w_frame, w_dict]),
+ ]
+ return space.newtuple(args)
+
+def setstate(self, w_args):
+ space = self.space
+ if self.sthread is not None:
+ raise geterror(space, "continulet.__setstate__() on an already-"
+ "initialized continulet")
+ w_frame, w_dict = space.fixedview(w_args, expected_length=2)
+ if not space.is_w(w_dict, space.w_None):
+ self.setdict(space, w_dict)
+ if space.is_w(w_frame, space.w_False):
+ return # not initialized
+ sthread = build_sthread(self.space)
+ self.sthread = sthread
+ self.bottomframe = space.interp_w(PyFrame, w_frame, can_be_None=True)
+ #
+ global_state.origin = self
+ if self.bottomframe is not None:
+ sthread.frame2continulet.set(self.bottomframe, self)
+ self.h = sthread.new(resume_trampoline_callback)
+ get_result() # propagate the eventual MemoryError
+
+# ____________________________________________________________
+
+def resume_trampoline_callback(h, arg):
+ self = global_state.origin
+ self.h = h
+ space = self.space
+ sthread = self.sthread
+ try:
+ global_state.clear()
+ if self.bottomframe is None:
+ w_result = space.w_None
+ else:
+ h = sthread.switch(self.h)
+ try:
+ w_result = post_switch(sthread, h)
+ operr = None
+ except OperationError, e:
+ w_result = None
+ operr = e
+ #
+ while True:
+ ec = sthread.ec
+ frame = ec.topframeref()
+ assert frame is not None # XXX better error message
+ exit_continulet = sthread.frame2continulet.get(frame)
+ #
+ continue_after_call(frame)
+ #
+ # small hack: unlink frame out of the execution context,
+ # because execute_frame will add it there again
+ ec.topframeref = frame.f_backref
+ #
+ try:
+ w_result = frame.execute_frame(w_result, operr)
+ operr = None
+ except OperationError, e:
+ w_result = None
+ operr = e
+ if exit_continulet is not None:
+ self = exit_continulet
+ break
+ sthread.ec.topframeref = jit.vref_None
+ if operr:
+ raise operr
+ except Exception, e:
+ global_state.propagate_exception = e
+ else:
+ global_state.w_value = w_result
+ global_state.origin = self
+ global_state.destination = self
+ return self.h
+
+def continue_after_call(frame):
+ code = frame.pycode.co_code
+ instr = frame.last_instr
+ opcode = ord(code[instr])
+ map = pythonopcode.opmap
+ call_ops = [map['CALL_FUNCTION'], map['CALL_FUNCTION_KW'],
+ map['CALL_FUNCTION_VAR'], map['CALL_FUNCTION_VAR_KW'],
+ map['CALL_METHOD']]
+ assert opcode in call_ops # XXX check better, and complain better
+ instr += 1
+ oparg = ord(code[instr]) | ord(code[instr + 1]) << 8
+ nargs = oparg & 0xff
+ nkwds = (oparg >> 8) & 0xff
+ if nkwds == 0: # only positional arguments
+ # fast paths leaves things on the stack, pop them
+ if (frame.space.config.objspace.opcodes.CALL_METHOD and
+ opcode == map['CALL_METHOD']):
+ frame.dropvalues(nargs + 2)
+ elif opcode == map['CALL_FUNCTION']:
+ frame.dropvalues(nargs + 1)
+ frame.last_instr = instr + 1 # continue after the call
diff --git a/pypy/module/_continuation/test/test_stacklet.py b/pypy/module/_continuation/test/test_stacklet.py
--- a/pypy/module/_continuation/test/test_stacklet.py
+++ b/pypy/module/_continuation/test/test_stacklet.py
@@ -13,7 +13,7 @@
from _continuation import continulet
#
def empty_callback(c):
- pass
+ never_called
#
c = continulet(empty_callback)
assert type(c) is continulet
@@ -36,7 +36,7 @@
from _continuation import continulet, error
#
def empty_callback(c1):
- pass
+ never_called
#
c = continulet(empty_callback)
raises(error, c.__init__, empty_callback)
diff --git a/pypy/module/_continuation/test/test_zpickle.py b/pypy/module/_continuation/test/test_zpickle.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_continuation/test/test_zpickle.py
@@ -0,0 +1,262 @@
+from pypy.conftest import gettestobjspace
+
+
+class AppTestCopy:
+ def setup_class(cls):
+ cls.space = gettestobjspace(usemodules=('_continuation',),
+ CALL_METHOD=True)
+ cls.space.config.translation.continuation = True
+
+ def test_basic_setup(self):
+ from _continuation import continulet
+ lst = [4]
+ co = continulet(lst.append)
+ assert lst == [4]
+ res = co.switch()
+ assert res is None
+ assert lst == [4, co]
+
+ def test_copy_continulet_not_started(self):
+ from _continuation import continulet, error
+ import copy
+ lst = []
+ co = continulet(lst.append)
+ co2, lst2 = copy.deepcopy((co, lst))
+ #
+ assert lst == []
+ co.switch()
+ assert lst == [co]
+ #
+ assert lst2 == []
+ co2.switch()
+ assert lst2 == [co2]
+
+ def test_copy_continulet_not_started_multiple(self):
+ from _continuation import continulet, error
+ import copy
+ lst = []
+ co = continulet(lst.append)
+ co2, lst2 = copy.deepcopy((co, lst))
+ co3, lst3 = copy.deepcopy((co, lst))
+ co4, lst4 = copy.deepcopy((co, lst))
+ #
+ assert lst == []
+ co.switch()
+ assert lst == [co]
+ #
+ assert lst2 == []
+ co2.switch()
+ assert lst2 == [co2]
+ #
+ assert lst3 == []
+ co3.switch()
+ assert lst3 == [co3]
+ #
+ assert lst4 == []
+ co4.switch()
+ assert lst4 == [co4]
+
+ def test_copy_continulet_real(self):
+ import new, sys
+ mod = new.module('test_copy_continulet_real')
+ sys.modules['test_copy_continulet_real'] = mod
+ exec '''if 1:
+ from _continuation import continulet
+ import copy
+ def f(co, x):
+ co.switch(x + 1)
+ co.switch(x + 2)
+ return x + 3
+ co = continulet(f, 40)
+ res = co.switch()
+ assert res == 41
+ co2 = copy.deepcopy(co)
+ #
+ res = co2.switch()
+ assert res == 42
+ assert co2.is_pending()
+ res = co2.switch()
+ assert res == 43
+ assert not co2.is_pending()
+ #
+ res = co.switch()
+ assert res == 42
+ assert co.is_pending()
+ res = co.switch()
+ assert res == 43
+ assert not co.is_pending()
+ ''' in mod.__dict__
+
+ def test_copy_continulet_already_finished(self):
+ from _continuation import continulet, error
+ import copy
+ lst = []
+ co = continulet(lst.append)
+ co.switch()
+ co2 = copy.deepcopy(co)
+ assert not co.is_pending()
+ assert not co2.is_pending()
+ raises(error, co.__init__, lst.append)
+ raises(error, co2.__init__, lst.append)
+ raises(error, co.switch)
+ raises(error, co2.switch)
+
+
+class AppTestPickle:
+ version = 0
+
+ def setup_class(cls):
+ cls.space = gettestobjspace(usemodules=('_continuation',),
+ CALL_METHOD=True)
+ cls.space.appexec([], """():
+ global continulet, A, __name__
+
+ import sys
+ __name__ = 'test_pickle_continulet'
+ thismodule = type(sys)(__name__)
+ sys.modules[__name__] = thismodule
+
+ from _continuation import continulet
+ class A(continulet):
+ pass
+
+ thismodule.__dict__.update(globals())
+ """)
+ cls.w_version = cls.space.wrap(cls.version)
+
+ def test_pickle_continulet_empty(self):
+ from _continuation import continulet
+ lst = [4]
+ co = continulet.__new__(continulet)
+ import pickle
+ pckl = pickle.dumps(co, self.version)
+ print repr(pckl)
+ co2 = pickle.loads(pckl)
+ assert co2 is not co
+ assert not co.is_pending()
+ assert not co2.is_pending()
+ # the empty unpickled coroutine can still be used:
+ result = [5]
+ co2.__init__(result.append)
+ res = co2.switch()
+ assert res is None
+ assert result == [5, co2]
+
+ def test_pickle_continulet_empty_subclass(self):
+ from test_pickle_continulet import continulet, A
+ lst = [4]
+ co = continulet.__new__(A)
+ co.foo = 'bar'
+ co.bar = 'baz'
+ import pickle
+ pckl = pickle.dumps(co, self.version)
+ print repr(pckl)
+ co2 = pickle.loads(pckl)
+ assert co2 is not co
+ assert not co.is_pending()
+ assert not co2.is_pending()
+ assert type(co) is type(co2) is A
+ assert co.foo == co2.foo == 'bar'
+ assert co.bar == co2.bar == 'baz'
+ # the empty unpickled coroutine can still be used:
+ result = [5]
+ co2.__init__(result.append)
+ res = co2.switch()
+ assert res is None
+ assert result == [5, co2]
+
+ def test_pickle_continulet_not_started(self):
+ from _continuation import continulet, error
+ import pickle
+ lst = []
+ co = continulet(lst.append)
+ pckl = pickle.dumps((co, lst))
+ print pckl
+ del co, lst
+ for i in range(2):
+ print 'resume...'
+ co2, lst2 = pickle.loads(pckl)
+ assert lst2 == []
+ co2.switch()
+ assert lst2 == [co2]
+
+ def test_pickle_continulet_real(self):
+ import new, sys
+ mod = new.module('test_pickle_continulet_real')
+ sys.modules['test_pickle_continulet_real'] = mod
+ mod.version = self.version
+ exec '''if 1:
+ from _continuation import continulet
+ import pickle
+ def f(co, x):
+ co.switch(x + 1)
+ co.switch(x + 2)
+ return x + 3
+ co = continulet(f, 40)
+ res = co.switch()
+ assert res == 41
+ pckl = pickle.dumps(co, version)
+ print repr(pckl)
+ co2 = pickle.loads(pckl)
+ #
+ res = co2.switch()
+ assert res == 42
+ assert co2.is_pending()
+ res = co2.switch()
+ assert res == 43
+ assert not co2.is_pending()
+ #
+ res = co.switch()
+ assert res == 42
+ assert co.is_pending()
+ res = co.switch()
+ assert res == 43
+ assert not co.is_pending()
+ ''' in mod.__dict__
+
+ def test_pickle_continulet_real_subclass(self):
+ import new, sys
+ mod = new.module('test_pickle_continulet_real_subclass')
+ sys.modules['test_pickle_continulet_real_subclass'] = mod
+ mod.version = self.version
+ exec '''if 1:
+ from _continuation import continulet
+ import pickle
+ class A(continulet):
+ def __init__(self):
+ crash
+ def f(co):
+ co.switch(co.x + 1)
+ co.switch(co.x + 2)
+ return co.x + 3
+ co = A.__new__(A)
+ continulet.__init__(co, f)
+ co.x = 40
+ res = co.switch()
+ assert res == 41
+ pckl = pickle.dumps(co, version)
+ print repr(pckl)
+ co2 = pickle.loads(pckl)
+ #
+ assert type(co2) is A
+ res = co2.switch()
+ assert res == 42
+ assert co2.is_pending()
+ res = co2.switch()
+ assert res == 43
+ assert not co2.is_pending()
+ #
+ res = co.switch()
+ assert res == 42
+ assert co.is_pending()
+ res = co.switch()
+ assert res == 43
+ assert not co.is_pending()
+ ''' in mod.__dict__
+
+
+class AppTestPickle_v1(AppTestPickle):
+ version = 1
+
+class AppTestPickle_v2(AppTestPickle):
+ version = 2
diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -1,5 +1,5 @@
from __future__ import with_statement
-import py
+import py, os, errno
from pypy.conftest import gettestobjspace, option
@@ -257,6 +257,35 @@
assert self.temppath in g.getvalue()
+class AppTestNonblocking(object):
+ def setup_class(cls):
+ from pypy.module._file.interp_file import W_File
+
+ cls.old_read = os.read
+ state = [0]
+ def read(fd, n=None):
+ if fd != 42:
+ return cls.old_read(fd, n)
+ if state[0] == 0:
+ state[0] += 1
+ return "xyz"
+ if state[0] < 3:
+ state[0] += 1
+ raise OSError(errno.EAGAIN, "xyz")
+ return ''
+ os.read = read
+ stdin = W_File(cls.space)
+ stdin.file_fdopen(42, "r", 1)
+ stdin.name = '<stdin>'
+ cls.w_stream = stdin
+
+ def teardown_class(cls):
+ os.read = cls.old_read
+
+ def test_nonblocking_file(self):
+ res = self.stream.read()
+ assert res == 'xyz'
+
class AppTestConcurrency(object):
# these tests only really make sense on top of a translated pypy-c,
# because on top of py.py the inner calls to os.write() don't
diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ b/pypy/module/_multiprocessing/interp_connection.py
@@ -225,7 +225,9 @@
except OSError:
pass
- def __init__(self, fd, flags):
+ def __init__(self, space, fd, flags):
+ if fd == self.INVALID_HANDLE_VALUE or fd < 0:
+ raise OperationError(space.w_IOError, space.wrap("invalid handle %d" % fd))
W_BaseConnection.__init__(self, flags)
self.fd = fd
@@ -234,7 +236,7 @@
flags = (readable and READABLE) | (writable and WRITABLE)
self = space.allocate_instance(W_FileConnection, w_subtype)
- W_FileConnection.__init__(self, fd, flags)
+ W_FileConnection.__init__(self, space, fd, flags)
return space.wrap(self)
def fileno(self, space):
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -468,6 +468,9 @@
self.count -= 1
+ def after_fork(self):
+ self.count = 0
+
@unwrap_spec(kind=int, maxvalue=int)
def rebuild(space, w_cls, w_handle, kind, maxvalue):
self = space.allocate_instance(W_SemLock, w_cls)
@@ -512,6 +515,7 @@
acquire = interp2app(W_SemLock.acquire),
release = interp2app(W_SemLock.release),
_rebuild = interp2app(W_SemLock.rebuild.im_func, as_classmethod=True),
+ _after_fork = interp2app(W_SemLock.after_fork),
__enter__=interp2app(W_SemLock.enter),
__exit__=interp2app(W_SemLock.exit),
SEM_VALUE_MAX=SEM_VALUE_MAX,
diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py
--- a/pypy/module/_multiprocessing/test/test_connection.py
+++ b/pypy/module/_multiprocessing/test/test_connection.py
@@ -145,3 +145,9 @@
else:
c.close()
space.delslice(w_connections, space.wrap(0), space.wrap(100))
+
+ def test_bad_fd(self):
+ import _multiprocessing
+
+ raises(IOError, _multiprocessing.Connection, -1)
+ raises(IOError, _multiprocessing.Connection, -15)
\ No newline at end of file
diff --git a/pypy/module/_multiprocessing/test/test_semaphore.py b/pypy/module/_multiprocessing/test/test_semaphore.py
--- a/pypy/module/_multiprocessing/test/test_semaphore.py
+++ b/pypy/module/_multiprocessing/test/test_semaphore.py
@@ -39,6 +39,10 @@
sem.release()
assert sem._count() == 0
+ sem.acquire()
+ sem._after_fork()
+ assert sem._count() == 0
+
def test_recursive(self):
from _multiprocessing import SemLock
kind = self.RECURSIVE
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -99,6 +99,7 @@
# SRE_Pattern class
class W_SRE_Pattern(Wrappable):
+ _immutable_fields_ = ["code", "flags"]
def cannot_copy_w(self):
space = self.space
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -57,6 +57,9 @@
compile_extra=['-DPy_BUILD_CORE'],
)
+class CConfig2:
+ _compilation_info_ = CConfig._compilation_info_
+
class CConfig_constants:
_compilation_info_ = CConfig._compilation_info_
@@ -300,9 +303,13 @@
return unwrapper_raise # used in 'normal' RPython code.
return decorate
-def cpython_struct(name, fields, forward=None):
+def cpython_struct(name, fields, forward=None, level=1):
configname = name.replace(' ', '__')
- setattr(CConfig, configname, rffi_platform.Struct(name, fields))
+ if level == 1:
+ config = CConfig
+ else:
+ config = CConfig2
+ setattr(config, configname, rffi_platform.Struct(name, fields))
if forward is None:
forward = lltype.ForwardReference()
TYPES[configname] = forward
@@ -445,9 +452,10 @@
More information about the pypy-commit
mailing list