[pypy-commit] pypy win32-cleanup2: merge from default
mattip
noreply at buildbot.pypy.org
Fri Mar 30 13:22:10 CEST 2012
Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54099:ddd8b35460f5
Date: 2012-03-27 11:29 +0200
http://bitbucket.org/pypy/pypy/changeset/ddd8b35460f5/
Log: merge from default
diff too long, truncating to 10000 out of 22022 lines
diff --git a/lib-python/2.7/SimpleXMLRPCServer.py b/lib-python/2.7/SimpleXMLRPCServer.py
--- a/lib-python/2.7/SimpleXMLRPCServer.py
+++ b/lib-python/2.7/SimpleXMLRPCServer.py
@@ -486,7 +486,10 @@
L = []
while size_remaining:
chunk_size = min(size_remaining, max_chunk_size)
- L.append(self.rfile.read(chunk_size))
+ chunk = self.rfile.read(chunk_size)
+ if not chunk:
+ break
+ L.append(chunk)
size_remaining -= len(L[-1])
data = ''.join(L)
diff --git a/lib-python/2.7/test/test_xmlrpc.py b/lib-python/2.7/test/test_xmlrpc.py
--- a/lib-python/2.7/test/test_xmlrpc.py
+++ b/lib-python/2.7/test/test_xmlrpc.py
@@ -308,7 +308,7 @@
global ADDR, PORT, URL
ADDR, PORT = serv.socket.getsockname()
#connect to IP address directly. This avoids socket.create_connection()
- #trying to connect to to "localhost" using all address families, which
+ #trying to connect to "localhost" using all address families, which
#causes slowdown e.g. on vista which supports AF_INET6. The server listens
#on AF_INET only.
URL = "http://%s:%d"%(ADDR, PORT)
@@ -367,7 +367,7 @@
global ADDR, PORT, URL
ADDR, PORT = serv.socket.getsockname()
#connect to IP address directly. This avoids socket.create_connection()
- #trying to connect to to "localhost" using all address families, which
+ #trying to connect to "localhost" using all address families, which
#causes slowdown e.g. on vista which supports AF_INET6. The server listens
#on AF_INET only.
URL = "http://%s:%d"%(ADDR, PORT)
@@ -472,6 +472,9 @@
# protocol error; provide additional information in test output
self.fail("%s\n%s" % (e, getattr(e, "headers", "")))
+ def test_unicode_host(self):
+ server = xmlrpclib.ServerProxy(u"http://%s:%d/RPC2"%(ADDR, PORT))
+ self.assertEqual(server.add("a", u"\xe9"), u"a\xe9")
# [ch] The test 404 is causing lots of false alarms.
def XXXtest_404(self):
@@ -586,6 +589,12 @@
# This avoids waiting for the socket timeout.
self.test_simple1()
+ def test_partial_post(self):
+ # Check that a partial POST doesn't make the server loop: issue #14001.
+ conn = httplib.HTTPConnection(ADDR, PORT)
+ conn.request('POST', '/RPC2 HTTP/1.0\r\nContent-Length: 100\r\n\r\nbye')
+ conn.close()
+
class MultiPathServerTestCase(BaseServerTestCase):
threadFunc = staticmethod(http_multi_server)
request_count = 2
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -311,7 +311,7 @@
RegrTest('test_mimetypes.py'),
RegrTest('test_MimeWriter.py', core=False),
RegrTest('test_minidom.py'),
- RegrTest('test_mmap.py'),
+ RegrTest('test_mmap.py', usemodules="mmap"),
RegrTest('test_module.py', core=True),
RegrTest('test_modulefinder.py'),
RegrTest('test_msilib.py', skip=only_win32),
diff --git a/lib-python/modified-2.7/distutils/command/bdist_wininst.py b/lib-python/modified-2.7/distutils/command/bdist_wininst.py
--- a/lib-python/modified-2.7/distutils/command/bdist_wininst.py
+++ b/lib-python/modified-2.7/distutils/command/bdist_wininst.py
@@ -298,7 +298,8 @@
bitmaplen, # number of bytes in bitmap
)
file.write(header)
- file.write(open(arcname, "rb").read())
+ with open(arcname, "rb") as arcfile:
+ file.write(arcfile.read())
# create_exe()
diff --git a/lib-python/modified-2.7/distutils/sysconfig_pypy.py b/lib-python/modified-2.7/distutils/sysconfig_pypy.py
--- a/lib-python/modified-2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/modified-2.7/distutils/sysconfig_pypy.py
@@ -60,6 +60,7 @@
g['EXE'] = ""
g['SO'] = _get_so_extension() or ".so"
g['SOABI'] = g['SO'].rsplit('.')[0]
+ g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
global _config_vars
_config_vars = g
diff --git a/lib-python/modified-2.7/opcode.py b/lib-python/modified-2.7/opcode.py
--- a/lib-python/modified-2.7/opcode.py
+++ b/lib-python/modified-2.7/opcode.py
@@ -192,5 +192,6 @@
def_op('LOOKUP_METHOD', 201) # Index in name list
hasname.append(201)
def_op('CALL_METHOD', 202) # #args not including 'self'
+def_op('BUILD_LIST_FROM_ARG', 203)
del def_op, name_op, jrel_op, jabs_op
diff --git a/lib-python/modified-2.7/site.py b/lib-python/modified-2.7/site.py
--- a/lib-python/modified-2.7/site.py
+++ b/lib-python/modified-2.7/site.py
@@ -550,9 +550,18 @@
"'import usercustomize' failed; use -v for traceback"
+def import_builtin_stuff():
+ """PyPy specific: pre-import a few built-in modules, because
+ some programs actually rely on them to be in sys.modules :-("""
+ import exceptions
+ if 'zipimport' in sys.builtin_module_names:
+ import zipimport
+
+
def main():
global ENABLE_USER_SITE
+ import_builtin_stuff()
abs__file__()
known_paths = removeduppaths()
if (os.name == "posix" and sys.path and
diff --git a/lib-python/modified-2.7/test/test_dis.py b/lib-python/modified-2.7/test/test_dis.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/test/test_dis.py
@@ -0,0 +1,150 @@
+# Minimal tests for dis module
+
+from test.test_support import run_unittest
+import unittest
+import sys
+import dis
+import StringIO
+
+
+def _f(a):
+ print a
+ return 1
+
+dis_f = """\
+ %-4d 0 LOAD_FAST 0 (a)
+ 3 PRINT_ITEM
+ 4 PRINT_NEWLINE
+
+ %-4d 5 LOAD_CONST 1 (1)
+ 8 RETURN_VALUE
+"""%(_f.func_code.co_firstlineno + 1,
+ _f.func_code.co_firstlineno + 2)
+
+
+def bug708901():
+ for res in range(1,
+ 10):
+ pass
+
+dis_bug708901 = """\
+ %-4d 0 SETUP_LOOP 23 (to 26)
+ 3 LOAD_GLOBAL 0 (range)
+ 6 LOAD_CONST 1 (1)
+
+ %-4d 9 LOAD_CONST 2 (10)
+ 12 CALL_FUNCTION 2
+ 15 GET_ITER
+ >> 16 FOR_ITER 6 (to 25)
+ 19 STORE_FAST 0 (res)
+
+ %-4d 22 JUMP_ABSOLUTE 16
+ >> 25 POP_BLOCK
+ >> 26 LOAD_CONST 0 (None)
+ 29 RETURN_VALUE
+"""%(bug708901.func_code.co_firstlineno + 1,
+ bug708901.func_code.co_firstlineno + 2,
+ bug708901.func_code.co_firstlineno + 3)
+
+
+def bug1333982(x=[]):
+ assert 0, ([s for s in x] +
+ 1)
+ pass
+
+dis_bug1333982 = """\
+ %-4d 0 LOAD_CONST 1 (0)
+ 3 POP_JUMP_IF_TRUE 38
+ 6 LOAD_GLOBAL 0 (AssertionError)
+ 9 LOAD_FAST 0 (x)
+ 12 BUILD_LIST_FROM_ARG 0
+ 15 GET_ITER
+ >> 16 FOR_ITER 12 (to 31)
+ 19 STORE_FAST 1 (s)
+ 22 LOAD_FAST 1 (s)
+ 25 LIST_APPEND 2
+ 28 JUMP_ABSOLUTE 16
+
+ %-4d >> 31 LOAD_CONST 2 (1)
+ 34 BINARY_ADD
+ 35 RAISE_VARARGS 2
+
+ %-4d >> 38 LOAD_CONST 0 (None)
+ 41 RETURN_VALUE
+"""%(bug1333982.func_code.co_firstlineno + 1,
+ bug1333982.func_code.co_firstlineno + 2,
+ bug1333982.func_code.co_firstlineno + 3)
+
+_BIG_LINENO_FORMAT = """\
+%3d 0 LOAD_GLOBAL 0 (spam)
+ 3 POP_TOP
+ 4 LOAD_CONST 0 (None)
+ 7 RETURN_VALUE
+"""
+
+class DisTests(unittest.TestCase):
+ def do_disassembly_test(self, func, expected):
+ s = StringIO.StringIO()
+ save_stdout = sys.stdout
+ sys.stdout = s
+ dis.dis(func)
+ sys.stdout = save_stdout
+ got = s.getvalue()
+ # Trim trailing blanks (if any).
+ lines = got.split('\n')
+ lines = [line.rstrip() for line in lines]
+ expected = expected.split("\n")
+ import difflib
+ if expected != lines:
+ self.fail(
+ "events did not match expectation:\n" +
+ "\n".join(difflib.ndiff(expected,
+ lines)))
+
+ def test_opmap(self):
+ self.assertEqual(dis.opmap["STOP_CODE"], 0)
+ self.assertIn(dis.opmap["LOAD_CONST"], dis.hasconst)
+ self.assertIn(dis.opmap["STORE_NAME"], dis.hasname)
+
+ def test_opname(self):
+ self.assertEqual(dis.opname[dis.opmap["LOAD_FAST"]], "LOAD_FAST")
+
+ def test_boundaries(self):
+ self.assertEqual(dis.opmap["EXTENDED_ARG"], dis.EXTENDED_ARG)
+ self.assertEqual(dis.opmap["STORE_NAME"], dis.HAVE_ARGUMENT)
+
+ def test_dis(self):
+ self.do_disassembly_test(_f, dis_f)
+
+ def test_bug_708901(self):
+ self.do_disassembly_test(bug708901, dis_bug708901)
+
+ def test_bug_1333982(self):
+ # This one is checking bytecodes generated for an `assert` statement,
+ # so fails if the tests are run with -O. Skip this test then.
+ if __debug__:
+ self.do_disassembly_test(bug1333982, dis_bug1333982)
+
+ def test_big_linenos(self):
+ def func(count):
+ namespace = {}
+ func = "def foo():\n " + "".join(["\n "] * count + ["spam\n"])
+ exec func in namespace
+ return namespace['foo']
+
+ # Test all small ranges
+ for i in xrange(1, 300):
+ expected = _BIG_LINENO_FORMAT % (i + 2)
+ self.do_disassembly_test(func(i), expected)
+
+ # Test some larger ranges too
+ for i in xrange(300, 5000, 10):
+ expected = _BIG_LINENO_FORMAT % (i + 2)
+ self.do_disassembly_test(func(i), expected)
+
+def test_main():
+ run_unittest(DisTests)
+
+
+if __name__ == "__main__":
+ test_main()
diff --git a/lib-python/modified-2.7/test/test_set.py b/lib-python/modified-2.7/test/test_set.py
--- a/lib-python/modified-2.7/test/test_set.py
+++ b/lib-python/modified-2.7/test/test_set.py
@@ -1568,7 +1568,7 @@
for meth in (s.union, s.intersection, s.difference, s.symmetric_difference, s.isdisjoint):
for g in (G, I, Ig, L, R):
expected = meth(data)
- actual = meth(G(data))
+ actual = meth(g(data))
if isinstance(expected, bool):
self.assertEqual(actual, expected)
else:
diff --git a/lib_pypy/_csv.py b/lib_pypy/_csv.py
--- a/lib_pypy/_csv.py
+++ b/lib_pypy/_csv.py
@@ -414,7 +414,7 @@
def _parse_add_char(self, c):
if len(self.field) + len(c) > _field_limit:
- raise Error("field larget than field limit (%d)" % (_field_limit))
+ raise Error("field larger than field limit (%d)" % (_field_limit))
self.field += c
diff --git a/lib_pypy/_ctypes/builtin.py b/lib_pypy/_ctypes/builtin.py
--- a/lib_pypy/_ctypes/builtin.py
+++ b/lib_pypy/_ctypes/builtin.py
@@ -31,24 +31,20 @@
arg = cobj._get_buffer_value()
return _rawffi.wcharp2rawunicode(arg, lgt)
-class ErrorObject(local):
- def __init__(self):
- self.errno = 0
- self.winerror = 0
-_error_object = ErrorObject()
+_err = local()
def get_errno():
- return _error_object.errno
+ return getattr(_err, "errno", 0)
def set_errno(errno):
- old_errno = _error_object.errno
- _error_object.errno = errno
+ old_errno = get_errno()
+ _err.errno = errno
return old_errno
def get_last_error():
- return _error_object.winerror
+ return getattr(_err, "winerror", 0)
def set_last_error(winerror):
- old_winerror = _error_object.winerror
- _error_object.winerror = winerror
+ old_winerror = get_last_error()
+ _err.winerror = winerror
return old_winerror
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -3,7 +3,7 @@
from _ctypes.primitive import SimpleType, _SimpleCData
from _ctypes.basics import ArgumentError, keepalive_key
from _ctypes.basics import is_struct_shape
-from _ctypes.builtin import set_errno, set_last_error
+from _ctypes.builtin import get_errno, set_errno, get_last_error, set_last_error
import _rawffi
import _ffi
import sys
@@ -350,16 +350,24 @@
def _call_funcptr(self, funcptr, *newargs):
if self._flags_ & _rawffi.FUNCFLAG_USE_ERRNO:
- set_errno(_rawffi.get_errno())
+ tmp = _rawffi.get_errno()
+ _rawffi.set_errno(get_errno())
+ set_errno(tmp)
if self._flags_ & _rawffi.FUNCFLAG_USE_LASTERROR:
- set_last_error(_rawffi.get_last_error())
+ tmp = _rawffi.get_last_error()
+ _rawffi.set_last_error(get_last_error())
+ set_last_error(tmp)
try:
result = funcptr(*newargs)
finally:
if self._flags_ & _rawffi.FUNCFLAG_USE_ERRNO:
- set_errno(_rawffi.get_errno())
+ tmp = _rawffi.get_errno()
+ _rawffi.set_errno(get_errno())
+ set_errno(tmp)
if self._flags_ & _rawffi.FUNCFLAG_USE_LASTERROR:
- set_last_error(_rawffi.get_last_error())
+ tmp = _rawffi.get_last_error()
+ _rawffi.set_last_error(get_last_error())
+ set_last_error(tmp)
#
try:
return self._build_result(self._restype_, result, newargs)
diff --git a/lib_pypy/_locale.py b/lib_pypy/_locale.py
deleted file mode 100644
--- a/lib_pypy/_locale.py
+++ /dev/null
@@ -1,337 +0,0 @@
-# ctypes implementation of _locale module by Victor Stinner, 2008-03-27
-
-# ------------------------------------------------------------
-# Note that we also have our own interp-level implementation
-# ------------------------------------------------------------
-
-"""
-Support for POSIX locales.
-"""
-
-from ctypes import (Structure, POINTER, create_string_buffer,
- c_ubyte, c_int, c_char_p, c_wchar_p, c_size_t)
-from ctypes_support import standard_c_lib as libc
-from ctypes_support import get_errno
-
-# load the platform-specific cache made by running locale.ctc.py
-from ctypes_config_cache._locale_cache import *
-
-try: from __pypy__ import builtinify
-except ImportError: builtinify = lambda f: f
-
-
-# Ubuntu Gusty i386 structure
-class lconv(Structure):
- _fields_ = (
- # Numeric (non-monetary) information.
- ("decimal_point", c_char_p), # Decimal point character.
- ("thousands_sep", c_char_p), # Thousands separator.
-
- # Each element is the number of digits in each group;
- # elements with higher indices are farther left.
- # An element with value CHAR_MAX means that no further grouping is done.
- # An element with value 0 means that the previous element is used
- # for all groups farther left. */
- ("grouping", c_char_p),
-
- # Monetary information.
-
- # First three chars are a currency symbol from ISO 4217.
- # Fourth char is the separator. Fifth char is '\0'.
- ("int_curr_symbol", c_char_p),
- ("currency_symbol", c_char_p), # Local currency symbol.
- ("mon_decimal_point", c_char_p), # Decimal point character.
- ("mon_thousands_sep", c_char_p), # Thousands separator.
- ("mon_grouping", c_char_p), # Like `grouping' element (above).
- ("positive_sign", c_char_p), # Sign for positive values.
- ("negative_sign", c_char_p), # Sign for negative values.
- ("int_frac_digits", c_ubyte), # Int'l fractional digits.
- ("frac_digits", c_ubyte), # Local fractional digits.
- # 1 if currency_symbol precedes a positive value, 0 if succeeds.
- ("p_cs_precedes", c_ubyte),
- # 1 iff a space separates currency_symbol from a positive value.
- ("p_sep_by_space", c_ubyte),
- # 1 if currency_symbol precedes a negative value, 0 if succeeds.
- ("n_cs_precedes", c_ubyte),
- # 1 iff a space separates currency_symbol from a negative value.
- ("n_sep_by_space", c_ubyte),
-
- # Positive and negative sign positions:
- # 0 Parentheses surround the quantity and currency_symbol.
- # 1 The sign string precedes the quantity and currency_symbol.
- # 2 The sign string follows the quantity and currency_symbol.
- # 3 The sign string immediately precedes the currency_symbol.
- # 4 The sign string immediately follows the currency_symbol.
- ("p_sign_posn", c_ubyte),
- ("n_sign_posn", c_ubyte),
- # 1 if int_curr_symbol precedes a positive value, 0 if succeeds.
- ("int_p_cs_precedes", c_ubyte),
- # 1 iff a space separates int_curr_symbol from a positive value.
- ("int_p_sep_by_space", c_ubyte),
- # 1 if int_curr_symbol precedes a negative value, 0 if succeeds.
- ("int_n_cs_precedes", c_ubyte),
- # 1 iff a space separates int_curr_symbol from a negative value.
- ("int_n_sep_by_space", c_ubyte),
- # Positive and negative sign positions:
- # 0 Parentheses surround the quantity and int_curr_symbol.
- # 1 The sign string precedes the quantity and int_curr_symbol.
- # 2 The sign string follows the quantity and int_curr_symbol.
- # 3 The sign string immediately precedes the int_curr_symbol.
- # 4 The sign string immediately follows the int_curr_symbol.
- ("int_p_sign_posn", c_ubyte),
- ("int_n_sign_posn", c_ubyte),
- )
-
-_setlocale = libc.setlocale
-_setlocale.argtypes = (c_int, c_char_p)
-_setlocale.restype = c_char_p
-
-_localeconv = libc.localeconv
-_localeconv.argtypes = None
-_localeconv.restype = POINTER(lconv)
-
-_strcoll = libc.strcoll
-_strcoll.argtypes = (c_char_p, c_char_p)
-_strcoll.restype = c_int
-
-_wcscoll = libc.wcscoll
-_wcscoll.argtypes = (c_wchar_p, c_wchar_p)
-_wcscoll.restype = c_int
-
-_strxfrm = libc.strxfrm
-_strxfrm.argtypes = (c_char_p, c_char_p, c_size_t)
-_strxfrm.restype = c_size_t
-
-HAS_LIBINTL = hasattr(libc, 'gettext')
-if HAS_LIBINTL:
- _gettext = libc.gettext
- _gettext.argtypes = (c_char_p,)
- _gettext.restype = c_char_p
-
- _dgettext = libc.dgettext
- _dgettext.argtypes = (c_char_p, c_char_p)
- _dgettext.restype = c_char_p
-
- _dcgettext = libc.dcgettext
- _dcgettext.argtypes = (c_char_p, c_char_p, c_int)
- _dcgettext.restype = c_char_p
-
- _textdomain = libc.textdomain
- _textdomain.argtypes = (c_char_p,)
- _textdomain.restype = c_char_p
-
- _bindtextdomain = libc.bindtextdomain
- _bindtextdomain.argtypes = (c_char_p, c_char_p)
- _bindtextdomain.restype = c_char_p
-
- HAS_BIND_TEXTDOMAIN_CODESET = hasattr(libc, 'bindtextdomain_codeset')
- if HAS_BIND_TEXTDOMAIN_CODESET:
- _bind_textdomain_codeset = libc.bindtextdomain_codeset
- _bind_textdomain_codeset.argtypes = (c_char_p, c_char_p)
- _bind_textdomain_codeset.restype = c_char_p
-
-class Error(Exception):
- pass
-
-def fixup_ulcase():
- import string
- #import strop
-
- # create uppercase map string
- ul = []
- for c in xrange(256):
- c = chr(c)
- if c.isupper():
- ul.append(c)
- ul = ''.join(ul)
- string.uppercase = ul
- #strop.uppercase = ul
-
- # create lowercase string
- ul = []
- for c in xrange(256):
- c = chr(c)
- if c.islower():
- ul.append(c)
- ul = ''.join(ul)
- string.lowercase = ul
- #strop.lowercase = ul
-
- # create letters string
- ul = []
- for c in xrange(256):
- c = chr(c)
- if c.isalpha():
- ul.append(c)
- ul = ''.join(ul)
- string.letters = ul
-
- at builtinify
-def setlocale(category, locale=None):
- "(integer,string=None) -> string. Activates/queries locale processing."
- if locale:
- # set locale
- result = _setlocale(category, locale)
- if not result:
- raise Error("unsupported locale setting")
-
- # record changes to LC_CTYPE
- if category in (LC_CTYPE, LC_ALL):
- fixup_ulcase()
- else:
- # get locale
- result = _setlocale(category, None)
- if not result:
- raise Error("locale query failed")
- return result
-
-def _copy_grouping(text):
- groups = [ ord(group) for group in text ]
- if groups:
- groups.append(0)
- return groups
-
- at builtinify
-def localeconv():
- "() -> dict. Returns numeric and monetary locale-specific parameters."
-
- # if LC_NUMERIC is different in the C library, use saved value
- lp = _localeconv()
- l = lp.contents
-
- # hopefully, the localeconv result survives the C library calls
- # involved herein
-
- # Numeric information
- result = {
- "decimal_point": l.decimal_point,
- "thousands_sep": l.thousands_sep,
- "grouping": _copy_grouping(l.grouping),
- "int_curr_symbol": l.int_curr_symbol,
- "currency_symbol": l.currency_symbol,
- "mon_decimal_point": l.mon_decimal_point,
- "mon_thousands_sep": l.mon_thousands_sep,
- "mon_grouping": _copy_grouping(l.mon_grouping),
- "positive_sign": l.positive_sign,
- "negative_sign": l.negative_sign,
- "int_frac_digits": l.int_frac_digits,
- "frac_digits": l.frac_digits,
- "p_cs_precedes": l.p_cs_precedes,
- "p_sep_by_space": l.p_sep_by_space,
- "n_cs_precedes": l.n_cs_precedes,
- "n_sep_by_space": l.n_sep_by_space,
- "p_sign_posn": l.p_sign_posn,
- "n_sign_posn": l.n_sign_posn,
- }
- return result
-
- at builtinify
-def strcoll(s1, s2):
- "string,string -> int. Compares two strings according to the locale."
-
- # If both arguments are byte strings, use strcoll.
- if isinstance(s1, str) and isinstance(s2, str):
- return _strcoll(s1, s2)
-
- # If neither argument is unicode, it's an error.
- if not isinstance(s1, unicode) and not isinstance(s2, unicode):
- raise ValueError("strcoll arguments must be strings")
-
- # Convert the non-unicode argument to unicode.
- s1 = unicode(s1)
- s2 = unicode(s2)
-
- # Collate the strings.
- return _wcscoll(s1, s2)
-
- at builtinify
-def strxfrm(s):
- "string -> string. Returns a string that behaves for cmp locale-aware."
-
- # assume no change in size, first
- n1 = len(s) + 1
- buf = create_string_buffer(n1)
- n2 = _strxfrm(buf, s, n1) + 1
- if n2 > n1:
- # more space needed
- buf = create_string_buffer(n2)
- _strxfrm(buf, s, n2)
- return buf.value
-
- at builtinify
-def getdefaultlocale():
- # TODO: Port code from CPython for Windows and Mac OS
- raise NotImplementedError()
-
-if HAS_LANGINFO:
- _nl_langinfo = libc.nl_langinfo
- _nl_langinfo.argtypes = (nl_item,)
- _nl_langinfo.restype = c_char_p
-
- def nl_langinfo(key):
- """nl_langinfo(key) -> string
- Return the value for the locale information associated with key."""
- # Check whether this is a supported constant. GNU libc sometimes
- # returns numeric values in the char* return value, which would
- # crash PyString_FromString.
- result = _nl_langinfo(key)
- if result is not None:
- return result
- raise ValueError("unsupported langinfo constant")
-
-if HAS_LIBINTL:
- @builtinify
- def gettext(msg):
- """gettext(msg) -> string
- Return translation of msg."""
- return _gettext(msg)
-
- @builtinify
- def dgettext(domain, msg):
- """dgettext(domain, msg) -> string
- Return translation of msg in domain."""
- return _dgettext(domain, msg)
-
- @builtinify
- def dcgettext(domain, msg, category):
- """dcgettext(domain, msg, category) -> string
- Return translation of msg in domain and category."""
- return _dcgettext(domain, msg, category)
-
- @builtinify
- def textdomain(domain):
- """textdomain(domain) -> string
- Set the C library's textdomain to domain, returning the new domain."""
- return _textdomain(domain)
-
- @builtinify
- def bindtextdomain(domain, dir):
- """bindtextdomain(domain, dir) -> string
- Bind the C library's domain to dir."""
- dirname = _bindtextdomain(domain, dir)
- if not dirname:
- errno = get_errno()
- raise OSError(errno)
- return dirname
-
- if HAS_BIND_TEXTDOMAIN_CODESET:
- @builtinify
- def bind_textdomain_codeset(domain, codeset):
- """bind_textdomain_codeset(domain, codeset) -> string
- Bind the C library's domain to codeset."""
- codeset = _bind_textdomain_codeset(domain, codeset)
- if codeset:
- return codeset
- return None
-
-__all__ = (
- 'Error',
- 'setlocale', 'localeconv', 'strxfrm', 'strcoll',
-) + ALL_CONSTANTS
-if HAS_LIBINTL:
- __all__ += ('gettext', 'dgettext', 'dcgettext', 'textdomain',
- 'bindtextdomain')
- if HAS_BIND_TEXTDOMAIN_CODESET:
- __all__ += ('bind_textdomain_codeset',)
-if HAS_LANGINFO:
- __all__ += ('nl_langinfo',)
diff --git a/lib_pypy/array.py b/lib_pypy/array.py
deleted file mode 100644
--- a/lib_pypy/array.py
+++ /dev/null
@@ -1,531 +0,0 @@
-"""This module defines an object type which can efficiently represent
-an array of basic values: characters, integers, floating point
-numbers. Arrays are sequence types and behave very much like lists,
-except that the type of objects stored in them is constrained. The
-type is specified at object creation time by using a type code, which
-is a single character. The following type codes are defined:
-
- Type code C Type Minimum size in bytes
- 'c' character 1
- 'b' signed integer 1
- 'B' unsigned integer 1
- 'u' Unicode character 2
- 'h' signed integer 2
- 'H' unsigned integer 2
- 'i' signed integer 2
- 'I' unsigned integer 2
- 'l' signed integer 4
- 'L' unsigned integer 4
- 'f' floating point 4
- 'd' floating point 8
-
-The constructor is:
-
-array(typecode [, initializer]) -- create a new array
-"""
-
-from struct import calcsize, pack, pack_into, unpack_from
-import operator
-
-# the buffer-like object to use internally: trying from
-# various places in order...
-try:
- import _rawffi # a reasonable implementation based
- _RAWARRAY = _rawffi.Array('c') # on raw_malloc, and providing a
- def bytebuffer(size): # real address
- return _RAWARRAY(size, autofree=True)
- def getbufaddress(buf):
- return buf.buffer
-except ImportError:
- try:
- from __pypy__ import bytebuffer # a reasonable implementation
- def getbufaddress(buf): # compatible with oo backends,
- return 0 # but no address
- except ImportError:
- # not running on PyPy. Fall back to ctypes...
- import ctypes
- bytebuffer = ctypes.create_string_buffer
- def getbufaddress(buf):
- voidp = ctypes.cast(ctypes.pointer(buf), ctypes.c_void_p)
- return voidp.value
-
-# ____________________________________________________________
-
-TYPECODES = "cbBuhHiIlLfd"
-
-class array(object):
- """array(typecode [, initializer]) -> array
-
- Return a new array whose items are restricted by typecode, and
- initialized from the optional initializer value, which must be a list,
- string. or iterable over elements of the appropriate type.
-
- Arrays represent basic values and behave very much like lists, except
- the type of objects stored in them is constrained.
-
- Methods:
-
- append() -- append a new item to the end of the array
- buffer_info() -- return information giving the current memory info
- byteswap() -- byteswap all the items of the array
- count() -- return number of occurences of an object
- extend() -- extend array by appending multiple elements from an iterable
- fromfile() -- read items from a file object
- fromlist() -- append items from the list
- fromstring() -- append items from the string
- index() -- return index of first occurence of an object
- insert() -- insert a new item into the array at a provided position
- pop() -- remove and return item (default last)
- read() -- DEPRECATED, use fromfile()
- remove() -- remove first occurence of an object
- reverse() -- reverse the order of the items in the array
- tofile() -- write all items to a file object
- tolist() -- return the array converted to an ordinary list
- tostring() -- return the array converted to a string
- write() -- DEPRECATED, use tofile()
-
- Attributes:
-
- typecode -- the typecode character used to create the array
- itemsize -- the length in bytes of one array item
- """
- __slots__ = ["typecode", "itemsize", "_data", "_descriptor", "__weakref__"]
-
- def __new__(cls, typecode, initializer=[], **extrakwds):
- self = object.__new__(cls)
- if cls is array and extrakwds:
- raise TypeError("array() does not take keyword arguments")
- if not isinstance(typecode, str) or len(typecode) != 1:
- raise TypeError(
- "array() argument 1 must be char, not %s" % type(typecode))
- if typecode not in TYPECODES:
- raise ValueError(
- "bad typecode (must be one of %s)" % ', '.join(TYPECODES))
- self._data = bytebuffer(0)
- self.typecode = typecode
- self.itemsize = calcsize(typecode)
- if isinstance(initializer, list):
- self.fromlist(initializer)
- elif isinstance(initializer, str):
- self.fromstring(initializer)
- elif isinstance(initializer, unicode) and self.typecode == "u":
- self.fromunicode(initializer)
- else:
- self.extend(initializer)
- return self
-
- def _clear(self):
- self._data = bytebuffer(0)
-
- ##### array-specific operations
-
- def fromfile(self, f, n):
- """Read n objects from the file object f and append them to the end of
- the array. Also called as read."""
- if not isinstance(f, file):
- raise TypeError("arg1 must be open file")
- size = self.itemsize * n
- item = f.read(size)
- if len(item) < size:
- raise EOFError("not enough items in file")
- self.fromstring(item)
-
- def fromlist(self, l):
- """Append items to array from list."""
- if not isinstance(l, list):
- raise TypeError("arg must be list")
- self._fromiterable(l)
-
- def fromstring(self, s):
- """Appends items from the string, interpreting it as an array of machine
- values, as if it had been read from a file using the fromfile()
- method."""
- if isinstance(s, unicode):
- s = str(s)
- self._frombuffer(s)
-
- def _frombuffer(self, s):
- length = len(s)
- if length % self.itemsize != 0:
- raise ValueError("string length not a multiple of item size")
- boundary = len(self._data)
- newdata = bytebuffer(boundary + length)
- newdata[:boundary] = self._data
- newdata[boundary:] = s
- self._data = newdata
-
- def fromunicode(self, ustr):
- """Extends this array with data from the unicode string ustr. The array
- must be a type 'u' array; otherwise a ValueError is raised. Use
- array.fromstring(ustr.encode(...)) to append Unicode data to an array of
- some other type."""
- if not self.typecode == "u":
- raise ValueError(
- "fromunicode() may only be called on type 'u' arrays")
- # XXX the following probable bug is not emulated:
- # CPython accepts a non-unicode string or a buffer, and then
- # behaves just like fromstring(), except that it strangely truncates
- # string arguments at multiples of the unicode byte size.
- # Let's only accept unicode arguments for now.
- if not isinstance(ustr, unicode):
- raise TypeError("fromunicode() argument should probably be "
- "a unicode string")
- # _frombuffer() does the currect thing using
- # the buffer behavior of unicode objects
- self._frombuffer(buffer(ustr))
-
- def tofile(self, f):
- """Write all items (as machine values) to the file object f. Also
- called as write."""
- if not isinstance(f, file):
- raise TypeError("arg must be open file")
- f.write(self.tostring())
-
- def tolist(self):
- """Convert array to an ordinary list with the same items."""
- count = len(self._data) // self.itemsize
- return list(unpack_from('%d%s' % (count, self.typecode), self._data))
-
- def tostring(self):
- return self._data[:]
-
- def __buffer__(self):
- return buffer(self._data)
-
- def tounicode(self):
- """Convert the array to a unicode string. The array must be a type 'u'
- array; otherwise a ValueError is raised. Use array.tostring().decode()
- to obtain a unicode string from an array of some other type."""
- if self.typecode != "u":
- raise ValueError("tounicode() may only be called on type 'u' arrays")
- # XXX performance is not too good
- return u"".join(self.tolist())
-
- def byteswap(self):
- """Byteswap all items of the array. If the items in the array are not
- 1, 2, 4, or 8 bytes in size, RuntimeError is raised."""
- if self.itemsize not in [1, 2, 4, 8]:
- raise RuntimeError("byteswap not supported for this array")
- # XXX slowish
- itemsize = self.itemsize
- bytes = self._data
- for start in range(0, len(bytes), itemsize):
- stop = start + itemsize
- bytes[start:stop] = bytes[start:stop][::-1]
-
- def buffer_info(self):
- """Return a tuple (address, length) giving the current memory address
- and the length in items of the buffer used to hold array's contents. The
- length should be multiplied by the itemsize attribute to calculate the
- buffer length in bytes. On PyPy the address might be meaningless
- (returned as 0), depending on the available modules."""
- return (getbufaddress(self._data), len(self))
-
- read = fromfile
-
- write = tofile
-
- ##### general object protocol
-
- def __repr__(self):
- if len(self._data) == 0:
- return "array('%s')" % self.typecode
- elif self.typecode == "c":
- return "array('%s', %s)" % (self.typecode, repr(self.tostring()))
- elif self.typecode == "u":
- return "array('%s', %s)" % (self.typecode, repr(self.tounicode()))
- else:
- return "array('%s', %s)" % (self.typecode, repr(self.tolist()))
-
- def __copy__(self):
- a = array(self.typecode)
- a._data = bytebuffer(len(self._data))
- a._data[:] = self._data
- return a
-
- def __eq__(self, other):
- if not isinstance(other, array):
- return NotImplemented
- if self.typecode == 'c':
- return buffer(self._data) == buffer(other._data)
- else:
- return self.tolist() == other.tolist()
-
- def __ne__(self, other):
- if not isinstance(other, array):
- return NotImplemented
- if self.typecode == 'c':
- return buffer(self._data) != buffer(other._data)
- else:
- return self.tolist() != other.tolist()
-
- def __lt__(self, other):
- if not isinstance(other, array):
- return NotImplemented
- if self.typecode == 'c':
- return buffer(self._data) < buffer(other._data)
- else:
- return self.tolist() < other.tolist()
-
- def __gt__(self, other):
- if not isinstance(other, array):
- return NotImplemented
- if self.typecode == 'c':
- return buffer(self._data) > buffer(other._data)
- else:
- return self.tolist() > other.tolist()
-
- def __le__(self, other):
- if not isinstance(other, array):
- return NotImplemented
- if self.typecode == 'c':
- return buffer(self._data) <= buffer(other._data)
- else:
- return self.tolist() <= other.tolist()
-
- def __ge__(self, other):
- if not isinstance(other, array):
- return NotImplemented
- if self.typecode == 'c':
- return buffer(self._data) >= buffer(other._data)
- else:
- return self.tolist() >= other.tolist()
-
- def __reduce__(self):
- dict = getattr(self, '__dict__', None)
- data = self.tostring()
- if data:
- initargs = (self.typecode, data)
- else:
- initargs = (self.typecode,)
- return (type(self), initargs, dict)
-
- ##### list methods
-
- def append(self, x):
- """Append new value x to the end of the array."""
- self._frombuffer(pack(self.typecode, x))
-
- def count(self, x):
- """Return number of occurences of x in the array."""
- return operator.countOf(self, x)
-
- def extend(self, iterable):
- """Append items to the end of the array."""
- if isinstance(iterable, array) \
- and not self.typecode == iterable.typecode:
- raise TypeError("can only extend with array of same kind")
- self._fromiterable(iterable)
-
- def index(self, x):
- """Return index of first occurence of x in the array."""
- return operator.indexOf(self, x)
-
- def insert(self, i, x):
- """Insert a new item x into the array before position i."""
- seqlength = len(self)
- if i < 0:
- i += seqlength
- if i < 0:
- i = 0
- elif i > seqlength:
- i = seqlength
- boundary = i * self.itemsize
- data = pack(self.typecode, x)
- newdata = bytebuffer(len(self._data) + len(data))
- newdata[:boundary] = self._data[:boundary]
- newdata[boundary:boundary+self.itemsize] = data
- newdata[boundary+self.itemsize:] = self._data[boundary:]
- self._data = newdata
-
- def pop(self, i=-1):
- """Return the i-th element and delete it from the array. i defaults to
- -1."""
- seqlength = len(self)
- if i < 0:
- i += seqlength
- if not (0 <= i < seqlength):
- raise IndexError(i)
- boundary = i * self.itemsize
- result = unpack_from(self.typecode, self._data, boundary)[0]
- newdata = bytebuffer(len(self._data) - self.itemsize)
- newdata[:boundary] = self._data[:boundary]
- newdata[boundary:] = self._data[boundary+self.itemsize:]
- self._data = newdata
- return result
-
- def remove(self, x):
- """Remove the first occurence of x in the array."""
- self.pop(self.index(x))
-
- def reverse(self):
- """Reverse the order of the items in the array."""
- lst = self.tolist()
- lst.reverse()
- self._clear()
- self.fromlist(lst)
-
- ##### list protocol
-
- def __len__(self):
- return len(self._data) // self.itemsize
-
- def __add__(self, other):
- if not isinstance(other, array):
- raise TypeError("can only append array to array")
- if self.typecode != other.typecode:
- raise TypeError("bad argument type for built-in operation")
- return array(self.typecode, buffer(self._data) + buffer(other._data))
-
- def __mul__(self, repeat):
- return array(self.typecode, buffer(self._data) * repeat)
-
- __rmul__ = __mul__
-
- def __getitem__(self, i):
- seqlength = len(self)
- if isinstance(i, slice):
- start, stop, step = i.indices(seqlength)
- if step != 1:
- sublist = self.tolist()[i] # fall-back
- return array(self.typecode, sublist)
- if start < 0:
- start = 0
- if stop < start:
- stop = start
- assert stop <= seqlength
- return array(self.typecode, self._data[start * self.itemsize :
- stop * self.itemsize])
- else:
- if i < 0:
- i += seqlength
- if self.typecode == 'c': # speed trick
- return self._data[i]
- if not (0 <= i < seqlength):
- raise IndexError(i)
- boundary = i * self.itemsize
- return unpack_from(self.typecode, self._data, boundary)[0]
-
- def __getslice__(self, i, j):
- return self.__getitem__(slice(i, j))
-
- def __setitem__(self, i, x):
- if isinstance(i, slice):
- if (not isinstance(x, array)
- or self.typecode != x.typecode):
- raise TypeError("can only assign array of same kind"
- " to array slice")
- seqlength = len(self)
- start, stop, step = i.indices(seqlength)
- if step != 1:
- sublist = self.tolist() # fall-back
- sublist[i] = x.tolist()
- self._clear()
- self.fromlist(sublist)
- return
- if start < 0:
- start = 0
- if stop < start:
- stop = start
- assert stop <= seqlength
- boundary1 = start * self.itemsize
- boundary2 = stop * self.itemsize
- boundary2new = boundary1 + len(x._data)
- if boundary2 == boundary2new:
- self._data[boundary1:boundary2] = x._data
- else:
- newdata = bytebuffer(len(self._data) + boundary2new-boundary2)
- newdata[:boundary1] = self._data[:boundary1]
- newdata[boundary1:boundary2new] = x._data
- newdata[boundary2new:] = self._data[boundary2:]
- self._data = newdata
- else:
- seqlength = len(self)
- if i < 0:
- i += seqlength
- if self.typecode == 'c': # speed trick
- self._data[i] = x
- return
- if not (0 <= i < seqlength):
- raise IndexError(i)
- boundary = i * self.itemsize
- pack_into(self.typecode, self._data, boundary, x)
-
- def __setslice__(self, i, j, x):
- self.__setitem__(slice(i, j), x)
-
- def __delitem__(self, i):
- if isinstance(i, slice):
- seqlength = len(self)
- start, stop, step = i.indices(seqlength)
- if start < 0:
- start = 0
- if stop < start:
- stop = start
- assert stop <= seqlength
- if step != 1:
- sublist = self.tolist() # fall-back
- del sublist[i]
- self._clear()
- self.fromlist(sublist)
- return
- dellength = stop - start
- boundary1 = start * self.itemsize
- boundary2 = stop * self.itemsize
- newdata = bytebuffer(len(self._data) - (boundary2-boundary1))
- newdata[:boundary1] = self._data[:boundary1]
- newdata[boundary1:] = self._data[boundary2:]
- self._data = newdata
- else:
- seqlength = len(self)
- if i < 0:
- i += seqlength
- if not (0 <= i < seqlength):
- raise IndexError(i)
- boundary = i * self.itemsize
- newdata = bytebuffer(len(self._data) - self.itemsize)
- newdata[:boundary] = self._data[:boundary]
- newdata[boundary:] = self._data[boundary+self.itemsize:]
- self._data = newdata
-
- def __delslice__(self, i, j):
- self.__delitem__(slice(i, j))
-
- def __contains__(self, item):
- for x in self:
- if x == item:
- return True
- return False
-
- def __iadd__(self, other):
- if not isinstance(other, array):
- raise TypeError("can only extend array with array")
- self.extend(other)
- return self
-
- def __imul__(self, repeat):
- newdata = buffer(self._data) * repeat
- self._data = bytebuffer(len(newdata))
- self._data[:] = newdata
- return self
-
- def __iter__(self):
- p = 0
- typecode = self.typecode
- itemsize = self.itemsize
- while p < len(self._data):
- yield unpack_from(typecode, self._data, p)[0]
- p += itemsize
-
- ##### internal methods
-
- def _fromiterable(self, iterable):
- iterable = tuple(iterable)
- n = len(iterable)
- boundary = len(self._data)
- newdata = bytebuffer(boundary + n * self.itemsize)
- newdata[:boundary] = self._data
- pack_into('%d%s' % (n, self.typecode), newdata, boundary, *iterable)
- self._data = newdata
-
-ArrayType = array
diff --git a/lib_pypy/binascii.py b/lib_pypy/binascii.py
deleted file mode 100644
--- a/lib_pypy/binascii.py
+++ /dev/null
@@ -1,720 +0,0 @@
-"""A pure Python implementation of binascii.
-
-Rather slow and buggy in corner cases.
-PyPy provides an RPython version too.
-"""
-
-class Error(Exception):
- pass
-
-class Done(Exception):
- pass
-
-class Incomplete(Exception):
- pass
-
-def a2b_uu(s):
- if not s:
- return ''
-
- length = (ord(s[0]) - 0x20) % 64
-
- def quadruplets_gen(s):
- while s:
- try:
- yield ord(s[0]), ord(s[1]), ord(s[2]), ord(s[3])
- except IndexError:
- s += ' '
- yield ord(s[0]), ord(s[1]), ord(s[2]), ord(s[3])
- return
- s = s[4:]
-
- try:
- result = [''.join(
- [chr((A - 0x20) << 2 | (((B - 0x20) >> 4) & 0x3)),
- chr(((B - 0x20) & 0xf) << 4 | (((C - 0x20) >> 2) & 0xf)),
- chr(((C - 0x20) & 0x3) << 6 | ((D - 0x20) & 0x3f))
- ]) for A, B, C, D in quadruplets_gen(s[1:].rstrip())]
- except ValueError:
- raise Error('Illegal char')
- result = ''.join(result)
- trailingdata = result[length:]
- if trailingdata.strip('\x00'):
- raise Error('Trailing garbage')
- result = result[:length]
- if len(result) < length:
- result += ((length - len(result)) * '\x00')
- return result
-
-
-def b2a_uu(s):
- length = len(s)
- if length > 45:
- raise Error('At most 45 bytes at once')
-
- def triples_gen(s):
- while s:
- try:
- yield ord(s[0]), ord(s[1]), ord(s[2])
- except IndexError:
- s += '\0\0'
- yield ord(s[0]), ord(s[1]), ord(s[2])
- return
- s = s[3:]
-
- result = [''.join(
- [chr(0x20 + (( A >> 2 ) & 0x3F)),
- chr(0x20 + (((A << 4) | ((B >> 4) & 0xF)) & 0x3F)),
- chr(0x20 + (((B << 2) | ((C >> 6) & 0x3)) & 0x3F)),
- chr(0x20 + (( C ) & 0x3F))])
- for A, B, C in triples_gen(s)]
- return chr(ord(' ') + (length & 077)) + ''.join(result) + '\n'
-
-
-table_a2b_base64 = {
- 'A': 0,
- 'B': 1,
- 'C': 2,
- 'D': 3,
- 'E': 4,
- 'F': 5,
- 'G': 6,
- 'H': 7,
- 'I': 8,
- 'J': 9,
- 'K': 10,
- 'L': 11,
- 'M': 12,
- 'N': 13,
- 'O': 14,
- 'P': 15,
- 'Q': 16,
- 'R': 17,
- 'S': 18,
- 'T': 19,
- 'U': 20,
- 'V': 21,
- 'W': 22,
- 'X': 23,
- 'Y': 24,
- 'Z': 25,
- 'a': 26,
- 'b': 27,
- 'c': 28,
- 'd': 29,
- 'e': 30,
- 'f': 31,
- 'g': 32,
- 'h': 33,
- 'i': 34,
- 'j': 35,
- 'k': 36,
- 'l': 37,
- 'm': 38,
- 'n': 39,
- 'o': 40,
- 'p': 41,
- 'q': 42,
- 'r': 43,
- 's': 44,
- 't': 45,
- 'u': 46,
- 'v': 47,
- 'w': 48,
- 'x': 49,
- 'y': 50,
- 'z': 51,
- '0': 52,
- '1': 53,
- '2': 54,
- '3': 55,
- '4': 56,
- '5': 57,
- '6': 58,
- '7': 59,
- '8': 60,
- '9': 61,
- '+': 62,
- '/': 63,
- '=': 0,
-}
-
-
-def a2b_base64(s):
- if not isinstance(s, (str, unicode)):
- raise TypeError("expected string or unicode, got %r" % (s,))
- s = s.rstrip()
- # clean out all invalid characters, this also strips the final '=' padding
- # check for correct padding
-
- def next_valid_char(s, pos):
- for i in range(pos + 1, len(s)):
- c = s[i]
- if c < '\x7f':
- try:
- table_a2b_base64[c]
- return c
- except KeyError:
- pass
- return None
-
- quad_pos = 0
- leftbits = 0
- leftchar = 0
- res = []
- for i, c in enumerate(s):
- if c > '\x7f' or c == '\n' or c == '\r' or c == ' ':
- continue
- if c == '=':
- if quad_pos < 2 or (quad_pos == 2 and next_valid_char(s, i) != '='):
- continue
- else:
- leftbits = 0
- break
- try:
- next_c = table_a2b_base64[c]
- except KeyError:
- continue
- quad_pos = (quad_pos + 1) & 0x03
- leftchar = (leftchar << 6) | next_c
- leftbits += 6
- if leftbits >= 8:
- leftbits -= 8
- res.append((leftchar >> leftbits & 0xff))
- leftchar &= ((1 << leftbits) - 1)
- if leftbits != 0:
- raise Error('Incorrect padding')
-
- return ''.join([chr(i) for i in res])
-
-table_b2a_base64 = \
-"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
-
-def b2a_base64(s):
- length = len(s)
- final_length = length % 3
-
- def triples_gen(s):
- while s:
- try:
- yield ord(s[0]), ord(s[1]), ord(s[2])
- except IndexError:
- s += '\0\0'
- yield ord(s[0]), ord(s[1]), ord(s[2])
- return
- s = s[3:]
-
-
- a = triples_gen(s[ :length - final_length])
-
- result = [''.join(
- [table_b2a_base64[( A >> 2 ) & 0x3F],
- table_b2a_base64[((A << 4) | ((B >> 4) & 0xF)) & 0x3F],
- table_b2a_base64[((B << 2) | ((C >> 6) & 0x3)) & 0x3F],
- table_b2a_base64[( C ) & 0x3F]])
- for A, B, C in a]
-
- final = s[length - final_length:]
- if final_length == 0:
- snippet = ''
- elif final_length == 1:
- a = ord(final[0])
- snippet = table_b2a_base64[(a >> 2 ) & 0x3F] + \
- table_b2a_base64[(a << 4 ) & 0x3F] + '=='
- else:
- a = ord(final[0])
- b = ord(final[1])
- snippet = table_b2a_base64[(a >> 2) & 0x3F] + \
- table_b2a_base64[((a << 4) | (b >> 4) & 0xF) & 0x3F] + \
- table_b2a_base64[(b << 2) & 0x3F] + '='
- return ''.join(result) + snippet + '\n'
-
-def a2b_qp(s, header=False):
- inp = 0
- odata = []
- while inp < len(s):
- if s[inp] == '=':
- inp += 1
- if inp >= len(s):
- break
- # Soft line breaks
- if (s[inp] == '\n') or (s[inp] == '\r'):
- if s[inp] != '\n':
- while inp < len(s) and s[inp] != '\n':
- inp += 1
- if inp < len(s):
- inp += 1
- elif s[inp] == '=':
- # broken case from broken python qp
- odata.append('=')
- inp += 1
- elif s[inp] in hex_numbers and s[inp + 1] in hex_numbers:
- ch = chr(int(s[inp:inp+2], 16))
- inp += 2
- odata.append(ch)
- else:
- odata.append('=')
- elif header and s[inp] == '_':
- odata.append(' ')
- inp += 1
- else:
- odata.append(s[inp])
- inp += 1
- return ''.join(odata)
-
-def b2a_qp(data, quotetabs=False, istext=True, header=False):
- """quotetabs=True means that tab and space characters are always
- quoted.
- istext=False means that \r and \n are treated as regular characters
- header=True encodes space characters with '_' and requires
- real '_' characters to be quoted.
- """
- MAXLINESIZE = 76
-
- # See if this string is using CRLF line ends
- lf = data.find('\n')
- crlf = lf > 0 and data[lf-1] == '\r'
-
- inp = 0
- linelen = 0
- odata = []
- while inp < len(data):
- c = data[inp]
- if (c > '~' or
- c == '=' or
- (header and c == '_') or
- (c == '.' and linelen == 0 and (inp+1 == len(data) or
- data[inp+1] == '\n' or
- data[inp+1] == '\r')) or
- (not istext and (c == '\r' or c == '\n')) or
- ((c == '\t' or c == ' ') and (inp + 1 == len(data))) or
- (c <= ' ' and c != '\r' and c != '\n' and
- (quotetabs or (not quotetabs and (c != '\t' and c != ' '))))):
- linelen += 3
- if linelen >= MAXLINESIZE:
- odata.append('=')
- if crlf: odata.append('\r')
- odata.append('\n')
- linelen = 3
- odata.append('=' + two_hex_digits(ord(c)))
- inp += 1
- else:
- if (istext and
- (c == '\n' or (inp+1 < len(data) and c == '\r' and
- data[inp+1] == '\n'))):
- linelen = 0
- # Protect against whitespace on end of line
- if (len(odata) > 0 and
- (odata[-1] == ' ' or odata[-1] == '\t')):
- ch = ord(odata[-1])
- odata[-1] = '='
- odata.append(two_hex_digits(ch))
-
- if crlf: odata.append('\r')
- odata.append('\n')
- if c == '\r':
- inp += 2
- else:
- inp += 1
- else:
- if (inp + 1 < len(data) and
- data[inp+1] != '\n' and
- (linelen + 1) >= MAXLINESIZE):
- odata.append('=')
- if crlf: odata.append('\r')
- odata.append('\n')
- linelen = 0
-
- linelen += 1
- if header and c == ' ':
- c = '_'
- odata.append(c)
- inp += 1
- return ''.join(odata)
-
-hex_numbers = '0123456789ABCDEF'
-def hex(n):
- if n == 0:
- return '0'
-
- if n < 0:
- n = -n
- sign = '-'
- else:
- sign = ''
- arr = []
-
- def hex_gen(n):
- """ Yield a nibble at a time. """
- while n:
- yield n % 0x10
- n = n / 0x10
-
- for nibble in hex_gen(n):
- arr = [hex_numbers[nibble]] + arr
- return sign + ''.join(arr)
-
-def two_hex_digits(n):
- return hex_numbers[n / 0x10] + hex_numbers[n % 0x10]
-
-
-def strhex_to_int(s):
- i = 0
- for c in s:
- i = i * 0x10 + hex_numbers.index(c)
- return i
-
-hqx_encoding = '!"#$%&\'()*+,-012345689 at ABCDEFGHIJKLMNPQRSTUVXYZ[`abcdefhijklmpqr'
-
-DONE = 0x7f
-SKIP = 0x7e
-FAIL = 0x7d
-
-table_a2b_hqx = [
- #^@ ^A ^B ^C ^D ^E ^F ^G
- FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
- #\b \t \n ^K ^L \r ^N ^O
- FAIL, FAIL, SKIP, FAIL, FAIL, SKIP, FAIL, FAIL,
- #^P ^Q ^R ^S ^T ^U ^V ^W
- FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
- #^X ^Y ^Z ^[ ^\ ^] ^^ ^_
- FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
- # ! " # $ % & '
- FAIL, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
- #( ) * + , - . /
- 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, FAIL, FAIL,
- #0 1 2 3 4 5 6 7
- 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, FAIL,
- #8 9 : ; < = > ?
- 0x14, 0x15, DONE, FAIL, FAIL, FAIL, FAIL, FAIL,
- #@ A B C D E F G
- 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D,
- #H I J K L M N O
- 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23, 0x24, FAIL,
- #P Q R S T U V W
- 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, FAIL,
- #X Y Z [ \ ] ^ _
- 0x2C, 0x2D, 0x2E, 0x2F, FAIL, FAIL, FAIL, FAIL,
- #` a b c d e f g
- 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, FAIL,
- #h i j k l m n o
- 0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, FAIL, FAIL,
- #p q r s t u v w
- 0x3D, 0x3E, 0x3F, FAIL, FAIL, FAIL, FAIL, FAIL,
- #x y z { | } ~ ^?
- FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
- FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
- FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
- FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
- FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
- FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
- FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
- FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
- FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
- FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
- FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
- FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
- FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
- FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
- FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
- FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
- FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-]
-
-def a2b_hqx(s):
- result = []
-
- def quadruples_gen(s):
- t = []
- for c in s:
- res = table_a2b_hqx[ord(c)]
- if res == SKIP:
- continue
- elif res == FAIL:
- raise Error('Illegal character')
- elif res == DONE:
- yield t
- raise Done
- else:
- t.append(res)
- if len(t) == 4:
- yield t
- t = []
- yield t
-
- done = 0
- try:
- for snippet in quadruples_gen(s):
- length = len(snippet)
- if length == 4:
- result.append(chr(((snippet[0] & 0x3f) << 2) | (snippet[1] >> 4)))
- result.append(chr(((snippet[1] & 0x0f) << 4) | (snippet[2] >> 2)))
- result.append(chr(((snippet[2] & 0x03) << 6) | (snippet[3])))
- elif length == 3:
- result.append(chr(((snippet[0] & 0x3f) << 2) | (snippet[1] >> 4)))
- result.append(chr(((snippet[1] & 0x0f) << 4) | (snippet[2] >> 2)))
- elif length == 2:
- result.append(chr(((snippet[0] & 0x3f) << 2) | (snippet[1] >> 4)))
- except Done:
- done = 1
- except Error:
- raise
- return (''.join(result), done)
-
-def b2a_hqx(s):
- result =[]
-
- def triples_gen(s):
- while s:
- try:
- yield ord(s[0]), ord(s[1]), ord(s[2])
- except IndexError:
- yield tuple([ord(c) for c in s])
- s = s[3:]
-
- for snippet in triples_gen(s):
- length = len(snippet)
- if length == 3:
- result.append(
- hqx_encoding[(snippet[0] & 0xfc) >> 2])
- result.append(hqx_encoding[
- ((snippet[0] & 0x03) << 4) | ((snippet[1] & 0xf0) >> 4)])
- result.append(hqx_encoding[
- (snippet[1] & 0x0f) << 2 | ((snippet[2] & 0xc0) >> 6)])
- result.append(hqx_encoding[snippet[2] & 0x3f])
- elif length == 2:
- result.append(
- hqx_encoding[(snippet[0] & 0xfc) >> 2])
- result.append(hqx_encoding[
- ((snippet[0] & 0x03) << 4) | ((snippet[1] & 0xf0) >> 4)])
- result.append(hqx_encoding[
- (snippet[1] & 0x0f) << 2])
- elif length == 1:
- result.append(
- hqx_encoding[(snippet[0] & 0xfc) >> 2])
- result.append(hqx_encoding[
- ((snippet[0] & 0x03) << 4)])
- return ''.join(result)
-
-crctab_hqx = [
- 0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7,
- 0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef,
- 0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6,
- 0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de,
- 0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485,
- 0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d,
- 0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4,
- 0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc,
- 0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823,
- 0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b,
- 0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12,
- 0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a,
- 0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41,
- 0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49,
- 0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70,
- 0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78,
- 0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f,
- 0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067,
- 0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e,
- 0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256,
- 0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d,
- 0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405,
- 0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c,
- 0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634,
- 0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab,
- 0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3,
- 0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a,
- 0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92,
- 0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9,
- 0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1,
- 0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8,
- 0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0,
-]
-
-def crc_hqx(s, crc):
- for c in s:
- crc = ((crc << 8) & 0xff00) ^ crctab_hqx[((crc >> 8) & 0xff) ^ ord(c)]
-
- return crc
-
-def rlecode_hqx(s):
- """
- Run length encoding for binhex4.
- The CPython implementation does not do run length encoding
- of \x90 characters. This implementation does.
- """
- if not s:
- return ''
- result = []
- prev = s[0]
- count = 1
- # Add a dummy character to get the loop to go one extra round.
- # The dummy must be different from the last character of s.
- # In the same step we remove the first character, which has
- # already been stored in prev.
- if s[-1] == '!':
- s = s[1:] + '?'
- else:
- s = s[1:] + '!'
-
- for c in s:
- if c == prev and count < 255:
- count += 1
- else:
- if count == 1:
- if prev != '\x90':
- result.append(prev)
- else:
- result.extend(['\x90', '\x00'])
- elif count < 4:
- if prev != '\x90':
- result.extend([prev] * count)
- else:
- result.extend(['\x90', '\x00'] * count)
- else:
- if prev != '\x90':
- result.extend([prev, '\x90', chr(count)])
- else:
- result.extend(['\x90', '\x00', '\x90', chr(count)])
- count = 1
- prev = c
-
- return ''.join(result)
-
-def rledecode_hqx(s):
- s = s.split('\x90')
- result = [s[0]]
- prev = s[0]
- for snippet in s[1:]:
- count = ord(snippet[0])
- if count > 0:
- result.append(prev[-1] * (count-1))
- prev = snippet
- else:
- result. append('\x90')
- prev = '\x90'
- result.append(snippet[1:])
-
- return ''.join(result)
-
-crc_32_tab = [
- 0x00000000L, 0x77073096L, 0xee0e612cL, 0x990951baL, 0x076dc419L,
- 0x706af48fL, 0xe963a535L, 0x9e6495a3L, 0x0edb8832L, 0x79dcb8a4L,
- 0xe0d5e91eL, 0x97d2d988L, 0x09b64c2bL, 0x7eb17cbdL, 0xe7b82d07L,
- 0x90bf1d91L, 0x1db71064L, 0x6ab020f2L, 0xf3b97148L, 0x84be41deL,
- 0x1adad47dL, 0x6ddde4ebL, 0xf4d4b551L, 0x83d385c7L, 0x136c9856L,
- 0x646ba8c0L, 0xfd62f97aL, 0x8a65c9ecL, 0x14015c4fL, 0x63066cd9L,
- 0xfa0f3d63L, 0x8d080df5L, 0x3b6e20c8L, 0x4c69105eL, 0xd56041e4L,
- 0xa2677172L, 0x3c03e4d1L, 0x4b04d447L, 0xd20d85fdL, 0xa50ab56bL,
- 0x35b5a8faL, 0x42b2986cL, 0xdbbbc9d6L, 0xacbcf940L, 0x32d86ce3L,
- 0x45df5c75L, 0xdcd60dcfL, 0xabd13d59L, 0x26d930acL, 0x51de003aL,
- 0xc8d75180L, 0xbfd06116L, 0x21b4f4b5L, 0x56b3c423L, 0xcfba9599L,
- 0xb8bda50fL, 0x2802b89eL, 0x5f058808L, 0xc60cd9b2L, 0xb10be924L,
- 0x2f6f7c87L, 0x58684c11L, 0xc1611dabL, 0xb6662d3dL, 0x76dc4190L,
- 0x01db7106L, 0x98d220bcL, 0xefd5102aL, 0x71b18589L, 0x06b6b51fL,
- 0x9fbfe4a5L, 0xe8b8d433L, 0x7807c9a2L, 0x0f00f934L, 0x9609a88eL,
- 0xe10e9818L, 0x7f6a0dbbL, 0x086d3d2dL, 0x91646c97L, 0xe6635c01L,
- 0x6b6b51f4L, 0x1c6c6162L, 0x856530d8L, 0xf262004eL, 0x6c0695edL,
- 0x1b01a57bL, 0x8208f4c1L, 0xf50fc457L, 0x65b0d9c6L, 0x12b7e950L,
- 0x8bbeb8eaL, 0xfcb9887cL, 0x62dd1ddfL, 0x15da2d49L, 0x8cd37cf3L,
- 0xfbd44c65L, 0x4db26158L, 0x3ab551ceL, 0xa3bc0074L, 0xd4bb30e2L,
- 0x4adfa541L, 0x3dd895d7L, 0xa4d1c46dL, 0xd3d6f4fbL, 0x4369e96aL,
- 0x346ed9fcL, 0xad678846L, 0xda60b8d0L, 0x44042d73L, 0x33031de5L,
- 0xaa0a4c5fL, 0xdd0d7cc9L, 0x5005713cL, 0x270241aaL, 0xbe0b1010L,
- 0xc90c2086L, 0x5768b525L, 0x206f85b3L, 0xb966d409L, 0xce61e49fL,
- 0x5edef90eL, 0x29d9c998L, 0xb0d09822L, 0xc7d7a8b4L, 0x59b33d17L,
- 0x2eb40d81L, 0xb7bd5c3bL, 0xc0ba6cadL, 0xedb88320L, 0x9abfb3b6L,
- 0x03b6e20cL, 0x74b1d29aL, 0xead54739L, 0x9dd277afL, 0x04db2615L,
- 0x73dc1683L, 0xe3630b12L, 0x94643b84L, 0x0d6d6a3eL, 0x7a6a5aa8L,
- 0xe40ecf0bL, 0x9309ff9dL, 0x0a00ae27L, 0x7d079eb1L, 0xf00f9344L,
- 0x8708a3d2L, 0x1e01f268L, 0x6906c2feL, 0xf762575dL, 0x806567cbL,
- 0x196c3671L, 0x6e6b06e7L, 0xfed41b76L, 0x89d32be0L, 0x10da7a5aL,
- 0x67dd4accL, 0xf9b9df6fL, 0x8ebeeff9L, 0x17b7be43L, 0x60b08ed5L,
- 0xd6d6a3e8L, 0xa1d1937eL, 0x38d8c2c4L, 0x4fdff252L, 0xd1bb67f1L,
- 0xa6bc5767L, 0x3fb506ddL, 0x48b2364bL, 0xd80d2bdaL, 0xaf0a1b4cL,
- 0x36034af6L, 0x41047a60L, 0xdf60efc3L, 0xa867df55L, 0x316e8eefL,
- 0x4669be79L, 0xcb61b38cL, 0xbc66831aL, 0x256fd2a0L, 0x5268e236L,
- 0xcc0c7795L, 0xbb0b4703L, 0x220216b9L, 0x5505262fL, 0xc5ba3bbeL,
- 0xb2bd0b28L, 0x2bb45a92L, 0x5cb36a04L, 0xc2d7ffa7L, 0xb5d0cf31L,
- 0x2cd99e8bL, 0x5bdeae1dL, 0x9b64c2b0L, 0xec63f226L, 0x756aa39cL,
- 0x026d930aL, 0x9c0906a9L, 0xeb0e363fL, 0x72076785L, 0x05005713L,
- 0x95bf4a82L, 0xe2b87a14L, 0x7bb12baeL, 0x0cb61b38L, 0x92d28e9bL,
- 0xe5d5be0dL, 0x7cdcefb7L, 0x0bdbdf21L, 0x86d3d2d4L, 0xf1d4e242L,
- 0x68ddb3f8L, 0x1fda836eL, 0x81be16cdL, 0xf6b9265bL, 0x6fb077e1L,
- 0x18b74777L, 0x88085ae6L, 0xff0f6a70L, 0x66063bcaL, 0x11010b5cL,
- 0x8f659effL, 0xf862ae69L, 0x616bffd3L, 0x166ccf45L, 0xa00ae278L,
- 0xd70dd2eeL, 0x4e048354L, 0x3903b3c2L, 0xa7672661L, 0xd06016f7L,
- 0x4969474dL, 0x3e6e77dbL, 0xaed16a4aL, 0xd9d65adcL, 0x40df0b66L,
- 0x37d83bf0L, 0xa9bcae53L, 0xdebb9ec5L, 0x47b2cf7fL, 0x30b5ffe9L,
- 0xbdbdf21cL, 0xcabac28aL, 0x53b39330L, 0x24b4a3a6L, 0xbad03605L,
- 0xcdd70693L, 0x54de5729L, 0x23d967bfL, 0xb3667a2eL, 0xc4614ab8L,
- 0x5d681b02L, 0x2a6f2b94L, 0xb40bbe37L, 0xc30c8ea1L, 0x5a05df1bL,
- 0x2d02ef8dL
-]
-
-def crc32(s, crc=0):
- result = 0
- crc = ~long(crc) & 0xffffffffL
- for c in s:
- crc = crc_32_tab[(crc ^ long(ord(c))) & 0xffL] ^ (crc >> 8)
- #/* Note: (crc >> 8) MUST zero fill on left
-
- result = crc ^ 0xffffffffL
-
- if result > 2**31:
- result = ((result + 2**31) % 2**32) - 2**31
-
- return result
-
-def b2a_hex(s):
- result = []
- for char in s:
- c = (ord(char) >> 4) & 0xf
- if c > 9:
- c = c + ord('a') - 10
- else:
- c = c + ord('0')
- result.append(chr(c))
- c = ord(char) & 0xf
- if c > 9:
- c = c + ord('a') - 10
- else:
- c = c + ord('0')
- result.append(chr(c))
- return ''.join(result)
-
-hexlify = b2a_hex
-
-table_hex = [
- -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
- -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
- -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
- 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,-1,-1, -1,-1,-1,-1,
- -1,10,11,12, 13,14,15,-1, -1,-1,-1,-1, -1,-1,-1,-1,
- -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
- -1,10,11,12, 13,14,15,-1, -1,-1,-1,-1, -1,-1,-1,-1,
- -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1
-]
-
-
-def a2b_hex(t):
- result = []
-
- def pairs_gen(s):
- while s:
- try:
- yield table_hex[ord(s[0])], table_hex[ord(s[1])]
- except IndexError:
- if len(s):
- raise TypeError('Odd-length string')
- return
- s = s[2:]
-
- for a, b in pairs_gen(t):
- if a < 0 or b < 0:
- raise TypeError('Non-hexadecimal digit found')
- result.append(chr((a << 4) + b))
- return ''.join(result)
-
-
-unhexlify = a2b_hex
diff --git a/lib_pypy/numpypy/core/numeric.py b/lib_pypy/numpypy/core/numeric.py
--- a/lib_pypy/numpypy/core/numeric.py
+++ b/lib_pypy/numpypy/core/numeric.py
@@ -6,7 +6,7 @@
import _numpypy as multiarray # ARGH
from numpypy.core.arrayprint import array2string
-
+newaxis = None
def asanyarray(a, dtype=None, order=None, maskna=None, ownmaskna=False):
"""
@@ -306,6 +306,125 @@
else:
return multiarray.set_string_function(f, repr)
+def array_equal(a1, a2):
+ """
+ True if two arrays have the same shape and elements, False otherwise.
+
+ Parameters
+ ----------
+ a1, a2 : array_like
+ Input arrays.
+
+ Returns
+ -------
+ b : bool
+ Returns True if the arrays are equal.
+
+ See Also
+ --------
+ allclose: Returns True if two arrays are element-wise equal within a
+ tolerance.
+ array_equiv: Returns True if input arrays are shape consistent and all
+ elements equal.
+
+ Examples
+ --------
+ >>> np.array_equal([1, 2], [1, 2])
+ True
+ >>> np.array_equal(np.array([1, 2]), np.array([1, 2]))
+ True
+ >>> np.array_equal([1, 2], [1, 2, 3])
+ False
+ >>> np.array_equal([1, 2], [1, 4])
+ False
+
+ """
+ try:
+ a1, a2 = asarray(a1), asarray(a2)
+ except:
+ return False
+ if a1.shape != a2.shape:
+ return False
+ return bool((a1 == a2).all())
+
+def asarray(a, dtype=None, order=None, maskna=None, ownmaskna=False):
+ """
+ Convert the input to an array.
+
+ Parameters
+ ----------
+ a : array_like
+ Input data, in any form that can be converted to an array. This
+ includes lists, lists of tuples, tuples, tuples of tuples, tuples
+ of lists and ndarrays.
+ dtype : data-type, optional
+ By default, the data-type is inferred from the input data.
+ order : {'C', 'F'}, optional
+ Whether to use row-major ('C') or column-major ('F' for FORTRAN)
+ memory representation. Defaults to 'C'.
+ maskna : bool or None, optional
+ If this is set to True, it forces the array to have an NA mask.
+ If this is set to False, it forces the array to not have an NA
+ mask.
+ ownmaskna : bool, optional
+ If this is set to True, forces the array to have a mask which
+ it owns.
+
+ Returns
+ -------
+ out : ndarray
+ Array interpretation of `a`. No copy is performed if the input
+ is already an ndarray. If `a` is a subclass of ndarray, a base
+ class ndarray is returned.
+
+ See Also
+ --------
+ asanyarray : Similar function which passes through subclasses.
+ ascontiguousarray : Convert input to a contiguous array.
+ asfarray : Convert input to a floating point ndarray.
+ asfortranarray : Convert input to an ndarray with column-major
+ memory order.
+ asarray_chkfinite : Similar function which checks input for NaNs and Infs.
+ fromiter : Create an array from an iterator.
+ fromfunction : Construct an array by executing a function on grid
+ positions.
+
+ Examples
+ --------
+ Convert a list into an array:
+
+ >>> a = [1, 2]
+ >>> np.asarray(a)
+ array([1, 2])
+
+ Existing arrays are not copied:
+
+ >>> a = np.array([1, 2])
+ >>> np.asarray(a) is a
+ True
+
+ If `dtype` is set, array is copied only if dtype does not match:
+
+ >>> a = np.array([1, 2], dtype=np.float32)
+ >>> np.asarray(a, dtype=np.float32) is a
+ True
+ >>> np.asarray(a, dtype=np.float64) is a
+ False
+
+ Contrary to `asanyarray`, ndarray subclasses are not passed through:
+
+ >>> issubclass(np.matrix, np.ndarray)
+ True
+ >>> a = np.matrix([[1, 2]])
+ >>> np.asarray(a) is a
+ False
+ >>> np.asanyarray(a) is a
+ True
+
+ """
+ return array(a, dtype, copy=False, order=order,
+ maskna=maskna, ownmaskna=ownmaskna)
+
set_string_function(array_str, 0)
set_string_function(array_repr, 1)
@@ -319,4 +438,4 @@
False_ = bool_(False)
True_ = bool_(True)
e = math.e
-pi = math.pi
\ No newline at end of file
+pi = math.pi
diff --git a/lib_pypy/pypy_test/test_binascii.py b/lib_pypy/pypy_test/test_binascii.py
deleted file mode 100644
--- a/lib_pypy/pypy_test/test_binascii.py
+++ /dev/null
@@ -1,168 +0,0 @@
-from __future__ import absolute_import
-import py
-from lib_pypy import binascii
-
-# Create binary test data
-data = "The quick brown fox jumps over the lazy dog.\r\n"
-# Be slow so we don't depend on other modules
-data += "".join(map(chr, xrange(256)))
-data += "\r\nHello world.\n"
-
-def test_exceptions():
- # Check module exceptions
- assert issubclass(binascii.Error, Exception)
- assert issubclass(binascii.Incomplete, Exception)
-
-def test_functions():
- # Check presence of all functions
- funcs = []
- for suffix in "base64", "hqx", "uu", "hex":
- prefixes = ["a2b_", "b2a_"]
- if suffix == "hqx":
- prefixes.extend(["crc_", "rlecode_", "rledecode_"])
- for prefix in prefixes:
- name = prefix + suffix
- assert callable(getattr(binascii, name))
- py.test.raises(TypeError, getattr(binascii, name))
- for name in ("hexlify", "unhexlify"):
- assert callable(getattr(binascii, name))
- py.test.raises(TypeError, getattr(binascii, name))
-
-def test_base64valid():
- # Test base64 with valid data
- MAX_BASE64 = 57
- lines = []
- for i in range(0, len(data), MAX_BASE64):
- b = data[i:i+MAX_BASE64]
- a = binascii.b2a_base64(b)
- lines.append(a)
- res = ""
- for line in lines:
- b = binascii.a2b_base64(line)
- res = res + b
- assert res == data
-
-def test_base64invalid():
- # Test base64 with random invalid characters sprinkled throughout
- # (This requires a new version of binascii.)
- MAX_BASE64 = 57
- lines = []
- for i in range(0, len(data), MAX_BASE64):
- b = data[i:i+MAX_BASE64]
- a = binascii.b2a_base64(b)
- lines.append(a)
-
- fillers = ""
- valid = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+/"
- for i in xrange(256):
- c = chr(i)
- if c not in valid:
- fillers += c
- def addnoise(line):
- noise = fillers
- ratio = len(line) // len(noise)
- res = ""
- while line and noise:
- if len(line) // len(noise) > ratio:
- c, line = line[0], line[1:]
- else:
- c, noise = noise[0], noise[1:]
- res += c
- return res + noise + line
- res = ""
- for line in map(addnoise, lines):
- b = binascii.a2b_base64(line)
- res += b
- assert res == data
-
- # Test base64 with just invalid characters, which should return
- # empty strings. TBD: shouldn't it raise an exception instead ?
- assert binascii.a2b_base64(fillers) == ''
-
-def test_uu():
- MAX_UU = 45
- lines = []
- for i in range(0, len(data), MAX_UU):
- b = data[i:i+MAX_UU]
- a = binascii.b2a_uu(b)
- lines.append(a)
- res = ""
- for line in lines:
- b = binascii.a2b_uu(line)
- res += b
- assert res == data
-
- assert binascii.a2b_uu("\x7f") == "\x00"*31
- assert binascii.a2b_uu("\x80") == "\x00"*32
- assert binascii.a2b_uu("\xff") == "\x00"*31
- py.test.raises(binascii.Error, binascii.a2b_uu, "\xff\x00")
- py.test.raises(binascii.Error, binascii.a2b_uu, "!!!!")
-
- py.test.raises(binascii.Error, binascii.b2a_uu, 46*"!")
-
-def test_crc32():
- crc = binascii.crc32("Test the CRC-32 of")
- crc = binascii.crc32(" this string.", crc)
- assert crc == 1571220330
-
- crc = binascii.crc32('frotz\n', 0)
- assert crc == -372923920
-
- py.test.raises(TypeError, binascii.crc32)
-
-def test_hex():
- # test hexlification
- s = '{s\005\000\000\000worldi\002\000\000\000s\005\000\000\000helloi\001\000\000\0000'
- t = binascii.b2a_hex(s)
- u = binascii.a2b_hex(t)
- assert s == u
- py.test.raises(TypeError, binascii.a2b_hex, t[:-1])
- py.test.raises(TypeError, binascii.a2b_hex, t[:-1] + 'q')
-
- # Verify the treatment of Unicode strings
- assert binascii.hexlify(unicode('a', 'ascii')) == '61'
-
-def test_qp():
- # A test for SF bug 534347 (segfaults without the proper fix)
- try:
- binascii.a2b_qp("", **{1:1})
- except TypeError:
- pass
- else:
- fail("binascii.a2b_qp(**{1:1}) didn't raise TypeError")
- assert binascii.a2b_qp("= ") == "= "
- assert binascii.a2b_qp("==") == "="
- assert binascii.a2b_qp("=AX") == "=AX"
- py.test.raises(TypeError, binascii.b2a_qp, foo="bar")
- assert binascii.a2b_qp("=00\r\n=00") == "\x00\r\n\x00"
- assert binascii.b2a_qp("\xff\r\n\xff\n\xff") == "=FF\r\n=FF\r\n=FF"
- target = "0"*75+"=\r\n=FF\r\n=FF\r\n=FF"
- assert binascii.b2a_qp("0"*75+"\xff\r\n\xff\r\n\xff") == target
-
-def test_empty_string():
- # A test for SF bug #1022953. Make sure SystemError is not raised.
- for n in ['b2a_qp', 'a2b_hex', 'b2a_base64', 'a2b_uu', 'a2b_qp',
- 'b2a_hex', 'unhexlify', 'hexlify', 'crc32', 'b2a_hqx',
- 'a2b_hqx', 'a2b_base64', 'rlecode_hqx', 'b2a_uu',
- 'rledecode_hqx']:
- f = getattr(binascii, n)
- f('')
- binascii.crc_hqx('', 0)
-
-def test_qp_bug_case():
- assert binascii.b2a_qp('y'*77, False, False) == 'y'*75 + '=\nyy'
- assert binascii.b2a_qp(' '*77, False, False) == ' '*75 + '=\n =20'
- assert binascii.b2a_qp('y'*76, False, False) == 'y'*76
- assert binascii.b2a_qp(' '*76, False, False) == ' '*75 + '=\n=20'
-
-def test_wrong_padding():
- s = 'CSixpLDtKSC/7Liuvsax4iC6uLmwMcijIKHaILzSwd/H0SC8+LCjwLsgv7W/+Mj3IQ'
- py.test.raises(binascii.Error, binascii.a2b_base64, s)
-
-def test_crap_after_padding():
- s = 'xxx=axxxx'
- assert binascii.a2b_base64(s) == '\xc7\x1c'
-
-def test_wrong_args():
- # this should grow as a way longer list
- py.test.raises(TypeError, binascii.a2b_base64, 42)
diff --git a/lib_pypy/pypy_test/test_locale.py b/lib_pypy/pypy_test/test_locale.py
deleted file mode 100644
--- a/lib_pypy/pypy_test/test_locale.py
+++ /dev/null
@@ -1,79 +0,0 @@
-from __future__ import absolute_import
-import py
-import sys
-
-from lib_pypy.ctypes_config_cache import rebuild
-rebuild.rebuild_one('locale.ctc.py')
-
-from lib_pypy import _locale
-
-
-def setup_module(mod):
- if sys.platform == 'darwin':
- py.test.skip("Locale support on MacOSX is minimal and cannot be tested")
-
-class TestLocale:
- def setup_class(cls):
- cls.oldlocale = _locale.setlocale(_locale.LC_NUMERIC)
- if sys.platform.startswith("win"):
- cls.tloc = "en"
- elif sys.platform.startswith("freebsd"):
- cls.tloc = "en_US.US-ASCII"
- else:
- cls.tloc = "en_US.UTF8"
- try:
- _locale.setlocale(_locale.LC_NUMERIC, cls.tloc)
- except _locale.Error:
- py.test.skip("test locale %s not supported" % cls.tloc)
-
- def teardown_class(cls):
- _locale.setlocale(_locale.LC_NUMERIC, cls.oldlocale)
-
- def test_format(self):
- py.test.skip("XXX fix or kill me")
-
- def testformat(formatstr, value, grouping = 0, output=None):
- if output:
- print "%s %% %s =? %s ..." %\
- (repr(formatstr), repr(value), repr(output)),
- else:
- print "%s %% %s works? ..." % (repr(formatstr), repr(value)),
- result = locale.format(formatstr, value, grouping = grouping)
- assert result == output
-
- testformat("%f", 1024, grouping=1, output='1,024.000000')
- testformat("%f", 102, grouping=1, output='102.000000')
- testformat("%f", -42, grouping=1, output='-42.000000')
- testformat("%+f", -42, grouping=1, output='-42.000000')
- testformat("%20.f", -42, grouping=1, output=' -42')
- testformat("%+10.f", -4200, grouping=1, output=' -4,200')
- testformat("%-10.f", 4200, grouping=1, output='4,200 ')
-
- def test_getpreferredencoding(self):
- py.test.skip("XXX fix or kill me")
- # Invoke getpreferredencoding to make sure it does not cause exceptions
- _locale.getpreferredencoding()
-
- # Test BSD Rune locale's bug for isctype functions.
- def test_bsd_bug(self):
- def teststrop(s, method, output):
- print "%s.%s() =? %s ..." % (repr(s), method, repr(output)),
- result = getattr(s, method)()
- assert result == output
-
- oldlocale = _locale.setlocale(_locale.LC_CTYPE)
- _locale.setlocale(_locale.LC_CTYPE, self.tloc)
- try:
- teststrop('\x20', 'isspace', True)
- teststrop('\xa0', 'isspace', False)
- teststrop('\xa1', 'isspace', False)
- teststrop('\xc0', 'isalpha', False)
- teststrop('\xc0', 'isalnum', False)
- teststrop('\xc0', 'isupper', False)
- teststrop('\xc0', 'islower', False)
- teststrop('\xec\xa0\xbc', 'split', ['\xec\xa0\xbc'])
- teststrop('\xed\x95\xa0', 'strip', '\xed\x95\xa0')
- teststrop('\xcc\x85', 'lower', '\xcc\x85')
- teststrop('\xed\x95\xa0', 'upper', '\xed\x95\xa0')
- finally:
- _locale.setlocale(_locale.LC_CTYPE, oldlocale)
diff --git a/lib_pypy/pypy_test/test_site_extra.py b/lib_pypy/pypy_test/test_site_extra.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/pypy_test/test_site_extra.py
@@ -0,0 +1,13 @@
+import sys, os
+
+
+def test_preimported_modules():
+ lst = ['__builtin__', '_codecs', '_warnings', 'codecs', 'encodings',
+ 'exceptions', 'signal', 'sys', 'zipimport']
+ g = os.popen("'%s' -c 'import sys; print sorted(sys.modules)'" %
+ (sys.executable,))
+ real_data = g.read()
+ g.close()
+ for name in lst:
+ quoted_name = repr(name)
+ assert quoted_name in real_data
diff --git a/lib_pypy/pypy_test/test_struct_extra.py b/lib_pypy/pypy_test/test_struct_extra.py
deleted file mode 100644
--- a/lib_pypy/pypy_test/test_struct_extra.py
+++ /dev/null
@@ -1,25 +0,0 @@
-from __future__ import absolute_import
-from lib_pypy import struct
-
-def test_simple():
- morezeros = '\x00' * (struct.calcsize('l')-4)
- assert struct.pack('<l', 16) == '\x10\x00\x00\x00' + morezeros
- assert struct.pack('4s', 'WAVE') == 'WAVE'
- assert struct.pack('<4sl', 'WAVE', 16) == 'WAVE\x10\x00\x00\x00' + morezeros
- s = 'ABCD01234567\x01\x00\x00\x00\x02\x00\x00\x00\x03\x00'
- assert struct.unpack('<4s4H2lH', s) == ('ABCD', 0x3130, 0x3332, 0x3534,
- 0x3736, 1, 2, 3)
-
-def test_infinity():
- INFINITY = 1e200 * 1e200
- assert str(struct.unpack("!d", struct.pack("!d", INFINITY))[0]) \
- == str(INFINITY)
- assert str(struct.unpack("!d", struct.pack("!d", -INFINITY))[0]) \
- == str(-INFINITY)
-
-def test_nan():
- INFINITY = 1e200 * 1e200
- NAN = INFINITY / INFINITY
- assert str(struct.unpack("!d", '\xff\xf8\x00\x00\x00\x00\x00\x00')[0]) \
- == str(NAN)
- assert str(struct.unpack("!d", struct.pack("!d", NAN))[0]) == str(NAN)
diff --git a/lib_pypy/struct.py b/lib_pypy/struct.py
deleted file mode 100644
--- a/lib_pypy/struct.py
+++ /dev/null
@@ -1,417 +0,0 @@
-#
-# This module is a pure Python version of pypy.module.struct.
-# It is only imported if the vastly faster pypy.module.struct is not
-# compiled in. For now we keep this version for reference and
-# because pypy.module.struct is not ootype-backend-friendly yet.
-#
-
-"""Functions to convert between Python values and C structs.
-Python strings are used to hold the data representing the C struct
-and also as format strings to describe the layout of data in the C struct.
-
-The optional first format char indicates byte order, size and alignment:
- @: native order, size & alignment (default)
- =: native order, std. size & alignment
- <: little-endian, std. size & alignment
- >: big-endian, std. size & alignment
- !: same as >
-
-The remaining chars indicate types of args and must match exactly;
-these can be preceded by a decimal repeat count:
- x: pad byte (no data);
- c:char;
- b:signed byte;
- B:unsigned byte;
- h:short;
- H:unsigned short;
- i:int;
- I:unsigned int;
- l:long;
- L:unsigned long;
- f:float;
- d:double.
-Special cases (preceding decimal count indicates length):
- s:string (array of char); p: pascal string (with count byte).
-Special case (only available in native format):
- P:an integer type that is wide enough to hold a pointer.
-Special case (not in native mode unless 'long long' in platform C):
- q:long long;
- Q:unsigned long long
-Whitespace between formats is ignored.
-
-The variable struct.error is an exception raised on errors."""
-
-import math, sys
-
-# TODO: XXX Find a way to get information on native sizes and alignments
-class StructError(Exception):
- pass
-error = StructError
-def unpack_int(data,index,size,le):
- bytes = [ord(b) for b in data[index:index+size]]
- if le == 'little':
- bytes.reverse()
- number = 0L
- for b in bytes:
- number = number << 8 | b
- return int(number)
-
-def unpack_signed_int(data,index,size,le):
- number = unpack_int(data,index,size,le)
- max = 2**(size*8)
- if number > 2**(size*8 - 1) - 1:
- number = int(-1*(max - number))
- return number
-
-INFINITY = 1e200 * 1e200
-NAN = INFINITY / INFINITY
-
-def unpack_char(data,index,size,le):
- return data[index:index+size]
-
-def pack_int(number,size,le):
- x=number
- res=[]
- for i in range(size):
- res.append(chr(x&0xff))
- x >>= 8
- if le == 'big':
- res.reverse()
- return ''.join(res)
-
-def pack_signed_int(number,size,le):
- if not isinstance(number, (int,long)):
- raise StructError,"argument for i,I,l,L,q,Q,h,H must be integer"
- if number > 2**(8*size-1)-1 or number < -1*2**(8*size-1):
- raise OverflowError,"Number:%i too large to convert" % number
- return pack_int(number,size,le)
-
-def pack_unsigned_int(number,size,le):
- if not isinstance(number, (int,long)):
- raise StructError,"argument for i,I,l,L,q,Q,h,H must be integer"
- if number < 0:
- raise TypeError,"can't convert negative long to unsigned"
- if number > 2**(8*size)-1:
- raise OverflowError,"Number:%i too large to convert" % number
- return pack_int(number,size,le)
-
-def pack_char(char,size,le):
- return str(char)
-
-def isinf(x):
- return x != 0.0 and x / 2 == x
-def isnan(v):
- return v != v*1.0 or (v == 1.0 and v == 2.0)
-
-def pack_float(x, size, le):
- unsigned = float_pack(x, size)
- result = []
- for i in range(8):
- result.append(chr((unsigned >> (i * 8)) & 0xFF))
- if le == "big":
- result.reverse()
- return ''.join(result)
-
-def unpack_float(data, index, size, le):
- binary = [data[i] for i in range(index, index + 8)]
- if le == "big":
- binary.reverse()
- unsigned = 0
- for i in range(8):
- unsigned |= ord(binary[i]) << (i * 8)
- return float_unpack(unsigned, size, le)
-
-def round_to_nearest(x):
- """Python 3 style round: round a float x to the nearest int, but
- unlike the builtin Python 2.x round function:
-
- - return an int, not a float
- - do round-half-to-even, not round-half-away-from-zero.
-
- We assume that x is finite and nonnegative; except wrong results
- if you use this for negative x.
-
- """
- int_part = int(x)
- frac_part = x - int_part
- if frac_part > 0.5 or frac_part == 0.5 and int_part & 1 == 1:
- int_part += 1
- return int_part
-
-def float_unpack(Q, size, le):
- """Convert a 32-bit or 64-bit integer created
- by float_pack into a Python float."""
-
- if size == 8:
- MIN_EXP = -1021 # = sys.float_info.min_exp
- MAX_EXP = 1024 # = sys.float_info.max_exp
- MANT_DIG = 53 # = sys.float_info.mant_dig
- BITS = 64
- elif size == 4:
- MIN_EXP = -125 # C's FLT_MIN_EXP
- MAX_EXP = 128 # FLT_MAX_EXP
- MANT_DIG = 24 # FLT_MANT_DIG
- BITS = 32
- else:
- raise ValueError("invalid size value")
-
- if Q >> BITS:
- raise ValueError("input out of range")
-
- # extract pieces
- sign = Q >> BITS - 1
- exp = (Q & ((1 << BITS - 1) - (1 << MANT_DIG - 1))) >> MANT_DIG - 1
- mant = Q & ((1 << MANT_DIG - 1) - 1)
-
- if exp == MAX_EXP - MIN_EXP + 2:
- # nan or infinity
- result = float('nan') if mant else float('inf')
- elif exp == 0:
- # subnormal or zero
- result = math.ldexp(float(mant), MIN_EXP - MANT_DIG)
- else:
- # normal
- mant += 1 << MANT_DIG - 1
- result = math.ldexp(float(mant), exp + MIN_EXP - MANT_DIG - 1)
- return -result if sign else result
-
-
-def float_pack(x, size):
- """Convert a Python float x into a 64-bit unsigned integer
- with the same byte representation."""
-
- if size == 8:
- MIN_EXP = -1021 # = sys.float_info.min_exp
- MAX_EXP = 1024 # = sys.float_info.max_exp
- MANT_DIG = 53 # = sys.float_info.mant_dig
- BITS = 64
- elif size == 4:
- MIN_EXP = -125 # C's FLT_MIN_EXP
- MAX_EXP = 128 # FLT_MAX_EXP
- MANT_DIG = 24 # FLT_MANT_DIG
- BITS = 32
- else:
- raise ValueError("invalid size value")
-
- sign = math.copysign(1.0, x) < 0.0
- if math.isinf(x):
- mant = 0
- exp = MAX_EXP - MIN_EXP + 2
- elif math.isnan(x):
- mant = 1 << (MANT_DIG-2) # other values possible
- exp = MAX_EXP - MIN_EXP + 2
- elif x == 0.0:
- mant = 0
- exp = 0
- else:
- m, e = math.frexp(abs(x)) # abs(x) == m * 2**e
- exp = e - (MIN_EXP - 1)
- if exp > 0:
- # Normal case.
- mant = round_to_nearest(m * (1 << MANT_DIG))
- mant -= 1 << MANT_DIG - 1
- else:
- # Subnormal case.
- if exp + MANT_DIG - 1 >= 0:
- mant = round_to_nearest(m * (1 << exp + MANT_DIG - 1))
- else:
- mant = 0
- exp = 0
-
- # Special case: rounding produced a MANT_DIG-bit mantissa.
- assert 0 <= mant <= 1 << MANT_DIG - 1
- if mant == 1 << MANT_DIG - 1:
- mant = 0
- exp += 1
-
- # Raise on overflow (in some circumstances, may want to return
- # infinity instead).
- if exp >= MAX_EXP - MIN_EXP + 2:
- raise OverflowError("float too large to pack in this format")
-
- # check constraints
- assert 0 <= mant < 1 << MANT_DIG - 1
- assert 0 <= exp <= MAX_EXP - MIN_EXP + 2
- assert 0 <= sign <= 1
- return ((sign << BITS - 1) | (exp << MANT_DIG - 1)) | mant
-
-
-big_endian_format = {
- 'x':{ 'size' : 1, 'alignment' : 0, 'pack' : None, 'unpack' : None},
- 'b':{ 'size' : 1, 'alignment' : 0, 'pack' : pack_signed_int, 'unpack' : unpack_signed_int},
- 'B':{ 'size' : 1, 'alignment' : 0, 'pack' : pack_unsigned_int, 'unpack' : unpack_int},
- 'c':{ 'size' : 1, 'alignment' : 0, 'pack' : pack_char, 'unpack' : unpack_char},
- 's':{ 'size' : 1, 'alignment' : 0, 'pack' : None, 'unpack' : None},
- 'p':{ 'size' : 1, 'alignment' : 0, 'pack' : None, 'unpack' : None},
- 'h':{ 'size' : 2, 'alignment' : 0, 'pack' : pack_signed_int, 'unpack' : unpack_signed_int},
- 'H':{ 'size' : 2, 'alignment' : 0, 'pack' : pack_unsigned_int, 'unpack' : unpack_int},
- 'i':{ 'size' : 4, 'alignment' : 0, 'pack' : pack_signed_int, 'unpack' : unpack_signed_int},
- 'I':{ 'size' : 4, 'alignment' : 0, 'pack' : pack_unsigned_int, 'unpack' : unpack_int},
- 'l':{ 'size' : 4, 'alignment' : 0, 'pack' : pack_signed_int, 'unpack' : unpack_signed_int},
- 'L':{ 'size' : 4, 'alignment' : 0, 'pack' : pack_unsigned_int, 'unpack' : unpack_int},
- 'q':{ 'size' : 8, 'alignment' : 0, 'pack' : pack_signed_int, 'unpack' : unpack_signed_int},
- 'Q':{ 'size' : 8, 'alignment' : 0, 'pack' : pack_unsigned_int, 'unpack' : unpack_int},
- 'f':{ 'size' : 4, 'alignment' : 0, 'pack' : pack_float, 'unpack' : unpack_float},
- 'd':{ 'size' : 8, 'alignment' : 0, 'pack' : pack_float, 'unpack' : unpack_float},
- }
-default = big_endian_format
-formatmode={ '<' : (default, 'little'),
- '>' : (default, 'big'),
- '!' : (default, 'big'),
- '=' : (default, sys.byteorder),
- '@' : (default, sys.byteorder)
- }
-
-def getmode(fmt):
- try:
- formatdef,endianness = formatmode[fmt[0]]
- index = 1
- except KeyError:
- formatdef,endianness = formatmode['@']
- index = 0
- return formatdef,endianness,index
-def getNum(fmt,i):
- num=None
- cur = fmt[i]
- while ('0'<= cur ) and ( cur <= '9'):
- if num == None:
- num = int(cur)
- else:
- num = 10*num + int(cur)
- i += 1
- cur = fmt[i]
- return num,i
-
-def calcsize(fmt):
- """calcsize(fmt) -> int
- Return size of C struct described by format string fmt.
- See struct.__doc__ for more on format strings."""
-
- formatdef,endianness,i = getmode(fmt)
- num = 0
- result = 0
- while i<len(fmt):
- num,i = getNum(fmt,i)
- cur = fmt[i]
- try:
- format = formatdef[cur]
- except KeyError:
- raise StructError,"%s is not a valid format"%cur
- if num != None :
- result += num*format['size']
- else:
- result += format['size']
- num = 0
- i += 1
- return result
-
-def pack(fmt,*args):
- """pack(fmt, v1, v2, ...) -> string
- Return string containing values v1, v2, ... packed according to fmt.
- See struct.__doc__ for more on format strings."""
- formatdef,endianness,i = getmode(fmt)
- args = list(args)
- n_args = len(args)
- result = []
- while i<len(fmt):
- num,i = getNum(fmt,i)
- cur = fmt[i]
- try:
- format = formatdef[cur]
- except KeyError:
- raise StructError,"%s is not a valid format"%cur
- if num == None :
- num_s = 0
- num = 1
- else:
- num_s = num
-
- if cur == 'x':
- result += ['\0'*num]
- elif cur == 's':
- if isinstance(args[0], str):
- padding = num - len(args[0])
- result += [args[0][:num] + '\0'*padding]
- args.pop(0)
- else:
- raise StructError,"arg for string format not a string"
- elif cur == 'p':
- if isinstance(args[0], str):
- padding = num - len(args[0]) - 1
-
- if padding > 0:
- result += [chr(len(args[0])) + args[0][:num-1] + '\0'*padding]
- else:
- if num<255:
- result += [chr(num-1) + args[0][:num-1]]
- else:
- result += [chr(255) + args[0][:num-1]]
- args.pop(0)
- else:
- raise StructError,"arg for string format not a string"
-
- else:
- if len(args) < num:
- raise StructError,"insufficient arguments to pack"
- for var in args[:num]:
- result += [format['pack'](var,format['size'],endianness)]
- args=args[num:]
- num = None
- i += 1
- if len(args) != 0:
- raise StructError,"too many arguments for pack format"
- return ''.join(result)
-
-def unpack(fmt,data):
- """unpack(fmt, string) -> (v1, v2, ...)
- Unpack the string, containing packed C structure data, according
- to fmt. Requires len(string)==calcsize(fmt).
- See struct.__doc__ for more on format strings."""
- formatdef,endianness,i = getmode(fmt)
- j = 0
- num = 0
- result = []
- length= calcsize(fmt)
- if length != len (data):
- raise StructError,"unpack str size does not match format"
- while i<len(fmt):
- num,i=getNum(fmt,i)
- cur = fmt[i]
- i += 1
- try:
- format = formatdef[cur]
- except KeyError:
- raise StructError,"%s is not a valid format"%cur
-
- if not num :
- num = 1
-
- if cur == 'x':
- j += num
- elif cur == 's':
- result.append(data[j:j+num])
- j += num
- elif cur == 'p':
- n=ord(data[j])
- if n >= num:
- n = num-1
- result.append(data[j+1:j+n+1])
- j += num
- else:
- for n in range(num):
- result += [format['unpack'](data,j,format['size'],endianness)]
- j += format['size']
-
- return tuple(result)
-
-def pack_into(fmt, buf, offset, *args):
- data = pack(fmt, *args)
- buffer(buf)[offset:offset+len(data)] = data
-
-def unpack_from(fmt, buf, offset=0):
- size = calcsize(fmt)
- data = buffer(buf)[offset:offset+size]
- if len(data) != size:
- raise error("unpack_from requires a buffer of at least %d bytes"
- % (size,))
- return unpack(fmt, data)
diff --git a/pypy/__init__.py b/pypy/__init__.py
--- a/pypy/__init__.py
+++ b/pypy/__init__.py
@@ -1,1 +1,16 @@
# Empty
+
+# XXX Should be empty again, soon.
+# XXX hack for win64:
+# This patch must stay here until the END OF STAGE 1
+# When all tests work, this branch will be merged
+# and the branch stage 2 is started, where we remove this patch.
+import sys
+if hasattr(sys, "maxsize"):
+ if sys.maxint != sys.maxsize:
+ sys.maxint = sys.maxsize
+ import warnings
+ warnings.warn("""\n
+---> This win64 port is now in stage 1: sys.maxint was modified.
+---> When pypy/__init__.py becomes empty again, we have reached stage 2.
+""")
diff --git a/pypy/annotation/builtin.py b/pypy/annotation/builtin.py
--- a/pypy/annotation/builtin.py
+++ b/pypy/annotation/builtin.py
@@ -37,7 +37,11 @@
try:
realresult = func(*args)
except (ValueError, OverflowError):
- return s_ImpossibleValue # no possible answer for this precise input
+ # no possible answer for this precise input. Be conservative
+ # and keep the computation non-constant. Example:
+ # unichr(constant-that-doesn't-fit-16-bits) on platforms where
+ # the underlying Python has sys.maxunicode == 0xffff.
+ return s_result
s_realresult = immutablevalue(realresult)
if not s_result.contains(s_realresult):
raise Exception("%s%r returned %r, which is not contained in %s" % (
@@ -163,7 +167,7 @@
r.const = False
return r
- assert not issubclass(typ, (int,long)) or typ in (bool, int), (
+ assert not issubclass(typ, (int, long)) or typ in (bool, int, long), (
"for integers only isinstance(.,int|r_uint) are supported")
if s_obj.is_constant():
@@ -297,7 +301,7 @@
def robjmodel_instantiate(s_clspbc):
assert isinstance(s_clspbc, SomePBC)
clsdef = None
- more_than_one = len(s_clspbc.descriptions)
+ more_than_one = len(s_clspbc.descriptions) > 1
for desc in s_clspbc.descriptions:
cdef = desc.getuniqueclassdef()
if more_than_one:
diff --git a/pypy/annotation/classdef.py b/pypy/annotation/classdef.py
--- a/pypy/annotation/classdef.py
+++ b/pypy/annotation/classdef.py
@@ -134,13 +134,19 @@
if self.name not in homedef.classdesc.all_enforced_attrs:
self.attr_allowed = False
if not self.readonly:
- raise NoSuchAttrError(homedef, self.name)
+ raise NoSuchAttrError(
+ "setting forbidden attribute %r on %r" % (
+ self.name, homedef))
def modified(self, classdef='?'):
self.readonly = False
if not self.attr_allowed:
- raise NoSuchAttrError(classdef, self.name)
-
+ raise NoSuchAttrError(
+ "Attribute %r on %r should be read-only.\n" % (self.name,
+ classdef) +
+ "This error can be caused by another 'getattr' that promoted\n"
+ "the attribute here; the list of read locations is:\n" +
+ '\n'.join([str(loc[0]) for loc in self.read_locations]))
class ClassDef(object):
"Wraps a user class."
diff --git a/pypy/annotation/description.py b/pypy/annotation/description.py
--- a/pypy/annotation/description.py
+++ b/pypy/annotation/description.py
@@ -398,7 +398,6 @@
cls = pyobj
base = object
baselist = list(cls.__bases__)
- baselist.reverse()
# special case: skip BaseException in Python 2.5, and pretend
# that all exceptions ultimately inherit from Exception instead
@@ -408,17 +407,27 @@
elif baselist == [py.builtin.BaseException]:
baselist = [Exception]
+ mixins_before = []
+ mixins_after = []
for b1 in baselist:
if b1 is object:
continue
if b1.__dict__.get('_mixin_', False):
- self.add_mixin(b1)
+ if base is object:
+ mixins_before.append(b1)
+ else:
+ mixins_after.append(b1)
else:
assert base is object, ("multiple inheritance only supported "
"with _mixin_: %r" % (cls,))
base = b1
+ if mixins_before and mixins_after:
+ raise Exception("unsupported: class %r has mixin bases both"
+ " before and after the regular base" % (self,))
+ self.add_mixins(mixins_after, check_not_in=base)
+ self.add_mixins(mixins_before)
+ self.add_sources_for_class(cls)
- self.add_sources_for_class(cls)
if base is not object:
self.basedesc = bookkeeper.getdesc(base)
@@ -480,14 +489,30 @@
return
self.classdict[name] = Constant(value)
- def add_mixin(self, base):
- for subbase in base.__bases__:
- if subbase is object:
- continue
- assert subbase.__dict__.get("_mixin_", False), ("Mixin class %r has non"
- "mixin base class %r" % (base, subbase))
- self.add_mixin(subbase)
- self.add_sources_for_class(base, mixin=True)
+ def add_mixins(self, mixins, check_not_in=object):
+ if not mixins:
+ return
+ A = type('tmp', tuple(mixins) + (object,), {})
+ mro = A.__mro__
+ assert mro[0] is A and mro[-1] is object
+ mro = mro[1:-1]
+ #
+ skip = set()
+ def add(cls):
+ if cls is not object:
+ for base in cls.__bases__:
+ add(base)
+ for name in cls.__dict__:
+ skip.add(name)
+ add(check_not_in)
+ #
+ for base in reversed(mro):
+ assert base.__dict__.get("_mixin_", False), ("Mixin class %r has non"
+ "mixin base class %r" % (mixins, base))
+ for name, value in base.__dict__.items():
+ if name in skip:
+ continue
+ self.add_source_attribute(name, value, mixin=True)
def add_sources_for_class(self, cls, mixin=False):
for name, value in cls.__dict__.items():
diff --git a/pypy/annotation/model.py b/pypy/annotation/model.py
--- a/pypy/annotation/model.py
+++ b/pypy/annotation/model.py
@@ -786,12 +786,15 @@
#
# safety check that no-one is trying to make annotation and translation
# faster by providing the -O option to Python.
-try:
- assert False
-except AssertionError:
- pass # fine
-else:
- raise RuntimeError("The annotator relies on 'assert' statements from the\n"
+import os
+if "WINGDB_PYTHON" not in os.environ:
+ # ...but avoiding this boring check in the IDE
+ try:
+ assert False
+ except AssertionError:
+ pass # fine
+ else:
+ raise RuntimeError("The annotator relies on 'assert' statements from the\n"
"\tannotated program: you cannot run it with 'python -O'.")
# this has the side-effect of registering the unary and binary operations
diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -1,15 +1,12 @@
from __future__ import with_statement
-import autopath
import py.test
import sys
from pypy import conftest
-from pypy.tool.udir import udir
from pypy.annotation import model as annmodel
from pypy.annotation.annrpython import RPythonAnnotator as _RPythonAnnotator
from pypy.translator.translator import graphof as tgraphof
from pypy.annotation import policy
-from pypy.annotation import specialize
from pypy.annotation.listdef import ListDef, ListChangeUnallowed
from pypy.annotation.dictdef import DictDef
from pypy.objspace.flow.model import *
@@ -2431,6 +2428,52 @@
assert isinstance(s.items[1], annmodel.SomeChar)
assert isinstance(s.items[2], annmodel.SomeChar)
+ def test_mixin_first(self):
+ class Mixin(object):
+ _mixin_ = True
+ def foo(self): return 4
+ class Base(object):
+ def foo(self): return 5
+ class Concrete(Mixin, Base):
+ pass
+ def f():
+ return Concrete().foo()
+
+ assert f() == 4
+ a = self.RPythonAnnotator()
+ s = a.build_types(f, [])
+ assert s.const == 4
+
+ def test_mixin_last(self):
+ class Mixin(object):
+ _mixin_ = True
+ def foo(self): return 4
+ class Base(object):
+ def foo(self): return 5
+ class Concrete(Base, Mixin):
+ pass
+ def f():
+ return Concrete().foo()
+
+ assert f() == 5
+ a = self.RPythonAnnotator()
+ s = a.build_types(f, [])
+ assert s.const == 5
+
+ def test_mixin_concrete(self):
+ class Mixin(object):
+ _mixin_ = True
+ def foo(self): return 4
+ class Concrete(Mixin):
+ def foo(self): return 5
+ def f():
+ return Concrete().foo()
+
+ assert f() == 5
+ a = self.RPythonAnnotator()
+ s = a.build_types(f, [])
+ assert s.const == 5
+
def test_multiple_mixins_mro(self):
# an obscure situation, but it occurred in module/micronumpy/types.py
class A(object):
@@ -2510,6 +2553,26 @@
s = a.build_types(f, [int])
assert s.knowntype == int
+ def test_slots_reads(self):
+ class A(object):
+ __slots__ = ()
+ class B(A):
+ def __init__(self, x):
+ self.x = x
+ def f(x):
+ if x:
+ a = A()
+ else:
+ a = B(x)
+ return a.x # should explode here
+
+ a = self.RPythonAnnotator()
+ e = py.test.raises(Exception, a.build_types, f, [int])
+ # this should explode on reading the attribute 'a.x', but it can
+ # sometimes explode on 'self.x = x', which does not make much sense.
+ # But it looks hard to fix in general: we don't know yet during 'a.x'
+ # if the attribute x will be read-only or read-write.
+
def test_unboxed_value(self):
class A(object):
__slots__ = ()
diff --git a/pypy/bin/rpython b/pypy/bin/rpython
new file mode 100644
--- /dev/null
+++ b/pypy/bin/rpython
@@ -0,0 +1,18 @@
+#!/usr/bin/env pypy
+
+"""RPython translation usage:
+
+rpython <translation options> target <targetoptions>
+
+run with --help for more information
+"""
+
+import sys
+from pypy.translator.goal.translate import main
+
+# no implicit targets
+if len(sys.argv) == 1:
+ print __doc__
+ sys.exit(1)
+
+main()
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -176,9 +176,6 @@
cmdline="--translationmodules",
suggests=[("objspace.allworkingmodules", False)]),
- BoolOption("geninterp", "specify whether geninterp should be used",
- default=False),
-
BoolOption("logbytecodes",
"keep track of bytecode usage",
default=False),
@@ -392,10 +389,6 @@
config.objspace.std.suggest(withsmalllong=True)
# xxx other options? ropes maybe?
- # completely disable geninterp in a level 0 translation
- if level == '0':
- config.objspace.suggest(geninterp=False)
-
# some optimizations have different effects depending on the typesystem
if type_system == 'ootype':
config.objspace.std.suggest(multimethods="doubledispatch")
diff --git a/pypy/config/translationoption.py b/pypy/config/translationoption.py
--- a/pypy/config/translationoption.py
+++ b/pypy/config/translationoption.py
@@ -182,11 +182,6 @@
# Flags of the TranslationContext:
BoolOption("simplifying", "Simplify flow graphs", default=True),
- BoolOption("builtins_can_raise_exceptions",
- "When true, assume any call to a 'simple' builtin such as "
- "'hex' can raise an arbitrary exception",
- default=False,
- cmdline=None),
BoolOption("list_comprehension_operations",
"When true, look for and special-case the sequence of "
"operations that results from a list comprehension and "
diff --git a/pypy/doc/discussion/win64_todo.txt b/pypy/doc/discussion/win64_todo.txt
new file mode 100644
--- /dev/null
+++ b/pypy/doc/discussion/win64_todo.txt
@@ -0,0 +1,9 @@
+2011-11-04
+ll_os.py has a problem with the file rwin32.py.
+Temporarily disabled for the win64_gborg branch. This needs to be
+investigated and re-enabled.
+Resolved, enabled.
+
+2011-11-05
+test_typed.py needs explicit tests to ensure that we
+handle word sizes right.
\ No newline at end of file
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -103,21 +103,13 @@
* A concurrent garbage collector (a lot of work)
-Remove the GIL
---------------
+STM, a.k.a. "remove the GIL"
+----------------------------
-This is a major task that requires lots of thinking. However, few subprojects
-can be potentially specified, unless a better plan can be thought out:
+Removing the GIL --- or more precisely, a GIL-less thread-less solution ---
+is `now work in progress.`__ Contributions welcome.
-* A thread-aware garbage collector
-
-* Better RPython primitives for dealing with concurrency
-
-* JIT passes to remove locks on objects
-
-* (maybe) implement locking in Python interpreter
-
-* alternatively, look at Software Transactional Memory
+.. __: http://pypy.org/tmdonate.html
Introduce new benchmarks
------------------------
diff --git a/pypy/doc/sandbox.rst b/pypy/doc/sandbox.rst
--- a/pypy/doc/sandbox.rst
+++ b/pypy/doc/sandbox.rst
@@ -82,7 +82,10 @@
In pypy/translator/goal::
- ./translate.py --sandbox targetpypystandalone.py
+ ./translate.py -O2 --sandbox targetpypystandalone.py
+
+If you don't have a regular PyPy installed, you should, because it's
+faster to translate, but you can also run ``python translate.py`` instead.
To run it, use the tools in the pypy/translator/sandbox directory::
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -18,7 +18,8 @@
Edition. Other configurations may work as well.
The translation scripts will set up the appropriate environment variables
-for the compiler. They will attempt to locate the same compiler version that
+for the compiler, so you do not need to run vcvars before translation.
+They will attempt to locate the same compiler version that
was used to build the Python interpreter doing the
translation. Failing that, they will pick the most recent Visual Studio
compiler they can find. In addition, the target architecture
@@ -26,7 +27,7 @@
using a 32 bit Python and vice versa.
**Note:** PyPy is currently not supported for 64 bit Windows, and translation
-will be aborted in this case.
+will fail in this case.
The compiler is all you need to build pypy-c, but it will miss some
modules that relies on third-party libraries. See below how to get
@@ -57,7 +58,8 @@
install third-party libraries. We chose to install them in the parent
directory of the pypy checkout. For example, if you installed pypy in
``d:\pypy\trunk\`` (This directory contains a README file), the base
-directory is ``d:\pypy``.
+directory is ``d:\pypy``. You may choose different values by setting the
+INCLUDE, LIB and PATH (for DLLs)
The Boehm garbage collector
~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -126,18 +128,54 @@
------------------------
You can compile pypy with the mingw compiler, using the --cc=mingw32 option;
-mingw.exe must be on the PATH.
+gcc.exe must be on the PATH. If the -cc flag does not begin with "ming", it should be
+the name of a valid gcc-derivative compiler, i.e. x86_64-w64-mingw32-gcc for the 64 bit
+compiler creating a 64 bit target.
-libffi for the mingw32 compiler
+libffi for the mingw compiler
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-To enable the _rawffi (and ctypes) module, you need to compile a mingw32
-version of libffi. I downloaded the `libffi source files`_, and extracted
-them in the base directory. Then run::
+To enable the _rawffi (and ctypes) module, you need to compile a mingw
+version of libffi. Here is one way to do this, wich should allow you to try
+to build for win64 or win32:
+
+#. Download and unzip a `mingw32 build`_ or `mingw64 build`_, say into c:\mingw
+#. If you do not use cygwin, you will need msys to provide make,
+ autoconf tools and other goodies.
+
+ #. Download and unzip a `msys for mingw`_, say into c:\msys
+ #. Edit the c:\msys\etc\fstab file to mount c:\mingw
+
+#. Download and unzip the `libffi source files`_, and extract
+ them in the base directory.
+#. Run c:\msys\msys.bat or a cygwin shell which should make you
+ feel better since it is a shell prompt with shell tools.
+#. From inside the shell, cd to the libffi directory and do::
sh ./configure
make
cp .libs/libffi-5.dll <somewhere on the PATH>
+If you can't find the dll, and the libtool issued a warning about
+"undefined symbols not allowed", you will need to edit the libffi
+Makefile in the toplevel directory. Add the flag -no-undefined to
+the definition of libffi_la_LDFLAGS
+
+If you wish to experiment with win64, you must run configure with flags::
+
+ sh ./configure --build=x86_64-w64-mingw32 --host=x86_64-w64-mingw32
+
+or such, depending on your mingw64 download.
+
+hacking on Pypy with the mingw compiler
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Since hacking on Pypy means running tests, you will need a way to specify
+the mingw compiler when hacking (as opposed to translating). As of
+March 2012, --cc is not a valid option for pytest.py. However if you set an
+environment variable CC it will allow you to choose a compiler.
+
+.. _'mingw32 build': http://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win32/Automated%20Builds
+.. _`mingw64 build`: http://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win64/Automated%20Builds
+.. _`msys for mingw`: http://sourceforge.net/projects/mingw-w64/files/External%20binary%20packages%20%28Win64%20hosted%29/MSYS%20%2832-bit%29
.. _`libffi source files`: http://sourceware.org/libffi/
.. _`RPython translation toolchain`: translation.html
diff --git a/pypy/doc/you-want-to-help.rst b/pypy/doc/you-want-to-help.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/you-want-to-help.rst
@@ -0,0 +1,86 @@
+
+You want to help with PyPy, now what?
+=====================================
+
+PyPy is a very large project that has a reputation of being hard to dive into.
+Some of this fame is warranted, some of it is purely accidental. There are three
+important lessons that everyone willing to contribute should learn:
+
+* PyPy has layers. There are many pieces of architecture that are very well
+ separated from each other. More about this below, but often the manifestation
+ of this is that things are at a different layer than you would expect them
+ to be. For example if you are looking for the JIT implementation, you will
+ not find it in the implementation of the Python programming language.
+
+* Because of the above, we are very serious about Test Driven Development.
+ It's not only what we believe in, but also that PyPy's architecture is
+ working very well with TDD in mind and not so well without it. Often
+ the development means progressing in an unrelated corner, one unittest
+ at a time; and then flipping a giant switch, bringing it all together.
+ (It generally works out of the box. If it doesn't, then we didn't
+ write enough unit tests.) It's worth repeating - PyPy
+ approach is great if you do TDD, not so great otherwise.
+
+* PyPy uses an entirely different set of tools - most of them included
+ in the PyPy repository. There is no Makefile, nor autoconf. More below
+
+Architecture
+============
+
+PyPy has layers. The 100 miles view:
+
+* `RPython`_ is the language in which we write interpreters. Not the entire
+ PyPy project is written in RPython, only the parts that are compiled in
+ the translation process. The interesting point is that RPython has no parser,
+ it's compiled from the live python objects, which make it possible to do
+ all kinds of metaprogramming during import time. In short, Python is a meta
+ programming language for RPython.
+
+ The RPython standard library is to be found in the ``rlib`` subdirectory.
+
+.. _`RPython`: coding-guide.html#RPython
+
+* The translation toolchain - this is the part that takes care about translating
+ RPython to flow graphs and then to C. There is more in the `architecture`_
+ document written about it.
+
+ It mostly lives in ``rpython``, ``annotator`` and ``objspace/flow``.
+
+.. _`architecture`: architecture.html
+
+* Python Interpreter
+
+ xxx
+
+* Python modules
+
+ xxx
+
+* Just-in-Time Compiler (JIT): `we have a tracing JIT`_ that traces the
+ interpreter written in RPython, rather than the user program that it
+ interprets. As a result it applies to any interpreter, i.e. any
+ language. But getting it to work correctly is not trivial: it
+ requires a small number of precise "hints" and possibly some small
+ refactorings of the interpreter. The JIT itself also has several
+ almost-independent parts: the tracer itself in ``jit/metainterp``, the
+ optimizer in ``jit/metainterp/optimizer`` that optimizes a list of
+ residual operations, and the backend in ``jit/backend/<machine-name>``
+ that turns it into machine code. Writing a new backend is a
+ traditional way to get into the project.
+
+.. _`we have a tracing JIT`: jit/index.html
+
+* Garbage Collectors (GC): as you can notice if you are used to CPython's
+ C code, there are no ``Py_INCREF/Py_DECREF`` equivalents in RPython code.
+ `Garbage collection in PyPy`_ is inserted
+ during translation. Moreover, this is not reference counting; it is a real
+ GC written as more RPython code. The best one we have so far is in
+ ``rpython/memory/gc/minimark.py``.
+
+.. _`Garbage collection in PyPy`: garbage_collection.html
+
+
+Toolset
+=======
+
+xxx
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -610,6 +610,8 @@
ops.JUMP_IF_FALSE_OR_POP : 0,
ops.POP_JUMP_IF_TRUE : -1,
ops.POP_JUMP_IF_FALSE : -1,
+
+ ops.BUILD_LIST_FROM_ARG: 1,
}
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -965,7 +965,7 @@
self.emit_op_arg(ops.CALL_METHOD, (kwarg_count << 8) | arg_count)
return True
- def _listcomp_generator(self, gens, gen_index, elt):
+ def _listcomp_generator(self, gens, gen_index, elt, single=False):
start = self.new_block()
skip = self.new_block()
if_cleanup = self.new_block()
@@ -973,6 +973,8 @@
gen = gens[gen_index]
assert isinstance(gen, ast.comprehension)
gen.iter.walkabout(self)
+ if single:
+ self.emit_op_arg(ops.BUILD_LIST_FROM_ARG, 0)
self.emit_op(ops.GET_ITER)
self.use_next_block(start)
self.emit_jump(ops.FOR_ITER, anchor)
@@ -998,8 +1000,12 @@
def visit_ListComp(self, lc):
self.update_position(lc.lineno)
- self.emit_op_arg(ops.BUILD_LIST, 0)
- self._listcomp_generator(lc.generators, 0, lc.elt)
+ if len(lc.generators) != 1 or lc.generators[0].ifs:
+ single = False
+ self.emit_op_arg(ops.BUILD_LIST, 0)
+ else:
+ single = True
+ self._listcomp_generator(lc.generators, 0, lc.elt, single=single)
def _comp_generator(self, node, generators, gen_index):
start = self.new_block()
diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -10,16 +10,6 @@
from pypy.interpreter.astcompiler import ast, consts
-try:
- all
-except NameError:
- def all(iterable):
- for x in iterable:
- if not x:
- return False
- return True
-
-
class TestAstBuilder:
def setup_class(cls):
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -58,7 +58,8 @@
w_res = pyco_expr.exec_host_bytecode(w_dict, w_dict)
res = space.str_w(space.repr(w_res))
if not isinstance(expected, float):
- assert res == repr(expected)
+ noL = lambda expr: expr.replace('L', '')
+ assert noL(res) == noL(repr(expected))
else:
# Float representation can vary a bit between interpreter
# versions, compare the numbers instead.
@@ -908,3 +909,17 @@
return d['f'](5)
""")
assert 'generator' in space.str_w(space.repr(w_generator))
+
+ def test_list_comprehension(self):
+ source = "def f(): [i for i in l]"
+ source2 = "def f(): [i for i in l for j in l]"
+ source3 = "def f(): [i for i in l if i]"
+ counts = self.count_instructions(source)
+ assert ops.BUILD_LIST not in counts
+ assert counts[ops.BUILD_LIST_FROM_ARG] == 1
+ counts = self.count_instructions(source2)
+ assert counts[ops.BUILD_LIST] == 1
+ assert ops.BUILD_LIST_FROM_ARG not in counts
+ counts = self.count_instructions(source3)
+ assert counts[ops.BUILD_LIST] == 1
+ assert ops.BUILD_LIST_FROM_ARG not in counts
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -7,7 +7,8 @@
from pypy.interpreter.miscutils import ThreadLocals
from pypy.tool.cache import Cache
from pypy.tool.uid import HUGEVAL_BYTES
-from pypy.rlib.objectmodel import we_are_translated, newlist, compute_unique_id
+from pypy.rlib.objectmodel import we_are_translated, newlist_hint,\
+ compute_unique_id
from pypy.rlib.debug import make_sure_not_resized
from pypy.rlib.timer import DummyTimer, Timer
from pypy.rlib.rarithmetic import r_uint
@@ -833,7 +834,7 @@
items = []
else:
try:
- items = newlist(lgt_estimate)
+ items = newlist_hint(lgt_estimate)
except MemoryError:
items = [] # it might have lied
#
@@ -1335,7 +1336,7 @@
if not self.is_true(self.isinstance(w_obj, self.w_str)):
raise OperationError(self.w_TypeError,
self.wrap('argument must be a string'))
- return self.str_w(w_obj)
+ return self.str_w(w_obj)
def unicode_w(self, w_obj):
return w_obj.unicode_w(self)
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -47,6 +47,11 @@
def async(self, space):
"Check if this is an exception that should better not be caught."
+ if not space.full_exceptions:
+ # flow objspace does not support such exceptions and more
+ # importantly, raises KeyboardInterrupt if you try to access
+ # space.w_KeyboardInterrupt
+ return False
return (self.match(space, space.w_SystemExit) or
self.match(space, space.w_KeyboardInterrupt))
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -901,24 +901,20 @@
def __init__(self, source, filename=None, modname='__builtin__'):
# HAAACK (but a good one)
+ self.filename = filename
+ self.source = str(py.code.Source(source).deindent())
+ self.modname = modname
if filename is None:
f = sys._getframe(1)
filename = '<%s:%d>' % (f.f_code.co_filename, f.f_lineno)
+ if not os.path.exists(filename):
+ # make source code available for tracebacks
+ lines = [x + "\n" for x in source.split("\n")]
+ py.std.linecache.cache[filename] = (1, None, lines, filename)
self.filename = filename
- self.source = str(py.code.Source(source).deindent())
- self.modname = modname
- # look at the first three lines for a NOT_RPYTHON tag
- first = "\n".join(source.split("\n", 3)[:3])
- if "NOT_RPYTHON" in first:
- self.can_use_geninterp = False
- else:
- self.can_use_geninterp = True
- # make source code available for tracebacks
- lines = [x + "\n" for x in source.split("\n")]
- py.std.linecache.cache[filename] = (1, None, lines, filename)
def __repr__(self):
- return "<ApplevelClass filename=%r can_use_geninterp=%r>" % (self.filename, self.can_use_geninterp)
+ return "<ApplevelClass filename=%r>" % (self.filename,)
def getwdict(self, space):
return space.fromcache(ApplevelCache).getorbuild(self)
@@ -979,10 +975,7 @@
def build(self, app):
"NOT_RPYTHON. Called indirectly by Applevel.getwdict()."
- if self.space.config.objspace.geninterp and app.can_use_geninterp:
- return PyPyCacheDir.build_applevelinterp_dict(app, self.space)
- else:
- return build_applevel_dict(app, self.space)
+ return build_applevel_dict(app, self.space)
# __________ pure applevel version __________
@@ -996,157 +989,6 @@
filename=self.filename)
return w_glob
-# __________ geninterplevel version __________
-
-class PyPyCacheDir:
- "NOT_RPYTHON"
- # similar to applevel, but using translation to interp-level.
- # This version maintains a cache folder with single files.
-
- def build_applevelinterp_dict(cls, self, space):
- "NOT_RPYTHON"
- # N.B. 'self' is the ApplevelInterp; this is a class method,
- # just so that we have a convenient place to store the global state.
- if not cls._setup_done:
- cls._setup()
-
- from pypy.translator.geninterplevel import translate_as_module
- import marshal
- scramble = md5(cls.seed)
- scramble.update(marshal.dumps(self.source))
- key = scramble.hexdigest()
- initfunc = cls.known_code.get(key)
- if not initfunc:
- # try to get it from file
- name = key
- if self.filename:
- prename = os.path.splitext(os.path.basename(self.filename))[0]
- else:
- prename = 'zznoname'
- name = "%s_%s" % (prename, name)
- try:
- __import__("pypy._cache."+name)
- except ImportError, x:
- # print x
- pass
- else:
- initfunc = cls.known_code[key]
- if not initfunc:
- # build it and put it into a file
- initfunc, newsrc = translate_as_module(
- self.source, self.filename, self.modname)
- fname = cls.cache_path.join(name+".py").strpath
- f = file(get_tmp_file_name(fname), "w")
- print >> f, """\
-# self-destruct on double-click:
-if __name__ == "__main__":
- from pypy import _cache
- import os
- namestart = os.path.join(os.path.split(_cache.__file__)[0], '%s')
- for ending in ('.py', '.pyc', '.pyo'):
- try:
- os.unlink(namestart+ending)
- except os.error:
- pass""" % name
- print >> f
- print >> f, newsrc
- print >> f, "from pypy._cache import known_code"
- print >> f, "known_code[%r] = %s" % (key, initfunc.__name__)
- f.close()
- rename_tmp_to_eventual_file_name(fname)
- w_glob = initfunc(space)
- return w_glob
- build_applevelinterp_dict = classmethod(build_applevelinterp_dict)
-
- _setup_done = False
-
- def _setup(cls):
- """NOT_RPYTHON"""
- lp = py.path.local
- import pypy, os
- p = lp(pypy.__file__).new(basename='_cache').ensure(dir=1)
- cls.cache_path = p
- ini = p.join('__init__.py')
- try:
- if not ini.check():
- raise ImportError # don't import if only a .pyc file left!!!
- from pypy._cache import known_code, \
- GI_VERSION_RENDERED
- except ImportError:
- GI_VERSION_RENDERED = 0
- from pypy.translator.geninterplevel import GI_VERSION
- cls.seed = md5(str(GI_VERSION)).digest()
- if GI_VERSION != GI_VERSION_RENDERED or GI_VERSION is None:
- for pth in p.listdir():
- if pth.check(file=1):
- try:
- pth.remove()
- except: pass
- f = file(get_tmp_file_name(str(ini)), "w")
- f.write("""\
-# This folder acts as a cache for code snippets which have been
-# compiled by compile_as_module().
-# It will get a new entry for every piece of code that has
-# not been seen, yet.
-#
-# Caution! Only the code snippet is checked. If something
-# is imported, changes are not detected. Also, changes
-# to geninterplevel or gateway are also not checked.
-# Exception: There is a checked version number in geninterplevel.py
-#
-# If in doubt, remove this file from time to time.
-
-GI_VERSION_RENDERED = %r
-
-known_code = {}
-
-# self-destruct on double-click:
-def harakiri():
- import pypy._cache as _c
- import py
- lp = py.path.local
- for pth in lp(_c.__file__).dirpath().listdir():
- try:
- pth.remove()
- except: pass
-
-if __name__ == "__main__":
- harakiri()
-
-del harakiri
-""" % GI_VERSION)
- f.close()
- rename_tmp_to_eventual_file_name(str(ini))
- import pypy._cache
- cls.known_code = pypy._cache.known_code
- cls._setup_done = True
- _setup = classmethod(_setup)
-
-
-def gethostname(_cache=[]):
- if not _cache:
- try:
- import socket
- hostname = socket.gethostname()
- except:
- hostname = ''
- _cache.append(hostname)
- return _cache[0]
-
-def get_tmp_file_name(fname):
- return '%s~%s~%d' % (fname, gethostname(), os.getpid())
-
-def rename_tmp_to_eventual_file_name(fname):
- # generated files are first written to the host- and process-specific
- # file 'tmpname', and then atomically moved to their final 'fname'
- # to avoid problems if py.py is started several times in parallel
- tmpname = get_tmp_file_name(fname)
- try:
- os.rename(tmpname, fname)
- except (OSError, IOError):
- os.unlink(fname) # necessary on Windows
- os.rename(tmpname, fname)
-
# ____________________________________________________________
def appdef(source, applevel=ApplevelClass, filename=None):
@@ -1184,11 +1026,6 @@
return build_applevel_dict(self, space)
-class applevelinterp_temp(ApplevelClass):
- hidden_applevel = False
- def getwdict(self, space): # no cache
- return PyPyCacheDir.build_applevelinterp_dict(self, space)
-
# app2interp_temp is used for testing mainly
def app2interp_temp(func, applevel_temp=applevel_temp, filename=None):
""" NOT_RPYTHON """
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -15,9 +15,8 @@
from pypy.rlib.rarithmetic import r_uint, intmask
from pypy.rlib.unroll import unrolling_iterable
from pypy.rlib.debug import check_nonneg
-from pypy.tool.stdlib_opcode import (bytecode_spec, host_bytecode_spec,
- unrolling_all_opcode_descs, opmap,
- host_opmap)
+from pypy.tool.stdlib_opcode import (bytecode_spec,
+ unrolling_all_opcode_descs)
def unaryoperation(operationname):
"""NOT_RPYTHON"""
@@ -713,6 +712,19 @@
w_list = self.space.newlist(items)
self.pushvalue(w_list)
+ def BUILD_LIST_FROM_ARG(self, _, next_instr):
+ # this is a little dance, because list has to be before the
+ # value
+ last_val = self.popvalue()
+ try:
+ lgt = self.space.len_w(last_val)
+ except OperationError, e:
+ if e.async(self.space):
+ raise
+ lgt = 0 # oh well
+ self.pushvalue(self.space.newlist([], sizehint=lgt))
+ self.pushvalue(last_val)
+
def LOAD_ATTR(self, nameindex, next_instr):
"obj.attributename"
w_obj = self.popvalue()
@@ -1419,11 +1431,9 @@
if lastchar.isspace() and lastchar != ' ':
return
file_softspace(stream, True)
- print_item_to._annspecialcase_ = "specialize:argtype(0)"
def print_item(x):
print_item_to(x, sys_stdout())
- print_item._annspecialcase_ = "flowspace:print_item"
def print_newline_to(stream):
stream.write("\n")
@@ -1431,7 +1441,6 @@
def print_newline():
print_newline_to(sys_stdout())
- print_newline._annspecialcase_ = "flowspace:print_newline"
def file_softspace(file, newflag):
try:
diff --git a/pypy/interpreter/test/test_appinterp.py b/pypy/interpreter/test/test_appinterp.py
--- a/pypy/interpreter/test/test_appinterp.py
+++ b/pypy/interpreter/test/test_appinterp.py
@@ -1,6 +1,6 @@
import py
-from pypy.interpreter.gateway import appdef, ApplevelClass, applevel_temp, applevelinterp_temp
+from pypy.interpreter.gateway import appdef, ApplevelClass, applevel_temp
from pypy.interpreter.error import OperationError
def test_execwith_novars(space):
@@ -82,9 +82,6 @@
w_res = g(space, space.wrap(10), space.wrap(1))
assert space.eq_w(w_res, space.wrap(-9))
-def test_applevelinterp_functions(space):
- test_applevel_functions(space, applevel_temp = applevelinterp_temp)
-
def test_applevel_class(space, applevel_temp = applevel_temp):
app = applevel_temp('''
class C(object):
@@ -99,9 +96,6 @@
w_clsattr = space.getattr(c, space.wrap('attr'))
assert space.eq_w(w_clsattr, space.wrap(17))
-def test_applevelinterp_class(space):
- test_applevel_class(space, applevel_temp = applevelinterp_temp)
-
def app_test_something_at_app_level():
x = 2
assert x/2 == 1
@@ -161,7 +155,7 @@
w_str = space1.getattr(w_mymod1, space1.wrap("hi"))
assert space1.str_w(w_str) == "hello"
- def test_geninterp_can_unfreeze(self):
+ def test_random_stuff_can_unfreeze(self):
# When a module contains an "import" statement in applevel code, the
# imported module is initialized, possibly after it has been already
# frozen.
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -101,14 +101,6 @@
g3 = gateway.app2interp_temp(noapp_g3, gateway.applevel_temp)
assert self.space.eq_w(g3(self.space, w('foo'), w('bar')), w('foobar'))
- def test_app2interp2(self):
- """same but using transformed code"""
- w = self.space.wrap
- def noapp_g3(a, b):
- return a+b
- g3 = gateway.app2interp_temp(noapp_g3, gateway.applevelinterp_temp)
- assert self.space.eq_w(g3(self.space, w('foo'), w('bar')), w('foobar'))
-
def test_app2interp_general_args(self):
w = self.space.wrap
def app_general(x, *args, **kwds):
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -312,8 +312,8 @@
mods = space.get_builtinmodule_to_install()
assert '__pypy__' in mods # real builtin
- assert 'array' not in mods # in lib_pypy
- assert 'faked+array' not in mods # in lib_pypy
+ assert '_functools' not in mods # in lib_pypy
+ assert 'faked+_functools' not in mods # in lib_pypy
assert 'this_doesnt_exist' not in mods # not in lib_pypy
assert 'faked+this_doesnt_exist' in mods # not in lib_pypy, but in
# ALL_BUILTIN_MODULES
diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py
--- a/pypy/interpreter/test/test_zzpickle_and_slow.py
+++ b/pypy/interpreter/test/test_zzpickle_and_slow.py
@@ -75,6 +75,7 @@
class AppTestInterpObjectPickling:
pytestmark = py.test.mark.skipif("config.option.runappdirect")
def setup_class(cls):
+ cls.space = gettestobjspace(usemodules=['struct'])
_attach_helpers(cls.space)
def teardown_class(cls):
diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -599,7 +599,7 @@
# if convenient for the backend, we compute the info about
# the flag as (byte-offset, single-byte-flag).
import struct
- value = struct.pack("l", flag_word)
+ value = struct.pack(lltype.SignedFmt, flag_word)
assert value.count('\x00') == len(value) - 1 # only one byte is != 0
i = 0
while value[i] == '\x00': i += 1
diff --git a/pypy/jit/backend/llsupport/regalloc.py b/pypy/jit/backend/llsupport/regalloc.py
--- a/pypy/jit/backend/llsupport/regalloc.py
+++ b/pypy/jit/backend/llsupport/regalloc.py
@@ -321,7 +321,7 @@
except KeyError:
pass # 'var' is already not in a register
- def loc(self, box):
+ def loc(self, box, must_exist=False):
""" Return the location of 'box'.
"""
self._check_type(box)
@@ -332,6 +332,8 @@
except KeyError:
if box in self.bindings_to_frame_reg:
return self.frame_reg
+ if must_exist:
+ return self.frame_manager.bindings[box]
return self.frame_manager.loc(box)
def return_constant(self, v, forbidden_vars=[], selected_reg=None):
@@ -360,7 +362,7 @@
self._check_type(v)
if isinstance(v, Const):
return self.return_constant(v, forbidden_vars, selected_reg)
- prev_loc = self.loc(v)
+ prev_loc = self.loc(v, must_exist=True)
if prev_loc is self.frame_reg and selected_reg is None:
return prev_loc
loc = self.force_allocate_reg(v, forbidden_vars, selected_reg,
diff --git a/pypy/jit/backend/llsupport/test/test_descr.py b/pypy/jit/backend/llsupport/test/test_descr.py
--- a/pypy/jit/backend/llsupport/test/test_descr.py
+++ b/pypy/jit/backend/llsupport/test/test_descr.py
@@ -148,7 +148,7 @@
#
def get_alignment(code):
# Retrieve default alignment for the compiler/platform
- return struct.calcsize('l' + code) - struct.calcsize(code)
+ return struct.calcsize(lltype.SignedFmt + code) - struct.calcsize(code)
assert descr1.basesize == get_alignment('c')
assert descr2.basesize == get_alignment('p')
assert descr3.basesize == get_alignment('p')
diff --git a/pypy/jit/backend/llsupport/test/test_ffisupport.py b/pypy/jit/backend/llsupport/test/test_ffisupport.py
--- a/pypy/jit/backend/llsupport/test/test_ffisupport.py
+++ b/pypy/jit/backend/llsupport/test/test_ffisupport.py
@@ -2,6 +2,7 @@
from pypy.jit.codewriter.longlong import is_64_bit
from pypy.jit.backend.llsupport.descr import *
from pypy.jit.backend.llsupport.ffisupport import *
+from pypy.rlib.rarithmetic import is_emulated_long
class FakeCPU:
@@ -43,7 +44,7 @@
assert descr.result_flag == FLAG_UNSIGNED
assert descr.is_result_signed() == False
- if not is_64_bit:
+ if not is_64_bit or is_emulated_long:
descr = get_call_descr_dynamic(FakeCPU(), [], types.slonglong,
None, 42)
assert descr is None # missing longlongs
diff --git a/pypy/jit/backend/llsupport/test/test_gc.py b/pypy/jit/backend/llsupport/test/test_gc.py
--- a/pypy/jit/backend/llsupport/test/test_gc.py
+++ b/pypy/jit/backend/llsupport/test/test_gc.py
@@ -11,6 +11,7 @@
from pypy.jit.tool.oparser import parse
from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE
from pypy.jit.metainterp.optimizeopt.util import equaloplists
+from pypy.rlib.rarithmetic import is_valid_int
def test_boehm():
gc_ll_descr = GcLLDescr_boehm(None, None, None)
@@ -103,7 +104,7 @@
gcrootmap.put(retaddr, shapeaddr)
assert gcrootmap._gcmap[0] == retaddr
assert gcrootmap._gcmap[1] == shapeaddr
- p = rffi.cast(rffi.LONGP, gcrootmap.gcmapstart())
+ p = rffi.cast(rffi.SIGNEDP, gcrootmap.gcmapstart())
assert p[0] == retaddr
assert (gcrootmap.gcmapend() ==
gcrootmap.gcmapstart() + rffi.sizeof(lltype.Signed) * 2)
@@ -419,9 +420,9 @@
assert newops[0].getarg(1) == v_value
assert newops[0].result is None
wbdescr = newops[0].getdescr()
- assert isinstance(wbdescr.jit_wb_if_flag, int)
- assert isinstance(wbdescr.jit_wb_if_flag_byteofs, int)
- assert isinstance(wbdescr.jit_wb_if_flag_singlebyte, int)
+ assert is_valid_int(wbdescr.jit_wb_if_flag)
+ assert is_valid_int(wbdescr.jit_wb_if_flag_byteofs)
+ assert is_valid_int(wbdescr.jit_wb_if_flag_singlebyte)
def test_get_rid_of_debug_merge_point(self):
operations = [
diff --git a/pypy/jit/backend/llsupport/test/test_regalloc.py b/pypy/jit/backend/llsupport/test/test_regalloc.py
--- a/pypy/jit/backend/llsupport/test/test_regalloc.py
+++ b/pypy/jit/backend/llsupport/test/test_regalloc.py
@@ -1,4 +1,4 @@
-
+import py
from pypy.jit.metainterp.history import BoxInt, ConstInt, BoxFloat, INT, FLOAT
from pypy.jit.backend.llsupport.regalloc import FrameManager
from pypy.jit.backend.llsupport.regalloc import RegisterManager as BaseRegMan
@@ -236,6 +236,16 @@
assert isinstance(loc, FakeFramePos)
assert len(asm.moves) == 1
+ def test_bogus_make_sure_var_in_reg(self):
+ b0, = newboxes(0)
+ longevity = {b0: (0, 1)}
+ fm = TFrameManager()
+ asm = MockAsm()
+ rm = RegisterManager(longevity, frame_manager=fm, assembler=asm)
+ rm.next_instruction()
+ # invalid call to make_sure_var_in_reg(): box unknown so far
+ py.test.raises(KeyError, rm.make_sure_var_in_reg, b0)
+
def test_return_constant(self):
asm = MockAsm()
boxes, longevity = boxes_and_longevity(5)
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -16,9 +16,11 @@
from pypy.rpython.annlowlevel import llhelper
from pypy.rpython.llinterp import LLException
from pypy.jit.codewriter import heaptracker, longlong
-from pypy.rlib.rarithmetic import intmask
+from pypy.rlib import longlong2float
+from pypy.rlib.rarithmetic import intmask, is_valid_int
from pypy.jit.backend.detect_cpu import autodetect_main_model_and_size
+
def boxfloat(x):
return BoxFloat(longlong.getfloatstorage(x))
@@ -493,7 +495,7 @@
if cpu.supports_floats:
def func(f, i):
assert isinstance(f, float)
- assert isinstance(i, int)
+ assert is_valid_int(i)
return f - float(i)
FPTR = self.Ptr(self.FuncType([lltype.Float, lltype.Signed],
lltype.Float))
@@ -1496,13 +1498,30 @@
c_nest, c_nest], 'void')
def test_read_timestamp(self):
+ if sys.platform == 'win32':
+ # windows quite often is very inexact (like the old Intel 8259 PIC),
+ # so we stretch the time a little bit.
+ # On my virtual Parallels machine in a 2GHz Core i7 Mac Mini,
+ # the test starts working at delay == 21670 and stops at 20600000.
+ # We take the geometric mean value.
+ from math import log, exp
+ delay_min = 21670
+ delay_max = 20600000
+ delay = int(exp((log(delay_min)+log(delay_max))/2))
+ def wait_a_bit():
+ for i in xrange(delay): pass
+ else:
+ def wait_a_bit():
+ pass
if longlong.is_64_bit:
got1 = self.execute_operation(rop.READ_TIMESTAMP, [], 'int')
+ wait_a_bit()
got2 = self.execute_operation(rop.READ_TIMESTAMP, [], 'int')
res1 = got1.getint()
res2 = got2.getint()
else:
got1 = self.execute_operation(rop.READ_TIMESTAMP, [], 'float')
+ wait_a_bit()
got2 = self.execute_operation(rop.READ_TIMESTAMP, [], 'float')
res1 = got1.getlonglong()
res2 = got2.getlonglong()
@@ -1598,6 +1617,12 @@
[BoxPtr(x)], 'int').value
assert res == -19
+ def test_convert_float_bytes(self):
+ t = 'int' if longlong.is_64_bit else 'float'
+ res = self.execute_operation(rop.CONVERT_FLOAT_BYTES_TO_LONGLONG,
+ [boxfloat(2.5)], t).value
+ assert res == longlong2float.float2longlong(2.5)
+
def test_ooops_non_gc(self):
x = lltype.malloc(lltype.Struct('x'), flavor='raw')
v = heaptracker.adr2int(llmemory.cast_ptr_to_adr(x))
diff --git a/pypy/jit/backend/test/support.py b/pypy/jit/backend/test/support.py
--- a/pypy/jit/backend/test/support.py
+++ b/pypy/jit/backend/test/support.py
@@ -3,6 +3,7 @@
from pypy.rlib.debug import debug_print
from pypy.translator.translator import TranslationContext, graphof
from pypy.jit.metainterp.optimizeopt import ALL_OPTS_NAMES
+from pypy.rlib.rarithmetic import is_valid_int
class BaseCompiledMixin(object):
@@ -24,7 +25,7 @@
from pypy.annotation import model as annmodel
for arg in args:
- assert isinstance(arg, int)
+ assert is_valid_int(arg)
self.pre_translation_hook()
t = self._get_TranslationContext()
diff --git a/pypy/jit/backend/test/test_random.py b/pypy/jit/backend/test/test_random.py
--- a/pypy/jit/backend/test/test_random.py
+++ b/pypy/jit/backend/test/test_random.py
@@ -449,6 +449,7 @@
OPERATIONS.append(CastFloatToIntOperation(rop.CAST_FLOAT_TO_INT))
OPERATIONS.append(CastIntToFloatOperation(rop.CAST_INT_TO_FLOAT))
+OPERATIONS.append(CastFloatToIntOperation(rop.CONVERT_FLOAT_BYTES_TO_LONGLONG))
OperationBuilder.OPERATIONS = OPERATIONS
@@ -502,11 +503,11 @@
else:
assert 0, "unknown backend %r" % pytest.config.option.backend
-# ____________________________________________________________
+# ____________________________________________________________
class RandomLoop(object):
dont_generate_more = False
-
+
def __init__(self, cpu, builder_factory, r, startvars=None):
self.cpu = cpu
if startvars is None:
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -606,7 +606,7 @@
else:
assert token
struct.number = compute_unique_id(token)
- self.loop_run_counters.append(struct)
+ self.loop_run_counters.append(struct)
return struct
def _find_failure_recovery_bytecode(self, faildescr):
@@ -665,7 +665,7 @@
ResOperation(rop.SETFIELD_RAW, [c_adr, box2],
None, descr=self.debug_counter_descr)]
operations.extend(ops)
-
+
@specialize.argtype(1)
def _inject_debugging_code(self, looptoken, operations, tp, number):
if self._debug:
@@ -836,8 +836,8 @@
self.mc.MOVSD_sx(0, loc.value)
elif WORD == 4 and isinstance(loc, StackLoc) and loc.get_width() == 8:
# XXX evil trick
- self.mc.PUSH_b(get_ebp_ofs(loc.position))
- self.mc.PUSH_b(get_ebp_ofs(loc.position + 1))
+ self.mc.PUSH_b(loc.value + 4)
+ self.mc.PUSH_b(loc.value)
else:
self.mc.PUSH(loc)
@@ -847,8 +847,8 @@
self.mc.ADD_ri(esp.value, 8) # = size of doubles
elif WORD == 4 and isinstance(loc, StackLoc) and loc.get_width() == 8:
# XXX evil trick
- self.mc.POP_b(get_ebp_ofs(loc.position + 1))
- self.mc.POP_b(get_ebp_ofs(loc.position))
+ self.mc.POP_b(loc.value)
+ self.mc.POP_b(loc.value + 4)
else:
self.mc.POP(loc)
@@ -1242,6 +1242,15 @@
self.mc.MOVD_xr(resloc.value, loc0.value)
self.mc.CVTSS2SD_xx(resloc.value, resloc.value)
+ def genop_convert_float_bytes_to_longlong(self, op, arglocs, resloc):
+ loc0, = arglocs
+ if longlong.is_64_bit:
+ assert isinstance(resloc, RegLoc)
+ assert isinstance(loc0, RegLoc)
+ self.mc.MOVD(resloc, loc0)
+ else:
+ self.mov(loc0, resloc)
+
def genop_guard_int_is_true(self, op, guard_op, guard_token, arglocs, resloc):
guard_opnum = guard_op.getopnum()
self.mc.CMP(arglocs[0], imm0)
@@ -1954,8 +1963,6 @@
mc.PUSH_r(ebx.value)
elif IS_X86_64:
mc.MOV_rr(edi.value, ebx.value)
- # XXX: Correct to only align the stack on 64-bit?
- mc.AND_ri(esp.value, -16)
else:
raise AssertionError("Shouldn't happen")
@@ -2117,9 +2124,12 @@
# First, we need to save away the registers listed in
# 'save_registers' that are not callee-save. XXX We assume that
# the XMM registers won't be modified. We store them in
- # [ESP+4], [ESP+8], etc., leaving enough room in [ESP] for the
- # single argument to closestack_addr below.
- p = WORD
+ # [ESP+4], [ESP+8], etc.; on x86-32 we leave enough room in [ESP]
+ # for the single argument to closestack_addr below.
+ if IS_X86_32:
+ p = WORD
+ elif IS_X86_64:
+ p = 0
for reg in self._regalloc.rm.save_around_call_regs:
if reg in save_registers:
self.mc.MOV_sr(p, reg.value)
@@ -2174,7 +2184,10 @@
#
self._emit_call(-1, imm(self.releasegil_addr), args)
# Finally, restore the registers saved above.
- p = WORD
+ if IS_X86_32:
+ p = WORD
+ elif IS_X86_64:
+ p = 0
for reg in self._regalloc.rm.save_around_call_regs:
if reg in save_registers:
self.mc.MOV_rs(reg.value, p)
diff --git a/pypy/jit/backend/x86/codebuf.py b/pypy/jit/backend/x86/codebuf.py
--- a/pypy/jit/backend/x86/codebuf.py
+++ b/pypy/jit/backend/x86/codebuf.py
@@ -19,8 +19,8 @@
class MachineCodeBlockWrapper(BlockBuilderMixin,
- codebuilder_cls,
- LocationCodeBuilder):
+ LocationCodeBuilder,
+ codebuilder_cls):
def __init__(self):
self.init_block_builder()
# a list of relative positions; for each position p, the bytes
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -766,6 +766,19 @@
consider_cast_singlefloat_to_float = consider_cast_int_to_float
+ def consider_convert_float_bytes_to_longlong(self, op):
+ if longlong.is_64_bit:
+ loc0 = self.xrm.make_sure_var_in_reg(op.getarg(0))
+ loc1 = self.rm.force_allocate_reg(op.result)
+ self.Perform(op, [loc0], loc1)
+ self.xrm.possibly_free_var(op.getarg(0))
+ else:
+ arg0 = op.getarg(0)
+ loc0 = self.xrm.loc(arg0)
+ loc1 = self.xrm.force_allocate_reg(op.result, forbidden_vars=[arg0])
+ self.Perform(op, [loc0], loc1)
+ self.xrm.possibly_free_var(op.getarg(0))
+
def _consider_llong_binop_xx(self, op):
# must force both arguments into xmm registers, because we don't
# know if they will be suitably aligned. Exception: if the second
diff --git a/pypy/jit/backend/x86/rx86.py b/pypy/jit/backend/x86/rx86.py
--- a/pypy/jit/backend/x86/rx86.py
+++ b/pypy/jit/backend/x86/rx86.py
@@ -601,9 +601,12 @@
CVTSS2SD_xb = xmminsn('\xF3', rex_nw, '\x0F\x5A',
register(1, 8), stack_bp(2))
- MOVD_rx = xmminsn('\x66', rex_nw, '\x0F\x7E', register(2, 8), register(1), '\xC0')
- MOVD_xr = xmminsn('\x66', rex_nw, '\x0F\x6E', register(1, 8), register(2), '\xC0')
- MOVD_xb = xmminsn('\x66', rex_nw, '\x0F\x6E', register(1, 8), stack_bp(2))
+ # These work on machine sized registers, so MOVD is actually MOVQ
+ # when running on 64 bits. Note a bug in the Intel documentation:
+ # http://lists.gnu.org/archive/html/bug-binutils/2007-07/msg00095.html
+ MOVD_rx = xmminsn('\x66', rex_w, '\x0F\x7E', register(2, 8), register(1), '\xC0')
+ MOVD_xr = xmminsn('\x66', rex_w, '\x0F\x6E', register(1, 8), register(2), '\xC0')
+ MOVD_xb = xmminsn('\x66', rex_w, '\x0F\x6E', register(1, 8), stack_bp(2))
PSRAD_xi = xmminsn('\x66', rex_nw, '\x0F\x72', register(1), '\xE0', immediate(2, 'b'))
diff --git a/pypy/jit/backend/x86/support.py b/pypy/jit/backend/x86/support.py
--- a/pypy/jit/backend/x86/support.py
+++ b/pypy/jit/backend/x86/support.py
@@ -36,15 +36,15 @@
# ____________________________________________________________
-if sys.platform == 'win32':
- ensure_sse2_floats = lambda : None
- # XXX check for SSE2 on win32 too
+if WORD == 4:
+ extra = ['-DPYPY_X86_CHECK_SSE2']
else:
- if WORD == 4:
- extra = ['-DPYPY_X86_CHECK_SSE2']
- else:
- extra = []
- ensure_sse2_floats = rffi.llexternal_use_eci(ExternalCompilationInfo(
- compile_extra = ['-msse2', '-mfpmath=sse',
- '-DPYPY_CPU_HAS_STANDARD_PRECISION'] + extra,
- ))
+ extra = []
+
+if sys.platform != 'win32':
+ extra = ['-msse2', '-mfpmath=sse',
+ '-DPYPY_CPU_HAS_STANDARD_PRECISION'] + extra
+
+ensure_sse2_floats = rffi.llexternal_use_eci(ExternalCompilationInfo(
+ compile_extra = extra,
+))
diff --git a/pypy/jit/backend/x86/test/conftest.py b/pypy/jit/backend/x86/test/conftest.py
--- a/pypy/jit/backend/x86/test/conftest.py
+++ b/pypy/jit/backend/x86/test/conftest.py
@@ -1,4 +1,4 @@
-import py
+import py, os
from pypy.jit.backend import detect_cpu
cpu = detect_cpu.autodetect()
@@ -6,5 +6,7 @@
if cpu not in ('x86', 'x86_64'):
py.test.skip("x86/x86_64 tests skipped: cpu is %r" % (cpu,))
if cpu == 'x86_64':
+ if os.name == "nt":
+ py.test.skip("Windows cannot allocate non-reserved memory")
from pypy.rpython.lltypesystem import ll2ctypes
ll2ctypes.do_allocation_in_far_regions()
diff --git a/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py b/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py
--- a/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py
+++ b/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py
@@ -182,6 +182,12 @@
filename = str(testdir.join(FILENAME % methname))
g = open(inputname, 'w')
g.write('\x09.string "%s"\n' % BEGIN_TAG)
+ #
+ if instrname == 'MOVD' and self.WORD == 8:
+ instrname = 'MOVQ'
+ if argmodes == 'xb':
+ py.test.skip('"as" uses an undocumented alternate encoding??')
+ #
for args in args_lists:
suffix = ""
## all = instr.as_all_suffixes
@@ -229,9 +235,6 @@
# movq $xxx, %rax => movl $xxx, %eax
suffix = 'l'
ops[1] = reduce_to_32bit(ops[1])
- if instrname.lower() == 'movd':
- ops[0] = reduce_to_32bit(ops[0])
- ops[1] = reduce_to_32bit(ops[1])
#
op = '\t%s%s %s%s' % (instrname.lower(), suffix,
', '.join(ops), following)
diff --git a/pypy/jit/backend/x86/test/test_zmath.py b/pypy/jit/backend/x86/test/test_zmath.py
--- a/pypy/jit/backend/x86/test/test_zmath.py
+++ b/pypy/jit/backend/x86/test/test_zmath.py
@@ -6,6 +6,8 @@
from pypy.translator.c.test.test_genc import compile
from pypy.jit.backend.x86.support import ensure_sse2_floats
from pypy.rlib import rfloat
+from pypy.rlib.unroll import unrolling_iterable
+from pypy.rlib.debug import debug_print
def get_test_case((fnname, args, expected)):
@@ -16,16 +18,32 @@
expect_valueerror = (expected == ValueError)
expect_overflowerror = (expected == OverflowError)
check = test_direct.get_tester(expected)
+ unroll_args = unrolling_iterable(args)
#
def testfn():
+ debug_print('calling', fnname, 'with arguments:')
+ for arg in unroll_args:
+ debug_print('\t', arg)
try:
got = fn(*args)
except ValueError:
- return expect_valueerror
+ if expect_valueerror:
+ return True
+ else:
+ debug_print('unexpected ValueError!')
+ return False
except OverflowError:
- return expect_overflowerror
+ if expect_overflowerror:
+ return True
+ else:
+ debug_print('unexpected OverflowError!')
+ return False
else:
- return check(got)
+ if check(got):
+ return True
+ else:
+ debug_print('unexpected result:', got)
+ return False
#
testfn.func_name = 'test_' + fnname
return testfn
diff --git a/pypy/jit/backend/x86/tool/viewcode.py b/pypy/jit/backend/x86/tool/viewcode.py
--- a/pypy/jit/backend/x86/tool/viewcode.py
+++ b/pypy/jit/backend/x86/tool/viewcode.py
@@ -34,7 +34,7 @@
# I am porting it in a lazy fashion... See py-utils/xam.py
if sys.platform == "win32":
- XXX # lots more in Psyco
+ pass # lots more in Psyco
def machine_code_dump(data, originaddr, backend_name, label_list=None):
objdump_backend_option = {
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -291,6 +291,11 @@
op1 = SpaceOperation('-live-', [], None)
return [op, op1]
+ def _noop_rewrite(self, op):
+ return op
+
+ rewrite_op_convert_float_bytes_to_longlong = _noop_rewrite
+
# ----------
# Various kinds of calls
@@ -365,7 +370,7 @@
def handle_builtin_call(self, op):
oopspec_name, args = support.decode_builtin_call(op)
# dispatch to various implementations depending on the oopspec_name
- if oopspec_name.startswith('list.') or oopspec_name == 'newlist':
+ if oopspec_name.startswith('list.') or oopspec_name.startswith('newlist'):
prepare = self._handle_list_call
elif oopspec_name.startswith('stroruni.'):
prepare = self._handle_stroruni_call
@@ -1494,6 +1499,14 @@
arraydescr, v_length],
op.result)
+ def do_resizable_newlist_hint(self, op, args, arraydescr, lengthdescr,
+ itemsdescr, structdescr):
+ v_hint = self._get_initial_newlist_length(op, args)
+ return SpaceOperation('newlist_hint',
+ [structdescr, lengthdescr, itemsdescr,
+ arraydescr, v_hint],
+ op.result)
+
def do_resizable_list_getitem(self, op, args, arraydescr, lengthdescr,
itemsdescr, structdescr):
v_index, extraop = self._prepare_list_getset(op, lengthdescr, args,
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -144,6 +144,10 @@
_ll_1_newlist.need_result_type = True
_ll_2_newlist.need_result_type = True
+def _ll_1_newlist_hint(LIST, hint):
+ return LIST.ll_newlist_hint(hint)
+_ll_1_newlist_hint.need_result_type = True
+
def _ll_1_list_len(l):
return l.ll_length()
def _ll_2_list_getitem(l, index):
diff --git a/pypy/jit/codewriter/test/test_flatten.py b/pypy/jit/codewriter/test/test_flatten.py
--- a/pypy/jit/codewriter/test/test_flatten.py
+++ b/pypy/jit/codewriter/test/test_flatten.py
@@ -968,6 +968,21 @@
int_return %i2
""", transform=True)
+ def test_convert_float_bytes_to_int(self):
+ from pypy.rlib.longlong2float import float2longlong
+ def f(x):
+ return float2longlong(x)
+ if longlong.is_64_bit:
+ result_var = "%i0"
+ return_op = "int_return"
+ else:
+ result_var = "%f1"
+ return_op = "float_return"
+ self.encoding_test(f, [25.0], """
+ convert_float_bytes_to_longlong %%f0 -> %(result_var)s
+ %(return_op)s %(result_var)s
+ """ % {"result_var": result_var, "return_op": return_op})
+
def check_force_cast(FROM, TO, operations, value):
"""Check that the test is correctly written..."""
diff --git a/pypy/jit/codewriter/test/test_longlong.py b/pypy/jit/codewriter/test/test_longlong.py
--- a/pypy/jit/codewriter/test/test_longlong.py
+++ b/pypy/jit/codewriter/test/test_longlong.py
@@ -1,6 +1,6 @@
import py, sys
-from pypy.rlib.rarithmetic import r_longlong, intmask
+from pypy.rlib.rarithmetic import r_longlong, intmask, is_valid_int
from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
from pypy.objspace.flow.model import Block, Link
from pypy.translator.unsimplify import varoftype
@@ -32,7 +32,7 @@
def test_functions():
xll = longlong.getfloatstorage(3.5)
assert longlong.getrealfloat(xll) == 3.5
- assert isinstance(longlong.gethash(xll), int)
+ assert is_valid_int(longlong.gethash(xll))
class TestLongLong:
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -1,15 +1,16 @@
+from pypy.jit.codewriter import heaptracker, longlong
+from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr
+from pypy.jit.metainterp.compile import ResumeAtPositionDescr
+from pypy.jit.metainterp.jitexc import JitException, get_llexception, reraise
+from pypy.rlib import longlong2float
+from pypy.rlib.debug import debug_start, debug_stop, ll_assert, make_sure_not_resized
+from pypy.rlib.objectmodel import we_are_translated
+from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck
+from pypy.rlib.rtimer import read_timestamp
from pypy.rlib.unroll import unrolling_iterable
-from pypy.rlib.rtimer import read_timestamp
-from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck
-from pypy.rlib.objectmodel import we_are_translated
-from pypy.rlib.debug import debug_start, debug_stop, ll_assert
-from pypy.rlib.debug import make_sure_not_resized
from pypy.rpython.lltypesystem import lltype, llmemory, rclass
from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr
-from pypy.jit.codewriter import heaptracker, longlong
-from pypy.jit.metainterp.jitexc import JitException, get_llexception, reraise
-from pypy.jit.metainterp.compile import ResumeAtPositionDescr
+
def arguments(*argtypes, **kwds):
resulttype = kwds.pop('returns', None)
@@ -20,6 +21,9 @@
return function
return decorate
+LONGLONG_TYPECODE = 'i' if longlong.is_64_bit else 'f'
+
+
class LeaveFrame(JitException):
pass
@@ -663,6 +667,11 @@
a = float(a)
return longlong.getfloatstorage(a)
+ @arguments("f", returns=LONGLONG_TYPECODE)
+ def bhimpl_convert_float_bytes_to_longlong(a):
+ a = longlong.getrealfloat(a)
+ return longlong2float.float2longlong(a)
+
# ----------
# control flow operations
@@ -982,6 +991,15 @@
cpu.bh_setfield_gc_r(result, itemsdescr, items)
return result
+ @arguments("cpu", "d", "d", "d", "d", "i", returns="r")
+ def bhimpl_newlist_hint(cpu, structdescr, lengthdescr, itemsdescr,
+ arraydescr, lengthhint):
+ result = cpu.bh_new(structdescr)
+ cpu.bh_setfield_gc_i(result, lengthdescr, 0)
+ items = cpu.bh_new_array(arraydescr, lengthhint)
+ cpu.bh_setfield_gc_r(result, itemsdescr, items)
+ return result
+
@arguments("cpu", "r", "d", "d", "i", returns="i")
def bhimpl_getlistitem_gc_i(cpu, lst, itemsdescr, arraydescr, index):
items = cpu.bh_getfield_gc_r(lst, itemsdescr)
@@ -1176,14 +1194,14 @@
def bhimpl_getinteriorfield_gc_f(cpu, array, index, descr):
return cpu.bh_getinteriorfield_gc_f(array, index, descr)
- @arguments("cpu", "r", "i", "d", "i")
- def bhimpl_setinteriorfield_gc_i(cpu, array, index, descr, value):
+ @arguments("cpu", "r", "i", "i", "d")
+ def bhimpl_setinteriorfield_gc_i(cpu, array, index, value, descr):
cpu.bh_setinteriorfield_gc_i(array, index, descr, value)
- @arguments("cpu", "r", "i", "d", "r")
- def bhimpl_setinteriorfield_gc_r(cpu, array, index, descr, value):
+ @arguments("cpu", "r", "i", "r", "d")
+ def bhimpl_setinteriorfield_gc_r(cpu, array, index, value, descr):
cpu.bh_setinteriorfield_gc_r(array, index, descr, value)
- @arguments("cpu", "r", "i", "d", "f")
- def bhimpl_setinteriorfield_gc_f(cpu, array, index, descr, value):
+ @arguments("cpu", "r", "i", "f", "d")
+ def bhimpl_setinteriorfield_gc_f(cpu, array, index, value, descr):
cpu.bh_setinteriorfield_gc_f(array, index, descr, value)
@arguments("cpu", "r", "d", returns="i")
@@ -1300,7 +1318,7 @@
def bhimpl_copyunicodecontent(cpu, src, dst, srcstart, dststart, length):
cpu.bh_copyunicodecontent(src, dst, srcstart, dststart, length)
- @arguments(returns=(longlong.is_64_bit and "i" or "f"))
+ @arguments(returns=LONGLONG_TYPECODE)
def bhimpl_ll_read_timestamp():
return read_timestamp()
diff --git a/pypy/jit/metainterp/executor.py b/pypy/jit/metainterp/executor.py
--- a/pypy/jit/metainterp/executor.py
+++ b/pypy/jit/metainterp/executor.py
@@ -2,7 +2,7 @@
"""
from pypy.rpython.lltypesystem import lltype, rstr
-from pypy.rlib.rarithmetic import ovfcheck, r_longlong
+from pypy.rlib.rarithmetic import ovfcheck, r_longlong, is_valid_int
from pypy.rlib.rtimer import read_timestamp
from pypy.rlib.unroll import unrolling_iterable
from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, check_descr
@@ -248,7 +248,7 @@
def do_read_timestamp(cpu, _):
x = read_timestamp()
if longlong.is_64_bit:
- assert isinstance(x, int) # 64-bit
+ assert is_valid_int(x) # 64-bit
return BoxInt(x)
else:
assert isinstance(x, r_longlong) # 32-bit
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -4,7 +4,8 @@
from pypy.rpython.ootypesystem import ootype
from pypy.rlib.objectmodel import we_are_translated, Symbolic
from pypy.rlib.objectmodel import compute_unique_id
-from pypy.rlib.rarithmetic import r_int64
+from pypy.rlib.rarithmetic import r_int64, is_valid_int
+
from pypy.conftest import option
from pypy.jit.metainterp.resoperation import ResOperation, rop
@@ -213,7 +214,7 @@
def __init__(self, value):
if not we_are_translated():
- if isinstance(value, int):
+ if is_valid_int(value):
value = int(value) # bool -> int
else:
assert isinstance(value, Symbolic)
@@ -448,7 +449,7 @@
def __init__(self, value=0):
if not we_are_translated():
- if isinstance(value, int):
+ if is_valid_int(value):
value = int(value) # bool -> int
else:
assert isinstance(value, Symbolic)
diff --git a/pypy/jit/metainterp/optimizeopt/intutils.py b/pypy/jit/metainterp/optimizeopt/intutils.py
--- a/pypy/jit/metainterp/optimizeopt/intutils.py
+++ b/pypy/jit/metainterp/optimizeopt/intutils.py
@@ -1,10 +1,9 @@
-from pypy.rlib.rarithmetic import ovfcheck, LONG_BIT
+from pypy.rlib.rarithmetic import ovfcheck, LONG_BIT, maxint, is_valid_int
from pypy.rlib.objectmodel import we_are_translated
from pypy.jit.metainterp.resoperation import rop, ResOperation
from pypy.jit.metainterp.history import BoxInt, ConstInt
-import sys
-MAXINT = sys.maxint
-MININT = -sys.maxint - 1
+MAXINT = maxint
+MININT = -maxint - 1
class IntBound(object):
_attrs_ = ('has_upper', 'has_lower', 'upper', 'lower')
@@ -16,8 +15,8 @@
self.lower = lower
# check for unexpected overflows:
if not we_are_translated():
- assert type(upper) is not long
- assert type(lower) is not long
+ assert type(upper) is not long or is_valid_int(upper)
+ assert type(lower) is not long or is_valid_int(lower)
# Returns True if the bound was updated
def make_le(self, other):
diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -10,6 +10,8 @@
from pypy.rlib.unroll import unrolling_iterable
from pypy.rpython import annlowlevel
from pypy.rpython.lltypesystem import lltype, rstr
+from pypy.rlib.rarithmetic import is_valid_int
+
class StrOrUnicode(object):
@@ -730,7 +732,7 @@
for name in dir(OptString):
if name.startswith(prefix):
value = getattr(EffectInfo, 'OS_' + name[len(prefix):])
- assert isinstance(value, int) and value != 0
+ assert is_valid_int(value) and value != 0
result.append((value, getattr(OptString, name)))
return unrolling_iterable(result)
opt_call_oopspec_ops = _findall_call_oopspec()
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -223,6 +223,7 @@
'cast_float_to_singlefloat', 'cast_singlefloat_to_float',
'float_neg', 'float_abs',
'cast_ptr_to_int', 'cast_int_to_ptr',
+ 'convert_float_bytes_to_longlong',
]:
exec py.code.Source('''
@arguments("box")
@@ -509,6 +510,15 @@
self._opimpl_setfield_gc_any(sbox, itemsdescr, abox)
return sbox
+ @arguments("descr", "descr", "descr", "descr", "box")
+ def opimpl_newlist_hint(self, structdescr, lengthdescr, itemsdescr,
+ arraydescr, sizehintbox):
+ sbox = self.opimpl_new(structdescr)
+ self._opimpl_setfield_gc_any(sbox, lengthdescr, history.CONST_FALSE)
+ abox = self.opimpl_new_array(arraydescr, sizehintbox)
+ self._opimpl_setfield_gc_any(sbox, itemsdescr, abox)
+ return sbox
+
@arguments("box", "descr", "descr", "box")
def _opimpl_getlistitem_gc_any(self, listbox, itemsdescr, arraydescr,
indexbox):
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -419,6 +419,7 @@
'CAST_INT_TO_FLOAT/1', # need some messy code in the backend
'CAST_FLOAT_TO_SINGLEFLOAT/1',
'CAST_SINGLEFLOAT_TO_FLOAT/1',
+ 'CONVERT_FLOAT_BYTES_TO_LONGLONG/1',
#
'INT_LT/2b',
'INT_LE/2b',
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3,6 +3,7 @@
import py
from pypy import conftest
+from pypy.jit.codewriter import longlong
from pypy.jit.codewriter.policy import JitPolicy, StopAtXPolicy
from pypy.jit.metainterp import pyjitpl, history
from pypy.jit.metainterp.optimizeopt import ALL_OPTS_DICT
@@ -14,7 +15,8 @@
loop_invariant, elidable, promote, jit_debug, assert_green,
AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
isconstant, isvirtual, promote_string, set_param, record_known_class)
-from pypy.rlib.rarithmetic import ovfcheck
+from pypy.rlib.longlong2float import float2longlong
+from pypy.rlib.rarithmetic import ovfcheck, is_valid_int
from pypy.rpython.lltypesystem import lltype, llmemory, rffi
from pypy.rpython.ootypesystem import ootype
@@ -292,7 +294,7 @@
assert res == f(6, sys.maxint, 32, 48)
res = self.meta_interp(f, [sys.maxint, 6, 32, 48])
assert res == f(sys.maxint, 6, 32, 48)
-
+
def test_loop_invariant_intbox(self):
myjitdriver = JitDriver(greens = [], reds = ['y', 'res', 'x'])
@@ -953,7 +955,7 @@
self.meta_interp(f, [20], repeat=7)
# the loop and the entry path as a single trace
self.check_jitcell_token_count(1)
-
+
# we get:
# ENTER - compile the new loop and the entry bridge
# ENTER - compile the leaving path
@@ -1470,7 +1472,7 @@
assert res == f(299)
self.check_resops(guard_class=0, guard_nonnull=4,
guard_nonnull_class=4, guard_isnull=2)
-
+
def test_merge_guardnonnull_guardvalue(self):
from pypy.rlib.objectmodel import instantiate
@@ -1499,7 +1501,7 @@
assert res == f(299)
self.check_resops(guard_value=4, guard_class=0, guard_nonnull=4,
guard_nonnull_class=0, guard_isnull=2)
-
+
def test_merge_guardnonnull_guardvalue_2(self):
from pypy.rlib.objectmodel import instantiate
@@ -1528,7 +1530,7 @@
assert res == f(299)
self.check_resops(guard_value=4, guard_class=0, guard_nonnull=4,
guard_nonnull_class=0, guard_isnull=2)
-
+
def test_merge_guardnonnull_guardclass_guardvalue(self):
from pypy.rlib.objectmodel import instantiate
@@ -2296,7 +2298,7 @@
self.check_resops(int_rshift=3)
bigval = 1
- while (bigval << 3).__class__ is int:
+ while is_valid_int(bigval << 3):
bigval = bigval << 1
assert self.meta_interp(f, [bigval, 5]) == 0
@@ -2341,7 +2343,7 @@
self.check_resops(int_rshift=3)
bigval = 1
- while (bigval << 3).__class__ is int:
+ while is_valid_int(bigval << 3):
bigval = bigval << 1
assert self.meta_interp(f, [bigval, 5]) == 0
@@ -2636,7 +2638,7 @@
return sa
assert self.meta_interp(f, [20]) == f(20)
self.check_resops(int_lt=6, int_le=2, int_ge=4, int_gt=3)
-
+
def test_intbounds_not_generalized2(self):
myjitdriver = JitDriver(greens = [], reds = ['n', 'i', 'sa', 'node'])
@@ -2677,7 +2679,7 @@
assert self.meta_interp(f, [20, 3]) == f(20, 3)
self.check_jitcell_token_count(1)
self.check_target_token_count(5)
-
+
def test_max_retrace_guards(self):
myjitdriver = JitDriver(greens = [], reds = ['n', 'i', 'sa', 'a'])
@@ -2815,7 +2817,7 @@
for cell in get_stats().get_all_jitcell_tokens():
# Initialal trace with two labels and 5 retraces
assert len(cell.target_tokens) <= 7
-
+
def test_nested_retrace(self):
myjitdriver = JitDriver(greens = ['pc'], reds = ['n', 'a', 'i', 'j', 'sa'])
@@ -3784,6 +3786,25 @@
assert res == 11 * 12 * 13
self.check_operations_history(int_add=3, int_mul=2)
+ def test_setinteriorfield(self):
+ A = lltype.GcArray(lltype.Struct('S', ('x', lltype.Signed)))
+ a = lltype.malloc(A, 5, immortal=True)
+ def g(n):
+ a[n].x = n + 2
+ return a[n].x
+ res = self.interp_operations(g, [1])
+ assert res == 3
+
+ def test_float2longlong(self):
+ def f(n):
+ return float2longlong(n)
+
+ for x in [2.5, float("nan"), -2.5, float("inf")]:
+ # There are tests elsewhere to verify the correctness of this.
+ expected = float2longlong(x)
+ res = self.interp_operations(f, [x])
+ assert longlong.getfloatstorage(res) == expected
+
class TestLLtype(BaseLLtypeTests, LLJitMixin):
def test_tagged(self):
diff --git a/pypy/jit/metainterp/test/test_list.py b/pypy/jit/metainterp/test/test_list.py
--- a/pypy/jit/metainterp/test/test_list.py
+++ b/pypy/jit/metainterp/test/test_list.py
@@ -1,4 +1,5 @@
import py
+from pypy.rlib.objectmodel import newlist_hint
from pypy.rlib.jit import JitDriver
from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
@@ -228,6 +229,28 @@
self.check_resops({'jump': 1, 'int_gt': 2, 'int_add': 2,
'guard_true': 2, 'int_sub': 2})
+ def test_newlist_hint(self):
+ def f(i):
+ l = newlist_hint(i)
+ l[0] = 55
+ return len(l)
+
+ r = self.interp_operations(f, [3])
+ assert r == 0
+
+ def test_newlist_hint_optimized(self):
+ driver = JitDriver(greens = [], reds = ['i'])
+
+ def f(i):
+ while i > 0:
+ driver.jit_merge_point(i=i)
+ l = newlist_hint(5)
+ l.append(1)
+ i -= l[0]
+
+ self.meta_interp(f, [10], listops=True)
+ self.check_resops(new_array=0, call=0)
+
class TestOOtype(ListTests, OOJitMixin):
pass
diff --git a/pypy/jit/tl/tlc.py b/pypy/jit/tl/tlc.py
--- a/pypy/jit/tl/tlc.py
+++ b/pypy/jit/tl/tlc.py
@@ -6,6 +6,8 @@
from pypy.jit.tl.tlopcode import *
from pypy.jit.tl import tlopcode
from pypy.rlib.jit import JitDriver, elidable
+from pypy.rlib.rarithmetic import is_valid_int
+
class Obj(object):
@@ -219,7 +221,7 @@
class Frame(object):
def __init__(self, args, pc):
- assert isinstance(pc, int)
+ assert is_valid_int(pc)
self.args = args
self.pc = pc
self.stack = []
@@ -239,7 +241,7 @@
return interp_eval(code, pc, args, pool).int_o()
def interp_eval(code, pc, args, pool):
- assert isinstance(pc, int)
+ assert is_valid_int(pc)
frame = Frame(args, pc)
pc = frame.pc
diff --git a/pypy/module/__builtin__/app_inspect.py b/pypy/module/__builtin__/app_inspect.py
--- a/pypy/module/__builtin__/app_inspect.py
+++ b/pypy/module/__builtin__/app_inspect.py
@@ -8,8 +8,6 @@
from __pypy__ import lookup_special
def _caller_locals():
- # note: the reason why this is working is because the functions in here are
- # compiled by geninterp, so they don't have a frame
return sys._getframe(0).f_locals
def vars(*obj):
@@ -26,17 +24,6 @@
except AttributeError:
raise TypeError, "vars() argument must have __dict__ attribute"
-# Replaced by the interp-level helper space.callable():
-##def callable(ob):
-## import __builtin__ # XXX this is insane but required for now for geninterp
-## for c in type(ob).__mro__:
-## if '__call__' in c.__dict__:
-## if isinstance(ob, __builtin__._instance): # old style instance!
-## return getattr(ob, '__call__', None) is not None
-## return True
-## else:
-## return False
-
def dir(*args):
"""dir([object]) -> list of strings
diff --git a/pypy/module/__builtin__/interp_memoryview.py b/pypy/module/__builtin__/interp_memoryview.py
--- a/pypy/module/__builtin__/interp_memoryview.py
+++ b/pypy/module/__builtin__/interp_memoryview.py
@@ -69,6 +69,10 @@
return W_MemoryView(buf)
def descr_buffer(self, space):
+ """Note that memoryview() objects in PyPy support buffer(), whereas
+ not in CPython; but CPython supports passing memoryview() to most
+ built-in functions that accept buffers, with the notable exception
+ of the buffer() built-in."""
return space.wrap(self.buf)
def descr_tobytes(self, space):
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -1,5 +1,5 @@
+import sys
-# Package initialisation
from pypy.interpreter.mixedmodule import MixedModule
from pypy.module.imp.importing import get_pyc_magic
@@ -12,6 +12,21 @@
"UnicodeBuilder": "interp_builders.W_UnicodeBuilder",
}
+class TimeModule(MixedModule):
+ appleveldefs = {}
+ interpleveldefs = {}
+ if sys.platform.startswith("linux"):
+ from pypy.module.__pypy__ import interp_time
+ interpleveldefs["clock_gettime"] = "interp_time.clock_gettime"
+ interpleveldefs["clock_getres"] = "interp_time.clock_getres"
+ for name in [
+ "CLOCK_REALTIME", "CLOCK_MONOTONIC", "CLOCK_MONOTONIC_RAW",
+ "CLOCK_PROCESS_CPUTIME_ID", "CLOCK_THREAD_CPUTIME_ID"
+ ]:
+ if getattr(interp_time, name) is not None:
+ interpleveldefs[name] = "space.wrap(interp_time.%s)" % name
+
+
class Module(MixedModule):
appleveldefs = {
}
@@ -32,6 +47,7 @@
submodules = {
"builders": BuildersModule,
+ "time": TimeModule,
}
def setup_after_space_initialization(self):
diff --git a/pypy/module/__pypy__/interp_time.py b/pypy/module/__pypy__/interp_time.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__pypy__/interp_time.py
@@ -0,0 +1,65 @@
+from __future__ import with_statement
+import sys
+
+from pypy.interpreter.error import exception_from_errno
+from pypy.interpreter.gateway import unwrap_spec
+from pypy.rpython.lltypesystem import rffi, lltype
+from pypy.rpython.tool import rffi_platform
+from pypy.translator.tool.cbuild import ExternalCompilationInfo
+
+
+class CConfig:
+ _compilation_info_ = ExternalCompilationInfo(
+ includes=["time.h"],
+ libraries=["rt"],
+ )
+
+ HAS_CLOCK_GETTIME = rffi_platform.Has('clock_gettime')
+
+ CLOCK_REALTIME = rffi_platform.DefinedConstantInteger("CLOCK_REALTIME")
+ CLOCK_MONOTONIC = rffi_platform.DefinedConstantInteger("CLOCK_MONOTONIC")
+ CLOCK_MONOTONIC_RAW = rffi_platform.DefinedConstantInteger("CLOCK_MONOTONIC_RAW")
+ CLOCK_PROCESS_CPUTIME_ID = rffi_platform.DefinedConstantInteger("CLOCK_PROCESS_CPUTIME_ID")
+ CLOCK_THREAD_CPUTIME_ID = rffi_platform.DefinedConstantInteger("CLOCK_THREAD_CPUTIME_ID")
+
+ TIMESPEC = rffi_platform.Struct("struct timespec", [
+ ("tv_sec", rffi.TIME_T),
+ ("tv_nsec", rffi.LONG),
+ ])
+
+cconfig = rffi_platform.configure(CConfig)
+
+HAS_CLOCK_GETTIME = cconfig["HAS_CLOCK_GETTIME"]
+
+CLOCK_REALTIME = cconfig["CLOCK_REALTIME"]
+CLOCK_MONOTONIC = cconfig["CLOCK_MONOTONIC"]
+CLOCK_MONOTONIC_RAW = cconfig["CLOCK_MONOTONIC_RAW"]
+CLOCK_PROCESS_CPUTIME_ID = cconfig["CLOCK_PROCESS_CPUTIME_ID"]
+CLOCK_THREAD_CPUTIME_ID = cconfig["CLOCK_THREAD_CPUTIME_ID"]
+
+TIMESPEC = cconfig["TIMESPEC"]
+
+c_clock_gettime = rffi.llexternal("clock_gettime",
+ [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT,
+ compilation_info=CConfig._compilation_info_, threadsafe=False
+)
+c_clock_getres = rffi.llexternal("clock_getres",
+ [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT,
+ compilation_info=CConfig._compilation_info_, threadsafe=False
+)
+
+ at unwrap_spec(clk_id="c_int")
+def clock_gettime(space, clk_id):
+ with lltype.scoped_alloc(TIMESPEC) as tp:
+ ret = c_clock_gettime(clk_id, tp)
+ if ret != 0:
+ raise exception_from_errno(space, space.w_IOError)
+ return space.wrap(tp.c_tv_sec + tp.c_tv_nsec * 1e-9)
+
+ at unwrap_spec(clk_id="c_int")
+def clock_getres(space, clk_id):
+ with lltype.scoped_alloc(TIMESPEC) as tp:
+ ret = c_clock_getres(clk_id, tp)
+ if ret != 0:
+ raise exception_from_errno(space, space.w_IOError)
+ return space.wrap(tp.c_tv_sec + tp.c_tv_nsec * 1e-9)
diff --git a/pypy/module/__pypy__/test/test_time.py b/pypy/module/__pypy__/test/test_time.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__pypy__/test/test_time.py
@@ -0,0 +1,26 @@
+import py
+
+from pypy.module.__pypy__.interp_time import HAS_CLOCK_GETTIME
+
+
+class AppTestTime(object):
+ def setup_class(cls):
+ if not HAS_CLOCK_GETTIME:
+ py.test.skip("need time.clock_gettime")
+
+ def test_clock_realtime(self):
+ from __pypy__ import time
+ res = time.clock_gettime(time.CLOCK_REALTIME)
+ assert isinstance(res, float)
+
+ def test_clock_monotonic(self):
+ from __pypy__ import time
+ a = time.clock_gettime(time.CLOCK_MONOTONIC)
+ b = time.clock_gettime(time.CLOCK_MONOTONIC)
+ assert a <= b
+
+ def test_clock_getres(self):
+ from __pypy__ import time
+ res = time.clock_getres(time.CLOCK_REALTIME)
+ assert res > 0.0
+ assert res <= 1.0
diff --git a/pypy/module/_ast/test/test_ast.py b/pypy/module/_ast/test/test_ast.py
--- a/pypy/module/_ast/test/test_ast.py
+++ b/pypy/module/_ast/test/test_ast.py
@@ -1,9 +1,10 @@
import py
-
+from pypy.conftest import gettestobjspace
class AppTestAST:
def setup_class(cls):
+ cls.space = gettestobjspace(usemodules=['struct'])
cls.w_ast = cls.space.appexec([], """():
import _ast
return _ast""")
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -4,7 +4,7 @@
class AppTestCodecs:
def setup_class(cls):
- space = gettestobjspace(usemodules=('unicodedata',))
+ space = gettestobjspace(usemodules=('unicodedata', 'struct'))
cls.space = space
def test_register_noncallable(self):
diff --git a/pypy/module/_continuation/test/test_zpickle.py b/pypy/module/_continuation/test/test_zpickle.py
--- a/pypy/module/_continuation/test/test_zpickle.py
+++ b/pypy/module/_continuation/test/test_zpickle.py
@@ -106,8 +106,9 @@
version = 0
def setup_class(cls):
- cls.space = gettestobjspace(usemodules=('_continuation',),
+ cls.space = gettestobjspace(usemodules=('_continuation', 'struct'),
CALL_METHOD=True)
+ cls.space.config.translation.continuation = True
cls.space.appexec([], """():
global continulet, A, __name__
diff --git a/pypy/module/_hashlib/test/test_hashlib.py b/pypy/module/_hashlib/test/test_hashlib.py
--- a/pypy/module/_hashlib/test/test_hashlib.py
+++ b/pypy/module/_hashlib/test/test_hashlib.py
@@ -3,7 +3,7 @@
class AppTestHashlib:
def setup_class(cls):
- cls.space = gettestobjspace(usemodules=['_hashlib'])
+ cls.space = gettestobjspace(usemodules=['_hashlib', 'array', 'struct'])
def test_simple(self):
import _hashlib
diff --git a/pypy/module/_io/test/test_io.py b/pypy/module/_io/test/test_io.py
--- a/pypy/module/_io/test/test_io.py
+++ b/pypy/module/_io/test/test_io.py
@@ -158,7 +158,7 @@
class AppTestOpen:
def setup_class(cls):
- cls.space = gettestobjspace(usemodules=['_io', '_locale'])
+ cls.space = gettestobjspace(usemodules=['_io', '_locale', 'array', 'struct'])
tmpfile = udir.join('tmpfile').ensure()
cls.w_tmpfile = cls.space.wrap(str(tmpfile))
diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py
--- a/pypy/module/_multiprocessing/test/test_connection.py
+++ b/pypy/module/_multiprocessing/test/test_connection.py
@@ -92,7 +92,8 @@
class AppTestSocketConnection(BaseConnectionTest):
def setup_class(cls):
- space = gettestobjspace(usemodules=('_multiprocessing', 'thread', 'signal'))
+ space = gettestobjspace(usemodules=('_multiprocessing', 'thread', 'signal',
+ 'struct', 'array'))
cls.space = space
cls.w_connections = space.newlist([])
diff --git a/pypy/module/_multiprocessing/test/test_semaphore.py b/pypy/module/_multiprocessing/test/test_semaphore.py
--- a/pypy/module/_multiprocessing/test/test_semaphore.py
+++ b/pypy/module/_multiprocessing/test/test_semaphore.py
@@ -2,6 +2,7 @@
from pypy.module._multiprocessing.interp_semaphore import (
RECURSIVE_MUTEX, SEMAPHORE)
+
class AppTestSemaphore:
def setup_class(cls):
space = gettestobjspace(usemodules=('_multiprocessing', 'thread'))
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -6,7 +6,7 @@
from pypy.rpython.lltypesystem import lltype, rffi
def setup_module(mod):
- mod.space = gettestobjspace(usemodules=['_socket', 'array'])
+ mod.space = gettestobjspace(usemodules=['_socket', 'array', 'struct'])
global socket
import socket
mod.w_socket = space.appexec([], "(): import _socket as m; return m")
@@ -372,10 +372,9 @@
def test_socket_connect(self):
import _socket, os
s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0)
- # XXX temporarily we use python.org to test, will have more robust tests
- # in the absence of a network connection later when more parts of the
- # socket API are implemented. Currently skip the test if there is no
- # connection.
+ # it would be nice to have a test which works even if there is no
+ # network connection. However, this one is "good enough" for now. Skip
+ # it if there is no connection.
try:
s.connect(("www.python.org", 80))
except _socket.gaierror, ex:
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -2,6 +2,7 @@
import os
import py
+
class AppTestSSL:
def setup_class(cls):
space = gettestobjspace(usemodules=('_ssl', '_socket'))
@@ -29,7 +30,6 @@
assert isinstance(_ssl.SSL_ERROR_EOF, int)
assert isinstance(_ssl.SSL_ERROR_INVALID_ERROR_CODE, int)
- assert isinstance(_ssl.OPENSSL_VERSION_NUMBER, (int, long))
assert isinstance(_ssl.OPENSSL_VERSION_INFO, tuple)
assert len(_ssl.OPENSSL_VERSION_INFO) == 5
assert isinstance(_ssl.OPENSSL_VERSION, str)
@@ -90,7 +90,7 @@
class AppTestConnectedSSL:
def setup_class(cls):
- space = gettestobjspace(usemodules=('_ssl', '_socket'))
+ space = gettestobjspace(usemodules=('_ssl', '_socket', 'struct'))
cls.space = space
def setup_method(self, method):
@@ -179,7 +179,7 @@
# to exercise the poll() calls
def setup_class(cls):
- space = gettestobjspace(usemodules=('_ssl', '_socket'))
+ space = gettestobjspace(usemodules=('_ssl', '_socket', 'struct'))
cls.space = space
cls.space.appexec([], """():
import socket; socket.setdefaulttimeout(1)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -824,6 +824,8 @@
pypy_decls.append("#ifdef __cplusplus")
pypy_decls.append("extern \"C\" {")
pypy_decls.append("#endif\n")
+ pypy_decls.append('#define Signed long /* xxx temporary fix */\n')
+ pypy_decls.append('#define Unsigned unsigned long /* xxx temporary fix */\n')
for decl in FORWARD_DECLS:
pypy_decls.append("%s;" % (decl,))
@@ -855,6 +857,8 @@
typ = 'PyObject*'
pypy_decls.append('PyAPI_DATA(%s) %s;' % (typ, name))
+ pypy_decls.append('#undef Signed /* xxx temporary fix */\n')
+ pypy_decls.append('#undef Unsigned /* xxx temporary fix */\n')
pypy_decls.append("#ifdef __cplusplus")
pypy_decls.append("}")
pypy_decls.append("#endif")
diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py
--- a/pypy/module/cpyext/pystate.py
+++ b/pypy/module/cpyext/pystate.py
@@ -10,7 +10,7 @@
[('next', PyInterpreterState)],
PyInterpreterStateStruct)
PyThreadState = lltype.Ptr(cpython_struct(
- "PyThreadState",
+ "PyThreadState",
[('interp', PyInterpreterState),
('dict', PyObject),
]))
@@ -19,12 +19,15 @@
def PyEval_SaveThread(space):
"""Release the global interpreter lock (if it has been created and thread
support is enabled) and reset the thread state to NULL, returning the
- previous thread state (which is not NULL except in PyPy). If the lock has been created,
+ previous thread state. If the lock has been created,
the current thread must have acquired it. (This function is available even
when thread support is disabled at compile time.)"""
+ state = space.fromcache(InterpreterState)
if rffi.aroundstate.before:
rffi.aroundstate.before()
- return lltype.nullptr(PyThreadState.TO)
+ tstate = state.swap_thread_state(
+ space, lltype.nullptr(PyThreadState.TO))
+ return tstate
@cpython_api([PyThreadState], lltype.Void)
def PyEval_RestoreThread(space, tstate):
@@ -35,6 +38,8 @@
when thread support is disabled at compile time.)"""
if rffi.aroundstate.after:
rffi.aroundstate.after()
+ state = space.fromcache(InterpreterState)
+ state.swap_thread_state(space, tstate)
@cpython_api([], lltype.Void)
def PyEval_InitThreads(space):
@@ -67,28 +72,91 @@
dealloc=ThreadState_dealloc)
from pypy.interpreter.executioncontext import ExecutionContext
+
+# Keep track of the ThreadStateCapsule for a particular execution context. The
+# default is for new execution contexts not to have one; it is allocated on the
+# first cpyext-based request for it.
ExecutionContext.cpyext_threadstate = ThreadStateCapsule(None)
+# Also keep track of whether it has been initialized yet or not (None is a valid
+# PyThreadState for an execution context to have, when the GIL has been
+# released, so a check against that can't be used to determine the need for
+# initialization).
+ExecutionContext.cpyext_initialized_threadstate = False
+
+def cleanup_cpyext_state(self):
+ try:
+ del self.cpyext_threadstate
+ except AttributeError:
+ pass
+ self.cpyext_initialized_threadstate = False
+ExecutionContext.cleanup_cpyext_state = cleanup_cpyext_state
+
class InterpreterState(object):
def __init__(self, space):
self.interpreter_state = lltype.malloc(
PyInterpreterState.TO, flavor='raw', zero=True, immortal=True)
def new_thread_state(self, space):
+ """
+ Create a new ThreadStateCapsule to hold the PyThreadState for a
+ particular execution context.
+
+ :param space: A space.
+
+ :returns: A new ThreadStateCapsule holding a newly allocated
+ PyThreadState and referring to this interpreter state.
+ """
capsule = ThreadStateCapsule(space)
ts = capsule.memory
ts.c_interp = self.interpreter_state
ts.c_dict = make_ref(space, space.newdict())
return capsule
+
def get_thread_state(self, space):
+ """
+ Get the current PyThreadState for the current execution context.
+
+ :param space: A space.
+
+ :returns: The current PyThreadState for the current execution context,
+ or None if it does not have one.
+ """
ec = space.getexecutioncontext()
return self._get_thread_state(space, ec).memory
+
+ def swap_thread_state(self, space, tstate):
+ """
+ Replace the current thread state of the current execution context with a
+ new thread state.
+
+ :param space: The space.
+
+ :param tstate: The new PyThreadState for the current execution context.
+
+ :returns: The old thread state for the current execution context, either
+ None or a PyThreadState.
+ """
+ ec = space.getexecutioncontext()
+ capsule = self._get_thread_state(space, ec)
+ old_tstate = capsule.memory
+ capsule.memory = tstate
+ return old_tstate
+
def _get_thread_state(self, space, ec):
- if ec.cpyext_threadstate.memory == lltype.nullptr(PyThreadState.TO):
+ """
+ Get the ThreadStateCapsule for the given execution context, possibly
+ creating a new one if it does not already have one.
+
+ :param space: The space.
+ :param ec: The ExecutionContext of which to get the thread state.
+ :returns: The ThreadStateCapsule for the given execution context.
+ """
+ if not ec.cpyext_initialized_threadstate:
ec.cpyext_threadstate = self.new_thread_state(space)
-
+ ec.cpyext_initialized_threadstate = True
return ec.cpyext_threadstate
@cpython_api([], PyThreadState, error=CANNOT_FAIL)
@@ -105,13 +173,8 @@
def PyThreadState_Swap(space, tstate):
"""Swap the current thread state with the thread state given by the argument
tstate, which may be NULL. The global interpreter lock must be held."""
- # All cpyext calls release and acquire the GIL, so this function has no
- # side-effects
- if tstate:
- return lltype.nullptr(PyThreadState.TO)
- else:
- state = space.fromcache(InterpreterState)
- return state.get_thread_state(space)
+ state = space.fromcache(InterpreterState)
+ return state.swap_thread_state(space, tstate)
@cpython_api([PyThreadState], lltype.Void)
def PyEval_AcquireThread(space, tstate):
diff --git a/pypy/module/cpyext/src/getargs.c b/pypy/module/cpyext/src/getargs.c
--- a/pypy/module/cpyext/src/getargs.c
+++ b/pypy/module/cpyext/src/getargs.c
@@ -23,16 +23,33 @@
#define FLAG_COMPAT 1
#define FLAG_SIZE_T 2
+typedef int (*destr_t)(PyObject *, void *);
+
+
+/* Keep track of "objects" that have been allocated or initialized and
+ which will need to be deallocated or cleaned up somehow if overall
+ parsing fails.
+*/
+typedef struct {
+ void *item;
+ destr_t destructor;
+} freelistentry_t;
+
+typedef struct {
+ int first_available;
+ freelistentry_t *entries;
+} freelist_t;
+
/* Forward */
static int vgetargs1(PyObject *, const char *, va_list *, int);
static void seterror(int, const char *, int *, const char *, const char *);
static char *convertitem(PyObject *, const char **, va_list *, int, int *,
- char *, size_t, PyObject **);
+ char *, size_t, freelist_t *);
static char *converttuple(PyObject *, const char **, va_list *, int,
- int *, char *, size_t, int, PyObject **);
+ int *, char *, size_t, int, freelist_t *);
static char *convertsimple(PyObject *, const char **, va_list *, int, char *,
- size_t, PyObject **);
+ size_t, freelist_t *);
static Py_ssize_t convertbuffer(PyObject *, void **p, char **);
static int getbuffer(PyObject *, Py_buffer *, char**);
@@ -129,57 +146,56 @@
/* Handle cleanup of allocated memory in case of exception */
-static void
-cleanup_ptr(void *ptr)
+static int
+cleanup_ptr(PyObject *self, void *ptr)
{
- PyMem_FREE(ptr);
-}
-
-static void
-cleanup_buffer(void *ptr)
-{
- PyBuffer_Release((Py_buffer *) ptr);
+ if (ptr) {
+ PyMem_FREE(ptr);
+ }
+ return 0;
}
static int
-addcleanup(void *ptr, PyObject **freelist, void (*destr)(void *))
+cleanup_buffer(PyObject *self, void *ptr)
{
- PyObject *cobj;
- if (!*freelist) {
- *freelist = PyList_New(0);
- if (!*freelist) {
- destr(ptr);
- return -1;
- }
- }
- cobj = PyCObject_FromVoidPtr(ptr, destr);
- if (!cobj) {
- destr(ptr);
- return -1;
- }
- if (PyList_Append(*freelist, cobj)) {
- Py_DECREF(cobj);
- return -1;
- }
- Py_DECREF(cobj);
- return 0;
+ Py_buffer *buf = (Py_buffer *)ptr;
+ if (buf) {
+ PyBuffer_Release(buf);
+ }
+ return 0;
}
static int
-cleanreturn(int retval, PyObject *freelist)
+addcleanup(void *ptr, freelist_t *freelist, destr_t destructor)
{
- if (freelist && retval != 0) {
- /* We were successful, reset the destructors so that they
- don't get called. */
- Py_ssize_t len = PyList_GET_SIZE(freelist), i;
- for (i = 0; i < len; i++)
- ((PyCObject *) PyList_GET_ITEM(freelist, i))
- ->destructor = NULL;
- }
- Py_XDECREF(freelist);
- return retval;
+ int index;
+
+ index = freelist->first_available;
+ freelist->first_available += 1;
+
+ freelist->entries[index].item = ptr;
+ freelist->entries[index].destructor = destructor;
+
+ return 0;
}
+static int
+cleanreturn(int retval, freelist_t *freelist)
+{
+ int index;
+
+ if (retval == 0) {
+ /* A failure occurred, therefore execute all of the cleanup
+ functions.
+ */
+ for (index = 0; index < freelist->first_available; ++index) {
+ freelist->entries[index].destructor(NULL,
+ freelist->entries[index].item);
+ }
+ }
+ PyMem_Free(freelist->entries);
+ return retval;
+}
static int
vgetargs1(PyObject *args, const char *format, va_list *p_va, int flags)
@@ -195,7 +211,7 @@
const char *formatsave = format;
Py_ssize_t i, len;
char *msg;
- PyObject *freelist = NULL;
+ freelist_t freelist = {0, NULL};
int compat = flags & FLAG_COMPAT;
assert(compat || (args != (PyObject*)NULL));
@@ -251,16 +267,18 @@
format = formatsave;
+ freelist.entries = PyMem_New(freelistentry_t, max);
+
if (compat) {
if (max == 0) {
if (args == NULL)
- return 1;
+ return cleanreturn(1, &freelist);
PyOS_snprintf(msgbuf, sizeof(msgbuf),
"%.200s%s takes no arguments",
fname==NULL ? "function" : fname,
fname==NULL ? "" : "()");
PyErr_SetString(PyExc_TypeError, msgbuf);
- return 0;
+ return cleanreturn(0, &freelist);
}
else if (min == 1 && max == 1) {
if (args == NULL) {
@@ -269,26 +287,26 @@
fname==NULL ? "function" : fname,
fname==NULL ? "" : "()");
PyErr_SetString(PyExc_TypeError, msgbuf);
- return 0;
+ return cleanreturn(0, &freelist);
}
msg = convertitem(args, &format, p_va, flags, levels,
msgbuf, sizeof(msgbuf), &freelist);
if (msg == NULL)
- return cleanreturn(1, freelist);
+ return cleanreturn(1, &freelist);
seterror(levels[0], msg, levels+1, fname, message);
- return cleanreturn(0, freelist);
+ return cleanreturn(0, &freelist);
}
else {
PyErr_SetString(PyExc_SystemError,
"old style getargs format uses new features");
- return 0;
+ return cleanreturn(0, &freelist);
}
}
if (!PyTuple_Check(args)) {
PyErr_SetString(PyExc_SystemError,
"new style getargs format but argument is not a tuple");
- return 0;
+ return cleanreturn(0, &freelist);
}
len = PyTuple_GET_SIZE(args);
@@ -308,7 +326,7 @@
message = msgbuf;
}
PyErr_SetString(PyExc_TypeError, message);
- return 0;
+ return cleanreturn(0, &freelist);
}
for (i = 0; i < len; i++) {
@@ -319,7 +337,7 @@
sizeof(msgbuf), &freelist);
if (msg) {
seterror(i+1, msg, levels, fname, message);
- return cleanreturn(0, freelist);
+ return cleanreturn(0, &freelist);
}
}
@@ -328,10 +346,10 @@
*format != '|' && *format != ':' && *format != ';') {
PyErr_Format(PyExc_SystemError,
"bad format string: %.200s", formatsave);
- return cleanreturn(0, freelist);
+ return cleanreturn(0, &freelist);
}
- return cleanreturn(1, freelist);
+ return cleanreturn(1, &freelist);
}
@@ -395,7 +413,7 @@
static char *
converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags,
int *levels, char *msgbuf, size_t bufsize, int toplevel,
- PyObject **freelist)
+ freelist_t *freelist)
{
int level = 0;
int n = 0;
@@ -472,7 +490,7 @@
static char *
convertitem(PyObject *arg, const char **p_format, va_list *p_va, int flags,
- int *levels, char *msgbuf, size_t bufsize, PyObject **freelist)
+ int *levels, char *msgbuf, size_t bufsize, freelist_t *freelist)
{
char *msg;
const char *format = *p_format;
@@ -539,7 +557,7 @@
static char *
convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags,
- char *msgbuf, size_t bufsize, PyObject **freelist)
+ char *msgbuf, size_t bufsize, freelist_t *freelist)
{
/* For # codes */
#define FETCH_SIZE int *q=NULL;Py_ssize_t *q2=NULL;\
@@ -1501,7 +1519,9 @@
const char *fname, *msg, *custom_msg, *keyword;
int min = INT_MAX;
int i, len, nargs, nkeywords;
- PyObject *freelist = NULL, *current_arg;
+ PyObject *current_arg;
+ freelist_t freelist = {0, NULL};
+
assert(args != NULL && PyTuple_Check(args));
assert(keywords == NULL || PyDict_Check(keywords));
@@ -1525,6 +1545,8 @@
for (len=0; kwlist[len]; len++)
continue;
+ freelist.entries = PyMem_New(freelistentry_t, len);
+
nargs = PyTuple_GET_SIZE(args);
nkeywords = (keywords == NULL) ? 0 : PyDict_Size(keywords);
if (nargs + nkeywords > len) {
@@ -1535,7 +1557,7 @@
len,
(len == 1) ? "" : "s",
nargs + nkeywords);
- return 0;
+ return cleanreturn(0, &freelist);
}
/* convert tuple args and keyword args in same loop, using kwlist to drive process */
@@ -1549,7 +1571,7 @@
PyErr_Format(PyExc_RuntimeError,
"More keyword list entries (%d) than "
"format specifiers (%d)", len, i);
- return cleanreturn(0, freelist);
+ return cleanreturn(0, &freelist);
}
current_arg = NULL;
if (nkeywords) {
@@ -1563,11 +1585,11 @@
"Argument given by name ('%s') "
"and position (%d)",
keyword, i+1);
- return cleanreturn(0, freelist);
+ return cleanreturn(0, &freelist);
}
}
else if (nkeywords && PyErr_Occurred())
- return cleanreturn(0, freelist);
+ return cleanreturn(0, &freelist);
else if (i < nargs)
current_arg = PyTuple_GET_ITEM(args, i);
@@ -1576,7 +1598,7 @@
levels, msgbuf, sizeof(msgbuf), &freelist);
if (msg) {
seterror(i+1, msg, levels, fname, custom_msg);
- return cleanreturn(0, freelist);
+ return cleanreturn(0, &freelist);
}
continue;
}
@@ -1585,14 +1607,14 @@
PyErr_Format(PyExc_TypeError, "Required argument "
"'%s' (pos %d) not found",
keyword, i+1);
- return cleanreturn(0, freelist);
+ return cleanreturn(0, &freelist);
}
/* current code reports success when all required args
* fulfilled and no keyword args left, with no further
* validation. XXX Maybe skip this in debug build ?
*/
if (!nkeywords)
- return cleanreturn(1, freelist);
+ return cleanreturn(1, &freelist);
/* We are into optional args, skip thru to any remaining
* keyword args */
@@ -1600,7 +1622,7 @@
if (msg) {
PyErr_Format(PyExc_RuntimeError, "%s: '%s'", msg,
format);
- return cleanreturn(0, freelist);
+ return cleanreturn(0, &freelist);
}
}
@@ -1608,7 +1630,7 @@
PyErr_Format(PyExc_RuntimeError,
"more argument specifiers than keyword list entries "
"(remaining format:'%s')", format);
- return cleanreturn(0, freelist);
+ return cleanreturn(0, &freelist);
}
/* make sure there are no extraneous keyword arguments */
@@ -1621,7 +1643,7 @@
if (!PyString_Check(key)) {
PyErr_SetString(PyExc_TypeError,
"keywords must be strings");
- return cleanreturn(0, freelist);
+ return cleanreturn(0, &freelist);
}
ks = PyString_AsString(key);
for (i = 0; i < len; i++) {
@@ -1635,12 +1657,12 @@
"'%s' is an invalid keyword "
"argument for this function",
ks);
- return cleanreturn(0, freelist);
+ return cleanreturn(0, &freelist);
}
}
}
- return cleanreturn(1, freelist);
+ return cleanreturn(1, &freelist);
}
diff --git a/pypy/module/cpyext/stringobject.py b/pypy/module/cpyext/stringobject.py
--- a/pypy/module/cpyext/stringobject.py
+++ b/pypy/module/cpyext/stringobject.py
@@ -130,6 +130,11 @@
@cpython_api([PyObject], rffi.CCHARP, error=0)
def PyString_AsString(space, ref):
+ if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_str:
+ pass # typecheck returned "ok" without forcing 'ref' at all
+ elif not PyString_Check(space, ref): # otherwise, use the alternate way
+ raise OperationError(space.w_TypeError, space.wrap(
+ "PyString_AsString only support strings"))
ref_str = rffi.cast(PyStringObject, ref)
if not ref_str.c_buffer:
# copy string buffer
diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py
--- a/pypy/module/cpyext/test/conftest.py
+++ b/pypy/module/cpyext/test/conftest.py
@@ -10,7 +10,7 @@
return False
def pytest_funcarg__space(request):
- return gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi'])
+ return gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi', 'array'])
def pytest_funcarg__api(request):
return request.cls.api
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -19,7 +19,8 @@
class BaseApiTest(LeakCheckingTest):
def setup_class(cls):
- cls.space = space = gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi'])
+ cls.space = space = gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi',
+ 'array'])
# warm up reference counts:
# - the posix module allocates a HCRYPTPROV on Windows
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -1,3 +1,4 @@
+from pypy.conftest import gettestobjspace
from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
import py
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -35,7 +35,7 @@
class AppTestApi:
def setup_class(cls):
- cls.space = gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi'])
+ cls.space = gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi', 'array'])
from pypy.rlib.libffi import get_libc_name
cls.w_libc = cls.space.wrap(get_libc_name())
@@ -106,10 +106,7 @@
del obj
import gc; gc.collect()
- try:
- del space.getexecutioncontext().cpyext_threadstate
- except AttributeError:
- pass
+ space.getexecutioncontext().cleanup_cpyext_state()
for w_obj in state.non_heaptypes_w:
Py_DecRef(space, w_obj)
@@ -168,8 +165,9 @@
return leaking
class AppTestCpythonExtensionBase(LeakCheckingTest):
+
def setup_class(cls):
- cls.space = gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi'])
+ cls.space = gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi', 'array'])
cls.space.getbuiltinmodule("cpyext")
from pypy.module.imp.importing import importhook
importhook(cls.space, "os") # warm up reference counts
diff --git a/pypy/module/cpyext/test/test_import.py b/pypy/module/cpyext/test/test_import.py
--- a/pypy/module/cpyext/test/test_import.py
+++ b/pypy/module/cpyext/test/test_import.py
@@ -19,7 +19,7 @@
space.wrap('__name__'))) == 'foobar'
def test_getmoduledict(self, space, api):
- testmod = "binascii"
+ testmod = "_functools"
w_pre_dict = api.PyImport_GetModuleDict()
assert not space.is_true(space.contains(w_pre_dict, space.wrap(testmod)))
diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py
--- a/pypy/module/cpyext/test/test_longobject.py
+++ b/pypy/module/cpyext/test/test_longobject.py
@@ -101,9 +101,9 @@
space.wrap((2, 7)))):
py.test.skip("unsupported before Python 2.7")
- assert api._PyLong_Sign(space.wrap(0L)) == 0
- assert api._PyLong_Sign(space.wrap(2L)) == 1
- assert api._PyLong_Sign(space.wrap(-2L)) == -1
+ assert api._PyLong_Sign(space.wraplong(0L)) == 0
+ assert api._PyLong_Sign(space.wraplong(2L)) == 1
+ assert api._PyLong_Sign(space.wraplong(-2L)) == -1
assert api._PyLong_NumBits(space.wrap(0)) == 0
assert api._PyLong_NumBits(space.wrap(1)) == 1
diff --git a/pypy/module/cpyext/test/test_number.py b/pypy/module/cpyext/test/test_number.py
--- a/pypy/module/cpyext/test/test_number.py
+++ b/pypy/module/cpyext/test/test_number.py
@@ -6,12 +6,12 @@
class TestIterator(BaseApiTest):
def test_check(self, space, api):
assert api.PyIndex_Check(space.wrap(12))
- assert api.PyIndex_Check(space.wrap(-12L))
+ assert api.PyIndex_Check(space.wraplong(-12L))
assert not api.PyIndex_Check(space.wrap(12.1))
assert not api.PyIndex_Check(space.wrap('12'))
assert api.PyNumber_Check(space.wrap(12))
- assert api.PyNumber_Check(space.wrap(-12L))
+ assert api.PyNumber_Check(space.wraplong(-12L))
assert api.PyNumber_Check(space.wrap(12.1))
assert not api.PyNumber_Check(space.wrap('12'))
assert not api.PyNumber_Check(space.wrap(1+3j))
@@ -21,7 +21,7 @@
assert api.PyLong_CheckExact(w_l)
def test_number_int(self, space, api):
- w_l = api.PyNumber_Int(space.wrap(123L))
+ w_l = api.PyNumber_Int(space.wraplong(123L))
assert api.PyInt_CheckExact(w_l)
w_l = api.PyNumber_Int(space.wrap(2 << 65))
assert api.PyLong_CheckExact(w_l)
@@ -29,7 +29,7 @@
assert api.PyInt_CheckExact(w_l)
def test_number_index(self, space, api):
- w_l = api.PyNumber_Index(space.wrap(123L))
+ w_l = api.PyNumber_Index(space.wraplong(123L))
assert api.PyLong_CheckExact(w_l)
w_l = api.PyNumber_Index(space.wrap(42.3))
assert w_l is None
diff --git a/pypy/module/cpyext/test/test_pystate.py b/pypy/module/cpyext/test/test_pystate.py
--- a/pypy/module/cpyext/test/test_pystate.py
+++ b/pypy/module/cpyext/test/test_pystate.py
@@ -3,6 +3,10 @@
from pypy.rpython.lltypesystem.lltype import nullptr
from pypy.module.cpyext.pystate import PyInterpreterState, PyThreadState
from pypy.module.cpyext.pyobject import from_ref
+from pypy.rpython.lltypesystem import lltype
+from pypy.module.cpyext.test.test_cpyext import LeakCheckingTest, freeze_refcnts
+from pypy.module.cpyext.pystate import PyThreadState_Get, PyInterpreterState_Head
+from pypy.tool import leakfinder
class AppTestThreads(AppTestCpythonExtensionBase):
def test_allow_threads(self):
@@ -21,6 +25,93 @@
# Should compile at least
module.test()
+
+ def test_thread_state_get(self):
+ module = self.import_extension('foo', [
+ ("get", "METH_NOARGS",
+ """
+ PyThreadState *tstate = PyThreadState_Get();
+ if (tstate == NULL) {
+ return PyLong_FromLong(0);
+ }
+ if (tstate->interp != PyInterpreterState_Head()) {
+ return PyLong_FromLong(1);
+ }
+ if (tstate->interp->next != NULL) {
+ return PyLong_FromLong(2);
+ }
+ return PyLong_FromLong(3);
+ """),
+ ])
+ assert module.get() == 3
+
+ def test_basic_threadstate_dance(self):
+ module = self.import_extension('foo', [
+ ("dance", "METH_NOARGS",
+ """
+ PyThreadState *old_tstate, *new_tstate;
+
+ old_tstate = PyThreadState_Swap(NULL);
+ if (old_tstate == NULL) {
+ return PyLong_FromLong(0);
+ }
+
+ new_tstate = PyThreadState_Get();
+ if (new_tstate != NULL) {
+ return PyLong_FromLong(1);
+ }
+
+ new_tstate = PyThreadState_Swap(old_tstate);
+ if (new_tstate != NULL) {
+ return PyLong_FromLong(2);
+ }
+
+ new_tstate = PyThreadState_Get();
+ if (new_tstate != old_tstate) {
+ return PyLong_FromLong(3);
+ }
+
+ return PyLong_FromLong(4);
+ """),
+ ])
+ assert module.dance() == 4
+
+ def test_threadstate_dict(self):
+ module = self.import_extension('foo', [
+ ("getdict", "METH_NOARGS",
+ """
+ PyObject *dict = PyThreadState_GetDict();
+ Py_INCREF(dict);
+ return dict;
+ """),
+ ])
+ assert isinstance(module.getdict(), dict)
+
+ def test_savethread(self):
+ module = self.import_extension('foo', [
+ ("bounce", "METH_NOARGS",
+ """
+ PyThreadState *tstate = PyEval_SaveThread();
+ if (tstate == NULL) {
+ return PyLong_FromLong(0);
+ }
+
+ if (PyThreadState_Get() != NULL) {
+ return PyLong_FromLong(1);
+ }
+
+ PyEval_RestoreThread(tstate);
+
+ if (PyThreadState_Get() != tstate) {
+ return PyLong_FromLong(2);
+ }
+
+ return PyLong_FromLong(3);
+ """),
+ ])
+
+
+
class TestInterpreterState(BaseApiTest):
def test_interpreter_head(self, space, api):
state = api.PyInterpreterState_Head()
@@ -29,31 +120,3 @@
def test_interpreter_next(self, space, api):
state = api.PyInterpreterState_Head()
assert nullptr(PyInterpreterState.TO) == api.PyInterpreterState_Next(state)
-
-class TestThreadState(BaseApiTest):
- def test_thread_state_get(self, space, api):
- ts = api.PyThreadState_Get()
- assert ts != nullptr(PyThreadState.TO)
-
- def test_thread_state_interp(self, space, api):
- ts = api.PyThreadState_Get()
- assert ts.c_interp == api.PyInterpreterState_Head()
- assert ts.c_interp.c_next == nullptr(PyInterpreterState.TO)
-
- def test_basic_threadstate_dance(self, space, api):
- # Let extension modules call these functions,
- # Not sure of the semantics in pypy though.
- # (cpyext always acquires and releases the GIL around calls)
- tstate = api.PyThreadState_Swap(None)
- assert tstate is not None
- assert not api.PyThreadState_Swap(tstate)
-
- api.PyEval_AcquireThread(tstate)
- api.PyEval_ReleaseThread(tstate)
-
- def test_threadstate_dict(self, space, api):
- ts = api.PyThreadState_Get()
- ref = ts.c_dict
- assert ref == api.PyThreadState_GetDict()
- w_obj = from_ref(space, ref)
- assert space.isinstance_w(w_obj, space.w_dict)
diff --git a/pypy/module/cpyext/test/test_stringobject.py b/pypy/module/cpyext/test/test_stringobject.py
--- a/pypy/module/cpyext/test/test_stringobject.py
+++ b/pypy/module/cpyext/test/test_stringobject.py
@@ -105,6 +105,15 @@
)])
assert module.string_as_string("huheduwe") == "huhe"
+ def test_py_string_as_string_None(self):
+ module = self.import_extension('foo', [
+ ("string_None", "METH_VARARGS",
+ '''
+ return PyString_AsString(Py_None);
+ '''
+ )])
+ raises(TypeError, module.string_None)
+
def test_AsStringAndSize(self):
module = self.import_extension('foo', [
("getstring", "METH_NOARGS",
diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py
--- a/pypy/module/fcntl/test/test_fcntl.py
+++ b/pypy/module/fcntl/test/test_fcntl.py
@@ -13,7 +13,7 @@
class AppTestFcntl:
def setup_class(cls):
- space = gettestobjspace(usemodules=('fcntl', 'array'))
+ space = gettestobjspace(usemodules=('fcntl', 'array', 'struct'))
cls.space = space
tmpprefix = str(udir.ensure('test_fcntl', dir=1).join('tmp_'))
cls.w_tmp = space.wrap(tmpprefix)
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -987,6 +987,10 @@
os.environ['LANG'] = oldlang
class AppTestImportHooks(object):
+
+ def setup_class(cls):
+ cls.space = gettestobjspace(usemodules=('struct',))
+
def test_meta_path(self):
tried_imports = []
class Importer(object):
diff --git a/pypy/module/itertools/test/test_itertools.py b/pypy/module/itertools/test/test_itertools.py
--- a/pypy/module/itertools/test/test_itertools.py
+++ b/pypy/module/itertools/test/test_itertools.py
@@ -891,7 +891,7 @@
class AppTestItertools27:
def setup_class(cls):
- cls.space = gettestobjspace(usemodules=['itertools'])
+ cls.space = gettestobjspace(usemodules=['itertools', 'struct'])
if cls.space.is_true(cls.space.appexec([], """():
import sys; return sys.version_info < (2, 7)
""")):
diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -327,8 +327,10 @@
# %r not supported in rpython
#u.raise_exc('invalid typecode in unmarshal: %r' % tc)
c = ord(tc)
- if c < 32 or c > 126:
- s = '\\x' + hex(c)
+ if c < 16:
+ s = '\\x0%x' % c
+ elif c < 32 or c > 126:
+ s = '\\x%x' % c
elif tc == '\\':
s = r'\\'
else:
diff --git a/pypy/module/marshal/test/make_test_marshal.py b/pypy/module/marshal/test/make_test_marshal.py
deleted file mode 100644
--- a/pypy/module/marshal/test/make_test_marshal.py
+++ /dev/null
@@ -1,78 +0,0 @@
-
-TESTCASES = """\
- None
- False
- True
- StopIteration
- Ellipsis
- 42
- -17
- sys.maxint
- -1.25
- -1.25 #2
- 2+5j
- 2+5j #2
- 42L
- -1234567890123456789012345678901234567890L
- hello # not interned
- "hello"
- ()
- (1, 2)
- []
- [3, 4]
- {}
- {5: 6, 7: 8}
- func.func_code
- scopefunc.func_code
- u'hello'
- set()
- set([1, 2])
- frozenset()
- frozenset([3, 4])
-""".strip().split('\n')
-
-def readable(s):
- for c, repl in (
- ("'", '_quote_'), ('"', '_Quote_'), (':', '_colon_'), ('.', '_dot_'),
- ('[', '_list_'), (']', '_tsil_'), ('{', '_dict_'), ('}', '_tcid_'),
- ('-', '_minus_'), ('+', '_plus_'),
- (',', '_comma_'), ('(', '_brace_'), (')', '_ecarb_') ):
- s = s.replace(c, repl)
- lis = list(s)
- for i, c in enumerate(lis):
- if c.isalnum() or c == '_':
- continue
- lis[i] = '_'
- return ''.join(lis)
-
-print """class AppTestMarshal:
-"""
-for line in TESTCASES:
- line = line.strip()
- name = readable(line)
- version = ''
- extra = ''
- if line.endswith('#2'):
- version = ', 2'
- extra = '; assert len(s) in (9, 17)'
- src = '''\
- def test_%(name)s(self):
- import sys
- hello = "he"
- hello += "llo"
- def func(x):
- return lambda y: x+y
- scopefunc = func(42)
- import marshal, StringIO
- case = %(line)s
- print "case: %%-30s func=%(name)s" %% (case, )
- s = marshal.dumps(case%(version)s)%(extra)s
- x = marshal.loads(s)
- assert x == case
- f = StringIO.StringIO()
- marshal.dump(case, f)
- f.seek(0)
- x = marshal.load(f)
- assert x == case
-''' % {'name': name, 'line': line, 'version' : version, 'extra': extra}
- print src
diff --git a/pypy/module/marshal/test/test_marshal.py b/pypy/module/marshal/test/test_marshal.py
--- a/pypy/module/marshal/test/test_marshal.py
+++ b/pypy/module/marshal/test/test_marshal.py
@@ -174,6 +174,11 @@
pass
raises(ValueError, marshal.dumps, subtype)
+ def test_bad_typecode(self):
+ import marshal
+ exc = raises(ValueError, marshal.loads, chr(1))
+ assert r"'\x01'" in exc.value.message
+
class AppTestRope(AppTestMarshal):
def setup_class(cls):
diff --git a/pypy/module/math/test/test_direct.py b/pypy/module/math/test/test_direct.py
--- a/pypy/module/math/test/test_direct.py
+++ b/pypy/module/math/test/test_direct.py
@@ -55,6 +55,15 @@
('frexp', (-1.25,), lambda x: x == (-0.625, 1)),
('modf', (4.25,), lambda x: x == (0.25, 4.0)),
('modf', (-4.25,), lambda x: x == (-0.25, -4.0)),
+ ('copysign', (1.5, 0.0), 1.5),
+ ('copysign', (1.5, -0.0), -1.5),
+ ('copysign', (1.5, INFINITY), 1.5),
+ ('copysign', (1.5, -INFINITY), -1.5),
+ ]
+ if sys.platform != 'win32': # all NaNs seem to be negative there...?
+ IRREGCASES += [
+ ('copysign', (1.5, NAN), 1.5),
+ ('copysign', (1.75, -NAN), -1.75), # special case for -NAN here
]
OVFCASES = [
diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py
--- a/pypy/module/math/test/test_math.py
+++ b/pypy/module/math/test/test_math.py
@@ -1,3 +1,4 @@
+from __future__ import with_statement
import sys
from pypy.conftest import gettestobjspace
from pypy.module.math.test import test_direct
@@ -5,7 +6,7 @@
class AppTestMath:
def setup_class(cls):
- cls.space = gettestobjspace(usemodules=['math'])
+ cls.space = gettestobjspace(usemodules=['math', 'struct'])
cls.w_cases = cls.space.wrap(test_direct.MathTests.TESTCASES)
cls.w_consistent_host = cls.space.wrap(test_direct.consistent_host)
@@ -268,3 +269,7 @@
def __trunc__(self):
return "truncated"
assert math.trunc(foo()) == "truncated"
+
+ def test_copysign_nan(self):
+ import math
+ assert math.copysign(1.0, float('-nan')) == -1.0
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -37,26 +37,44 @@
'True_': 'types.Bool.True',
'False_': 'types.Bool.False',
+ 'typeinfo': 'interp_dtype.get_dtype_cache(space).w_typeinfo',
+
'generic': 'interp_boxes.W_GenericBox',
'number': 'interp_boxes.W_NumberBox',
'integer': 'interp_boxes.W_IntegerBox',
'signedinteger': 'interp_boxes.W_SignedIntegerBox',
'unsignedinteger': 'interp_boxes.W_UnsignedIntegerBox',
'bool_': 'interp_boxes.W_BoolBox',
+ 'bool8': 'interp_boxes.W_BoolBox',
'int8': 'interp_boxes.W_Int8Box',
+ 'byte': 'interp_boxes.W_Int8Box',
'uint8': 'interp_boxes.W_UInt8Box',
+ 'ubyte': 'interp_boxes.W_UInt8Box',
'int16': 'interp_boxes.W_Int16Box',
+ 'short': 'interp_boxes.W_Int16Box',
'uint16': 'interp_boxes.W_UInt16Box',
+ 'ushort': 'interp_boxes.W_UInt16Box',
'int32': 'interp_boxes.W_Int32Box',
+ 'intc': 'interp_boxes.W_Int32Box',
'uint32': 'interp_boxes.W_UInt32Box',
+ 'uintc': 'interp_boxes.W_UInt32Box',
'int64': 'interp_boxes.W_Int64Box',
'uint64': 'interp_boxes.W_UInt64Box',
+ 'longlong': 'interp_boxes.W_LongLongBox',
+ 'ulonglong': 'interp_boxes.W_ULongLongBox',
'int_': 'interp_boxes.W_LongBox',
'inexact': 'interp_boxes.W_InexactBox',
'floating': 'interp_boxes.W_FloatingBox',
'float_': 'interp_boxes.W_Float64Box',
'float32': 'interp_boxes.W_Float32Box',
'float64': 'interp_boxes.W_Float64Box',
+ 'intp': 'types.IntP.BoxType',
+ 'uintp': 'types.UIntP.BoxType',
+ 'flexible': 'interp_boxes.W_FlexibleBox',
+ 'character': 'interp_boxes.W_CharacterBox',
+ 'str_': 'interp_boxes.W_StringBox',
+ 'unicode_': 'interp_boxes.W_UnicodeBox',
+ 'void': 'interp_boxes.W_VoidBox',
}
# ufuncs
@@ -67,6 +85,7 @@
("arccos", "arccos"),
("arcsin", "arcsin"),
("arctan", "arctan"),
+ ("arctan2", "arctan2"),
("arccosh", "arccosh"),
("arcsinh", "arcsinh"),
("arctanh", "arctanh"),
@@ -77,7 +96,10 @@
("true_divide", "true_divide"),
("equal", "equal"),
("exp", "exp"),
+ ("exp2", "exp2"),
+ ("expm1", "expm1"),
("fabs", "fabs"),
+ ("fmod", "fmod"),
("floor", "floor"),
("ceil", "ceil"),
("greater", "greater"),
@@ -89,8 +111,13 @@
("multiply", "multiply"),
("negative", "negative"),
("not_equal", "not_equal"),
+ ("radians", "radians"),
+ ("degrees", "degrees"),
+ ("deg2rad", "radians"),
+ ("rad2deg", "degrees"),
("reciprocal", "reciprocal"),
("sign", "sign"),
+ ("signbit", "signbit"),
("sin", "sin"),
("sinh", "sinh"),
("subtract", "subtract"),
@@ -103,10 +130,21 @@
('bitwise_not', 'invert'),
('isnan', 'isnan'),
('isinf', 'isinf'),
+ ('isneginf', 'isneginf'),
+ ('isposinf', 'isposinf'),
+ ('isfinite', 'isfinite'),
('logical_and', 'logical_and'),
('logical_xor', 'logical_xor'),
('logical_not', 'logical_not'),
('logical_or', 'logical_or'),
+ ('log', 'log'),
+ ('log2', 'log2'),
+ ('log10', 'log10'),
+ ('log1p', 'log1p'),
+ ('power', 'power'),
+ ('floor_divide', 'floor_divide'),
+ ('logaddexp', 'logaddexp'),
+ ('logaddexp2', 'logaddexp2'),
]:
interpleveldefs[exposed] = "interp_ufuncs.get(space).%s" % impl
diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py
--- a/pypy/module/micronumpy/app_numpy.py
+++ b/pypy/module/micronumpy/app_numpy.py
@@ -16,7 +16,7 @@
a[i][i] = 1
return a
-def sum(a,axis=None):
+def sum(a,axis=None, out=None):
'''sum(a, axis=None)
Sum of array elements over a given axis.
@@ -43,17 +43,17 @@
# TODO: add to doc (once it's implemented): cumsum : Cumulative sum of array elements.
if not hasattr(a, "sum"):
a = _numpypy.array(a)
- return a.sum(axis)
+ return a.sum(axis=axis, out=out)
-def min(a, axis=None):
+def min(a, axis=None, out=None):
if not hasattr(a, "min"):
a = _numpypy.array(a)
- return a.min(axis)
+ return a.min(axis=axis, out=out)
-def max(a, axis=None):
+def max(a, axis=None, out=None):
if not hasattr(a, "max"):
a = _numpypy.array(a)
- return a.max(axis)
+ return a.max(axis=axis, out=out)
def arange(start, stop=None, step=1, dtype=None):
'''arange([start], stop[, step], dtype=None)
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -33,7 +33,7 @@
pass
SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any",
- "unegative", "flat"]
+ "unegative", "flat", "tostring"]
TWO_ARG_FUNCTIONS = ["dot", 'take']
class FakeSpace(object):
@@ -51,6 +51,8 @@
w_long = "long"
w_tuple = 'tuple'
w_slice = "slice"
+ w_str = "str"
+ w_unicode = "unicode"
def __init__(self):
"""NOT_RPYTHON"""
@@ -91,8 +93,12 @@
return BoolObject(obj)
elif isinstance(obj, int):
return IntObject(obj)
+ elif isinstance(obj, long):
+ return LongObject(obj)
elif isinstance(obj, W_Root):
return obj
+ elif isinstance(obj, str):
+ return StringObject(obj)
raise NotImplementedError
def newlist(self, items):
@@ -120,6 +126,11 @@
return int(w_obj.floatval)
raise NotImplementedError
+ def str_w(self, w_obj):
+ if isinstance(w_obj, StringObject):
+ return w_obj.v
+ raise NotImplementedError
+
def int(self, w_obj):
if isinstance(w_obj, IntObject):
return w_obj
@@ -151,7 +162,13 @@
return instantiate(klass)
def newtuple(self, list_w):
- raise ValueError
+ return ListObject(list_w)
+
+ def newdict(self):
+ return {}
+
+ def setitem(self, dict, item, value):
+ dict[item] = value
def len_w(self, w_obj):
if isinstance(w_obj, ListObject):
@@ -178,6 +195,11 @@
def __init__(self, intval):
self.intval = intval
+class LongObject(W_Root):
+ tp = FakeSpace.w_long
+ def __init__(self, intval):
+ self.intval = intval
+
class ListObject(W_Root):
tp = FakeSpace.w_list
def __init__(self, items):
@@ -190,6 +212,11 @@
self.stop = stop
self.step = step
+class StringObject(W_Root):
+ tp = FakeSpace.w_str
+ def __init__(self, v):
+ self.v = v
+
class InterpreterState(object):
def __init__(self, code):
self.code = code
@@ -407,6 +434,9 @@
w_res = neg.call(interp.space, [arr])
elif self.name == "flat":
w_res = arr.descr_get_flatiter(interp.space)
+ elif self.name == "tostring":
+ arr.descr_tostring(interp.space)
+ w_res = None
else:
assert False # unreachable code
elif self.name in TWO_ARG_FUNCTIONS:
diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -1,24 +1,25 @@
from pypy.interpreter.baseobjspace import Wrappable
-from pypy.interpreter.error import operationerrfmt
+from pypy.interpreter.error import operationerrfmt, OperationError
from pypy.interpreter.gateway import interp2app, unwrap_spec
from pypy.interpreter.typedef import TypeDef
from pypy.objspace.std.floattype import float_typedef
+from pypy.objspace.std.stringtype import str_typedef
+from pypy.objspace.std.unicodetype import unicode_typedef, unicode_from_object
from pypy.objspace.std.inttype import int_typedef
from pypy.rlib.rarithmetic import LONG_BIT
from pypy.tool.sourcetools import func_with_new_name
-
MIXIN_32 = (int_typedef,) if LONG_BIT == 32 else ()
MIXIN_64 = (int_typedef,) if LONG_BIT == 64 else ()
def new_dtype_getter(name):
- def get_dtype(space):
+ def _get_dtype(space):
from pypy.module.micronumpy.interp_dtype import get_dtype_cache
return getattr(get_dtype_cache(space), "w_%sdtype" % name)
def new(space, w_subtype, w_value):
- dtype = get_dtype(space)
+ dtype = _get_dtype(space)
return dtype.itemtype.coerce_subtype(space, w_subtype, w_value)
- return func_with_new_name(new, name + "_box_new"), staticmethod(get_dtype)
+ return func_with_new_name(new, name + "_box_new"), staticmethod(_get_dtype)
class PrimitiveBox(object):
_mixin_ = True
@@ -37,6 +38,9 @@
w_subtype.getname(space, '?')
)
+ def get_dtype(self, space):
+ return self._get_dtype(space)
+
def descr_str(self, space):
return space.wrap(self.get_dtype(space).itemtype.str_format(self))
@@ -44,12 +48,12 @@
return space.format(self.item(space), w_spec)
def descr_int(self, space):
- box = self.convert_to(W_LongBox.get_dtype(space))
+ box = self.convert_to(W_LongBox._get_dtype(space))
assert isinstance(box, W_LongBox)
return space.wrap(box.value)
def descr_float(self, space):
- box = self.convert_to(W_Float64Box.get_dtype(space))
+ box = self.convert_to(W_Float64Box._get_dtype(space))
assert isinstance(box, W_Float64Box)
return space.wrap(box.value)
@@ -58,21 +62,24 @@
return space.wrap(dtype.itemtype.bool(self))
def _binop_impl(ufunc_name):
- def impl(self, space, w_other):
+ def impl(self, space, w_other, w_out=None):
from pypy.module.micronumpy import interp_ufuncs
- return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [self, w_other])
+ return getattr(interp_ufuncs.get(space), ufunc_name).call(space,
+ [self, w_other, w_out])
return func_with_new_name(impl, "binop_%s_impl" % ufunc_name)
def _binop_right_impl(ufunc_name):
- def impl(self, space, w_other):
+ def impl(self, space, w_other, w_out=None):
from pypy.module.micronumpy import interp_ufuncs
- return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [w_other, self])
+ return getattr(interp_ufuncs.get(space), ufunc_name).call(space,
+ [w_other, self, w_out])
return func_with_new_name(impl, "binop_right_%s_impl" % ufunc_name)
def _unaryop_impl(ufunc_name):
- def impl(self, space):
+ def impl(self, space, w_out=None):
from pypy.module.micronumpy import interp_ufuncs
- return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [self])
+ return getattr(interp_ufuncs.get(space), ufunc_name).call(space,
+ [self, w_out])
return func_with_new_name(impl, "unaryop_%s_impl" % ufunc_name)
descr_add = _binop_impl("add")
@@ -80,6 +87,7 @@
descr_mul = _binop_impl("multiply")
descr_div = _binop_impl("divide")
descr_truediv = _binop_impl("true_divide")
+ descr_floordiv = _binop_impl("floor_divide")
descr_mod = _binop_impl("mod")
descr_pow = _binop_impl("power")
descr_lshift = _binop_impl("left_shift")
@@ -100,6 +108,7 @@
descr_rmul = _binop_right_impl("multiply")
descr_rdiv = _binop_right_impl("divide")
descr_rtruediv = _binop_right_impl("true_divide")
+ descr_rfloordiv = _binop_right_impl("floor_divide")
descr_rmod = _binop_right_impl("mod")
descr_rpow = _binop_right_impl("power")
descr_rlshift = _binop_right_impl("left_shift")
@@ -128,7 +137,7 @@
class W_BoolBox(W_GenericBox, PrimitiveBox):
- descr__new__, get_dtype = new_dtype_getter("bool")
+ descr__new__, _get_dtype = new_dtype_getter("bool")
class W_NumberBox(W_GenericBox):
_attrs_ = ()
@@ -144,34 +153,40 @@
pass
class W_Int8Box(W_SignedIntegerBox, PrimitiveBox):
- descr__new__, get_dtype = new_dtype_getter("int8")
+ descr__new__, _get_dtype = new_dtype_getter("int8")
class W_UInt8Box(W_UnsignedIntegerBox, PrimitiveBox):
- descr__new__, get_dtype = new_dtype_getter("uint8")
+ descr__new__, _get_dtype = new_dtype_getter("uint8")
class W_Int16Box(W_SignedIntegerBox, PrimitiveBox):
- descr__new__, get_dtype = new_dtype_getter("int16")
+ descr__new__, _get_dtype = new_dtype_getter("int16")
class W_UInt16Box(W_UnsignedIntegerBox, PrimitiveBox):
- descr__new__, get_dtype = new_dtype_getter("uint16")
+ descr__new__, _get_dtype = new_dtype_getter("uint16")
class W_Int32Box(W_SignedIntegerBox, PrimitiveBox):
- descr__new__, get_dtype = new_dtype_getter("int32")
+ descr__new__, _get_dtype = new_dtype_getter("int32")
class W_UInt32Box(W_UnsignedIntegerBox, PrimitiveBox):
- descr__new__, get_dtype = new_dtype_getter("uint32")
+ descr__new__, _get_dtype = new_dtype_getter("uint32")
class W_LongBox(W_SignedIntegerBox, PrimitiveBox):
- descr__new__, get_dtype = new_dtype_getter("long")
+ descr__new__, _get_dtype = new_dtype_getter("long")
class W_ULongBox(W_UnsignedIntegerBox, PrimitiveBox):
- descr__new__, get_dtype = new_dtype_getter("ulong")
+ descr__new__, _get_dtype = new_dtype_getter("ulong")
class W_Int64Box(W_SignedIntegerBox, PrimitiveBox):
- descr__new__, get_dtype = new_dtype_getter("int64")
+ descr__new__, _get_dtype = new_dtype_getter("int64")
+
+class W_LongLongBox(W_SignedIntegerBox, PrimitiveBox):
+ descr__new__, _get_dtype = new_dtype_getter('longlong')
class W_UInt64Box(W_UnsignedIntegerBox, PrimitiveBox):
- descr__new__, get_dtype = new_dtype_getter("uint64")
+ descr__new__, _get_dtype = new_dtype_getter("uint64")
+
+class W_ULongLongBox(W_SignedIntegerBox, PrimitiveBox):
+ descr__new__, _get_dtype = new_dtype_getter('ulonglong')
class W_InexactBox(W_NumberBox):
_attrs_ = ()
@@ -180,16 +195,71 @@
_attrs_ = ()
class W_Float32Box(W_FloatingBox, PrimitiveBox):
- descr__new__, get_dtype = new_dtype_getter("float32")
+ descr__new__, _get_dtype = new_dtype_getter("float32")
class W_Float64Box(W_FloatingBox, PrimitiveBox):
- descr__new__, get_dtype = new_dtype_getter("float64")
+ descr__new__, _get_dtype = new_dtype_getter("float64")
+class W_FlexibleBox(W_GenericBox):
+ def __init__(self, arr, ofs, dtype):
+ self.arr = arr # we have to keep array alive
+ self.ofs = ofs
+ self.dtype = dtype
+
+ def get_dtype(self, space):
+ return self.arr.dtype
+
@unwrap_spec(self=W_GenericBox)
def descr_index(space, self):
return space.index(self.item(space))
+class W_VoidBox(W_FlexibleBox):
+ @unwrap_spec(item=str)
+ def descr_getitem(self, space, item):
+ try:
+ ofs, dtype = self.dtype.fields[item]
+ except KeyError:
+ raise OperationError(space.w_IndexError,
+ space.wrap("Field %s does not exist" % item))
+ return dtype.itemtype.read(self.arr, 1, self.ofs, ofs, dtype)
+
+ @unwrap_spec(item=str)
+ def descr_setitem(self, space, item, w_value):
+ try:
+ ofs, dtype = self.dtype.fields[item]
+ except KeyError:
+ raise OperationError(space.w_IndexError,
+ space.wrap("Field %s does not exist" % item))
+ dtype.itemtype.store(self.arr, 1, self.ofs, ofs,
+ dtype.coerce(space, w_value))
+
+class W_CharacterBox(W_FlexibleBox):
+ pass
+
+class W_StringBox(W_CharacterBox):
+ def descr__new__string_box(space, w_subtype, w_arg):
+ from pypy.module.micronumpy.interp_numarray import W_NDimArray
+ from pypy.module.micronumpy.interp_dtype import new_string_dtype
+
+ arg = space.str_w(space.str(w_arg))
+ arr = W_NDimArray([1], new_string_dtype(space, len(arg)))
+ for i in range(len(arg)):
+ arr.storage[i] = arg[i]
+ return W_StringBox(arr, 0, arr.dtype)
+
+
+class W_UnicodeBox(W_CharacterBox):
+ def descr__new__unicode_box(space, w_subtype, w_arg):
+ from pypy.module.micronumpy.interp_numarray import W_NDimArray
+ from pypy.module.micronumpy.interp_dtype import new_unicode_dtype
+
+ arg = space.unicode_w(unicode_from_object(space, w_arg))
+ arr = W_NDimArray([1], new_unicode_dtype(space, len(arg)))
+ # XXX not this way, we need store
+ #for i in range(len(arg)):
+ # arr.storage[i] = arg[i]
+ return W_UnicodeBox(arr, 0, arr.dtype)
W_GenericBox.typedef = TypeDef("generic",
__module__ = "numpypy",
@@ -208,6 +278,7 @@
__mul__ = interp2app(W_GenericBox.descr_mul),
__div__ = interp2app(W_GenericBox.descr_div),
__truediv__ = interp2app(W_GenericBox.descr_truediv),
+ __floordiv__ = interp2app(W_GenericBox.descr_floordiv),
__mod__ = interp2app(W_GenericBox.descr_mod),
__divmod__ = interp2app(W_GenericBox.descr_divmod),
__pow__ = interp2app(W_GenericBox.descr_pow),
@@ -222,6 +293,7 @@
__rmul__ = interp2app(W_GenericBox.descr_rmul),
__rdiv__ = interp2app(W_GenericBox.descr_rdiv),
__rtruediv__ = interp2app(W_GenericBox.descr_rtruediv),
+ __rfloordiv__ = interp2app(W_GenericBox.descr_rfloordiv),
__rmod__ = interp2app(W_GenericBox.descr_rmod),
__rdivmod__ = interp2app(W_GenericBox.descr_rdivmod),
__rpow__ = interp2app(W_GenericBox.descr_rpow),
@@ -344,3 +416,28 @@
__new__ = interp2app(W_Float64Box.descr__new__.im_func),
)
+
+W_FlexibleBox.typedef = TypeDef("flexible", W_GenericBox.typedef,
+ __module__ = "numpypy",
+)
+
+W_VoidBox.typedef = TypeDef("void", W_FlexibleBox.typedef,
+ __module__ = "numpypy",
+ __getitem__ = interp2app(W_VoidBox.descr_getitem),
+ __setitem__ = interp2app(W_VoidBox.descr_setitem),
+)
+
+W_CharacterBox.typedef = TypeDef("character", W_FlexibleBox.typedef,
+ __module__ = "numpypy",
+)
+
+W_StringBox.typedef = TypeDef("string_", (str_typedef, W_CharacterBox.typedef),
+ __module__ = "numpypy",
+ __new__ = interp2app(W_StringBox.descr__new__string_box.im_func),
+)
+
+W_UnicodeBox.typedef = TypeDef("unicode_", (unicode_typedef, W_CharacterBox.typedef),
+ __module__ = "numpypy",
+ __new__ = interp2app(W_UnicodeBox.descr__new__unicode_box.im_func),
+)
+
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -1,26 +1,29 @@
+
+import sys
from pypy.interpreter.baseobjspace import Wrappable
from pypy.interpreter.error import OperationError
-from pypy.interpreter.gateway import interp2app
+from pypy.interpreter.gateway import interp2app, unwrap_spec
from pypy.interpreter.typedef import (TypeDef, GetSetProperty,
interp_attrproperty, interp_attrproperty_w)
-from pypy.module.micronumpy import types, signature, interp_boxes
+from pypy.module.micronumpy import types, interp_boxes
from pypy.rlib.objectmodel import specialize
-from pypy.rlib.rarithmetic import LONG_BIT
-from pypy.rpython.lltypesystem import lltype, rffi
+from pypy.rlib.rarithmetic import LONG_BIT, r_longlong, r_ulonglong
UNSIGNEDLTR = "u"
SIGNEDLTR = "i"
BOOLLTR = "b"
FLOATINGLTR = "f"
-
-
-VOID_STORAGE = lltype.Array(lltype.Char, hints={'nolength': True, 'render_as_void': True})
+VOIDLTR = 'V'
+STRINGLTR = 'S'
+UNICODELTR = 'U'
class W_Dtype(Wrappable):
_immutable_fields_ = ["itemtype", "num", "kind"]
- def __init__(self, itemtype, num, kind, name, char, w_box_type, alternate_constructors=[], aliases=[]):
+ def __init__(self, itemtype, num, kind, name, char, w_box_type,
+ alternate_constructors=[], aliases=[],
+ fields=None, fieldnames=None, native=True):
self.itemtype = itemtype
self.num = num
self.kind = kind
@@ -29,53 +32,28 @@
self.w_box_type = w_box_type
self.alternate_constructors = alternate_constructors
self.aliases = aliases
-
- def malloc(self, length):
- # XXX find out why test_zjit explodes with tracking of allocations
- return lltype.malloc(VOID_STORAGE, self.itemtype.get_element_size() * length,
- zero=True, flavor="raw",
- track_allocation=False, add_memory_pressure=True
- )
+ self.fields = fields
+ self.fieldnames = fieldnames
+ self.native = native
@specialize.argtype(1)
def box(self, value):
return self.itemtype.box(value)
def coerce(self, space, w_item):
- return self.itemtype.coerce(space, w_item)
+ return self.itemtype.coerce(space, self, w_item)
- def getitem(self, storage, i):
- return self.itemtype.read(storage, self.itemtype.get_element_size(), i, 0)
+ def getitem(self, arr, i):
+ return self.itemtype.read(arr, 1, i, 0)
- def getitem_bool(self, storage, i):
- isize = self.itemtype.get_element_size()
- return self.itemtype.read_bool(storage, isize, i, 0)
+ def getitem_bool(self, arr, i):
+ return self.itemtype.read_bool(arr, 1, i, 0)
- def setitem(self, storage, i, box):
- self.itemtype.store(storage, self.itemtype.get_element_size(), i, 0, box)
+ def setitem(self, arr, i, box):
+ self.itemtype.store(arr, 1, i, 0, box)
def fill(self, storage, box, start, stop):
- self.itemtype.fill(storage, self.itemtype.get_element_size(), box, start, stop, 0)
-
- def descr__new__(space, w_subtype, w_dtype):
- cache = get_dtype_cache(space)
-
- if space.is_w(w_dtype, space.w_None):
- return cache.w_float64dtype
- elif space.isinstance_w(w_dtype, w_subtype):
- return w_dtype
- elif space.isinstance_w(w_dtype, space.w_str):
- name = space.str_w(w_dtype)
- for dtype in cache.builtin_dtypes:
- if dtype.name == name or dtype.char == name or name in dtype.aliases:
- return dtype
- else:
- for dtype in cache.builtin_dtypes:
- if w_dtype in dtype.alternate_constructors:
- return dtype
- if w_dtype is dtype.w_box_type:
- return dtype
- raise OperationError(space.w_TypeError, space.wrap("data type not understood"))
+ self.itemtype.fill(storage, self.get_size(), box, start, stop, 0)
def descr_str(self, space):
return space.wrap(self.name)
@@ -86,6 +64,14 @@
def descr_get_itemsize(self, space):
return space.wrap(self.itemtype.get_element_size())
+ def descr_get_byteorder(self, space):
+ if self.native:
+ return space.wrap('=')
+ return space.wrap(nonnative_byteorder_prefix)
+
+ def descr_get_alignment(self, space):
+ return space.wrap(self.itemtype.alignment)
+
def descr_get_shape(self, space):
return space.newtuple([])
@@ -99,31 +85,193 @@
def descr_ne(self, space, w_other):
return space.wrap(not self.eq(space, w_other))
+ def descr_get_fields(self, space):
+ if self.fields is None:
+ return space.w_None
+ w_d = space.newdict()
+ for name, (offset, subdtype) in self.fields.iteritems():
+ space.setitem(w_d, space.wrap(name), space.newtuple([subdtype,
+ space.wrap(offset)]))
+ return w_d
+
+ def descr_get_names(self, space):
+ if self.fieldnames is None:
+ return space.w_None
+ return space.newtuple([space.wrap(name) for name in self.fieldnames])
+
+ @unwrap_spec(item=str)
+ def descr_getitem(self, space, item):
+ if self.fields is None:
+ raise OperationError(space.w_KeyError, space.wrap("There are no keys in dtypes %s" % self.name))
+ try:
+ return self.fields[item][1]
+ except KeyError:
+ raise OperationError(space.w_KeyError, space.wrap("Field named %s not found" % item))
+
def is_int_type(self):
return (self.kind == SIGNEDLTR or self.kind == UNSIGNEDLTR or
self.kind == BOOLLTR)
+ def is_signed(self):
+ return self.kind == SIGNEDLTR
+
def is_bool_type(self):
return self.kind == BOOLLTR
+ def is_record_type(self):
+ return self.fields is not None
+
+ def __repr__(self):
+ if self.fields is not None:
+ return '<DType %r>' % self.fields
+ return '<DType %r>' % self.itemtype
+
+ def get_size(self):
+ return self.itemtype.get_element_size()
+
+def dtype_from_list(space, w_lst):
+ lst_w = space.listview(w_lst)
+ fields = {}
+ offset = 0
+ ofs_and_items = []
+ fieldnames = []
+ for w_elem in lst_w:
+ w_fldname, w_flddesc = space.fixedview(w_elem, 2)
+ subdtype = descr__new__(space, space.gettypefor(W_Dtype), w_flddesc)
+ fldname = space.str_w(w_fldname)
+ if fldname in fields:
+ raise OperationError(space.w_ValueError, space.wrap("two fields with the same name"))
+ assert isinstance(subdtype, W_Dtype)
+ fields[fldname] = (offset, subdtype)
+ ofs_and_items.append((offset, subdtype.itemtype))
+ offset += subdtype.itemtype.get_element_size()
+ fieldnames.append(fldname)
+ itemtype = types.RecordType(ofs_and_items, offset)
+ return W_Dtype(itemtype, 20, VOIDLTR, "void" + str(8 * itemtype.get_element_size()),
+ "V", space.gettypefor(interp_boxes.W_VoidBox), fields=fields,
+ fieldnames=fieldnames)
+
+def dtype_from_dict(space, w_dict):
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ "dtype from dict"))
+
+def variable_dtype(space, name):
+ if name[0] in '<>=':
+ name = name[1:]
+ char = name[0]
+ if len(name) == 1:
+ size = 0
+ else:
+ try:
+ size = int(name[1:])
+ except ValueError:
+ raise OperationError(space.w_TypeError, space.wrap("data type not understood"))
+ if char == 'S':
+ itemtype = types.StringType(size)
+ basename = 'string'
+ num = 18
+ w_box_type = space.gettypefor(interp_boxes.W_StringBox)
+ elif char == 'V':
+ num = 20
+ basename = 'void'
+ w_box_type = space.gettypefor(interp_boxes.W_VoidBox)
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ "pure void dtype"))
+ else:
+ assert char == 'U'
+ basename = 'unicode'
+ itemtype = types.UnicodeType(size)
+ num = 19
+ w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox)
+ return W_Dtype(itemtype, num, char,
+ basename + str(8 * itemtype.get_element_size()),
+ char, w_box_type)
+
+def dtype_from_spec(space, name):
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ "dtype from spec"))
+
+def descr__new__(space, w_subtype, w_dtype):
+ cache = get_dtype_cache(space)
+
+ if space.is_w(w_dtype, space.w_None):
+ return cache.w_float64dtype
+ elif space.isinstance_w(w_dtype, w_subtype):
+ return w_dtype
+ elif space.isinstance_w(w_dtype, space.w_str):
+ name = space.str_w(w_dtype)
+ if ',' in name:
+ return dtype_from_spec(space, name)
+ try:
+ return cache.dtypes_by_name[name]
+ except KeyError:
+ pass
+ if name[0] in 'VSU' or name[0] in '<>=' and name[1] in 'VSU':
+ return variable_dtype(space, name)
+ elif space.isinstance_w(w_dtype, space.w_list):
+ return dtype_from_list(space, w_dtype)
+ elif space.isinstance_w(w_dtype, space.w_dict):
+ return dtype_from_dict(space, w_dtype)
+ else:
+ for dtype in cache.builtin_dtypes:
+ if w_dtype in dtype.alternate_constructors:
+ return dtype
+ if w_dtype is dtype.w_box_type:
+ return dtype
+ raise OperationError(space.w_TypeError, space.wrap("data type not understood"))
+
W_Dtype.typedef = TypeDef("dtype",
__module__ = "numpypy",
- __new__ = interp2app(W_Dtype.descr__new__.im_func),
+ __new__ = interp2app(descr__new__),
__str__= interp2app(W_Dtype.descr_str),
__repr__ = interp2app(W_Dtype.descr_repr),
__eq__ = interp2app(W_Dtype.descr_eq),
__ne__ = interp2app(W_Dtype.descr_ne),
+ __getitem__ = interp2app(W_Dtype.descr_getitem),
num = interp_attrproperty("num", cls=W_Dtype),
kind = interp_attrproperty("kind", cls=W_Dtype),
+ char = interp_attrproperty("char", cls=W_Dtype),
type = interp_attrproperty_w("w_box_type", cls=W_Dtype),
+ byteorder = GetSetProperty(W_Dtype.descr_get_byteorder),
itemsize = GetSetProperty(W_Dtype.descr_get_itemsize),
+ alignment = GetSetProperty(W_Dtype.descr_get_alignment),
shape = GetSetProperty(W_Dtype.descr_get_shape),
name = interp_attrproperty('name', cls=W_Dtype),
+ fields = GetSetProperty(W_Dtype.descr_get_fields),
+ names = GetSetProperty(W_Dtype.descr_get_names),
)
W_Dtype.typedef.acceptable_as_base_class = False
+if sys.byteorder == 'little':
+ byteorder_prefix = '<'
+ nonnative_byteorder_prefix = '>'
+else:
+ byteorder_prefix = '>'
+ nonnative_byteorder_prefix = '<'
+
+def new_string_dtype(space, size):
+ return W_Dtype(
+ types.StringType(size),
+ num=18,
+ kind=STRINGLTR,
+ name='string',
+ char='S' + str(size),
+ w_box_type = space.gettypefor(interp_boxes.W_StringBox),
+ )
+
+def new_unicode_dtype(space, size):
+ return W_Dtype(
+ types.UnicodeType(size),
+ num=19,
+ kind=UNICODELTR,
+ name='unicode',
+ char='U' + str(size),
+ w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox),
+ )
+
+
class DtypeCache(object):
def __init__(self, space):
self.w_booldtype = W_Dtype(
@@ -239,18 +387,134 @@
alternate_constructors=[space.w_float],
aliases=["float"],
)
-
+ self.w_stringdtype = W_Dtype(
+ types.StringType(1),
+ num=18,
+ kind=STRINGLTR,
+ name='string',
+ char='S',
+ w_box_type = space.gettypefor(interp_boxes.W_StringBox),
+ alternate_constructors=[space.w_str],
+ )
+ self.w_unicodedtype = W_Dtype(
+ types.UnicodeType(1),
+ num=19,
+ kind=UNICODELTR,
+ name='unicode',
+ char='U',
+ w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox),
+ alternate_constructors=[space.w_unicode],
+ )
+ self.w_voiddtype = W_Dtype(
+ types.VoidType(0),
+ num=20,
+ kind=VOIDLTR,
+ name='void',
+ char='V',
+ w_box_type = space.gettypefor(interp_boxes.W_VoidBox),
+ #alternate_constructors=[space.w_buffer],
+ # XXX no buffer in space
+ )
self.builtin_dtypes = [
self.w_booldtype, self.w_int8dtype, self.w_uint8dtype,
self.w_int16dtype, self.w_uint16dtype, self.w_int32dtype,
self.w_uint32dtype, self.w_longdtype, self.w_ulongdtype,
- self.w_int64dtype, self.w_uint64dtype, self.w_float32dtype,
- self.w_float64dtype
+ self.w_int64dtype, self.w_uint64dtype,
+ self.w_float32dtype,
+ self.w_float64dtype, self.w_stringdtype, self.w_unicodedtype,
+ self.w_voiddtype,
]
- self.dtypes_by_num_bytes = sorted(
+ self.float_dtypes_by_num_bytes = sorted(
(dtype.itemtype.get_element_size(), dtype)
- for dtype in self.builtin_dtypes
+ for dtype in [self.w_float32dtype, self.w_float64dtype]
)
+ self.dtypes_by_name = {}
+ # we reverse, so the stuff with lower numbers override stuff with
+ # higher numbers
+ for dtype in reversed(self.builtin_dtypes):
+ self.dtypes_by_name[dtype.name] = dtype
+ can_name = dtype.kind + str(dtype.itemtype.get_element_size())
+ self.dtypes_by_name[can_name] = dtype
+ self.dtypes_by_name[byteorder_prefix + can_name] = dtype
+ self.dtypes_by_name['=' + can_name] = dtype
+ new_name = nonnative_byteorder_prefix + can_name
+ itemtypename = dtype.itemtype.__class__.__name__
+ itemtype = getattr(types, 'NonNative' + itemtypename)()
+ self.dtypes_by_name[new_name] = W_Dtype(
+ itemtype,
+ dtype.num, dtype.kind, new_name, dtype.char, dtype.w_box_type,
+ native=False)
+ for alias in dtype.aliases:
+ self.dtypes_by_name[alias] = dtype
+ self.dtypes_by_name[dtype.char] = dtype
+
+ typeinfo_full = {
+ 'LONGLONG': self.w_int64dtype,
+ 'SHORT': self.w_int16dtype,
+ 'VOID': self.w_voiddtype,
+ #'LONGDOUBLE':,
+ 'UBYTE': self.w_uint8dtype,
+ 'UINTP': self.w_ulongdtype,
+ 'ULONG': self.w_ulongdtype,
+ 'LONG': self.w_longdtype,
+ 'UNICODE': self.w_unicodedtype,
+ #'OBJECT',
+ 'ULONGLONG': self.w_uint64dtype,
+ 'STRING': self.w_stringdtype,
+ #'CDOUBLE',
+ #'DATETIME',
+ 'UINT': self.w_uint32dtype,
+ 'INTP': self.w_longdtype,
+ #'HALF',
+ 'BYTE': self.w_int8dtype,
+ #'CFLOAT': ,
+ #'TIMEDELTA',
+ 'INT': self.w_int32dtype,
+ 'DOUBLE': self.w_float64dtype,
+ 'USHORT': self.w_uint16dtype,
+ 'FLOAT': self.w_float32dtype,
+ 'BOOL': self.w_booldtype,
+ #, 'CLONGDOUBLE']
+ }
+ typeinfo_partial = {
+ 'Generic': interp_boxes.W_GenericBox,
+ 'Character': interp_boxes.W_CharacterBox,
+ 'Flexible': interp_boxes.W_FlexibleBox,
+ 'Inexact': interp_boxes.W_InexactBox,
+ 'Integer': interp_boxes.W_IntegerBox,
+ 'SignedInteger': interp_boxes.W_SignedIntegerBox,
+ 'UnsignedInteger': interp_boxes.W_UnsignedIntegerBox,
+ #'ComplexFloating',
+ 'Number': interp_boxes.W_NumberBox,
+ 'Floating': interp_boxes.W_FloatingBox
+ }
+ w_typeinfo = space.newdict()
+ for k, v in typeinfo_partial.iteritems():
+ space.setitem(w_typeinfo, space.wrap(k), space.gettypefor(v))
+ for k, dtype in typeinfo_full.iteritems():
+ itemsize = dtype.itemtype.get_element_size()
+ items_w = [space.wrap(dtype.char),
+ space.wrap(dtype.num),
+ space.wrap(itemsize * 8), # in case of changing
+ # number of bits per byte in the future
+ space.wrap(itemsize or 1)]
+ if dtype.is_int_type():
+ if dtype.kind == BOOLLTR:
+ w_maxobj = space.wrap(1)
+ w_minobj = space.wrap(0)
+ elif dtype.is_signed():
+ w_maxobj = space.wrap(r_longlong((1 << (itemsize*8 - 1))
+ - 1))
+ w_minobj = space.wrap(r_longlong(-1) << (itemsize*8 - 1))
+ else:
+ w_maxobj = space.wrap(r_ulonglong(1 << (itemsize*8)) - 1)
+ w_minobj = space.wrap(0)
+ items_w = items_w + [w_maxobj, w_minobj]
+ items_w = items_w + [dtype.w_box_type]
+
+ w_tuple = space.newtuple(items_w)
+ space.setitem(w_typeinfo, space.wrap(k), w_tuple)
+ self.w_typeinfo = w_typeinfo
def get_dtype_cache(space):
return space.fromcache(DtypeCache)
diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py
--- a/pypy/module/micronumpy/interp_iter.py
+++ b/pypy/module/micronumpy/interp_iter.py
@@ -2,7 +2,7 @@
from pypy.rlib import jit
from pypy.rlib.objectmodel import instantiate
from pypy.module.micronumpy.strides import calculate_broadcast_strides,\
- calculate_slice_strides, calculate_dot_strides
+ calculate_slice_strides, calculate_dot_strides, enumerate_chunks
""" This is a mini-tutorial on iterators, strides, and
memory layout. It assumes you are familiar with the terms, see
@@ -42,28 +42,81 @@
we can go faster.
All the calculations happen in next()
-next_step_x() tries to do the iteration for a number of steps at once,
+next_skip_x() tries to do the iteration for a number of steps at once,
but then we cannot gaurentee that we only overflow one single shape
dimension, perhaps we could overflow times in one big step.
"""
# structures to describe slicing
-class Chunk(object):
+class BaseChunk(object):
+ pass
+
+class RecordChunk(BaseChunk):
+ def __init__(self, name):
+ self.name = name
+
+ def apply(self, arr):
+ from pypy.module.micronumpy.interp_numarray import W_NDimSlice
+
+ arr = arr.get_concrete()
+ ofs, subdtype = arr.dtype.fields[self.name]
+ # strides backstrides are identical, ofs only changes start
+ return W_NDimSlice(arr.start + ofs, arr.strides[:], arr.backstrides[:],
+ arr.shape[:], arr, subdtype)
+
+class Chunks(BaseChunk):
+ def __init__(self, l):
+ self.l = l
+
+ @jit.unroll_safe
+ def extend_shape(self, old_shape):
+ shape = []
+ i = -1
+ for i, c in enumerate_chunks(self.l):
+ if c.step != 0:
+ shape.append(c.lgt)
+ s = i + 1
+ assert s >= 0
+ return shape[:] + old_shape[s:]
+
+ def apply(self, arr):
+ from pypy.module.micronumpy.interp_numarray import W_NDimSlice,\
+ VirtualSlice, ConcreteArray
+
+ shape = self.extend_shape(arr.shape)
+ if not isinstance(arr, ConcreteArray):
+ return VirtualSlice(arr, self, shape)
+ r = calculate_slice_strides(arr.shape, arr.start, arr.strides,
+ arr.backstrides, self.l)
+ _, start, strides, backstrides = r
+ return W_NDimSlice(start, strides[:], backstrides[:],
+ shape[:], arr)
+
+
+class Chunk(BaseChunk):
+ axis_step = 1
+
def __init__(self, start, stop, step, lgt):
self.start = start
self.stop = stop
self.step = step
self.lgt = lgt
- def extend_shape(self, shape):
- if self.step != 0:
- shape.append(self.lgt)
-
def __repr__(self):
return 'Chunk(%d, %d, %d, %d)' % (self.start, self.stop, self.step,
self.lgt)
+class NewAxisChunk(Chunk):
+ start = 0
+ stop = 1
+ step = 1
+ lgt = 1
+ axis_step = 0
+
+ def __init__(self):
+ pass
+
class BaseTransform(object):
pass
@@ -95,17 +148,19 @@
raise NotImplementedError
class ArrayIterator(BaseIterator):
- def __init__(self, size):
+ def __init__(self, size, element_size):
self.offset = 0
self.size = size
+ self.element_size = element_size
def next(self, shapelen):
return self.next_skip_x(1)
- def next_skip_x(self, ofs):
+ def next_skip_x(self, x):
arr = instantiate(ArrayIterator)
arr.size = self.size
- arr.offset = self.offset + ofs
+ arr.offset = self.offset + x * self.element_size
+ arr.element_size = self.element_size
return arr
def next_no_increase(self, shapelen):
@@ -152,7 +207,7 @@
elif isinstance(t, ViewTransform):
r = calculate_slice_strides(self.res_shape, self.offset,
self.strides,
- self.backstrides, t.chunks)
+ self.backstrides, t.chunks.l)
return ViewIterator(r[1], r[2], r[3], r[0])
@jit.unroll_safe
@@ -214,7 +269,7 @@
def apply_transformations(self, arr, transformations):
v = BaseIterator.apply_transformations(self, arr, transformations)
- if len(arr.shape) == 1:
+ if len(arr.shape) == 1 and len(v.res_shape) == 1:
return OneDimIterator(self.offset, self.strides[0],
self.res_shape[0])
return v
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -7,10 +7,10 @@
from pypy.module.micronumpy.appbridge import get_appbridge_cache
from pypy.module.micronumpy.dot import multidim_dot, match_dot_shapes
from pypy.module.micronumpy.interp_iter import (ArrayIterator,
- SkipLastAxisIterator, Chunk, ViewIterator)
-from pypy.module.micronumpy.strides import (calculate_slice_strides,
- shape_agreement, find_shape_and_elems, get_shape_from_iterable,
- calc_new_strides, to_coords)
+ SkipLastAxisIterator, Chunk, ViewIterator, Chunks, RecordChunk,
+ NewAxisChunk)
+from pypy.module.micronumpy.strides import (shape_agreement,
+ find_shape_and_elems, get_shape_from_iterable, calc_new_strides, to_coords)
from pypy.rlib import jit
from pypy.rlib.rstring import StringBuilder
from pypy.rpython.lltypesystem import lltype, rffi
@@ -47,7 +47,7 @@
)
flat_set_driver = jit.JitDriver(
greens=['shapelen', 'base'],
- reds=['step', 'ai', 'lngth', 'arr', 'basei'],
+ reds=['step', 'lngth', 'ri', 'arr', 'basei'],
name='numpy_flatset',
)
@@ -79,12 +79,13 @@
dtype = space.interp_w(interp_dtype.W_Dtype,
space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
)
- size, shape = _find_size_and_shape(space, w_size)
- return space.wrap(W_NDimArray(size, shape[:], dtype=dtype))
+ shape = _find_shape(space, w_size)
+ return space.wrap(W_NDimArray(shape[:], dtype=dtype))
def _unaryop_impl(ufunc_name):
- def impl(self, space):
- return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [self])
+ def impl(self, space, w_out=None):
+ return getattr(interp_ufuncs.get(space), ufunc_name).call(space,
+ [self, w_out])
return func_with_new_name(impl, "unaryop_%s_impl" % ufunc_name)
descr_pos = _unaryop_impl("positive")
@@ -93,8 +94,9 @@
descr_invert = _unaryop_impl("invert")
def _binop_impl(ufunc_name):
- def impl(self, space, w_other):
- return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [self, w_other])
+ def impl(self, space, w_other, w_out=None):
+ return getattr(interp_ufuncs.get(space), ufunc_name).call(space,
+ [self, w_other, w_out])
return func_with_new_name(impl, "binop_%s_impl" % ufunc_name)
descr_add = _binop_impl("add")
@@ -102,6 +104,7 @@
descr_mul = _binop_impl("multiply")
descr_div = _binop_impl("divide")
descr_truediv = _binop_impl("true_divide")
+ descr_floordiv = _binop_impl("floor_divide")
descr_mod = _binop_impl("mod")
descr_pow = _binop_impl("power")
descr_lshift = _binop_impl("left_shift")
@@ -123,12 +126,12 @@
return space.newtuple([w_quotient, w_remainder])
def _binop_right_impl(ufunc_name):
- def impl(self, space, w_other):
+ def impl(self, space, w_other, w_out=None):
w_other = scalar_w(space,
interp_ufuncs.find_dtype_for_scalar(space, w_other, self.find_dtype()),
w_other
)
- return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [w_other, self])
+ return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [w_other, self, w_out])
return func_with_new_name(impl, "binop_right_%s_impl" % ufunc_name)
descr_radd = _binop_right_impl("add")
@@ -136,6 +139,7 @@
descr_rmul = _binop_right_impl("multiply")
descr_rdiv = _binop_right_impl("divide")
descr_rtruediv = _binop_right_impl("true_divide")
+ descr_rfloordiv = _binop_right_impl("floor_divide")
descr_rmod = _binop_right_impl("mod")
descr_rpow = _binop_right_impl("power")
descr_rlshift = _binop_right_impl("left_shift")
@@ -150,13 +154,21 @@
return space.newtuple([w_quotient, w_remainder])
def _reduce_ufunc_impl(ufunc_name, promote_to_largest=False):
- def impl(self, space, w_axis=None):
+ def impl(self, space, w_axis=None, w_out=None):
if space.is_w(w_axis, space.w_None):
axis = -1
else:
axis = space.int_w(w_axis)
+ if space.is_w(w_out, space.w_None) or not w_out:
+ out = None
+ elif not isinstance(w_out, BaseArray):
+ raise OperationError(space.w_TypeError, space.wrap(
+ 'output must be an array'))
+ else:
+ out = w_out
return getattr(interp_ufuncs.get(space), ufunc_name).reduce(space,
- self, True, promote_to_largest, axis)
+ self, True, promote_to_largest, axis,
+ False, out)
return func_with_new_name(impl, "reduce_%s_impl" % ufunc_name)
descr_sum = _reduce_ufunc_impl("add")
@@ -211,6 +223,7 @@
def descr_dot(self, space, w_other):
other = convert_to_array(space, w_other)
if isinstance(other, Scalar):
+ #Note: w_out is not modified, this is numpy compliant.
return self.descr_mul(space, other)
elif len(self.shape) < 2 and len(other.shape) < 2:
w_res = self.descr_mul(space, other)
@@ -223,8 +236,7 @@
return scalar_w(space, dtype, space.wrap(0))
# Do the dims match?
out_shape, other_critical_dim = match_dot_shapes(space, self, other)
- out_size = support.product(out_shape)
- result = W_NDimArray(out_size, out_shape, dtype)
+ result = W_NDimArray(out_shape, dtype)
# This is the place to add fpypy and blas
return multidim_dot(space, self.get_concrete(),
other.get_concrete(), result, dtype,
@@ -243,7 +255,7 @@
return space.wrap(self.find_dtype().itemtype.get_element_size())
def descr_get_nbytes(self, space):
- return space.wrap(self.size * self.find_dtype().itemtype.get_element_size())
+ return space.wrap(self.size)
@jit.unroll_safe
def descr_get_shape(self, space):
@@ -251,13 +263,16 @@
def descr_set_shape(self, space, w_iterable):
new_shape = get_shape_from_iterable(space,
- self.size, w_iterable)
+ support.product(self.shape), w_iterable)
if isinstance(self, Scalar):
return
self.get_concrete().setshape(space, new_shape)
def descr_get_size(self, space):
- return space.wrap(self.size)
+ return space.wrap(self.get_size())
+
+ def get_size(self):
+ return self.size // self.find_dtype().get_size()
def descr_copy(self, space):
return self.copy(space)
@@ -277,7 +292,7 @@
def empty_copy(self, space, dtype):
shape = self.shape
- return W_NDimArray(support.product(shape), shape[:], dtype, 'C')
+ return W_NDimArray(shape[:], dtype, 'C')
def descr_len(self, space):
if len(self.shape):
@@ -318,7 +333,16 @@
""" The result of getitem/setitem is a single item if w_idx
is a list of scalars that match the size of shape
"""
+ if space.isinstance_w(w_idx, space.w_str):
+ return False
shape_len = len(self.shape)
+ if space.isinstance_w(w_idx, space.w_tuple):
+ for w_item in space.fixedview(w_idx):
+ if (space.isinstance_w(w_item, space.w_slice) or
+ space.is_w(w_item, space.w_None)):
+ return False
+ elif space.is_w(w_idx, space.w_None):
+ return False
if shape_len == 0:
raise OperationError(space.w_IndexError, space.wrap(
"0-d arrays can't be indexed"))
@@ -334,43 +358,55 @@
if lgt > shape_len:
raise OperationError(space.w_IndexError,
space.wrap("invalid index"))
- if lgt < shape_len:
- return False
- for w_item in space.fixedview(w_idx):
- if space.isinstance_w(w_item, space.w_slice):
- return False
- return True
+ return lgt == shape_len
@jit.unroll_safe
def _prepare_slice_args(self, space, w_idx):
+ if space.isinstance_w(w_idx, space.w_str):
+ idx = space.str_w(w_idx)
+ dtype = self.find_dtype()
+ if not dtype.is_record_type() or idx not in dtype.fields:
+ raise OperationError(space.w_ValueError, space.wrap(
+ "field named %s not defined" % idx))
+ return RecordChunk(idx)
if (space.isinstance_w(w_idx, space.w_int) or
space.isinstance_w(w_idx, space.w_slice)):
- return [Chunk(*space.decode_index4(w_idx, self.shape[0]))]
- return [Chunk(*space.decode_index4(w_item, self.shape[i])) for i, w_item in
- enumerate(space.fixedview(w_idx))]
+ return Chunks([Chunk(*space.decode_index4(w_idx, self.shape[0]))])
+ elif space.is_w(w_idx, space.w_None):
+ return Chunks([NewAxisChunk()])
+ result = []
+ i = 0
+ for w_item in space.fixedview(w_idx):
+ if space.is_w(w_item, space.w_None):
+ result.append(NewAxisChunk())
+ else:
+ result.append(Chunk(*space.decode_index4(w_item,
+ self.shape[i])))
+ i += 1
+ return Chunks(result)
- def count_all_true(self, arr):
- sig = arr.find_sig()
- frame = sig.create_frame(arr)
- shapelen = len(arr.shape)
+ def count_all_true(self):
+ sig = self.find_sig()
+ frame = sig.create_frame(self)
+ shapelen = len(self.shape)
s = 0
iter = None
while not frame.done():
- count_driver.jit_merge_point(arr=arr, frame=frame, iter=iter, s=s,
+ count_driver.jit_merge_point(arr=self, frame=frame, iter=iter, s=s,
shapelen=shapelen)
iter = frame.get_final_iter()
- s += arr.dtype.getitem_bool(arr.storage, iter.offset)
+ s += self.dtype.getitem_bool(self, iter.offset)
frame.next(shapelen)
return s
def getitem_filter(self, space, arr):
concr = arr.get_concrete()
- if concr.size > self.size:
+ if concr.get_size() > self.get_size():
raise OperationError(space.w_IndexError,
space.wrap("index out of range for array"))
- size = self.count_all_true(concr)
- res = W_NDimArray(size, [size], self.find_dtype())
- ri = ArrayIterator(size)
+ size = concr.count_all_true()
+ res = W_NDimArray([size], self.find_dtype())
+ ri = res.create_iter()
shapelen = len(self.shape)
argi = concr.create_iter()
sig = self.find_sig()
@@ -380,7 +416,7 @@
filter_driver.jit_merge_point(concr=concr, argi=argi, ri=ri,
frame=frame, v=v, res=res, sig=sig,
shapelen=shapelen, self=self)
- if concr.dtype.getitem_bool(concr.storage, argi.offset):
+ if concr.dtype.getitem_bool(concr, argi.offset):
v = sig.eval(frame, self)
res.setitem(ri.offset, v)
ri = ri.next(1)
@@ -390,23 +426,6 @@
frame.next(shapelen)
return res
- def setitem_filter(self, space, idx, val):
- size = self.count_all_true(idx)
- arr = SliceArray([size], self.dtype, self, val)
- sig = arr.find_sig()
- shapelen = len(self.shape)
- frame = sig.create_frame(arr)
- idxi = idx.create_iter()
- while not frame.done():
- filter_set_driver.jit_merge_point(idx=idx, idxi=idxi, sig=sig,
- frame=frame, arr=arr,
- shapelen=shapelen)
- if idx.dtype.getitem_bool(idx.storage, idxi.offset):
- sig.eval(frame, arr)
- frame.next_from_second(1)
- frame.next_first(shapelen)
- idxi = idxi.next(shapelen)
-
def descr_getitem(self, space, w_idx):
if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and
w_idx.find_dtype().is_bool_type()):
@@ -416,7 +435,24 @@
item = concrete._index_of_single_item(space, w_idx)
return concrete.getitem(item)
chunks = self._prepare_slice_args(space, w_idx)
- return self.create_slice(chunks)
+ return chunks.apply(self)
+
+ def setitem_filter(self, space, idx, val):
+ size = idx.count_all_true()
+ arr = SliceArray([size], self.dtype, self, val)
+ sig = arr.find_sig()
+ shapelen = len(self.shape)
+ frame = sig.create_frame(arr)
+ idxi = idx.create_iter()
+ while not frame.done():
+ filter_set_driver.jit_merge_point(idx=idx, idxi=idxi, sig=sig,
+ frame=frame, arr=arr,
+ shapelen=shapelen)
+ if idx.dtype.getitem_bool(idx, idxi.offset):
+ sig.eval(frame, arr)
+ frame.next_from_second(1)
+ frame.next_first(shapelen)
+ idxi = idxi.next(shapelen)
def descr_setitem(self, space, w_idx, w_value):
self.invalidated()
@@ -434,26 +470,9 @@
if not isinstance(w_value, BaseArray):
w_value = convert_to_array(space, w_value)
chunks = self._prepare_slice_args(space, w_idx)
- view = self.create_slice(chunks).get_concrete()
+ view = chunks.apply(self).get_concrete()
view.setslice(space, w_value)
- @jit.unroll_safe
- def create_slice(self, chunks):
- shape = []
- i = -1
- for i, chunk in enumerate(chunks):
- chunk.extend_shape(shape)
- s = i + 1
- assert s >= 0
- shape += self.shape[s:]
- if not isinstance(self, ConcreteArray):
- return VirtualSlice(self, chunks, shape)
- r = calculate_slice_strides(self.shape, self.start, self.strides,
- self.backstrides, chunks)
- _, start, strides, backstrides = r
- return W_NDimSlice(start, strides[:], backstrides[:],
- shape[:], self)
-
def descr_reshape(self, space, args_w):
"""reshape(...)
a.reshape(shape)
@@ -470,13 +489,16 @@
w_shape = args_w[0]
else:
w_shape = space.newtuple(args_w)
- new_shape = get_shape_from_iterable(space, self.size, w_shape)
+ new_shape = get_shape_from_iterable(space, support.product(self.shape),
+ w_shape)
return self.reshape(space, new_shape)
def reshape(self, space, new_shape):
concrete = self.get_concrete()
# Since we got to here, prod(new_shape) == self.size
- new_strides = calc_new_strides(new_shape, concrete.shape,
+ new_strides = None
+ if self.size > 0:
+ new_strides = calc_new_strides(new_shape, concrete.shape,
concrete.strides, concrete.order)
if new_strides:
# We can create a view, strides somehow match up.
@@ -503,14 +525,14 @@
)
return w_result
- def descr_mean(self, space, w_axis=None):
+ def descr_mean(self, space, w_axis=None, w_out=None):
if space.is_w(w_axis, space.w_None):
w_axis = space.wrap(-1)
- w_denom = space.wrap(self.size)
+ w_denom = space.wrap(support.product(self.shape))
else:
dim = space.int_w(w_axis)
w_denom = space.wrap(self.shape[dim])
- return space.div(self.descr_sum_promote(space, w_axis), w_denom)
+ return space.div(self.descr_sum_promote(space, w_axis, w_out), w_denom)
def descr_var(self, space, w_axis=None):
return get_appbridge_cache(space).call_method(space, '_var', self,
@@ -525,7 +547,7 @@
concr.fill(space, w_value)
def descr_nonzero(self, space):
- if self.size > 1:
+ if self.get_size() > 1:
raise OperationError(space.w_ValueError, space.wrap(
"The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()"))
concr = self.get_concrete_or_scalar()
@@ -604,8 +626,7 @@
space.wrap("axis unsupported for take"))
index_i = index.create_iter()
res_shape = index.shape
- size = support.product(res_shape)
- res = W_NDimArray(size, res_shape[:], concr.dtype, concr.order)
+ res = W_NDimArray(res_shape[:], concr.dtype, concr.order)
res_i = res.create_iter()
shapelen = len(index.shape)
sig = concr.find_sig()
@@ -644,6 +665,11 @@
raise OperationError(space.w_NotImplementedError, space.wrap(
"non-int arg not supported"))
+ def descr_tostring(self, space):
+ ra = ToStringArray(self)
+ loop.compute(ra)
+ return space.wrap(ra.s.build())
+
def compute_first_step(self, sig, frame):
pass
@@ -665,8 +691,7 @@
"""
Intermediate class representing a literal.
"""
- size = 1
- _attrs_ = ["dtype", "value", "shape"]
+ _attrs_ = ["dtype", "value", "shape", "size"]
def __init__(self, dtype, value):
self.shape = []
@@ -674,6 +699,7 @@
self.dtype = dtype
assert isinstance(value, interp_boxes.W_GenericBox)
self.value = value
+ self.size = dtype.get_size()
def find_dtype(self):
return self.dtype
@@ -691,8 +717,7 @@
return self
def reshape(self, space, new_shape):
- size = support.product(new_shape)
- res = W_NDimArray(size, new_shape, self.dtype, 'C')
+ res = W_NDimArray(new_shape, self.dtype, 'C')
res.setitem(0, self.value)
return res
@@ -700,11 +725,13 @@
"""
Class for representing virtual arrays, such as binary ops or ufuncs
"""
- def __init__(self, name, shape, res_dtype):
+ def __init__(self, name, shape, res_dtype, out_arg=None):
BaseArray.__init__(self, shape)
self.forced_result = None
self.res_dtype = res_dtype
self.name = name
+ self.res = out_arg
+ self.size = support.product(self.shape) * res_dtype.get_size()
def _del_sources(self):
# Function for deleting references to source arrays,
@@ -712,13 +739,18 @@
raise NotImplementedError
def compute(self):
- ra = ResultArray(self, self.size, self.shape, self.res_dtype)
+ ra = ResultArray(self, self.shape, self.res_dtype, self.res)
loop.compute(ra)
+ if self.res:
+ broadcast_dims = len(self.res.shape) - len(self.shape)
+ chunks = [Chunk(0,0,0,0)] * broadcast_dims + \
+ [Chunk(0, i, 1, i) for i in self.shape]
+ return Chunks(chunks).apply(self.res)
return ra.left
def force_if_needed(self):
if self.forced_result is None:
- self.forced_result = self.compute()
+ self.forced_result = self.compute().get_concrete()
self._del_sources()
def get_concrete(self):
@@ -740,7 +772,6 @@
def __init__(self, child, chunks, shape):
self.child = child
self.chunks = chunks
- self.size = support.product(shape)
VirtualArray.__init__(self, 'slice', shape, child.find_dtype())
def create_sig(self):
@@ -752,15 +783,16 @@
def force_if_needed(self):
if self.forced_result is None:
concr = self.child.get_concrete()
- self.forced_result = concr.create_slice(self.chunks)
+ self.forced_result = self.chunks.apply(concr)
def _del_sources(self):
self.child = None
class Call1(VirtualArray):
- def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, values):
- VirtualArray.__init__(self, name, shape, res_dtype)
+ def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, values,
+ out_arg=None):
+ VirtualArray.__init__(self, name, shape, res_dtype, out_arg)
self.values = values
self.size = values.size
self.ufunc = ufunc
@@ -772,6 +804,12 @@
def create_sig(self):
if self.forced_result is not None:
return self.forced_result.create_sig()
+ if self.shape != self.values.shape:
+ #This happens if out arg is used
+ return signature.BroadcastUfunc(self.ufunc, self.name,
+ self.calc_dtype,
+ self.values.create_sig(),
+ self.res.create_sig())
return signature.Call1(self.ufunc, self.name, self.calc_dtype,
self.values.create_sig())
@@ -779,13 +817,13 @@
"""
Intermediate class for performing binary operations.
"""
- def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, left, right):
- VirtualArray.__init__(self, name, shape, res_dtype)
+ def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, left, right,
+ out_arg=None):
+ VirtualArray.__init__(self, name, shape, res_dtype, out_arg)
self.ufunc = ufunc
self.left = left
self.right = right
self.calc_dtype = calc_dtype
- self.size = support.product(self.shape)
def _del_sources(self):
self.left = None
@@ -813,14 +851,34 @@
self.left.create_sig(), self.right.create_sig())
class ResultArray(Call2):
- def __init__(self, child, size, shape, dtype, res=None, order='C'):
+ def __init__(self, child, shape, dtype, res=None, order='C'):
if res is None:
- res = W_NDimArray(size, shape, dtype, order)
+ res = W_NDimArray(shape, dtype, order)
Call2.__init__(self, None, 'assign', shape, dtype, dtype, res, child)
def create_sig(self):
- return signature.ResultSignature(self.res_dtype, self.left.create_sig(),
- self.right.create_sig())
+ if self.left.shape != self.right.shape:
+ sig = signature.BroadcastResultSignature(self.res_dtype,
+ self.left.create_sig(), self.right.create_sig())
+ else:
+ sig = signature.ResultSignature(self.res_dtype,
+ self.left.create_sig(), self.right.create_sig())
+ return sig
+
+class ToStringArray(Call1):
+ def __init__(self, child):
+ dtype = child.find_dtype()
+ self.item_size = dtype.itemtype.get_element_size()
+ self.s = StringBuilder(child.size * self.item_size)
+ Call1.__init__(self, None, 'tostring', child.shape, dtype, dtype,
+ child)
+ self.res_str = W_NDimArray([1], dtype, order='C')
+ self.res_str_casted = rffi.cast(rffi.CArrayPtr(lltype.Char),
+ self.res_str.storage)
+
+ def create_sig(self):
+ return signature.ToStringSignature(self.calc_dtype,
+ self.values.create_sig())
def done_if_true(dtype, val):
return dtype.itemtype.bool(val)
@@ -893,13 +951,13 @@
"""
_immutable_fields_ = ['storage']
- def __init__(self, size, shape, dtype, order='C', parent=None):
- self.size = size
+ def __init__(self, shape, dtype, order='C', parent=None):
self.parent = parent
+ self.size = support.product(shape) * dtype.get_size()
if parent is not None:
self.storage = parent.storage
else:
- self.storage = dtype.malloc(size)
+ self.storage = dtype.itemtype.malloc(self.size)
self.order = order
self.dtype = dtype
if self.strides is None:
@@ -918,13 +976,14 @@
return self.dtype
def getitem(self, item):
- return self.dtype.getitem(self.storage, item)
+ return self.dtype.getitem(self, item)
def setitem(self, item, value):
self.invalidated()
- self.dtype.setitem(self.storage, item, value)
+ self.dtype.setitem(self, item, value.convert_to(self.dtype))
def calc_strides(self, shape):
+ dtype = self.find_dtype()
strides = []
backstrides = []
s = 1
@@ -932,8 +991,8 @@
if self.order == 'C':
shape_rev.reverse()
for sh in shape_rev:
- strides.append(s)
- backstrides.append(s * (sh - 1))
+ strides.append(s * dtype.get_size())
+ backstrides.append(s * (sh - 1) * dtype.get_size())
s *= sh
if self.order == 'C':
strides.reverse()
@@ -981,9 +1040,9 @@
shapelen = len(self.shape)
if shapelen == 1:
rffi.c_memcpy(
- rffi.ptradd(self.storage, self.start * itemsize),
- rffi.ptradd(w_value.storage, w_value.start * itemsize),
- self.size * itemsize
+ rffi.ptradd(self.storage, self.start),
+ rffi.ptradd(w_value.storage, w_value.start),
+ self.size
)
else:
dest = SkipLastAxisIterator(self)
@@ -998,7 +1057,7 @@
dest.next()
def copy(self, space):
- array = W_NDimArray(self.size, self.shape[:], self.dtype, self.order)
+ array = W_NDimArray(self.shape[:], self.dtype, self.order)
array.setslice(space, self)
return array
@@ -1012,14 +1071,15 @@
class W_NDimSlice(ViewArray):
- def __init__(self, start, strides, backstrides, shape, parent):
+ def __init__(self, start, strides, backstrides, shape, parent, dtype=None):
assert isinstance(parent, ConcreteArray)
if isinstance(parent, W_NDimSlice):
parent = parent.parent
self.strides = strides
self.backstrides = backstrides
- ViewArray.__init__(self, support.product(shape), shape, parent.dtype,
- parent.order, parent)
+ if dtype is None:
+ dtype = parent.dtype
+ ViewArray.__init__(self, shape, dtype, parent.order, parent)
self.start = start
def create_iter(self, transforms=None):
@@ -1029,18 +1089,19 @@
def setshape(self, space, new_shape):
if len(self.shape) < 1:
return
- elif len(self.shape) < 2:
+ elif len(self.shape) < 2 or self.size < 1:
# TODO: this code could be refactored into calc_strides
# but then calc_strides would have to accept a stepping factor
strides = []
backstrides = []
- s = self.strides[0]
+ dtype = self.find_dtype()
+ s = self.strides[0] // dtype.get_size()
if self.order == 'C':
new_shape.reverse()
for sh in new_shape:
- strides.append(s)
- backstrides.append(s * (sh - 1))
- s *= sh
+ strides.append(s * dtype.get_size())
+ backstrides.append(s * (sh - 1) * dtype.get_size())
+ s *= max(1, sh)
if self.order == 'C':
strides.reverse()
backstrides.reverse()
@@ -1067,14 +1128,16 @@
"""
def setitem(self, item, value):
self.invalidated()
- self.dtype.setitem(self.storage, item, value)
+ self.dtype.setitem(self, item, value)
def setshape(self, space, new_shape):
self.shape = new_shape
self.calc_strides(new_shape)
def create_iter(self, transforms=None):
- return ArrayIterator(self.size).apply_transformations(self, transforms)
+ esize = self.find_dtype().get_size()
+ return ArrayIterator(self.size, esize).apply_transformations(self,
+ transforms)
def create_sig(self):
return signature.ArraySignature(self.dtype)
@@ -1082,22 +1145,18 @@
def __del__(self):
lltype.free(self.storage, flavor='raw', track_allocation=False)
-def _find_size_and_shape(space, w_size):
+def _find_shape(space, w_size):
if space.isinstance_w(w_size, space.w_int):
- size = space.int_w(w_size)
- shape = [size]
- else:
- size = 1
- shape = []
- for w_item in space.fixedview(w_size):
- item = space.int_w(w_item)
- size *= item
- shape.append(item)
- return size, shape
+ return [space.int_w(w_size)]
+ shape = []
+ for w_item in space.fixedview(w_size):
+ shape.append(space.int_w(w_item))
+ return shape
@unwrap_spec(subok=bool, copy=bool, ownmaskna=bool)
def array(space, w_item_or_iterable, w_dtype=None, w_order=None,
- subok=True, copy=True, w_maskna=None, ownmaskna=False):
+ subok=True, copy=True, w_maskna=None, ownmaskna=False,
+ w_ndmin=None):
# find scalar
if w_maskna is None:
w_maskna = space.w_None
@@ -1127,28 +1186,33 @@
if copy:
return w_item_or_iterable.copy(space)
return w_item_or_iterable
- shape, elems_w = find_shape_and_elems(space, w_item_or_iterable)
+ if w_dtype is None or space.is_w(w_dtype, space.w_None):
+ dtype = None
+ else:
+ dtype = space.interp_w(interp_dtype.W_Dtype,
+ space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype))
+ shape, elems_w = find_shape_and_elems(space, w_item_or_iterable, dtype)
# they come back in C order
- size = len(elems_w)
- if w_dtype is None or space.is_w(w_dtype, space.w_None):
- w_dtype = None
+ if dtype is None:
for w_elem in elems_w:
- w_dtype = interp_ufuncs.find_dtype_for_scalar(space, w_elem,
- w_dtype)
- if w_dtype is interp_dtype.get_dtype_cache(space).w_float64dtype:
+ dtype = interp_ufuncs.find_dtype_for_scalar(space, w_elem,
+ dtype)
+ if dtype is interp_dtype.get_dtype_cache(space).w_float64dtype:
break
- if w_dtype is None:
- w_dtype = space.w_None
- dtype = space.interp_w(interp_dtype.W_Dtype,
- space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
- )
- arr = W_NDimArray(size, shape[:], dtype=dtype, order=order)
+ if dtype is None:
+ dtype = interp_dtype.get_dtype_cache(space).w_float64dtype
shapelen = len(shape)
- arr_iter = ArrayIterator(arr.size)
+ if w_ndmin is not None and not space.is_w(w_ndmin, space.w_None):
+ ndmin = space.int_w(w_ndmin)
+ if ndmin > shapelen:
+ shape = [1] * (ndmin - shapelen) + shape
+ shapelen = ndmin
+ arr = W_NDimArray(shape[:], dtype=dtype, order=order)
+ arr_iter = arr.create_iter()
# XXX we might want to have a jitdriver here
for i in range(len(elems_w)):
w_elem = elems_w[i]
- dtype.setitem(arr.storage, arr_iter.offset,
+ dtype.setitem(arr, arr_iter.offset,
dtype.coerce(space, w_elem))
arr_iter = arr_iter.next(shapelen)
return arr
@@ -1157,22 +1221,22 @@
dtype = space.interp_w(interp_dtype.W_Dtype,
space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
)
- size, shape = _find_size_and_shape(space, w_size)
+ shape = _find_shape(space, w_size)
if not shape:
return scalar_w(space, dtype, space.wrap(0))
- return space.wrap(W_NDimArray(size, shape[:], dtype=dtype))
+ return space.wrap(W_NDimArray(shape[:], dtype=dtype))
def ones(space, w_size, w_dtype=None):
dtype = space.interp_w(interp_dtype.W_Dtype,
space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
)
- size, shape = _find_size_and_shape(space, w_size)
+ shape = _find_shape(space, w_size)
if not shape:
return scalar_w(space, dtype, space.wrap(1))
- arr = W_NDimArray(size, shape[:], dtype=dtype)
+ arr = W_NDimArray(shape[:], dtype=dtype)
one = dtype.box(1)
- arr.dtype.fill(arr.storage, one, 0, size)
+ arr.dtype.fill(arr.storage, one, 0, arr.size)
return space.wrap(arr)
@unwrap_spec(arr=BaseArray, skipna=bool, keepdims=bool)
@@ -1220,13 +1284,13 @@
"array dimensions must agree except for axis being concatenated"))
elif i == axis:
shape[i] += axis_size
- res = W_NDimArray(support.product(shape), shape, dtype, 'C')
+ res = W_NDimArray(shape, dtype, 'C')
chunks = [Chunk(0, i, 1, i) for i in shape]
axis_start = 0
for arr in args_w:
chunks[axis] = Chunk(axis_start, axis_start + arr.shape[axis], 1,
arr.shape[axis])
- res.create_slice(chunks).setslice(space, arr)
+ Chunks(chunks).apply(res).setslice(space, arr)
axis_start += arr.shape[axis]
return res
@@ -1250,6 +1314,7 @@
__mul__ = interp2app(BaseArray.descr_mul),
__div__ = interp2app(BaseArray.descr_div),
__truediv__ = interp2app(BaseArray.descr_truediv),
+ __floordiv__ = interp2app(BaseArray.descr_floordiv),
__mod__ = interp2app(BaseArray.descr_mod),
__divmod__ = interp2app(BaseArray.descr_divmod),
__pow__ = interp2app(BaseArray.descr_pow),
@@ -1264,6 +1329,7 @@
__rmul__ = interp2app(BaseArray.descr_rmul),
__rdiv__ = interp2app(BaseArray.descr_rdiv),
__rtruediv__ = interp2app(BaseArray.descr_rtruediv),
+ __rfloordiv__ = interp2app(BaseArray.descr_rfloordiv),
__rmod__ = interp2app(BaseArray.descr_rmod),
__rdivmod__ = interp2app(BaseArray.descr_rdivmod),
__rpow__ = interp2app(BaseArray.descr_rpow),
@@ -1312,6 +1378,7 @@
std = interp2app(BaseArray.descr_std),
fill = interp2app(BaseArray.descr_fill),
+ tostring = interp2app(BaseArray.descr_tostring),
copy = interp2app(BaseArray.descr_copy),
flatten = interp2app(BaseArray.descr_flatten),
@@ -1334,7 +1401,7 @@
self.iter = sig.create_frame(arr).get_final_iter()
self.base = arr
self.index = 0
- ViewArray.__init__(self, arr.size, [arr.size], arr.dtype, arr.order,
+ ViewArray.__init__(self, [arr.get_size()], arr.dtype, arr.order,
arr)
def descr_next(self, space):
@@ -1349,7 +1416,7 @@
return self
def descr_len(self, space):
- return space.wrap(self.size)
+ return space.wrap(self.get_size())
def descr_index(self, space):
return space.wrap(self.index)
@@ -1367,28 +1434,26 @@
raise OperationError(space.w_IndexError,
space.wrap('unsupported iterator index'))
base = self.base
- start, stop, step, lngth = space.decode_index4(w_idx, base.size)
+ start, stop, step, lngth = space.decode_index4(w_idx, base.get_size())
# setslice would have been better, but flat[u:v] for arbitrary
# shapes of array a cannot be represented as a[x1:x2, y1:y2]
basei = ViewIterator(base.start, base.strides,
- base.backstrides,base.shape)
+ base.backstrides, base.shape)
shapelen = len(base.shape)
basei = basei.next_skip_x(shapelen, start)
if lngth <2:
return base.getitem(basei.offset)
- ri = ArrayIterator(lngth)
- res = W_NDimArray(lngth, [lngth], base.dtype,
- base.order)
+ res = W_NDimArray([lngth], base.dtype, base.order)
+ ri = res.create_iter()
while not ri.done():
flat_get_driver.jit_merge_point(shapelen=shapelen,
base=base,
basei=basei,
step=step,
res=res,
- ri=ri,
- )
+ ri=ri)
w_val = base.getitem(basei.offset)
- res.setitem(ri.offset,w_val)
+ res.setitem(ri.offset, w_val)
basei = basei.next_skip_x(shapelen, step)
ri = ri.next(shapelen)
return res
@@ -1399,27 +1464,28 @@
raise OperationError(space.w_IndexError,
space.wrap('unsupported iterator index'))
base = self.base
- start, stop, step, lngth = space.decode_index4(w_idx, base.size)
+ start, stop, step, lngth = space.decode_index4(w_idx, base.get_size())
arr = convert_to_array(space, w_value)
- ai = 0
+ ri = arr.create_iter()
basei = ViewIterator(base.start, base.strides,
- base.backstrides,base.shape)
+ base.backstrides, base.shape)
shapelen = len(base.shape)
basei = basei.next_skip_x(shapelen, start)
while lngth > 0:
flat_set_driver.jit_merge_point(shapelen=shapelen,
- basei=basei,
- base=base,
- step=step,
- arr=arr,
- ai=ai,
- lngth=lngth,
- )
- v = arr.getitem(ai).convert_to(base.dtype)
+ basei=basei,
+ base=base,
+ step=step,
+ arr=arr,
+ lngth=lngth,
+ ri=ri)
+ v = arr.getitem(ri.offset).convert_to(base.dtype)
base.setitem(basei.offset, v)
# need to repeat input values until all assignments are done
- ai = (ai + 1) % arr.size
basei = basei.next_skip_x(shapelen, step)
+ ri = ri.next(shapelen)
+ # WTF is numpy thinking?
+ ri.offset %= arr.size
lngth -= 1
def create_sig(self):
@@ -1427,9 +1493,9 @@
def create_iter(self, transforms=None):
return ViewIterator(self.base.start, self.base.strides,
- self.base.backstrides,
- self.base.shape).apply_transformations(self.base,
- transforms)
+ self.base.backstrides,
+ self.base.shape).apply_transformations(self.base,
+ transforms)
def descr_base(self, space):
return space.wrap(self.base)
diff --git a/pypy/module/micronumpy/interp_support.py b/pypy/module/micronumpy/interp_support.py
--- a/pypy/module/micronumpy/interp_support.py
+++ b/pypy/module/micronumpy/interp_support.py
@@ -51,9 +51,11 @@
raise OperationError(space.w_ValueError, space.wrap(
"string is smaller than requested size"))
- a = W_NDimArray(num_items, [num_items], dtype=dtype)
- for i, val in enumerate(items):
- a.dtype.setitem(a.storage, i, val)
+ a = W_NDimArray([num_items], dtype=dtype)
+ ai = a.create_iter()
+ for val in items:
+ a.dtype.setitem(a, ai.offset, val)
+ ai = ai.next(1)
return space.wrap(a)
@@ -61,6 +63,7 @@
from pypy.module.micronumpy.interp_numarray import W_NDimArray
itemsize = dtype.itemtype.get_element_size()
+ assert itemsize >= 0
if count == -1:
count = length / itemsize
if length % itemsize != 0:
@@ -71,20 +74,23 @@
raise OperationError(space.w_ValueError, space.wrap(
"string is smaller than requested size"))
- a = W_NDimArray(count, [count], dtype=dtype)
- fromstring_loop(a, count, dtype, itemsize, s)
+ a = W_NDimArray([count], dtype=dtype)
+ fromstring_loop(a, dtype, itemsize, s)
return space.wrap(a)
-fromstring_driver = jit.JitDriver(greens=[], reds=['count', 'i', 'itemsize',
- 'dtype', 's', 'a'])
+fromstring_driver = jit.JitDriver(greens=[], reds=['i', 'itemsize',
+ 'dtype', 'ai', 's', 'a'])
-def fromstring_loop(a, count, dtype, itemsize, s):
+def fromstring_loop(a, dtype, itemsize, s):
i = 0
- while i < count:
- fromstring_driver.jit_merge_point(a=a, count=count, dtype=dtype,
- itemsize=itemsize, s=s, i=i)
+ ai = a.create_iter()
+ while not ai.done():
+ fromstring_driver.jit_merge_point(a=a, dtype=dtype,
+ itemsize=itemsize, s=s, i=i,
+ ai=ai)
val = dtype.itemtype.runpack_str(s[i*itemsize:i*itemsize + itemsize])
- a.dtype.setitem(a.storage, i, val)
+ a.dtype.setitem(a, ai.offset, val)
+ ai = ai.next(1)
i += 1
@unwrap_spec(s=str, count=int, sep=str)
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -28,26 +28,38 @@
return self.identity
def descr_call(self, space, __args__):
+ from interp_numarray import BaseArray
args_w, kwds_w = __args__.unpack()
# it occurs to me that we don't support any datatypes that
# require casting, change it later when we do
kwds_w.pop('casting', None)
w_subok = kwds_w.pop('subok', None)
w_out = kwds_w.pop('out', space.w_None)
- if ((w_subok is not None and space.is_true(w_subok)) or
- not space.is_w(w_out, space.w_None)):
+ # Setup a default value for out
+ if space.is_w(w_out, space.w_None):
+ out = None
+ else:
+ out = w_out
+ if (w_subok is not None and space.is_true(w_subok)):
raise OperationError(space.w_NotImplementedError,
space.wrap("parameters unsupported"))
if kwds_w or len(args_w) < self.argcount:
raise OperationError(space.w_ValueError,
space.wrap("invalid number of arguments")
)
- elif len(args_w) > self.argcount:
- # The extra arguments should actually be the output array, but we
- # don't support that yet.
+ elif (len(args_w) > self.argcount and out is not None) or \
+ (len(args_w) > self.argcount + 1):
raise OperationError(space.w_TypeError,
space.wrap("invalid number of arguments")
)
+ # Override the default out value, if it has been provided in w_wargs
+ if len(args_w) > self.argcount:
+ out = args_w[-1]
+ else:
+ args_w = args_w[:] + [out]
+ if out is not None and not isinstance(out, BaseArray):
+ raise OperationError(space.w_TypeError, space.wrap(
+ 'output must be an array'))
return self.call(space, args_w)
@unwrap_spec(skipna=bool, keepdims=bool)
@@ -105,28 +117,33 @@
array([[ 1, 5],
[ 9, 13]])
"""
- if not space.is_w(w_out, space.w_None):
- raise OperationError(space.w_NotImplementedError, space.wrap(
- "out not supported"))
+ from pypy.module.micronumpy.interp_numarray import BaseArray
if w_axis is None:
axis = 0
elif space.is_w(w_axis, space.w_None):
axis = -1
else:
axis = space.int_w(w_axis)
- return self.reduce(space, w_obj, False, False, axis, keepdims)
+ if space.is_w(w_out, space.w_None):
+ out = None
+ elif not isinstance(w_out, BaseArray):
+ raise OperationError(space.w_TypeError, space.wrap(
+ 'output must be an array'))
+ else:
+ out = w_out
+ return self.reduce(space, w_obj, False, False, axis, keepdims, out)
- def reduce(self, space, w_obj, multidim, promote_to_largest, dim,
- keepdims=False):
+ def reduce(self, space, w_obj, multidim, promote_to_largest, axis,
+ keepdims=False, out=None):
from pypy.module.micronumpy.interp_numarray import convert_to_array, \
- Scalar, ReduceArray
+ Scalar, ReduceArray, W_NDimArray
if self.argcount != 2:
raise OperationError(space.w_ValueError, space.wrap("reduce only "
"supported for binary functions"))
assert isinstance(self, W_Ufunc2)
obj = convert_to_array(space, w_obj)
- if dim >= len(obj.shape):
- raise OperationError(space.w_ValueError, space.wrap("axis(=%d) out of bounds" % dim))
+ if axis >= len(obj.shape):
+ raise OperationError(space.w_ValueError, space.wrap("axis(=%d) out of bounds" % axis))
if isinstance(obj, Scalar):
raise OperationError(space.w_TypeError, space.wrap("cannot reduce "
"on a scalar"))
@@ -144,21 +161,55 @@
if self.identity is None and size == 0:
raise operationerrfmt(space.w_ValueError, "zero-size array to "
"%s.reduce without identity", self.name)
- if shapelen > 1 and dim >= 0:
- return self.do_axis_reduce(obj, dtype, dim, keepdims)
- arr = ReduceArray(self.func, self.name, self.identity, obj, dtype)
- return loop.compute(arr)
+ if shapelen > 1 and axis >= 0:
+ if keepdims:
+ shape = obj.shape[:axis] + [1] + obj.shape[axis + 1:]
+ else:
+ shape = obj.shape[:axis] + obj.shape[axis + 1:]
+ if out:
+ #Test for shape agreement
+ if len(out.shape) > len(shape):
+ raise operationerrfmt(space.w_ValueError,
+ 'output parameter for reduction operation %s' +
+ ' has too many dimensions', self.name)
+ elif len(out.shape) < len(shape):
+ raise operationerrfmt(space.w_ValueError,
+ 'output parameter for reduction operation %s' +
+ ' does not have enough dimensions', self.name)
+ elif out.shape != shape:
+ raise operationerrfmt(space.w_ValueError,
+ 'output parameter shape mismatch, expecting [%s]' +
+ ' , got [%s]',
+ ",".join([str(x) for x in shape]),
+ ",".join([str(x) for x in out.shape]),
+ )
+ #Test for dtype agreement, perhaps create an itermediate
+ #if out.dtype != dtype:
+ # raise OperationError(space.w_TypeError, space.wrap(
+ # "mismatched dtypes"))
+ return self.do_axis_reduce(obj, out.find_dtype(), axis, out)
+ else:
+ result = W_NDimArray(shape, dtype)
+ return self.do_axis_reduce(obj, dtype, axis, result)
+ if out:
+ if len(out.shape)>0:
+ raise operationerrfmt(space.w_ValueError, "output parameter "
+ "for reduction operation %s has too many"
+ " dimensions",self.name)
+ arr = ReduceArray(self.func, self.name, self.identity, obj,
+ out.find_dtype())
+ val = loop.compute(arr)
+ assert isinstance(out, Scalar)
+ out.value = val
+ else:
+ arr = ReduceArray(self.func, self.name, self.identity, obj, dtype)
+ val = loop.compute(arr)
+ return val
- def do_axis_reduce(self, obj, dtype, dim, keepdims):
- from pypy.module.micronumpy.interp_numarray import AxisReduce,\
- W_NDimArray
- if keepdims:
- shape = obj.shape[:dim] + [1] + obj.shape[dim + 1:]
- else:
- shape = obj.shape[:dim] + obj.shape[dim + 1:]
- result = W_NDimArray(support.product(shape), shape, dtype)
+ def do_axis_reduce(self, obj, dtype, axis, result):
+ from pypy.module.micronumpy.interp_numarray import AxisReduce
arr = AxisReduce(self.func, self.name, self.identity, obj.shape, dtype,
- result, obj, dim)
+ result, obj, axis)
loop.compute(arr)
return arr.left
@@ -176,24 +227,55 @@
self.bool_result = bool_result
def call(self, space, args_w):
- from pypy.module.micronumpy.interp_numarray import (Call1,
- convert_to_array, Scalar)
-
- [w_obj] = args_w
+ from pypy.module.micronumpy.interp_numarray import (Call1, BaseArray,
+ convert_to_array, Scalar, shape_agreement)
+ if len(args_w)<2:
+ [w_obj] = args_w
+ out = None
+ else:
+ [w_obj, out] = args_w
+ if space.is_w(out, space.w_None):
+ out = None
w_obj = convert_to_array(space, w_obj)
calc_dtype = find_unaryop_result_dtype(space,
w_obj.find_dtype(),
promote_to_float=self.promote_to_float,
promote_bools=self.promote_bools)
- if self.bool_result:
+ if out:
+ if not isinstance(out, BaseArray):
+ raise OperationError(space.w_TypeError, space.wrap(
+ 'output must be an array'))
+ res_dtype = out.find_dtype()
+ elif self.bool_result:
res_dtype = interp_dtype.get_dtype_cache(space).w_booldtype
else:
res_dtype = calc_dtype
if isinstance(w_obj, Scalar):
- return space.wrap(self.func(calc_dtype, w_obj.value.convert_to(calc_dtype)))
-
- w_res = Call1(self.func, self.name, w_obj.shape, calc_dtype, res_dtype,
- w_obj)
+ arr = self.func(calc_dtype, w_obj.value.convert_to(calc_dtype))
+ if isinstance(out,Scalar):
+ out.value=arr
+ elif isinstance(out, BaseArray):
+ out.fill(space, arr)
+ else:
+ out = arr
+ return space.wrap(out)
+ if out:
+ assert isinstance(out, BaseArray) # For translation
+ broadcast_shape = shape_agreement(space, w_obj.shape, out.shape)
+ if not broadcast_shape or broadcast_shape != out.shape:
+ raise operationerrfmt(space.w_ValueError,
+ 'output parameter shape mismatch, could not broadcast [%s]' +
+ ' to [%s]',
+ ",".join([str(x) for x in w_obj.shape]),
+ ",".join([str(x) for x in out.shape]),
+ )
+ w_res = Call1(self.func, self.name, out.shape, calc_dtype,
+ res_dtype, w_obj, out)
+ #Force it immediately
+ w_res.get_concrete()
+ else:
+ w_res = Call1(self.func, self.name, w_obj.shape, calc_dtype,
+ res_dtype, w_obj)
w_obj.add_invalidates(w_res)
return w_res
@@ -212,32 +294,61 @@
def call(self, space, args_w):
from pypy.module.micronumpy.interp_numarray import (Call2,
- convert_to_array, Scalar, shape_agreement)
-
- [w_lhs, w_rhs] = args_w
+ convert_to_array, Scalar, shape_agreement, BaseArray)
+ if len(args_w)>2:
+ [w_lhs, w_rhs, w_out] = args_w
+ else:
+ [w_lhs, w_rhs] = args_w
+ w_out = None
w_lhs = convert_to_array(space, w_lhs)
w_rhs = convert_to_array(space, w_rhs)
- calc_dtype = find_binop_result_dtype(space,
- w_lhs.find_dtype(), w_rhs.find_dtype(),
- int_only=self.int_only,
- promote_to_float=self.promote_to_float,
- promote_bools=self.promote_bools,
- )
+ if space.is_w(w_out, space.w_None) or w_out is None:
+ out = None
+ calc_dtype = find_binop_result_dtype(space,
+ w_lhs.find_dtype(), w_rhs.find_dtype(),
+ int_only=self.int_only,
+ promote_to_float=self.promote_to_float,
+ promote_bools=self.promote_bools,
+ )
+ elif not isinstance(w_out, BaseArray):
+ raise OperationError(space.w_TypeError, space.wrap(
+ 'output must be an array'))
+ else:
+ out = w_out
+ calc_dtype = out.find_dtype()
if self.comparison_func:
res_dtype = interp_dtype.get_dtype_cache(space).w_booldtype
else:
res_dtype = calc_dtype
if isinstance(w_lhs, Scalar) and isinstance(w_rhs, Scalar):
- return space.wrap(self.func(calc_dtype,
+ arr = self.func(calc_dtype,
w_lhs.value.convert_to(calc_dtype),
w_rhs.value.convert_to(calc_dtype)
- ))
+ )
+ if isinstance(out,Scalar):
+ out.value=arr
+ elif isinstance(out, BaseArray):
+ out.fill(space, arr)
+ else:
+ out = arr
+ return space.wrap(out)
new_shape = shape_agreement(space, w_lhs.shape, w_rhs.shape)
+ # Test correctness of out.shape
+ if out and out.shape != shape_agreement(space, new_shape, out.shape):
+ raise operationerrfmt(space.w_ValueError,
+ 'output parameter shape mismatch, could not broadcast [%s]' +
+ ' to [%s]',
+ ",".join([str(x) for x in new_shape]),
+ ",".join([str(x) for x in out.shape]),
+ )
w_res = Call2(self.func, self.name,
new_shape, calc_dtype,
- res_dtype, w_lhs, w_rhs)
+ res_dtype, w_lhs, w_rhs, out)
w_lhs.add_invalidates(w_res)
w_rhs.add_invalidates(w_res)
+ if out:
+ #out.add_invalidates(w_res) #causes a recursion loop
+ w_res.get_concrete()
return w_res
@@ -314,7 +425,7 @@
return dt
if dt.num >= 5:
return interp_dtype.get_dtype_cache(space).w_float64dtype
- for bytes, dtype in interp_dtype.get_dtype_cache(space).dtypes_by_num_bytes:
+ for bytes, dtype in interp_dtype.get_dtype_cache(space).float_dtypes_by_num_bytes:
if (dtype.kind == interp_dtype.FLOATINGLTR and
dtype.itemtype.get_element_size() > dt.itemtype.get_element_size()):
return dtype
@@ -388,6 +499,7 @@
"int_only": True}),
("bitwise_xor", "bitwise_xor", 2, {"int_only": True}),
("invert", "invert", 1, {"int_only": True}),
+ ("floor_divide", "floordiv", 2, {"promote_bools": True}),
("divide", "div", 2, {"promote_bools": True}),
("true_divide", "div", 2, {"promote_to_float": True}),
("mod", "mod", 2, {"promote_bools": True}),
@@ -403,6 +515,9 @@
("greater_equal", "ge", 2, {"comparison_func": True}),
("isnan", "isnan", 1, {"bool_result": True}),
("isinf", "isinf", 1, {"bool_result": True}),
+ ("isneginf", "isneginf", 1, {"bool_result": True}),
+ ("isposinf", "isposinf", 1, {"bool_result": True}),
+ ("isfinite", "isfinite", 1, {"bool_result": True}),
('logical_and', 'logical_and', 2, {'comparison_func': True,
'identity': 1}),
@@ -420,12 +535,16 @@
("negative", "neg", 1),
("absolute", "abs", 1),
("sign", "sign", 1, {"promote_bools": True}),
+ ("signbit", "signbit", 1, {"bool_result": True}),
("reciprocal", "reciprocal", 1),
("fabs", "fabs", 1, {"promote_to_float": True}),
+ ("fmod", "fmod", 2, {"promote_to_float": True}),
("floor", "floor", 1, {"promote_to_float": True}),
("ceil", "ceil", 1, {"promote_to_float": True}),
("exp", "exp", 1, {"promote_to_float": True}),
+ ("exp2", "exp2", 1, {"promote_to_float": True}),
+ ("expm1", "expm1", 1, {"promote_to_float": True}),
('sqrt', 'sqrt', 1, {'promote_to_float': True}),
@@ -435,12 +554,23 @@
("arcsin", "arcsin", 1, {"promote_to_float": True}),
("arccos", "arccos", 1, {"promote_to_float": True}),
("arctan", "arctan", 1, {"promote_to_float": True}),
+ ("arctan2", "arctan2", 2, {"promote_to_float": True}),
("sinh", "sinh", 1, {"promote_to_float": True}),
("cosh", "cosh", 1, {"promote_to_float": True}),
("tanh", "tanh", 1, {"promote_to_float": True}),
("arcsinh", "arcsinh", 1, {"promote_to_float": True}),
("arccosh", "arccosh", 1, {"promote_to_float": True}),
("arctanh", "arctanh", 1, {"promote_to_float": True}),
+
+ ("radians", "radians", 1, {"promote_to_float": True}),
+ ("degrees", "degrees", 1, {"promote_to_float": True}),
+
+ ("log", "log", 1, {"promote_to_float": True}),
+ ("log2", "log2", 1, {"promote_to_float": True}),
+ ("log10", "log10", 1, {"promote_to_float": True}),
+ ("log1p", "log1p", 1, {"promote_to_float": True}),
+ ("logaddexp", "logaddexp", 2, {"promote_to_float": True}),
+ ("logaddexp2", "logaddexp2", 2, {"promote_to_float": True}),
]:
self.add_ufunc(space, *ufunc_def)
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -4,6 +4,7 @@
ViewTransform, BroadcastTransform
from pypy.tool.pairtype import extendabletype
from pypy.module.micronumpy.loop import ComputationDone
+from pypy.rlib import jit
""" Signature specifies both the numpy expression that has been constructed
and the assembler to be compiled. This is a very important observation -
@@ -142,11 +143,10 @@
from pypy.module.micronumpy.interp_numarray import ConcreteArray
concr = arr.get_concrete()
assert isinstance(concr, ConcreteArray)
- storage = concr.storage
if self.iter_no >= len(iterlist):
iterlist.append(concr.create_iter(transforms))
if self.array_no >= len(arraylist):
- arraylist.append(storage)
+ arraylist.append(concr)
def eval(self, frame, arr):
iter = frame.iterators[self.iter_no]
@@ -216,13 +216,14 @@
return self.child.eval(frame, arr.child)
class Call1(Signature):
- _immutable_fields_ = ['unfunc', 'name', 'child', 'dtype']
+ _immutable_fields_ = ['unfunc', 'name', 'child', 'res', 'dtype']
- def __init__(self, func, name, dtype, child):
+ def __init__(self, func, name, dtype, child, res=None):
self.unfunc = func
self.child = child
self.name = name
self.dtype = dtype
+ self.res = res
def hash(self):
return compute_hash(self.name) ^ intmask(self.child.hash() << 1)
@@ -256,6 +257,29 @@
v = self.child.eval(frame, arr.values).convert_to(arr.calc_dtype)
return self.unfunc(arr.calc_dtype, v)
+
+class BroadcastUfunc(Call1):
+ def _invent_numbering(self, cache, allnumbers):
+ self.res._invent_numbering(cache, allnumbers)
+ self.child._invent_numbering(new_cache(), allnumbers)
+
+ def debug_repr(self):
+ return 'BroadcastUfunc(%s, %s)' % (self.name, self.child.debug_repr())
+
+ def _create_iter(self, iterlist, arraylist, arr, transforms):
+ from pypy.module.micronumpy.interp_numarray import Call1
+
+ assert isinstance(arr, Call1)
+ vtransforms = transforms + [BroadcastTransform(arr.values.shape)]
+ self.child._create_iter(iterlist, arraylist, arr.values, vtransforms)
+ self.res._create_iter(iterlist, arraylist, arr.res, transforms)
+
+ def eval(self, frame, arr):
+ from pypy.module.micronumpy.interp_numarray import Call1
+ assert isinstance(arr, Call1)
+ v = self.child.eval(frame, arr.values).convert_to(arr.calc_dtype)
+ return self.unfunc(arr.calc_dtype, v)
+
class Call2(Signature):
_immutable_fields_ = ['binfunc', 'name', 'calc_dtype', 'left', 'right']
@@ -316,7 +340,31 @@
assert isinstance(arr, ResultArray)
offset = frame.get_final_iter().offset
- arr.left.setitem(offset, self.right.eval(frame, arr.right))
+ val = self.right.eval(frame, arr.right)
+ arr.left.setitem(offset, val)
+
+class BroadcastResultSignature(ResultSignature):
+ def _create_iter(self, iterlist, arraylist, arr, transforms):
+ from pypy.module.micronumpy.interp_numarray import ResultArray
+
+ assert isinstance(arr, ResultArray)
+ rtransforms = transforms + [BroadcastTransform(arr.left.shape)]
+ self.left._create_iter(iterlist, arraylist, arr.left, transforms)
+ self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
+
+class ToStringSignature(Call1):
+ def __init__(self, dtype, child):
+ Call1.__init__(self, None, 'tostring', dtype, child)
+
+ @jit.unroll_safe
+ def eval(self, frame, arr):
+ from pypy.module.micronumpy.interp_numarray import ToStringArray
+
+ assert isinstance(arr, ToStringArray)
+ arr.res_str.setitem(0, self.child.eval(frame, arr.values).convert_to(
+ self.dtype))
+ for i in range(arr.item_size):
+ arr.s.append(arr.res_str_casted[i])
class BroadcastLeft(Call2):
def _invent_numbering(self, cache, allnumbers):
@@ -441,6 +489,5 @@
cur = arr.left.getitem(iterator.offset)
value = self.binfunc(self.calc_dtype, cur, v)
arr.left.setitem(iterator.offset, value)
-
def debug_repr(self):
return 'AxisReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -1,6 +1,14 @@
from pypy.rlib import jit
from pypy.interpreter.error import OperationError
+def enumerate_chunks(chunks):
+ result = []
+ i = -1
+ for chunk in chunks:
+ i += chunk.axis_step
+ result.append((i, chunk))
+ return result
+
@jit.look_inside_iff(lambda shape, start, strides, backstrides, chunks:
jit.isconstant(len(chunks))
)
@@ -10,7 +18,7 @@
rstart = start
rshape = []
i = -1
- for i, chunk in enumerate(chunks):
+ for i, chunk in enumerate_chunks(chunks):
if chunk.step != 0:
rstrides.append(strides[i] * chunk.step)
rbackstrides.append(strides[i] * (chunk.lgt - 1) * chunk.step)
@@ -38,22 +46,31 @@
rbackstrides = [0] * (len(res_shape) - len(orig_shape)) + rbackstrides
return rstrides, rbackstrides
-def find_shape_and_elems(space, w_iterable):
+def is_single_elem(space, w_elem, is_rec_type):
+ if (is_rec_type and space.isinstance_w(w_elem, space.w_tuple)):
+ return True
+ if space.issequence_w(w_elem):
+ return False
+ return True
+
+def find_shape_and_elems(space, w_iterable, dtype):
shape = [space.len_w(w_iterable)]
batch = space.listview(w_iterable)
+ is_rec_type = dtype is not None and dtype.is_record_type()
while True:
new_batch = []
if not batch:
return shape, []
- if not space.issequence_w(batch[0]):
- for elem in batch:
- if space.issequence_w(elem):
+ if is_single_elem(space, batch[0], is_rec_type):
+ for w_elem in batch:
+ if not is_single_elem(space, w_elem, is_rec_type):
raise OperationError(space.w_ValueError, space.wrap(
"setting an array element with a sequence"))
return shape, batch
size = space.len_w(batch[0])
for w_elem in batch:
- if not space.issequence_w(w_elem) or space.len_w(w_elem) != size:
+ if (is_single_elem(space, w_elem, is_rec_type) or
+ space.len_w(w_elem) != size):
raise OperationError(space.w_ValueError, space.wrap(
"setting an array element with a sequence"))
new_batch += space.listview(w_elem)
diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -4,6 +4,8 @@
from pypy.module.micronumpy.interp_ufuncs import (find_binop_result_dtype,
find_unaryop_result_dtype)
from pypy.module.micronumpy.interp_boxes import W_Float64Box
+from pypy.module.micronumpy.interp_dtype import nonnative_byteorder_prefix,\
+ byteorder_prefix
from pypy.conftest import option
import sys
@@ -15,14 +17,16 @@
sys.modules['numpypy'] = numpy
sys.modules['_numpypy'] = numpy
cls.space = gettestobjspace(usemodules=['micronumpy'])
+ cls.w_non_native_prefix = cls.space.wrap(nonnative_byteorder_prefix)
+ cls.w_native_prefix = cls.space.wrap(byteorder_prefix)
class TestSignature(object):
def test_binop_signature(self, space):
float64_dtype = get_dtype_cache(space).w_float64dtype
bool_dtype = get_dtype_cache(space).w_booldtype
- ar = W_NDimArray(10, [10], dtype=float64_dtype)
- ar2 = W_NDimArray(10, [10], dtype=float64_dtype)
+ ar = W_NDimArray([10], dtype=float64_dtype)
+ ar2 = W_NDimArray([10], dtype=float64_dtype)
v1 = ar.descr_add(space, ar)
v2 = ar.descr_add(space, Scalar(float64_dtype, W_Float64Box(2.0)))
sig1 = v1.find_sig()
@@ -40,7 +44,7 @@
v4 = ar.descr_add(space, ar)
assert v1.find_sig() is v4.find_sig()
- bool_ar = W_NDimArray(10, [10], dtype=bool_dtype)
+ bool_ar = W_NDimArray([10], dtype=bool_dtype)
v5 = ar.descr_add(space, bool_ar)
assert v5.find_sig() is not v1.find_sig()
assert v5.find_sig() is not v2.find_sig()
@@ -57,7 +61,7 @@
def test_slice_signature(self, space):
float64_dtype = get_dtype_cache(space).w_float64dtype
- ar = W_NDimArray(10, [10], dtype=float64_dtype)
+ ar = W_NDimArray([10], dtype=float64_dtype)
v1 = ar.descr_getitem(space, space.wrap(slice(1, 3, 1)))
v2 = ar.descr_getitem(space, space.wrap(slice(4, 6, 1)))
assert v1.find_sig() is v2.find_sig()
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -1,5 +1,7 @@
+import py
+from pypy.conftest import option
from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
-
+from pypy.interpreter.gateway import interp2app
class AppTestDtypes(BaseNumpyAppTest):
def test_dtype(self):
@@ -12,7 +14,10 @@
assert dtype(d) is d
assert dtype(None) is dtype(float)
assert dtype('int8').name == 'int8'
+ assert dtype(int).fields is None
+ assert dtype(int).names is None
raises(TypeError, dtype, 1042)
+ raises(KeyError, 'dtype(int)["asdasd"]')
def test_dtype_eq(self):
from _numpypy import dtype
@@ -53,13 +58,13 @@
assert a[i] is True_
def test_copy_array_with_dtype(self):
- from _numpypy import array, False_, True_, int64
+ from _numpypy import array, False_, longlong
a = array([0, 1, 2, 3], dtype=long)
# int on 64-bit, long in 32-bit
- assert isinstance(a[0], int64)
+ assert isinstance(a[0], longlong)
b = a.copy()
- assert isinstance(b[0], int64)
+ assert isinstance(b[0], longlong)
a = array([0, 1, 2, 3], dtype=bool)
assert a[0] is False_
@@ -81,17 +86,17 @@
assert a[i] is True_
def test_zeros_long(self):
- from _numpypy import zeros, int64
+ from _numpypy import zeros, longlong
a = zeros(10, dtype=long)
for i in range(10):
- assert isinstance(a[i], int64)
+ assert isinstance(a[i], longlong)
assert a[1] == 0
def test_ones_long(self):
- from _numpypy import ones, int64
+ from _numpypy import ones, longlong
a = ones(10, dtype=long)
for i in range(10):
- assert isinstance(a[i], int64)
+ assert isinstance(a[i], longlong)
assert a[1] == 1
def test_overflow(self):
@@ -181,17 +186,18 @@
assert dtype("float") is dtype(float)
-class AppTestTypes(BaseNumpyAppTest):
+class AppTestTypes(BaseNumpyAppTest):
def test_abstract_types(self):
import _numpypy as numpy
raises(TypeError, numpy.generic, 0)
raises(TypeError, numpy.number, 0)
raises(TypeError, numpy.integer, 0)
exc = raises(TypeError, numpy.signedinteger, 0)
- assert str(exc.value) == "cannot create 'signedinteger' instances"
+ assert 'cannot create' in str(exc.value)
+ assert 'signedinteger' in str(exc.value)
exc = raises(TypeError, numpy.unsignedinteger, 0)
- assert str(exc.value) == "cannot create 'unsignedinteger' instances"
-
+ assert 'cannot create' in str(exc.value)
+ assert 'unsignedinteger' in str(exc.value)
raises(TypeError, numpy.floating, 0)
raises(TypeError, numpy.inexact, 0)
@@ -296,6 +302,7 @@
else:
raises(OverflowError, numpy.int32, 2147483648)
raises(OverflowError, numpy.int32, '2147483648')
+ assert numpy.dtype('int32') is numpy.dtype(numpy.int32)
def test_uint32(self):
import sys
@@ -327,15 +334,11 @@
assert numpy.dtype(numpy.int64).type is numpy.int64
assert numpy.int64(3) == 3
- if sys.maxint >= 2 ** 63 - 1:
- assert numpy.int64(9223372036854775807) == 9223372036854775807
- assert numpy.int64('9223372036854775807') == 9223372036854775807
- else:
- raises(OverflowError, numpy.int64, 9223372036854775807)
- raises(OverflowError, numpy.int64, '9223372036854775807')
+ assert numpy.int64(9223372036854775807) == 9223372036854775807
+ assert numpy.int64(9223372036854775807) == 9223372036854775807
raises(OverflowError, numpy.int64, 9223372036854775808)
- raises(OverflowError, numpy.int64, '9223372036854775808')
+ raises(OverflowError, numpy.int64, 9223372036854775808L)
def test_uint64(self):
import sys
@@ -404,10 +407,29 @@
assert issubclass(int64, int)
assert int_ is int64
+ def test_various_types(self):
+ import _numpypy as numpy
+ import sys
+
+ assert numpy.int16 is numpy.short
+ assert numpy.int8 is numpy.byte
+ assert numpy.bool_ is numpy.bool8
+ if sys.maxint == (1 << 63) - 1:
+ assert numpy.intp is numpy.int64
+ else:
+ assert numpy.intp is numpy.int32
+
+ def test_mro(self):
+ import _numpypy as numpy
+
+ assert numpy.int16.__mro__ == (numpy.int16, numpy.signedinteger,
+ numpy.integer, numpy.number,
+ numpy.generic, object)
+ assert numpy.bool_.__mro__ == (numpy.bool_, numpy.generic, object)
+
def test_operators(self):
from operator import truediv
from _numpypy import float64, int_, True_, False_
-
assert 5 / int_(2) == int_(2)
assert truediv(int_(3), int_(2)) == float64(1.5)
assert truediv(3, int_(2)) == float64(1.5)
@@ -427,9 +449,115 @@
assert int_(3) ^ int_(5) == int_(6)
assert True_ ^ False_ is True_
assert 5 ^ int_(3) == int_(6)
-
assert +int_(3) == int_(3)
assert ~int_(3) == int_(-4)
-
raises(TypeError, lambda: float64(3) & 1)
+ def test_alternate_constructs(self):
+ from _numpypy import dtype
+ nnp = self.non_native_prefix
+ byteorder = self.native_prefix
+ assert dtype('i8') == dtype(byteorder + 'i8') == dtype('=i8') # XXX should be equal == dtype(long)
+ assert dtype(nnp + 'i8') != dtype('i8')
+ assert dtype(nnp + 'i8').byteorder == nnp
+ assert dtype('=i8').byteorder == '='
+ assert dtype(byteorder + 'i8').byteorder == '='
+
+ def test_alignment(self):
+ from _numpypy import dtype
+ assert dtype('i4').alignment == 4
+
+ def test_typeinfo(self):
+ from _numpypy import typeinfo, void, number, int64, bool_
+ assert typeinfo['Number'] == number
+ assert typeinfo['LONGLONG'] == ('q', 9, 64, 8, 9223372036854775807L, -9223372036854775808L, int64)
+ assert typeinfo['VOID'] == ('V', 20, 0, 1, void)
+ assert typeinfo['BOOL'] == ('?', 0, 8, 1, 1, 0, bool_)
+
+class AppTestStrUnicodeDtypes(BaseNumpyAppTest):
+ def test_str_unicode(self):
+ from _numpypy import str_, unicode_, character, flexible, generic
+
+ assert str_.mro() == [str_, str, basestring, character, flexible, generic, object]
+ assert unicode_.mro() == [unicode_, unicode, basestring, character, flexible, generic, object]
+
+ def test_str_dtype(self):
+ from _numpypy import dtype, str_
+
+ raises(TypeError, "dtype('Sx')")
+ d = dtype('S8')
+ assert d.itemsize == 8
+ assert dtype(str) == dtype('S')
+ assert d.kind == 'S'
+ assert d.type is str_
+ assert d.name == "string64"
+ assert d.num == 18
+
+ def test_unicode_dtype(self):
+ from _numpypy import dtype, unicode_
+
+ raises(TypeError, "dtype('Ux')")
+ d = dtype('U8')
+ assert d.itemsize == 8 * 4
+ assert dtype(unicode) == dtype('U')
+ assert d.kind == 'U'
+ assert d.type is unicode_
+ assert d.name == "unicode256"
+ assert d.num == 19
+
+ def test_string_boxes(self):
+ from _numpypy import str_
+ assert isinstance(str_(3), str_)
+
+ def test_unicode_boxes(self):
+ from _numpypy import unicode_
+ assert isinstance(unicode_(3), unicode)
+
+class AppTestRecordDtypes(BaseNumpyAppTest):
+ def test_create(self):
+ from _numpypy import dtype, void
+
+ raises(ValueError, "dtype([('x', int), ('x', float)])")
+ d = dtype([("x", "int32"), ("y", "int32"), ("z", "int32"), ("value", float)])
+ assert d.fields['x'] == (dtype('int32'), 0)
+ assert d.fields['value'] == (dtype(float), 12)
+ assert d['x'] == dtype('int32')
+ assert d.name == "void160"
+ assert d.num == 20
+ assert d.itemsize == 20
+ assert d.kind == 'V'
+ assert d.type is void
+ assert d.char == 'V'
+ assert d.names == ("x", "y", "z", "value")
+ raises(KeyError, 'd["xyz"]')
+ raises(KeyError, 'd.fields["xyz"]')
+
+ def test_create_from_dict(self):
+ skip("not yet")
+ from _numpypy import dtype
+ d = dtype({'names': ['a', 'b', 'c'],
+ })
+
+class AppTestNotDirect(BaseNumpyAppTest):
+ def setup_class(cls):
+ BaseNumpyAppTest.setup_class.im_func(cls)
+ def check_non_native(w_obj, w_obj2):
+ assert w_obj.storage[0] == w_obj2.storage[1]
+ assert w_obj.storage[1] == w_obj2.storage[0]
+ if w_obj.storage[0] == '\x00':
+ assert w_obj2.storage[1] == '\x00'
+ assert w_obj2.storage[0] == '\x01'
+ else:
+ assert w_obj2.storage[1] == '\x01'
+ assert w_obj2.storage[0] == '\x00'
+ cls.w_check_non_native = cls.space.wrap(interp2app(check_non_native))
+ if option.runappdirect:
+ py.test.skip("not a direct test")
+
+ def test_non_native(self):
+ from _numpypy import array
+ a = array([1, 2, 3], dtype=self.non_native_prefix + 'i2')
+ assert a[0] == 1
+ assert (a + a)[1] == 4
+ self.check_non_native(a, array([1, 2, 3], 'i2'))
+
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -5,15 +5,23 @@
from pypy.interpreter.error import OperationError
from pypy.module.micronumpy import signature
from pypy.module.micronumpy.appbridge import get_appbridge_cache
-from pypy.module.micronumpy.interp_iter import Chunk
+from pypy.module.micronumpy.interp_iter import Chunk, Chunks
from pypy.module.micronumpy.interp_numarray import W_NDimArray, shape_agreement
from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
class MockDtype(object):
- def malloc(self, size):
- return None
+ class itemtype(object):
+ @staticmethod
+ def malloc(size):
+ return None
+ def get_size(self):
+ return 1
+
+
+def create_slice(a, chunks):
+ return Chunks(chunks).apply(a)
class TestNumArrayDirect(object):
def newslice(self, *args):
@@ -29,116 +37,116 @@
return self.space.newtuple(args_w)
def test_strides_f(self):
- a = W_NDimArray(100, [10, 5, 3], MockDtype(), 'F')
+ a = W_NDimArray([10, 5, 3], MockDtype(), 'F')
assert a.strides == [1, 10, 50]
assert a.backstrides == [9, 40, 100]
def test_strides_c(self):
- a = W_NDimArray(100, [10, 5, 3], MockDtype(), 'C')
+ a = W_NDimArray([10, 5, 3], MockDtype(), 'C')
assert a.strides == [15, 3, 1]
assert a.backstrides == [135, 12, 2]
def test_create_slice_f(self):
- a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
- s = a.create_slice([Chunk(3, 0, 0, 1)])
+ a = W_NDimArray([10, 5, 3], MockDtype(), 'F')
+ s = create_slice(a, [Chunk(3, 0, 0, 1)])
assert s.start == 3
assert s.strides == [10, 50]
assert s.backstrides == [40, 100]
- s = a.create_slice([Chunk(1, 9, 2, 4)])
+ s = create_slice(a, [Chunk(1, 9, 2, 4)])
assert s.start == 1
assert s.strides == [2, 10, 50]
assert s.backstrides == [6, 40, 100]
- s = a.create_slice([Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1), Chunk(1, 0, 0, 1)])
+ s = create_slice(a, [Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1), Chunk(1, 0, 0, 1)])
assert s.shape == [2, 1]
assert s.strides == [3, 10]
assert s.backstrides == [3, 0]
- s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
+ s = create_slice(a, [Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
assert s.start == 20
assert s.shape == [10, 3]
def test_create_slice_c(self):
- a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'C')
- s = a.create_slice([Chunk(3, 0, 0, 1)])
+ a = W_NDimArray([10, 5, 3], MockDtype(), 'C')
+ s = create_slice(a, [Chunk(3, 0, 0, 1)])
assert s.start == 45
assert s.strides == [3, 1]
assert s.backstrides == [12, 2]
- s = a.create_slice([Chunk(1, 9, 2, 4)])
+ s = create_slice(a, [Chunk(1, 9, 2, 4)])
assert s.start == 15
assert s.strides == [30, 3, 1]
assert s.backstrides == [90, 12, 2]
- s = a.create_slice([Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1),
+ s = create_slice(a, [Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1),
Chunk(1, 0, 0, 1)])
assert s.start == 19
assert s.shape == [2, 1]
assert s.strides == [45, 3]
assert s.backstrides == [45, 0]
- s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
+ s = create_slice(a, [Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
assert s.start == 6
assert s.shape == [10, 3]
def test_slice_of_slice_f(self):
- a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
- s = a.create_slice([Chunk(5, 0, 0, 1)])
+ a = W_NDimArray([10, 5, 3], MockDtype(), 'F')
+ s = create_slice(a, [Chunk(5, 0, 0, 1)])
assert s.start == 5
- s2 = s.create_slice([Chunk(3, 0, 0, 1)])
+ s2 = create_slice(s, [Chunk(3, 0, 0, 1)])
assert s2.shape == [3]
assert s2.strides == [50]
assert s2.parent is a
assert s2.backstrides == [100]
assert s2.start == 35
- s = a.create_slice([Chunk(1, 5, 3, 2)])
- s2 = s.create_slice([Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)])
+ s = create_slice(a, [Chunk(1, 5, 3, 2)])
+ s2 = create_slice(s, [Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)])
assert s2.shape == [2, 3]
assert s2.strides == [3, 50]
assert s2.backstrides == [3, 100]
assert s2.start == 1 * 15 + 2 * 3
def test_slice_of_slice_c(self):
- a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), order='C')
- s = a.create_slice([Chunk(5, 0, 0, 1)])
+ a = W_NDimArray([10, 5, 3], MockDtype(), order='C')
+ s = create_slice(a, [Chunk(5, 0, 0, 1)])
assert s.start == 15 * 5
- s2 = s.create_slice([Chunk(3, 0, 0, 1)])
+ s2 = create_slice(s, [Chunk(3, 0, 0, 1)])
assert s2.shape == [3]
assert s2.strides == [1]
assert s2.parent is a
assert s2.backstrides == [2]
assert s2.start == 5 * 15 + 3 * 3
- s = a.create_slice([Chunk(1, 5, 3, 2)])
- s2 = s.create_slice([Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)])
+ s = create_slice(a, [Chunk(1, 5, 3, 2)])
+ s2 = create_slice(s, [Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)])
assert s2.shape == [2, 3]
assert s2.strides == [45, 1]
assert s2.backstrides == [45, 2]
assert s2.start == 1 * 15 + 2 * 3
def test_negative_step_f(self):
- a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
- s = a.create_slice([Chunk(9, -1, -2, 5)])
+ a = W_NDimArray([10, 5, 3], MockDtype(), 'F')
+ s = create_slice(a, [Chunk(9, -1, -2, 5)])
assert s.start == 9
assert s.strides == [-2, 10, 50]
assert s.backstrides == [-8, 40, 100]
def test_negative_step_c(self):
- a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), order='C')
- s = a.create_slice([Chunk(9, -1, -2, 5)])
+ a = W_NDimArray([10, 5, 3], MockDtype(), order='C')
+ s = create_slice(a, [Chunk(9, -1, -2, 5)])
assert s.start == 135
assert s.strides == [-30, 3, 1]
assert s.backstrides == [-120, 12, 2]
def test_index_of_single_item_f(self):
- a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
+ a = W_NDimArray([10, 5, 3], MockDtype(), 'F')
r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2))
assert r == 1 + 2 * 10 + 2 * 50
- s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
+ s = create_slice(a, [Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
r = s._index_of_single_item(self.space, self.newtuple(1, 0))
assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0))
r = s._index_of_single_item(self.space, self.newtuple(1, 1))
assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 1))
def test_index_of_single_item_c(self):
- a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'C')
+ a = W_NDimArray([10, 5, 3], MockDtype(), 'C')
r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2))
assert r == 1 * 3 * 5 + 2 * 3 + 2
- s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
+ s = create_slice(a, [Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
r = s._index_of_single_item(self.space, self.newtuple(1, 0))
assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0))
r = s._index_of_single_item(self.space, self.newtuple(1, 1))
@@ -203,6 +211,18 @@
assert a.shape == (3,)
assert a.dtype is dtype(int)
+ def test_ndmin(self):
+ from _numpypy import array
+
+ arr = array([[[1]]], ndmin=1)
+ assert arr.shape == (1, 1, 1)
+
+ def test_noop_ndmin(self):
+ from _numpypy import array
+
+ arr = array([1], ndmin=3)
+ assert arr.shape == (1, 1, 1)
+
def test_type(self):
from _numpypy import array
ar = array(range(5))
@@ -374,6 +394,58 @@
assert a[1] == 0.
assert a[3] == 0.
+ def test_newaxis(self):
+ from _numpypy import array
+ from numpypy.core.numeric import newaxis
+ a = array(range(5))
+ b = array([range(5)])
+ assert (a[newaxis] == b).all()
+
+ def test_newaxis_slice(self):
+ from _numpypy import array
+ from numpypy.core.numeric import newaxis
+
+ a = array(range(5))
+ b = array(range(1,5))
+ c = array([range(1,5)])
+ d = array([[x] for x in range(1,5)])
+
+ assert (a[1:] == b).all()
+ assert (a[1:,newaxis] == d).all()
+ assert (a[newaxis,1:] == c).all()
+
+ def test_newaxis_assign(self):
+ from _numpypy import array
+ from numpypy.core.numeric import newaxis
+
+ a = array(range(5))
+ a[newaxis,1] = [2]
+ assert a[1] == 2
+
+ def test_newaxis_virtual(self):
+ from _numpypy import array
+ from numpypy.core.numeric import newaxis
+
+ a = array(range(5))
+ b = (a + a)[newaxis]
+ c = array([[0, 2, 4, 6, 8]])
+ assert (b == c).all()
+
+ def test_newaxis_then_slice(self):
+ from _numpypy import array
+ from numpypy.core.numeric import newaxis
+ a = array(range(5))
+ b = a[newaxis]
+ assert b.shape == (1, 5)
+ assert (b[0,1:] == a[1:]).all()
+
+ def test_slice_then_newaxis(self):
+ from _numpypy import array
+ from numpypy.core.numeric import newaxis
+ a = array(range(5))
+ b = a[2:]
+ assert (b[newaxis] == [[2, 3, 4]]).all()
+
def test_scalar(self):
from _numpypy import array, dtype
a = array(3)
@@ -434,6 +506,8 @@
a = zeros((4, 2, 3))
a.shape = (12, 2)
(a + a).reshape(2, 12) # assert did not explode
+ a = array([[[[]]]])
+ assert a.reshape((0,)).shape == (0,)
def test_slice_reshape(self):
from _numpypy import zeros, arange
@@ -625,6 +699,56 @@
for i in range(5):
assert b[i] == i / 5.0
+ def test_floordiv(self):
+ from math import isnan
+ from _numpypy import array, dtype
+
+ a = array(range(1, 6))
+ b = a // a
+ assert (b == [1, 1, 1, 1, 1]).all()
+
+ a = array(range(1, 6), dtype=bool)
+ b = a // a
+ assert b.dtype is dtype("int8")
+ assert (b == [1, 1, 1, 1, 1]).all()
+
+ a = array([-1, 0, 1])
+ b = array([0, 0, 0])
+ c = a // b
+ assert (c == [0, 0, 0]).all()
+
+ a = array([-1.0, 0.0, 1.0])
+ b = array([0.0, 0.0, 0.0])
+ c = a // b
+ assert c[0] == float('-inf')
+ assert isnan(c[1])
+ assert c[2] == float('inf')
+
+ b = array([-0.0, -0.0, -0.0])
+ c = a // b
+ assert c[0] == float('inf')
+ assert isnan(c[1])
+ assert c[2] == float('-inf')
+
+ def test_floordiv_other(self):
+ from _numpypy import array
+ a = array(range(5))
+ b = array([2, 2, 2, 2, 2], float)
+ c = a // b
+ assert (c == [0, 0, 1, 1, 2]).all()
+
+ def test_rfloordiv(self):
+ from _numpypy import array
+ a = array(range(1, 6))
+ b = 3 // a
+ assert (b == [3, 1, 1, 0, 0]).all()
+
+ def test_floordiv_constant(self):
+ from _numpypy import array
+ a = array(range(5))
+ b = a // 2
+ assert (b == [0, 0, 1, 1, 2]).all()
+
def test_truediv(self):
from operator import truediv
from _numpypy import arange
@@ -871,6 +995,10 @@
assert a.sum() == 5
raises(TypeError, 'a.sum(2, 3)')
+ d = array(0.)
+ b = a.sum(out=d)
+ assert b == d
+ assert isinstance(b, float)
def test_reduce_nd(self):
from numpypy import arange, array, multiply
@@ -1037,7 +1165,7 @@
assert array([True, False]).dtype is dtype(bool)
assert array([True, 1]).dtype is dtype(int)
assert array([1, 2, 3]).dtype is dtype(int)
- assert array([1L, 2, 3]).dtype is dtype(long)
+ #assert array([1L, 2, 3]).dtype is dtype(long)
assert array([1.2, True]).dtype is dtype(float)
assert array([1.2, 5]).dtype is dtype(float)
assert array([]).dtype is dtype(float)
@@ -1371,8 +1499,6 @@
a = array([[1, 2], [3, 4], [5, 6], [7, 8],
[9, 10], [11, 12], [13, 14]])
b = a[::2]
- print a
- print b
assert (b == [[1, 2], [5, 6], [9, 10], [13, 14]]).all()
c = b + b
assert c[1][1] == 12
@@ -1544,6 +1670,7 @@
a = arange(12).reshape(3,4)
b = a.T.flat
b[6::2] = [-1, -2]
+ print a == [[0, 1, -1, 3], [4, 5, 6, -1], [8, 9, -2, 11]]
assert (a == [[0, 1, -1, 3], [4, 5, 6, -1], [8, 9, -2, 11]]).all()
b[0:2] = [[[100]]]
assert(a[0,0] == 100)
@@ -1818,6 +1945,12 @@
#5 bytes is larger than 3 bytes
raises(ValueError, fromstring, "\x01\x02\x03", count=5, dtype=uint8)
+ def test_tostring(self):
+ from _numpypy import array
+ assert array([1, 2, 3], 'i2').tostring() == '\x01\x00\x02\x00\x03\x00'
+ assert array([1, 2, 3], 'i2')[::2].tostring() == '\x01\x00\x03\x00'
+ assert array([1, 2, 3], '<i2')[::2].tostring() == '\x01\x00\x03\x00'
+ assert array([1, 2, 3], '>i2')[::2].tostring() == '\x00\x01\x00\x03'
class AppTestRanges(BaseNumpyAppTest):
def test_arange(self):
@@ -1863,3 +1996,57 @@
cache = get_appbridge_cache(cls.space)
cache.w_array_repr = cls.old_array_repr
cache.w_array_str = cls.old_array_str
+
+class AppTestRecordDtype(BaseNumpyAppTest):
+ def test_zeros(self):
+ from _numpypy import zeros
+ a = zeros(2, dtype=[('x', int), ('y', float)])
+ raises(IndexError, 'a[0]["xyz"]')
+ assert a[0]['x'] == 0
+ assert a[0]['y'] == 0
+ raises(ValueError, "a[0] = (1, 2, 3)")
+ a[0]['x'] = 13
+ assert a[0]['x'] == 13
+ a[1] = (1, 2)
+ assert a[1]['y'] == 2
+ b = zeros(2, dtype=[('x', int), ('y', float)])
+ b[1] = a[1]
+ assert a[1]['y'] == 2
+
+ def test_views(self):
+ from _numpypy import array
+ a = array([(1, 2), (3, 4)], dtype=[('x', int), ('y', float)])
+ raises(ValueError, 'array([1])["x"]')
+ raises(ValueError, 'a["z"]')
+ assert a['x'][1] == 3
+ assert a['y'][1] == 4
+ a['x'][0] = 15
+ assert a['x'][0] == 15
+ b = a['x'] + a['y']
+ assert (b == [15+2, 3+4]).all()
+ assert b.dtype == float
+
+ def test_assign_tuple(self):
+ from _numpypy import zeros
+ a = zeros((2, 3), dtype=[('x', int), ('y', float)])
+ a[1, 2] = (1, 2)
+ assert a['x'][1, 2] == 1
+ assert a['y'][1, 2] == 2
+
+ def test_creation_and_repr(self):
+ from _numpypy import array
+ a = array([(1, 2), (3, 4)], dtype=[('x', int), ('y', float)])
+ assert repr(a[0]) == '(1, 2.0)'
+
+ def test_nested_dtype(self):
+ from _numpypy import zeros
+ a = [('x', int), ('y', float)]
+ b = [('x', int), ('y', a)]
+ arr = zeros(3, dtype=b)
+ arr[1]['x'] = 15
+ assert arr[1]['x'] == 15
+ arr[1]['y']['y'] = 3.5
+ assert arr[1]['y']['y'] == 3.5
+ assert arr[1]['y']['x'] == 0.0
+ assert arr[1]['x'] == 15
+
diff --git a/pypy/module/micronumpy/test/test_outarg.py b/pypy/module/micronumpy/test/test_outarg.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_outarg.py
@@ -0,0 +1,126 @@
+import py
+from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+
+class AppTestOutArg(BaseNumpyAppTest):
+ def test_reduce_out(self):
+ from numpypy import arange, zeros, array
+ a = arange(15).reshape(5, 3)
+ b = arange(12).reshape(4,3)
+ c = a.sum(0, out=b[1])
+ assert (c == [30, 35, 40]).all()
+ assert (c == b[1]).all()
+ raises(ValueError, 'a.prod(0, out=arange(10))')
+ a=arange(12).reshape(3,2,2)
+ raises(ValueError, 'a.sum(0, out=arange(12).reshape(3,2,2))')
+ raises(ValueError, 'a.sum(0, out=arange(3))')
+ c = array([-1, 0, 1]).sum(out=zeros([], dtype=bool))
+ #You could argue that this should product False, but
+ # that would require an itermediate result. Cpython numpy
+ # gives True.
+ assert c == True
+ a = array([[-1, 0, 1], [1, 0, -1]])
+ c = a.sum(0, out=zeros((3,), dtype=bool))
+ assert (c == [True, False, True]).all()
+ c = a.sum(1, out=zeros((2,), dtype=bool))
+ assert (c == [True, True]).all()
+
+ def test_reduce_intermediary(self):
+ from numpypy import arange, array
+ a = arange(15).reshape(5, 3)
+ b = array(range(3), dtype=bool)
+ c = a.prod(0, out=b)
+ assert(b == [False, True, True]).all()
+
+ def test_ufunc_out(self):
+ from _numpypy import array, negative, zeros, sin
+ from math import sin as msin
+ a = array([[1, 2], [3, 4]])
+ c = zeros((2,2,2))
+ b = negative(a + a, out=c[1])
+ #test for view, and also test that forcing out also forces b
+ assert (c[:, :, 1] == [[0, 0], [-4, -8]]).all()
+ assert (b == [[-2, -4], [-6, -8]]).all()
+ #Test broadcast, type promotion
+ b = negative(3, out=a)
+ assert (a == -3).all()
+ c = zeros((2, 2), dtype=float)
+ b = negative(3, out=c)
+ assert b.dtype.kind == c.dtype.kind
+ assert b.shape == c.shape
+ a = array([1, 2])
+ b = sin(a, out=c)
+ assert(c == [[msin(1), msin(2)]] * 2).all()
+ b = sin(a, out=c+c)
+ assert (c == b).all()
+
+ #Test shape agreement
+ a = zeros((3,4))
+ b = zeros((3,5))
+ raises(ValueError, 'negative(a, out=b)')
+ b = zeros((1,4))
+ raises(ValueError, 'negative(a, out=b)')
+
+ def test_binfunc_out(self):
+ from _numpypy import array, add
+ a = array([[1, 2], [3, 4]])
+ out = array([[1, 2], [3, 4]])
+ c = add(a, a, out=out)
+ assert (c == out).all()
+ assert c.shape == a.shape
+ assert c.dtype is a.dtype
+ c[0,0] = 100
+ assert out[0, 0] == 100
+ out[:] = 100
+ raises(ValueError, 'c = add(a, a, out=out[1])')
+ c = add(a[0], a[1], out=out[1])
+ assert (c == out[1]).all()
+ assert (c == [4, 6]).all()
+ assert (out[0] == 100).all()
+ c = add(a[0], a[1], out=out)
+ assert (c == out[1]).all()
+ assert (c == out[0]).all()
+ out = array(16, dtype=int)
+ b = add(10, 10, out=out)
+ assert b==out
+ assert b.dtype == out.dtype
+
+ def test_applevel(self):
+ from _numpypy import array, sum, max, min
+ a = array([[1, 2], [3, 4]])
+ out = array([[0, 0], [0, 0]])
+ c = sum(a, axis=0, out=out[0])
+ assert (c == [4, 6]).all()
+ assert (c == out[0]).all()
+ assert (c != out[1]).all()
+ c = max(a, axis=1, out=out[0])
+ assert (c == [2, 4]).all()
+ assert (c == out[0]).all()
+ assert (c != out[1]).all()
+
+ def test_ufunc_cast(self):
+ from _numpypy import array, negative, add, sum
+ a = array(16, dtype = int)
+ c = array(0, dtype = float)
+ b = negative(a, out=c)
+ assert b == c
+ b = add(a, a, out=c)
+ assert b == c
+ d = array([16, 16], dtype=int)
+ b = sum(d, out=c)
+ assert b == c
+ try:
+ from _numpypy import version
+ v = version.version.split('.')
+ except:
+ v = ['1', '6', '0'] # numpypy is api compatable to what version?
+ if v[0]<'2':
+ b = negative(c, out=a)
+ assert b == a
+ b = add(c, c, out=a)
+ assert b == a
+ b = sum(array([16, 16], dtype=float), out=a)
+ assert b == a
+ else:
+ cast_error = raises(TypeError, negative, c, a)
+ assert str(cast_error.value) == \
+ "Cannot cast ufunc negative output from dtype('float64') to dtype('int64') with casting rule 'same_kind'"
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -113,14 +113,37 @@
assert (divide(array([-10]), array([2])) == array([-5])).all()
+ def test_true_divide(self):
+ from _numpypy import array, true_divide
+
+ a = array([0, 1, 2, 3, 4, 1, -1])
+ b = array([4, 4, 4, 4, 4, 0, 0])
+ c = true_divide(a, b)
+ assert (c == [0.0, 0.25, 0.5, 0.75, 1.0, float('inf'), float('-inf')]).all()
+
+ assert math.isnan(true_divide(0, 0))
+
def test_fabs(self):
from _numpypy import array, fabs
- from math import fabs as math_fabs
+ from math import fabs as math_fabs, isnan
a = array([-5.0, -0.0, 1.0])
b = fabs(a)
for i in range(3):
assert b[i] == math_fabs(a[i])
+ assert fabs(float('inf')) == float('inf')
+ assert fabs(float('-inf')) == float('inf')
+ assert isnan(fabs(float('nan')))
+
+ def test_fmod(self):
+ from _numpypy import fmod
+ import math
+
+ assert fmod(-1e-100, 1e100) == -1e-100
+ assert fmod(3, float('inf')) == 3
+ assert (fmod([-3, -2, -1, 1, 2, 3], 2) == [-1, 0, -1, 1, 0, 1]).all()
+ for v in [float('inf'), float('-inf'), float('nan'), float('-nan')]:
+ assert math.isnan(fmod(v, 2))
def test_minimum(self):
from _numpypy import array, minimum
@@ -172,6 +195,14 @@
assert a[0] == 1
assert a[1] == 0
+ def test_signbit(self):
+ from _numpypy import signbit, copysign
+
+ assert (signbit([0, 0.0, 1, 1.0, float('inf'), float('nan')]) ==
+ [False, False, False, False, False, False]).all()
More information about the pypy-commit
mailing list