From agaynor at codespeak.net Sun Aug 1 12:26:15 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sun, 1 Aug 2010 12:26:15 +0200 (CEST) Subject: [pypy-svn] r76422 - pypy/trunk/pypy/objspace/std Message-ID: <20100801102615.12EDC282B90@codespeak.net> Author: agaynor Date: Sun Aug 1 12:26:13 2010 New Revision: 76422 Modified: pypy/trunk/pypy/objspace/std/callmethod.py Log: Possible fix for stackless Modified: pypy/trunk/pypy/objspace/std/callmethod.py ============================================================================== --- pypy/trunk/pypy/objspace/std/callmethod.py (original) +++ pypy/trunk/pypy/objspace/std/callmethod.py Sun Aug 1 12:26:13 2010 @@ -89,7 +89,7 @@ try: w_result = f.space.call_args(w_callable, args) - rstack.resume_point("CALL_METHOD", f, returns=w_result) + rstack.resume_point("CALL_METHOD", f, w_self, returns=w_result) finally: f.dropvalues(1 + (w_self is None)) f.pushvalue(w_result) From agaynor at codespeak.net Sun Aug 1 12:55:04 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sun, 1 Aug 2010 12:55:04 +0200 (CEST) Subject: [pypy-svn] r76423 - pypy/trunk/pypy/objspace/std Message-ID: <20100801105504.4BDAA282B90@codespeak.net> Author: agaynor Date: Sun Aug 1 12:55:02 2010 New Revision: 76423 Modified: pypy/trunk/pypy/objspace/std/callmethod.py Log: Revert 76422, it did not fix stackless. Modified: pypy/trunk/pypy/objspace/std/callmethod.py ============================================================================== --- pypy/trunk/pypy/objspace/std/callmethod.py (original) +++ pypy/trunk/pypy/objspace/std/callmethod.py Sun Aug 1 12:55:02 2010 @@ -89,7 +89,7 @@ try: w_result = f.space.call_args(w_callable, args) - rstack.resume_point("CALL_METHOD", f, w_self, returns=w_result) + rstack.resume_point("CALL_METHOD", f, returns=w_result) finally: f.dropvalues(1 + (w_self is None)) f.pushvalue(w_result) From agaynor at codespeak.net Sun Aug 1 15:52:42 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sun, 1 Aug 2010 15:52:42 +0200 (CEST) Subject: [pypy-svn] r76424 - pypy/trunk/pypy/objspace/std Message-ID: <20100801135242.30D4F282B90@codespeak.net> Author: agaynor Date: Sun Aug 1 15:52:40 2010 New Revision: 76424 Modified: pypy/trunk/pypy/objspace/std/callmethod.py Log: Another attempt to fix stackless. Modified: pypy/trunk/pypy/objspace/std/callmethod.py ============================================================================== --- pypy/trunk/pypy/objspace/std/callmethod.py (original) +++ pypy/trunk/pypy/objspace/std/callmethod.py Sun Aug 1 15:52:40 2010 @@ -68,7 +68,7 @@ if not n_kwargs: try: w_result = f.space.call_valuestack(w_callable, n, f) - rstack.resume_point("CALL_METHOD", f, n_args, returns=w_result) + rstack.resume_point("CALL_METHOD_no_kwargs", f, n_args, returns=w_result) finally: f.dropvalues(n_args + 2) else: @@ -89,7 +89,7 @@ try: w_result = f.space.call_args(w_callable, args) - rstack.resume_point("CALL_METHOD", f, returns=w_result) + rstack.resume_point("CALL_METHOD", f, w_self, returns=w_result) finally: f.dropvalues(1 + (w_self is None)) f.pushvalue(w_result) From hakanardo at codespeak.net Sun Aug 1 16:02:53 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Sun, 1 Aug 2010 16:02:53 +0200 (CEST) Subject: [pypy-svn] r76425 - in pypy/branch/interplevel-array/pypy/module/array: . test Message-ID: <20100801140253.4E949282B90@codespeak.net> Author: hakanardo Date: Sun Aug 1 16:02:51 2010 New Revision: 76425 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: corect behaviour with overriden methods Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Sun Aug 1 16:02:51 2010 @@ -30,13 +30,13 @@ if w_initializer is not None: if not space.is_w(w_initializer, space.w_None): if space.type(w_initializer) is space.w_str: - space.call_method(a, 'fromstring', w_initializer) + a.fromstring(w_initializer) elif space.type(w_initializer) is space.w_unicode: - space.call_method(a, 'fromunicode', w_initializer) + a.fromsequence(w_initializer) elif space.type(w_initializer) is space.w_list: - space.call_method(a, 'fromlist', w_initializer) + a.fromlist(w_initializer) else: - space.call_method(a, 'extend', w_initializer) + a.extend(w_initializer) break else: msg = 'bad typecode (must be c, b, B, u, h, H, i, I, l, L, f or d)' @@ -241,6 +241,46 @@ raise self.setlen(oldlen + i) + def fromstring(self, w_s): + space = self.space + s = space.str_w(w_s) + if len(s)%self.itemsize !=0: + msg = 'string length not a multiple of item size' + raise OperationError(space.w_ValueError, space.wrap(msg)) + oldlen = self.len + new = len(s) / mytype.bytes + self.setlen(oldlen + new) + cbuf =self.charbuf() + for i in range(len(s)): + cbuf[oldlen * mytype.bytes + i] = s[i] + + def fromlist(self, w_lst): + s = self.len + try: + self.fromsequence(w_lst) + except OperationError: + self.setlen(s) + raise + + def extend(self, w_iterable): + space = self.space + if isinstance(w_iterable, W_Array): + oldlen = self.len + new = w_iterable.len + self.setlen(self.len + new) + for i in range(new): + if oldlen + i >= self.len: + self.setlen(oldlen + i + 1) + self.buffer[oldlen + i] = w_iterable.buffer[i] + self.setlen(oldlen + i + 1) + elif isinstance(w_iterable, W_ArrayBase): + msg = "can only extend with array of same kind" + raise OperationError(space.w_TypeError, space.wrap(msg)) + else: + self.fromsequence(w_iterable) + + + def charbuf(self): return rffi.cast(rffi.CCHARP, self.buffer) @@ -314,20 +354,7 @@ def array_extend__Array_ANY(space, self, w_iterable): - if isinstance(w_iterable, W_Array): - oldlen = self.len - new = w_iterable.len - self.setlen(self.len + new) - for i in range(new): - if oldlen + i >= self.len: - self.setlen(oldlen + i + 1) - self.buffer[oldlen + i] = w_iterable.buffer[i] - self.setlen(oldlen + i + 1) - elif isinstance(w_iterable, W_ArrayBase): - msg = "can only extend with array of same kind" - raise OperationError(space.w_TypeError, space.wrap(msg)) - else: - self.fromsequence(w_iterable) + self.extend(w_iterable) # List interface @@ -448,24 +475,10 @@ return w_l def array_fromlist__Array_List(space, self, w_lst): - s = self.len - try: - self.fromsequence(w_lst) - except OperationError: - self.setlen(s) - raise + self.fromlist(w_lst) def array_fromstring__Array_ANY(space, self, w_s): - s = space.str_w(w_s) - if len(s)%mytype.bytes !=0: - msg = 'string length not a multiple of item size' - raise OperationError(self.space.w_ValueError, self.space.wrap(msg)) - oldlen = self.len - new = len(s) / mytype.bytes - self.setlen(oldlen + new) - cbuf =self.charbuf() - for i in range(len(s)): - cbuf[oldlen * mytype.bytes + i] = s[i] + self.fromstring(w_s) def array_tostring__Array(space, self): cbuf = self.charbuf() @@ -577,15 +590,15 @@ if self.len == 0: return space.wrap("array('%s')" % self.typecode) elif self.typecode == "c": - r = space.repr(space.call_method(self, 'tostring')) + r = space.repr(array_tostring__Array(space, self)) s = "array('%s', %s)" % (self.typecode, space.str_w(r)) return space.wrap(s) elif self.typecode == "u": - r = space.repr(space.call_method(self, 'tounicode')) + r = space.repr(array_tounicode__Array(space, self)) s = "array('%s', %s)" % (self.typecode, space.str_w(r)) return space.wrap(s) else: - r = space.repr(space.call_method(self, 'tolist')) + r = space.repr(array_tolist__Array(space, self)) s = "array('%s', %s)" % (self.typecode, space.str_w(r)) return space.wrap(s) Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Sun Aug 1 16:02:51 2010 @@ -600,17 +600,22 @@ except TypeError: raised = True assert raised - - try: - raised = False - a = self.array('i').__add__(2) - print a - except TypeError: - raised = True - assert raised - raises(TypeError, self.array('i').__add__, (2,)) - raises(TypeError, self.array('i').__add__, self.array('b')) + # Calling __add__ directly raises TypeError in cpython but + # returns NotImplemented in pypy if placed within a + # try: except TypeError: construction. + # + # raises(TypeError, self.array('i').__add__, (2,)) + # raises(TypeError, self.array('i').__add__, self.array('b')) + + class addable(object): + def __add__(self, other): + return "add" + def __radd__(self, other): + return "radd" + + assert addable() + self.array('i') == 'add' + assert self.array('i') + addable() == 'radd' def test_delitem(self): a = self.array('i', [1, 2, 3]) @@ -679,6 +684,57 @@ assert len(a) == 3 assert a[0] == 2 + def test_override_from(self): + class mya(self.array): + def fromlist(self, lst): + self.append(7) + def fromstring(self, lst): + self.append('8') + def fromunicode(self, lst): + self.append(u'9') + def extend(self, lst): + self.append(10) + + assert repr(mya('c', 'hi')) == "array('c', 'hi')" + assert repr(mya('u', u'hi')) == "array('u', u'hi')" + assert repr(mya('i', [1, 2, 3])) == "array('i', [1, 2, 3])" + assert repr(mya('i', (1, 2, 3))) == "array('i', [1, 2, 3])" + + a = mya('i') + a.fromlist([1, 2, 3]) + assert repr(a) == "array('i', [7])" + + a = mya('c') + a.fromstring('hi') + assert repr(a) == "array('c', '8')" + + a = mya('u') + a.fromunicode(u'hi') + assert repr(a) == "array('u', u'9')" + + a = mya('i') + a.extend([1, 2, 3]) + assert repr(a) == "array('i', [10])" + + def test_override_to(self): + class mya(self.array): + def tolist(self): + return 'list' + def tostring(self): + return 'str' + def tounicode(self): + return 'unicode' + + assert mya('i', [1, 2, 3]).tolist() == 'list' + assert mya('c', 'hi').tostring() == 'str' + assert mya('u', u'hi').tounicode() == 'unicode' + + assert repr(mya('c', 'hi')) == "array('c', 'hi')" + assert repr(mya('u', u'hi')) == "array('u', u'hi')" + assert repr(mya('i', [1, 2, 3])) == "array('i', [1, 2, 3])" + assert repr(mya('i', (1, 2, 3))) == "array('i', [1, 2, 3])" + + class TestCPythonsOwnArray(BaseArrayTests): def setup_class(cls): From hakanardo at codespeak.net Sun Aug 1 16:47:59 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Sun, 1 Aug 2010 16:47:59 +0200 (CEST) Subject: [pypy-svn] r76426 - pypy/branch/interplevel-array/pypy/module/array Message-ID: <20100801144759.B4E93282B90@codespeak.net> Author: hakanardo Date: Sun Aug 1 16:47:58 2010 New Revision: 76426 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py Log: rpythonized Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Sun Aug 1 16:47:58 2010 @@ -268,11 +268,13 @@ oldlen = self.len new = w_iterable.len self.setlen(self.len + new) - for i in range(new): + i = 0 + while i < new: if oldlen + i >= self.len: self.setlen(oldlen + i + 1) self.buffer[oldlen + i] = w_iterable.buffer[i] - self.setlen(oldlen + i + 1) + i += 1 + self.setlen(oldlen + i) elif isinstance(w_iterable, W_ArrayBase): msg = "can only extend with array of same kind" raise OperationError(space.w_TypeError, space.wrap(msg)) From jcreigh at codespeak.net Mon Aug 2 15:59:34 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Mon, 2 Aug 2010 15:59:34 +0200 (CEST) Subject: [pypy-svn] r76430 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86 Message-ID: <20100802135934.8AF8B282BDB@codespeak.net> Author: jcreigh Date: Mon Aug 2 15:59:32 2010 New Revision: 76430 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Log: fix obscure GC issue related to pending_guard_tokens being a prebuilt GC object Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Mon Aug 2 15:59:32 2010 @@ -337,14 +337,22 @@ self.write_pending_failure_recoveries() def write_pending_failure_recoveries(self): - for tok in self.pending_guard_tokens: + # NB: We must be careful to only modify pending_guard_tokens, not + # re-assign it to a different list. Since it is a prebuilt gc object, + # if we were to re-assign it, the original list would never be freed. + # This is particularly bad in the case where the failargs of a token + # contains a BoxPtr to an object, which would also never be freed. + # + # This is why we pop() off the end instead of iterating and then + # saying "self.pending_guard_tokens = []" + while len(self.pending_guard_tokens) > 0: + tok = self.pending_guard_tokens.pop() # Okay to write to _mc because we've already made sure that # there's enough space by "reserving" bytes. addr = self.generate_quick_failure(self.mc._mc, tok.faildescr, tok.failargs, tok.fail_locs, tok.exc, tok.desc_bytes) tok.faildescr._x86_adr_recovery_stub = addr self.patch_jump_for_descr(tok.faildescr, addr) - self.pending_guard_tokens = [] self.mc.reset_reserved_bytes() self.mc.done() From jcreigh at codespeak.net Mon Aug 2 17:26:56 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Mon, 2 Aug 2010 17:26:56 +0200 (CEST) Subject: [pypy-svn] r76432 - in pypy/branch/x86-64-jit-backend: . lib-python/modified-2.5.2/ctypes lib_pypy/_ctypes lib_pypy/pypy_test pypy/interpreter/astcompiler pypy/interpreter/test pypy/jit/backend/x86 pypy/jit/backend/x86/test pypy/jit/metainterp pypy/jit/metainterp/test pypy/module/__builtin__ pypy/module/_ast pypy/module/_ast/test pypy/objspace/std pypy/objspace/std/test pypy/rlib pypy/rlib/test pypy/rpython/lltypesystem Message-ID: <20100802152656.5689D282BDB@codespeak.net> Author: jcreigh Date: Mon Aug 2 17:26:53 2010 New Revision: 76432 Modified: pypy/branch/x86-64-jit-backend/ (props changed) pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/ctypes/__init__.py pypy/branch/x86-64-jit-backend/lib_pypy/_ctypes/__init__.py pypy/branch/x86-64-jit-backend/lib_pypy/_ctypes/array.py pypy/branch/x86-64-jit-backend/lib_pypy/_ctypes/builtin.py pypy/branch/x86-64-jit-backend/lib_pypy/_ctypes/primitive.py pypy/branch/x86-64-jit-backend/lib_pypy/pypy_test/test_ctypes_support.py pypy/branch/x86-64-jit-backend/pypy/interpreter/astcompiler/assemble.py pypy/branch/x86-64-jit-backend/pypy/interpreter/astcompiler/codegen.py pypy/branch/x86-64-jit-backend/pypy/interpreter/astcompiler/consts.py pypy/branch/x86-64-jit-backend/pypy/interpreter/test/test_compiler.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/runner.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_runner.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/optimizeopt.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_loop.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_loop_spec.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/x86-64-jit-backend/pypy/module/__builtin__/compiling.py pypy/branch/x86-64-jit-backend/pypy/module/_ast/__init__.py pypy/branch/x86-64-jit-backend/pypy/module/_ast/test/test_ast.py pypy/branch/x86-64-jit-backend/pypy/objspace/std/callmethod.py pypy/branch/x86-64-jit-backend/pypy/objspace/std/test/test_callmethod.py pypy/branch/x86-64-jit-backend/pypy/rlib/objectmodel.py pypy/branch/x86-64-jit-backend/pypy/rlib/test/test_objectmodel.py pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/rstr.py Log: merged changes from trunk through r76430 Modified: pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/ctypes/__init__.py ============================================================================== --- pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/ctypes/__init__.py (original) +++ pypy/branch/x86-64-jit-backend/lib-python/modified-2.5.2/ctypes/__init__.py Mon Aug 2 17:26:53 2010 @@ -471,7 +471,7 @@ # functions -from _ctypes import _memmove_addr, _memset_addr, _string_at_addr, _cast_addr +from _ctypes import _memmove_addr, _memset_addr, _cast_addr ## void *memmove(void *, const void *, size_t); memmove = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)(_memmove_addr) @@ -490,24 +490,34 @@ def cast(obj, typ): return _cast(obj, obj, typ) -_string_at = CFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr) +try: + from _ctypes import _string_at_addr +except ImportError: + from _ctypes import _string_at +else: + _string_at = CFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr) + def string_at(ptr, size=-1): """string_at(addr[, size]) -> string Return the string at addr.""" return _string_at(ptr, size) +def wstring_at(ptr, size=-1): + """wstring_at(addr[, size]) -> string + + Return the string at addr.""" + return _wstring_at(ptr, size) + try: from _ctypes import _wstring_at_addr except ImportError: - pass + try: + from _ctypes import _wstring_at + except ImportError: + del wstring_at else: _wstring_at = CFUNCTYPE(py_object, c_void_p, c_int)(_wstring_at_addr) - def wstring_at(ptr, size=-1): - """wstring_at(addr[, size]) -> string - - Return the string at addr.""" - return _wstring_at(ptr, size) if _os.name in ("nt", "ce"): # COM stuff Modified: pypy/branch/x86-64-jit-backend/lib_pypy/_ctypes/__init__.py ============================================================================== --- pypy/branch/x86-64-jit-backend/lib_pypy/_ctypes/__init__.py (original) +++ pypy/branch/x86-64-jit-backend/lib_pypy/_ctypes/__init__.py Mon Aug 2 17:26:53 2010 @@ -7,8 +7,8 @@ from _ctypes.dll import dlopen from _ctypes.structure import Structure from _ctypes.array import Array -from _ctypes.builtin import _memmove_addr, _string_at_addr, _memset_addr,\ - set_conversion_mode, _wstring_at_addr +from _ctypes.builtin import _memmove_addr, _string_at, _memset_addr,\ + set_conversion_mode, _wstring_at from _ctypes.union import Union import os as _os Modified: pypy/branch/x86-64-jit-backend/lib_pypy/_ctypes/array.py ============================================================================== --- pypy/branch/x86-64-jit-backend/lib_pypy/_ctypes/array.py (original) +++ pypy/branch/x86-64-jit-backend/lib_pypy/_ctypes/array.py Mon Aug 2 17:26:53 2010 @@ -4,7 +4,6 @@ from _ctypes.basics import _CData, cdata_from_address, _CDataMeta, sizeof from _ctypes.basics import keepalive_key, store_reference, ensure_objects from _ctypes.basics import CArgObject -from _ctypes.builtin import _string_at_addr, _wstring_at_addr def _create_unicode(buffer, maxlength): res = [] Modified: pypy/branch/x86-64-jit-backend/lib_pypy/_ctypes/builtin.py ============================================================================== --- pypy/branch/x86-64-jit-backend/lib_pypy/_ctypes/builtin.py (original) +++ pypy/branch/x86-64-jit-backend/lib_pypy/_ctypes/builtin.py Mon Aug 2 17:26:53 2010 @@ -8,10 +8,10 @@ _memmove_addr = _rawffi.get_libc().getaddressindll('memmove') _memset_addr = _rawffi.get_libc().getaddressindll('memset') -def _string_at_addr(addr, lgt): +def _string_at(addr, lgt): # address here can be almost anything import ctypes - arg = ctypes.c_char_p._CData_value(addr) + arg = ctypes.c_void_p._CData_value(addr) return _rawffi.charp2rawstring(arg, lgt) def set_conversion_mode(encoding, errors): @@ -20,9 +20,9 @@ ConvMode.encoding = encoding return old_cm -def _wstring_at_addr(addr, lgt): +def _wstring_at(addr, lgt): import ctypes - arg = ctypes.c_wchar_p._CData_value(addr) + arg = ctypes.c_void_p._CData_value(addr) # XXX purely applevel if lgt == -1: lgt = sys.maxint Modified: pypy/branch/x86-64-jit-backend/lib_pypy/_ctypes/primitive.py ============================================================================== --- pypy/branch/x86-64-jit-backend/lib_pypy/_ctypes/primitive.py (original) +++ pypy/branch/x86-64-jit-backend/lib_pypy/_ctypes/primitive.py Mon Aug 2 17:26:53 2010 @@ -86,6 +86,8 @@ return value if isinstance(value, _Pointer): return cls.from_address(value._buffer.buffer) + if isinstance(value, (int, long)): + return cls(value) FROM_PARAM_BY_TYPE = { 'z': from_param_char_p, @@ -141,13 +143,13 @@ result.value = property(_getvalue, _setvalue) elif tp == 'Z': # c_wchar_p - from _ctypes import Array, _Pointer, _wstring_at_addr + from _ctypes import Array, _Pointer, _wstring_at def _getvalue(self): addr = self._buffer[0] if addr == 0: return None else: - return _wstring_at_addr(addr, -1) + return _wstring_at(addr, -1) def _setvalue(self, value): if isinstance(value, basestring): @@ -216,14 +218,14 @@ SysAllocStringLen = windll.oleaut32.SysAllocStringLen SysStringLen = windll.oleaut32.SysStringLen SysFreeString = windll.oleaut32.SysFreeString - from _ctypes import _wstring_at_addr + from _ctypes import _wstring_at def _getvalue(self): addr = self._buffer[0] if addr == 0: return None else: size = SysStringLen(addr) - return _wstring_at_addr(addr, size) + return _wstring_at(addr, size) def _setvalue(self, value): if isinstance(value, basestring): @@ -254,18 +256,21 @@ from_address = cdata_from_address def from_param(self, value): + if isinstance(value, self): + return value + from_param_f = FROM_PARAM_BY_TYPE.get(self._type_) if from_param_f: res = from_param_f(self, value) if res is not None: return res - - if isinstance(value, self): - return value - try: - return self(value) - except (TypeError, ValueError): - return super(SimpleType, self).from_param(value) + else: + try: + return self(value) + except (TypeError, ValueError): + pass + + return super(SimpleType, self).from_param(value) def _CData_output(self, resbuffer, base=None, index=-1): output = super(SimpleType, self)._CData_output(resbuffer, base, index) Modified: pypy/branch/x86-64-jit-backend/lib_pypy/pypy_test/test_ctypes_support.py ============================================================================== --- pypy/branch/x86-64-jit-backend/lib_pypy/pypy_test/test_ctypes_support.py (original) +++ pypy/branch/x86-64-jit-backend/lib_pypy/pypy_test/test_ctypes_support.py Mon Aug 2 17:26:53 2010 @@ -20,3 +20,14 @@ assert get_errno() != 0 set_errno(0) assert get_errno() == 0 + +def test_argument_conversion_and_checks(): + import ctypes + libc = ctypes.cdll.LoadLibrary("libc.so.6") + libc.strlen.argtypes = ctypes.c_char_p, + libc.strlen.restype = ctypes.c_size_t + assert libc.strlen("eggs") == 4 + + # Should raise ArgumentError, not segfault + py.test.raises(ctypes.ArgumentError, libc.strlen, False) + Modified: pypy/branch/x86-64-jit-backend/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/interpreter/astcompiler/assemble.py Mon Aug 2 17:26:53 2010 @@ -566,22 +566,22 @@ return (oparg % 256) + 2 * (oparg / 256) def _compute_CALL_FUNCTION(arg): - return _num_args(arg) + return -_num_args(arg) def _compute_CALL_FUNCTION_VAR(arg): - return _num_args(arg) - 1 + return -_num_args(arg) - 1 def _compute_CALL_FUNCTION_KW(arg): - return _num_args(arg) - 1 + return -_num_args(arg) - 1 def _compute_CALL_FUNCTION_VAR_KW(arg): - return _num_args(arg) - 2 + return -_num_args(arg) - 2 def _compute_CALL_LIKELY_BUILTIN(arg): return -(arg & 0xFF) + 1 def _compute_CALL_METHOD(arg): - return -arg - 1 + return -_num_args(arg) - 1 _stack_effect_computers = {} Modified: pypy/branch/x86-64-jit-backend/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/interpreter/astcompiler/codegen.py Mon Aug 2 17:26:53 2010 @@ -960,9 +960,12 @@ elif call_type == 3: op = ops.CALL_FUNCTION_VAR_KW self.emit_op_arg(op, arg) + + def _call_has_no_star_args(self, call): + return not call.starargs and not call.kwargs def _call_has_simple_args(self, call): - return not call.starargs and not call.kwargs and not call.keywords + return self._call_has_no_star_args(call) and not call.keywords def _optimize_builtin_call(self, call): if not self.space.config.objspace.opcodes.CALL_LIKELY_BUILTIN or \ @@ -988,7 +991,7 @@ def _optimize_method_call(self, call): if not self.space.config.objspace.opcodes.CALL_METHOD or \ - not self._call_has_simple_args(call) or \ + not self._call_has_no_star_args(call) or \ not isinstance(call.func, ast.Attribute): return False attr_lookup = call.func @@ -1000,7 +1003,12 @@ arg_count = len(call.args) else: arg_count = 0 - self.emit_op_arg(ops.CALL_METHOD, arg_count) + if call.keywords: + self.visit_sequence(call.keywords) + kwarg_count = len(call.keywords) + else: + kwarg_count = 0 + self.emit_op_arg(ops.CALL_METHOD, (kwarg_count << 8) | arg_count) return True def _listcomp_generator(self, list_name, gens, gen_index, elt): Modified: pypy/branch/x86-64-jit-backend/pypy/interpreter/astcompiler/consts.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/interpreter/astcompiler/consts.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/interpreter/astcompiler/consts.py Mon Aug 2 17:26:53 2010 @@ -18,4 +18,4 @@ PyCF_SOURCE_IS_UTF8 = 0x0100 PyCF_DONT_IMPLY_DEDENT = 0x0200 -PyCF_AST_ONLY = 0x0400 +PyCF_ONLY_AST = 0x0400 Modified: pypy/branch/x86-64-jit-backend/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/interpreter/test/test_compiler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/interpreter/test/test_compiler.py Mon Aug 2 17:26:53 2010 @@ -4,6 +4,7 @@ from pypy.interpreter.pycode import PyCode from pypy.interpreter.error import OperationError from pypy.interpreter.argument import Arguments +from pypy.conftest import gettestobjspace class BaseTestCompiler: def setup_method(self, method): @@ -848,14 +849,38 @@ import StringIO, sys, dis s = StringIO.StringIO() + out = sys.stdout sys.stdout = s try: dis.dis(code) finally: - sys.stdout = sys.__stdout__ + sys.stdout = out output = s.getvalue() assert "LOAD_GLOBAL" not in output +class AppTestCallMethod(object): + def setup_class(cls): + cls.space = gettestobjspace(**{'objspace.opcodes.CALL_METHOD': True}) + + def test_call_method_kwargs(self): + source = """def _f(a): + return a.f(a=a) + """ + exec source + code = _f.func_code + + import StringIO, sys, dis + s = StringIO.StringIO() + out = sys.stdout + sys.stdout = s + try: + dis.dis(code) + finally: + sys.stdout = out + output = s.getvalue() + assert "CALL_METHOD" in output + + class AppTestExceptions: def test_indentation_error(self): source = """if 1: Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Mon Aug 2 17:26:53 2010 @@ -1,4 +1,4 @@ -import sys +import sys, os from pypy.jit.backend.llsupport import symbolic from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxFloat from pypy.jit.metainterp.history import AbstractFailDescr, INT, REF, FLOAT,\ @@ -30,6 +30,7 @@ from pypy.rlib.debug import debug_print from pypy.rlib import rgc from pypy.jit.backend.x86.jump import remap_frame_layout +from pypy.rlib.streamio import open_file_as_stream # darwin requires the stack to be 16 bytes aligned on calls. Same for gcc 4.5.0, # better safe than sorry @@ -154,6 +155,7 @@ mc_size = MachineCodeBlockWrapper.MC_DEFAULT_SIZE _float_constants = None _regalloc = None + _output_loop_log = None def __init__(self, cpu, translate_support_code=False, failargs_limit=1000): @@ -168,18 +170,26 @@ self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) self.fail_boxes_float = values_array(lltype.Float, failargs_limit) self.fail_ebp = 0 + self.loop_run_counter = values_array(lltype.Signed, 10000) + self.loop_names = [] + # if we have 10000 loops, we have some other problems I guess self.float_const_neg_addr = 0 self.float_const_abs_addr = 0 self.malloc_fixedsize_slowpath1 = 0 self.malloc_fixedsize_slowpath2 = 0 self.pending_guard_tokens = [] self.setup_failure_recovery() + self._loop_counter = 0 + self._debug = False def leave_jitted_hook(self): ptrs = self.fail_boxes_ptr.ar llop.gc_assume_young_pointers(lltype.Void, llmemory.cast_ptr_to_adr(ptrs)) + def set_debug(self, v): + self._debug = v + def make_sure_mc_exists(self): if self.mc is None: # the address of the function called by 'new' @@ -209,6 +219,22 @@ self._build_float_constants() if hasattr(gc_ll_descr, 'get_malloc_fixedsize_slowpath_addr'): self._build_malloc_fixedsize_slowpath() + s = os.environ.get('PYPYLOG') + if s: + if s.find(':') != -1: + s = s.split(':')[-1] + self.set_debug(True) + self._output_loop_log = s + ".count" + + def finish_once(self): + if self._debug: + output_log = self._output_loop_log + assert output_log is not None + f = open_file_as_stream(output_log, "w") + for i in range(self._loop_counter): + f.write(self.loop_names[i] + ":" + + str(self.loop_run_counter.getitem(i)) + "\n") + f.close() def _build_float_constants(self): # 44 bytes: 32 bytes for the data, and up to 12 bytes for alignment @@ -267,9 +293,9 @@ # Arguments should be unique assert len(set(inputargs)) == len(inputargs) + self.make_sure_mc_exists() funcname = self._find_debug_merge_point(operations) - self.make_sure_mc_exists() regalloc = RegAlloc(self, self.cpu.translate_support_code) arglocs = regalloc.prepare_loop(inputargs, operations, looptoken) @@ -302,9 +328,9 @@ # Arguments should be unique assert len(set(inputargs)) == len(inputargs) + self.make_sure_mc_exists() funcname = self._find_debug_merge_point(operations) - self.make_sure_mc_exists() arglocs = self.rebuild_faillocs_from_descr( faildescr._x86_failure_recovery_bytecode) if not we_are_translated(): @@ -357,10 +383,18 @@ self.mc.done() def _find_debug_merge_point(self, operations): + for op in operations: if op.opnum == rop.DEBUG_MERGE_POINT: - return op.args[0]._get_str() - return "" + funcname = op.args[0]._get_str() + break + else: + funcname = "" % self._loop_counter + # invent the counter, so we don't get too confused + if self._debug: + self.loop_names.append(funcname) + self._loop_counter += 1 + return funcname def patch_jump_for_descr(self, faildescr, adr_new_target): adr_jump_offset = faildescr._x86_adr_jump_offset @@ -384,6 +418,15 @@ def _assemble(self, regalloc, operations): self._regalloc = regalloc + if self._debug: + # before doing anything, let's increase a counter + # we need one register free (a bit of a hack, but whatever) + self.mc.PUSH(eax) + adr = self.loop_run_counter.get_addr_for_num(self._loop_counter - 1) + self.mc.MOV(eax, heap(adr)) + self.mc.ADD(eax, imm(1)) + self.mc.MOV(heap(adr), eax) + self.mc.POP(eax) regalloc.walk_operations(operations) self.mc.done() if we_are_translated() or self.cpu.dont_keepalive_stuff: Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/runner.py Mon Aug 2 17:26:53 2010 @@ -46,6 +46,7 @@ self.profile_agent.startup() def finish_once(self): + self.assembler.finish_once() self.profile_agent.shutdown() def compile_loop(self, inputargs, operations, looptoken): Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_runner.py Mon Aug 2 17:26:53 2010 @@ -9,8 +9,11 @@ from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.executor import execute from pypy.jit.backend.test.runner_test import LLtypeBackendTest +from pypy.jit.metainterp.test.oparser import parse +from pypy.tool.udir import udir import ctypes import sys +import os CPU = getcpuclass() @@ -453,3 +456,36 @@ # Really just a sanity check. We're actually interested in # whether the test segfaults. assert self.cpu.get_latest_value_int(0) == finished.value + + +class TestDebuggingAssembler(object): + def setup_method(self, meth): + self.pypylog = os.environ.get('PYPYLOG', None) + self.logfile = str(udir.join('x86_runner.log')) + os.environ['PYPYLOG'] = "mumble:" + self.logfile + self.cpu = CPU(rtyper=None, stats=FakeStats()) + + def teardown_method(self, meth): + if self.pypylog is not None: + os.environ['PYPYLOG'] = self.pypylog + + def test_debugger_on(self): + loop = """ + [i0] + debug_merge_point('xyz') + i1 = int_add(i0, 1) + i2 = int_ge(i1, 10) + guard_false(i2) [] + jump(i1) + """ + ops = parse(loop) + self.cpu.assembler.set_debug(True) + self.cpu.compile_loop(ops.inputargs, ops.operations, ops.token) + self.cpu.set_future_value_int(0, 0) + self.cpu.execute_token(ops.token) + # check debugging info + assert self.cpu.assembler.loop_names == ["xyz"] + assert self.cpu.assembler.loop_run_counter.getitem(0) == 10 + self.cpu.finish_once() + lines = py.path.local(self.logfile + ".count").readlines() + assert lines[0] == 'xyz:10\n' Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/optimizeopt.py Mon Aug 2 17:26:53 2010 @@ -1001,6 +1001,17 @@ self.make_equal_to(op.result, v1) else: return self.optimize_default(op) + + def optimize_INT_ADD(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + # If one side of the op is 0 the result is the other side. + if v1.is_constant() and v1.box.getint() == 0: + self.make_equal_to(op.result, v2) + elif v2.is_constant() and v2.box.getint() == 0: + self.make_equal_to(op.result, v1) + else: + self.optimize_default(op) optimize_ops = _findall(Optimizer, 'optimize_') Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_loop.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_loop.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_loop.py Mon Aug 2 17:26:53 2010 @@ -9,6 +9,10 @@ class LoopTest(object): optimizer = OPTIMIZER_SIMPLE + automatic_promotion_result = { + 'int_add' : 6, 'int_gt' : 1, 'guard_false' : 1, 'jump' : 1, + 'guard_value' : 3 + } def meta_interp(self, f, args, policy=None): return ll_meta_interp(f, args, optimizer=self.optimizer, @@ -477,9 +481,9 @@ res = self.meta_interp(main_interpreter_loop, [1]) assert res == main_interpreter_loop(1) self.check_loop_count(1) - # XXX maybe later optimize guard_value away - self.check_loops({'int_add' : 6, 'int_gt' : 1, - 'guard_false' : 1, 'jump' : 1, 'guard_value' : 3}) + # These loops do different numbers of ops based on which optimizer we + # are testing with. + self.check_loops(self.automatic_promotion_result) def test_can_enter_jit_outside_main_loop(self): myjitdriver = JitDriver(greens=[], reds=['i', 'j', 'a']) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_loop_spec.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_loop_spec.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_loop_spec.py Mon Aug 2 17:26:53 2010 @@ -5,6 +5,10 @@ class LoopSpecTest(test_loop.LoopTest): optimizer = OPTIMIZER_FULL + automatic_promotion_result = { + 'int_add' : 3, 'int_gt' : 1, 'guard_false' : 1, 'jump' : 1, + 'guard_value' : 1 + } # ====> test_loop.py Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_optimizeopt.py Mon Aug 2 17:26:53 2010 @@ -2063,6 +2063,28 @@ jump(i0) """ self.optimize_loop(ops, 'Not', expected) + + ops = """ + [i0] + i1 = int_add(i0, 0) + jump(i1) + """ + expected = """ + [i0] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + ops = """ + [i0] + i1 = int_add(0, i0) + jump(i1) + """ + expected = """ + [i0] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) # ---------- Modified: pypy/branch/x86-64-jit-backend/pypy/module/__builtin__/compiling.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/__builtin__/compiling.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/__builtin__/compiling.py Mon Aug 2 17:26:53 2010 @@ -38,7 +38,7 @@ str_ = space.str_w(w_source) ec = space.getexecutioncontext() - if flags & ~(ec.compiler.compiler_flags | consts.PyCF_AST_ONLY | + if flags & ~(ec.compiler.compiler_flags | consts.PyCF_ONLY_AST | consts.PyCF_DONT_IMPLY_DEDENT | consts.PyCF_SOURCE_IS_UTF8): raise OperationError(space.w_ValueError, space.wrap("compile() unrecognized flags")) @@ -53,7 +53,7 @@ "or 'eval' or 'single'")) if ast_node is None: - if flags & consts.PyCF_AST_ONLY: + if flags & consts.PyCF_ONLY_AST: mod = ec.compiler.compile_to_ast(str_, filename, mode, flags) return space.wrap(mod) else: Modified: pypy/branch/x86-64-jit-backend/pypy/module/_ast/__init__.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/_ast/__init__.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/_ast/__init__.py Mon Aug 2 17:26:53 2010 @@ -5,7 +5,7 @@ class Module(MixedModule): interpleveldefs = { - "PyCF_AST_ONLY" : "space.wrap(%s)" % consts.PyCF_AST_ONLY + "PyCF_ONLY_AST" : "space.wrap(%s)" % consts.PyCF_ONLY_AST } appleveldefs = {} Modified: pypy/branch/x86-64-jit-backend/pypy/module/_ast/test/test_ast.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/_ast/test/test_ast.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/_ast/test/test_ast.py Mon Aug 2 17:26:53 2010 @@ -10,7 +10,7 @@ cls.w_get_ast = cls.space.appexec([], """(): def get_ast(source, mode="exec"): import _ast as ast - mod = compile(source, "", mode, ast.PyCF_AST_ONLY) + mod = compile(source, "", mode, ast.PyCF_ONLY_AST) assert isinstance(mod, ast.mod) return mod return get_ast""") Modified: pypy/branch/x86-64-jit-backend/pypy/objspace/std/callmethod.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/objspace/std/callmethod.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/objspace/std/callmethod.py Mon Aug 2 17:26:53 2010 @@ -12,7 +12,7 @@ from pypy.interpreter import function from pypy.objspace.descroperation import object_getattribute -from pypy.rlib import rstack # for resume points +from pypy.rlib import jit, rstack # for resume points # This module exports two extra methods for StdObjSpaceFrame implementing # the LOOKUP_METHOD and CALL_METHOD opcodes in an efficient way, as well @@ -56,16 +56,42 @@ f.pushvalue(w_value) f.pushvalue(None) -def CALL_METHOD(f, nargs, *ignored): - # 'nargs' is the argument count excluding the implicit 'self' - w_self = f.peekvalue(nargs) - w_callable = f.peekvalue(nargs + 1) - n = nargs + (w_self is not None) - try: - w_result = f.space.call_valuestack(w_callable, n, f) - rstack.resume_point("CALL_METHOD", f, nargs, returns=w_result) - finally: - f.dropvalues(nargs + 2) + at jit.unroll_safe +def CALL_METHOD(f, oparg, *ignored): + # opargs contains the arg, and kwarg count, excluding the implicit 'self' + n_args = oparg & 0xff + n_kwargs = (oparg >> 8) & 0xff + w_self = f.peekvalue(n_args + (2 * n_kwargs)) + w_callable = f.peekvalue(n_args + (2 * n_kwargs) + 1) + n = n_args + (w_self is not None) + + if not n_kwargs: + try: + w_result = f.space.call_valuestack(w_callable, n, f) + rstack.resume_point("CALL_METHOD_no_kwargs", f, n_args, returns=w_result) + finally: + f.dropvalues(n_args + 2) + else: + keywords = [None] * n_kwargs + keywords_w = [None] * n_kwargs + while True: + n_kwargs -= 1 + if n_kwargs < 0: + break + w_value = f.popvalue() + w_key = f.popvalue() + key = f.space.str_w(w_key) + keywords[n_kwargs] = key + keywords_w[n_kwargs] = w_value + + arguments = f.popvalues(n) + args = f.argument_factory(arguments, keywords, keywords_w, None, None) + + try: + w_result = f.space.call_args(w_callable, args) + rstack.resume_point("CALL_METHOD", f, w_self, returns=w_result) + finally: + f.dropvalues(1 + (w_self is None)) f.pushvalue(w_result) Modified: pypy/branch/x86-64-jit-backend/pypy/objspace/std/test/test_callmethod.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/objspace/std/test/test_callmethod.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/objspace/std/test/test_callmethod.py Mon Aug 2 17:26:53 2010 @@ -106,6 +106,15 @@ else: raise Exception("did not raise?") """ + + def test_kwargs(self): + exec """if 1: + class C(object): + def f(self, a): + return a + 2 + + assert C().f(a=3) == 5 + """ class AppTestCallMethodWithGetattributeShortcut(AppTestCallMethod): Modified: pypy/branch/x86-64-jit-backend/pypy/rlib/objectmodel.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rlib/objectmodel.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rlib/objectmodel.py Mon Aug 2 17:26:53 2010 @@ -26,7 +26,7 @@ by a call result then. """ def decorated_func(func): - func._annspecialcase_ = 'specialize:memo()' + func._annspecialcase_ = 'specialize:memo' return func return decorated_func @@ -63,7 +63,7 @@ for example). Same warnings about exponential behavior apply. """ def decorated_func(func): - func._annspecialcase_ = 'specialize:ll()' + func._annspecialcase_ = 'specialize:ll' return func return decorated_func @@ -82,6 +82,17 @@ specialize = _Specialize() +def enforceargs(*args): + """ Decorate a function with forcing of RPython-level types on arguments. + None means no enforcing. + + XXX shouldn't we also add asserts in function body? + """ + def decorator(f): + f._annenforceargs_ = args + return f + return decorator + # ____________________________________________________________ class Symbolic(object): Modified: pypy/branch/x86-64-jit-backend/pypy/rlib/test/test_objectmodel.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rlib/test/test_objectmodel.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rlib/test/test_objectmodel.py Mon Aug 2 17:26:53 2010 @@ -404,6 +404,13 @@ assert f._annspecialcase_ == 'specialize:arg(1)' +def test_enforceargs_decorator(): + @enforceargs(int, str, None) + def f(a, b, c): + pass + + assert f._annenforceargs_ == (int, str, None) + def getgraph(f, argtypes): from pypy.translator.translator import TranslationContext, graphof from pypy.translator.backendopt.all import backend_optimizations Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/rstr.py Mon Aug 2 17:26:53 2010 @@ -2,7 +2,7 @@ from pypy.tool.pairtype import pairtype from pypy.rpython.error import TyperError from pypy.rlib.objectmodel import malloc_zero_filled, we_are_translated -from pypy.rlib.objectmodel import _hash_string +from pypy.rlib.objectmodel import _hash_string, enforceargs from pypy.rlib.debug import ll_assert from pypy.rlib.jit import purefunction from pypy.rpython.robject import PyObjRepr, pyobj_repr @@ -56,6 +56,7 @@ llmemory.itemoffsetof(TP.chars, 0) + llmemory.sizeof(CHAR_TP) * item) + @enforceargs(None, None, int, int, int) def copy_string_contents(src, dst, srcstart, dststart, length): assert srcstart >= 0 assert dststart >= 0 From hakanardo at codespeak.net Mon Aug 2 18:01:45 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Mon, 2 Aug 2010 18:01:45 +0200 (CEST) Subject: [pypy-svn] r76433 - in pypy/branch/interplevel-array/pypy: jit/tl module/array objspace/std Message-ID: <20100802160145.7DCCD282BDB@codespeak.net> Author: hakanardo Date: Mon Aug 2 18:01:43 2010 New Revision: 76433 Modified: pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/objspace/std/model.py Log: Single type passes pypyjit.py Modified: pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py (original) +++ pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py Mon Aug 2 18:01:43 2010 @@ -37,18 +37,22 @@ ## t2 = time.time() ## print t2 - t1 -from array import array -def f(img): - i=0 - sa=0 - while i<4: - sa+=img[i] - i+=1 - return sa - -img=array('i',(1,2,3,4)) -print f(img) +try: + from array import array + def f(img): + i=0 + sa=0 + while i<4: + sa+=img[i] + i+=1 + return sa + img=array('i',(1,2,3,4)) + print f(img) +except Exception, e: + print "Exception: ", type(e) + print e + ## def f(): ## a=7 ## i=0 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Mon Aug 2 18:01:43 2010 @@ -127,7 +127,7 @@ 'I': TypeCode(rffi.UINT, 'int_w', True), 'l': TypeCode(rffi.LONG, 'int_w', True, True), 'L': TypeCode(rffi.ULONG, 'bigint_w'), # Overflow handled by rbigint.touint() which - # corresponds to the C-type unsigned long + # corresponds to the C-type unsigned long 'f': TypeCode(lltype.SingleFloat, 'float_w'), 'd': TypeCode(lltype.Float, 'float_w'), } @@ -311,13 +311,17 @@ w_lst = space.getitem(w_lst, w_slice) w_a=mytype.w_class(self.space) w_a.fromsequence(w_lst) + elif step == 0: + raise ValueError, 'getitem__Array_Slice with step zero' else: size = (stop - start) / step if (stop - start) % step > 0: size += 1 w_a=mytype.w_class(self.space) w_a.setlen(size) - for j,i in enumerate(range(start, stop, step)): - w_a.buffer[j]=self.buffer[i] + j = 0 + for i in range(start, stop, step): + w_a.buffer[j] = self.buffer[i] + j += 1 return w_a def getslice__Array_ANY_ANY(space, self, w_i, w_j): @@ -341,9 +345,13 @@ space.setitem(w_lst, w_idx, w_item) self.setlen(0) self.fromsequence(w_lst) + elif step == 0: + raise ValueError, 'setitem__Array_Slice with step zero' else: - for j,i in enumerate(range(start, stop, step)): - self.buffer[i]=w_item.buffer[j] + j = 0 + for i in range(start, stop, step): + self.buffer[i] = w_item.buffer[j] + j += 1 def setslice__Array_ANY_ANY_ANY(space, self, w_i, w_j, w_x): space.setitem(self, space.newslice(w_i, w_j, space.w_None), w_x) @@ -361,19 +369,19 @@ # List interface def array_count__Array_ANY(space, self, w_val): - val = self.item_w(w_val) cnt = 0 for i in range(self.len): - if self.buffer[i] == val: + w_item = getitem__Array_ANY(space, self, space.wrap(i)) + if space.is_true(space.eq(w_item, w_val)): cnt += 1 return space.wrap(cnt) def array_index__Array_ANY(space, self, w_val): - val = self.item_w(w_val) cnt = 0 for i in range(self.len): - if self.buffer[i] == val: + w_item = getitem__Array_ANY(space, self, space.wrap(i)) + if space.is_true(space.eq(w_item, w_val)): return space.wrap(i) msg = 'array.index(x): x not in list' raise OperationError(space.w_ValueError, space.wrap(msg)) @@ -504,7 +512,8 @@ item = space.str_w(w_item) if len(item) < size: n = len(item) % self.itemsize - if n != 0: item = item[0:-(len(item) % self.itemsize)] + elems = max(0,len(item)-(len(item) % self.itemsize)) + if n != 0: item = item[0:elems] w_item = space.wrap(item) array_fromstring__Array_ANY(space, self, w_item) msg = "not enough items in file" @@ -610,9 +619,17 @@ def init__Array(space, self, args): args.parse_obj(None, 'array', init_signature, init_defaults) - W_Array.__name__ = 'W_ArrayType_'+mytype.typecode mytype.w_class = W_Array + # Annotator seems to mess up if the names are not unique + name = 'ArrayType'+mytype.typecode + W_Array.__name__ = 'W_' + name + import re + for n,f in locals().items(): + new,n = re.subn('_Array_', '_%s_' % name, n) + if n>0: + f.__name__ = new + from pypy.objspace.std.sliceobject import W_SliceObject from pypy.objspace.std.listobject import W_ListObject from pypy.objspace.std.unicodeobject import W_UnicodeObject Modified: pypy/branch/interplevel-array/pypy/objspace/std/model.py ============================================================================== --- pypy/branch/interplevel-array/pypy/objspace/std/model.py (original) +++ pypy/branch/interplevel-array/pypy/objspace/std/model.py Mon Aug 2 18:01:43 2010 @@ -88,6 +88,7 @@ import pypy.objspace.std.default # register a few catch-all multimethods import pypy.objspace.std.marshal_impl # install marshal multimethods + import pypy.module.array # the set of implementation types self.typeorder = { From jcreigh at codespeak.net Mon Aug 2 18:04:58 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Mon, 2 Aug 2010 18:04:58 +0200 (CEST) Subject: [pypy-svn] r76434 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86 Message-ID: <20100802160458.527DF282BDB@codespeak.net> Author: jcreigh Date: Mon Aug 2 18:04:56 2010 New Revision: 76434 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Log: revert r76430 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Mon Aug 2 18:04:56 2010 @@ -363,22 +363,14 @@ self.write_pending_failure_recoveries() def write_pending_failure_recoveries(self): - # NB: We must be careful to only modify pending_guard_tokens, not - # re-assign it to a different list. Since it is a prebuilt gc object, - # if we were to re-assign it, the original list would never be freed. - # This is particularly bad in the case where the failargs of a token - # contains a BoxPtr to an object, which would also never be freed. - # - # This is why we pop() off the end instead of iterating and then - # saying "self.pending_guard_tokens = []" - while len(self.pending_guard_tokens) > 0: - tok = self.pending_guard_tokens.pop() + for tok in self.pending_guard_tokens: # Okay to write to _mc because we've already made sure that # there's enough space by "reserving" bytes. addr = self.generate_quick_failure(self.mc._mc, tok.faildescr, tok.failargs, tok.fail_locs, tok.exc, tok.desc_bytes) tok.faildescr._x86_adr_recovery_stub = addr self.patch_jump_for_descr(tok.faildescr, addr) + self.pending_guard_tokens = [] self.mc.reset_reserved_bytes() self.mc.done() From jcreigh at codespeak.net Mon Aug 2 18:24:06 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Mon, 2 Aug 2010 18:24:06 +0200 (CEST) Subject: [pypy-svn] r76435 - in pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86: . test Message-ID: <20100802162406.39BC9282BDB@codespeak.net> Author: jcreigh Date: Mon Aug 2 18:24:04 2010 New Revision: 76435 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_runner.py Log: rename make_sure_mc_exists() to setup(), as it does more than that now Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Mon Aug 2 18:24:04 2010 @@ -190,7 +190,7 @@ def set_debug(self, v): self._debug = v - def make_sure_mc_exists(self): + def setup(self): if self.mc is None: # the address of the function called by 'new' gc_ll_descr = self.cpu.gc_ll_descr @@ -293,7 +293,7 @@ # Arguments should be unique assert len(set(inputargs)) == len(inputargs) - self.make_sure_mc_exists() + self.setup() funcname = self._find_debug_merge_point(operations) @@ -328,7 +328,7 @@ # Arguments should be unique assert len(set(inputargs)) == len(inputargs) - self.make_sure_mc_exists() + self.setup() funcname = self._find_debug_merge_point(operations) arglocs = self.rebuild_faillocs_from_descr( Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_runner.py Mon Aug 2 18:24:04 2010 @@ -80,7 +80,7 @@ # relative addressing to work properly. addr = rffi.cast(lltype.Signed, addr) - self.cpu.assembler.make_sure_mc_exists() + self.cpu.assembler.setup() self.cpu.assembler.malloc_func_addr = addr ofs = symbolic.get_field_token(rstr.STR, 'chars', False)[0] @@ -393,7 +393,7 @@ ops.append(ResOperation(rop.FINISH, [v], None, descr=BasicFailDescr())) looptoken = LoopToken() - self.cpu.assembler.make_sure_mc_exists() + self.cpu.assembler.setup() old_mc_mc = self.cpu.assembler.mc._mc self.cpu.compile_loop([base_v], ops, looptoken) assert self.cpu.assembler.mc._mc != old_mc_mc # overflowed From jcreigh at codespeak.net Mon Aug 2 18:40:07 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Mon, 2 Aug 2010 18:40:07 +0200 (CEST) Subject: [pypy-svn] r76436 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86 Message-ID: <20100802164007.95792282BDB@codespeak.net> Author: jcreigh Date: Mon Aug 2 18:40:06 2010 New Revision: 76436 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Log: fix GC issue with pending_guard_tokens by initializing in setup() instead of __init__ (see also r76430, which was later reverted) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Mon Aug 2 18:40:06 2010 @@ -177,7 +177,7 @@ self.float_const_abs_addr = 0 self.malloc_fixedsize_slowpath1 = 0 self.malloc_fixedsize_slowpath2 = 0 - self.pending_guard_tokens = [] + self.pending_guard_tokens = None self.setup_failure_recovery() self._loop_counter = 0 self._debug = False @@ -225,6 +225,12 @@ s = s.split(':')[-1] self.set_debug(True) self._output_loop_log = s + ".count" + # Intialize here instead of __init__ to prevent + # pending_guard_tokens from being considered a prebuilt object, + # which sometimes causes memory leaks since the prebuilt list is + # still considered a GC root after we re-assign + # pending_guard_tokens in write_pending_failure_recoveries + self.pending_guard_tokens = [] def finish_once(self): if self._debug: From getxsick at codespeak.net Mon Aug 2 19:43:17 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Mon, 2 Aug 2010 19:43:17 +0200 (CEST) Subject: [pypy-svn] r76437 - in pypy/branch/fast-ctypes/pypy: module/jitffi rlib rlib/test Message-ID: <20100802174317.510DE282BDB@codespeak.net> Author: getxsick Date: Mon Aug 2 19:43:15 2010 New Revision: 76437 Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Log: pass wrappers for result in better way Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Mon Aug 2 19:43:15 2010 @@ -65,7 +65,21 @@ class W_Get(Wrappable): def __init__(self, space, cpu, lib, func, args_type, res_type='v'): self.space = space - self.rget = rjitffi._Get(cpu, lib, func, args_type, res_type) + + if res_type == 'i': + self.rget = rjitffi._Get(cpu, lib, func, args_type, + res_type, self.wrap_int) + elif res_type == 'f': + self.rget = rjitffi._Get(cpu, lib, func, args_type, + res_type, self.wrap_float) + elif res_type == 'v': + self.rget = rjitffi._Get(cpu, lib, func, args_type, + res_type, self.wrap_void) + else: + raise OperationError( + space.w_ValueError, + space.wrap('Unsupported type of result: %s' + % res_type)) def call_w(self, space, w_args=None): if not space.is_w(w_args, space.w_None): @@ -90,7 +104,11 @@ space.wrap('Unsupported type of argument: %s' % self.rget.args_type[0])) i += 1 - return self.rget.call(space.wrap) + return self.rget.call() + + wrap_int = lambda self, value: self.space.wrap(value) + wrap_float = lambda self, value: self.space.wrap(value) + wrap_void = lambda self, value: self.space.wrap(value) #def W_Get___new__(space, w_type, cpu, lib, func, args_type, res_type): # try: Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Mon Aug 2 19:43:15 2010 @@ -19,8 +19,8 @@ self.name = name self.cpu = GLOBAL_CPU - def get(self, func, args_type, res_type='v'): - return _Get(self.cpu, self.lib, func, args_type, res_type) + def get(self, func, args_type, res_type='v', push_result=None): + return _Get(self.cpu, self.lib, func, args_type, res_type, push_result) class _LibHandler(object): def __init__(self, name): @@ -33,10 +33,11 @@ rffi.free_charp(name_ptr) class _Get(object): - def __init__(self, cpu, lib, func, args_type, res_type='v'): + def __init__(self, cpu, lib, func, args_type, res_type='v', push_result=None): assert isinstance(args_type, list) self.args_type = args_type self.res_type = res_type + self.push_result = push_result self.cpu = cpu lib = lib.handler bargs = [] @@ -98,13 +99,13 @@ calldescr = cls(arg_classes) return calldescr - def call(self, push_result): + def call(self): self.cpu.execute_token(self.looptoken) if self.res_type == 'i': - r = push_result(self.cpu.get_latest_value_int(0)) + r = self.push_result(self.cpu.get_latest_value_int(0)) elif self.res_type == 'f': - r = push_result(self.cpu.get_latest_value_float(0)) + r = self.push_result(self.cpu.get_latest_value_float(0)) elif self.res_type == 'v': r = None else: Modified: pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Mon Aug 2 19:43:15 2010 @@ -71,60 +71,60 @@ def test_get(self): lib = rjitffi.CDLL(self.lib_name) - func = lib.get('add_integers', ['i', 'i'], 'i') + func = lib.get('add_integers', ['i', 'i'], 'i', self.push_result) func.push_int(1) func.push_int(2) - assert func.call(self.push_result) == 3 + assert func.call() == 3 - func = lib.get('add_integers', ['i', 'i'], 'i') + func = lib.get('add_integers', ['i', 'i'], 'i', self.push_result) func.push_int(-1) func.push_int(2) - assert func.call(self.push_result) == 1 + assert func.call() == 1 - func = lib.get('add_integers', ['i', 'i'], 'i') + func = lib.get('add_integers', ['i', 'i'], 'i', self.push_result) func.push_int(0) func.push_int(0) - assert func.call(self.push_result) == 0 + assert func.call() == 0 - func = lib.get('max3', ['i', 'i', 'i'], 'i') + func = lib.get('max3', ['i', 'i', 'i'], 'i', self.push_result) func.push_int(2) func.push_int(8) func.push_int(3) - assert func.call(self.push_result) == 8 + assert func.call() == 8 - func = lib.get('add_floats', ['f', 'f'], 'f') + func = lib.get('add_floats', ['f', 'f'], 'f', self.push_result) func.push_float(1.2) func.push_float(1.5) - assert func.call(self.push_result) == 2.7 + assert func.call() == 2.7 def test_get_void(self): lib = rjitffi.CDLL(self.lib_name) - func = lib.get('fvoid', [], 'i') - assert func.call(self.push_result) == 1 + func = lib.get('fvoid', [], 'i', self.push_result) + assert func.call() == 1 - func = lib.get('return_void', ['i', 'i'], 'v') + func = lib.get('return_void', ['i', 'i'], 'v', self.push_result) func.push_int(1) func.push_int(2) - assert func.call(self.push_result) is None + assert func.call() is None - func = lib.get('return_void', ['i', 'i']) + func = lib.get('return_void', ['i', 'i'], push_result=self.push_result) func.push_int(1) func.push_int(2) - assert func.call(self.push_result) is None + assert func.call() is None def test_various_type_args(self): lib = rjitffi.CDLL(self.lib_name) - func = lib.get('add_intfloat', ['i', 'f'], 'i') + func = lib.get('add_intfloat', ['i', 'f'], 'i', self.push_result) func.push_int(1) func.push_float(2.9) - assert func.call(self.push_result) == 3 + assert func.call() == 3 # stack is cleaned up after calling func.push_int(0) func.push_float(1.3) - assert func.call(self.push_result) == 1 + assert func.call() == 1 def test_undefined_func(self): lib = rjitffi.CDLL(self.lib_name) From jcreigh at codespeak.net Mon Aug 2 19:52:18 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Mon, 2 Aug 2010 19:52:18 +0200 (CEST) Subject: [pypy-svn] r76438 - in pypy/trunk: . pypy/config pypy/jit/backend pypy/jit/backend/llsupport pypy/jit/backend/test pypy/jit/backend/x86 pypy/jit/backend/x86/test pypy/jit/backend/x86/tool pypy/module/signal Message-ID: <20100802175218.ABA3B282BDB@codespeak.net> Author: jcreigh Date: Mon Aug 2 19:52:15 2010 New Revision: 76438 Added: pypy/trunk/pypy/jit/backend/x86/arch.py - copied unchanged from r76437, pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/arch.py pypy/trunk/pypy/jit/backend/x86/regloc.py - copied unchanged from r76437, pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py pypy/trunk/pypy/jit/backend/x86/rx86.py - copied unchanged from r76437, pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py pypy/trunk/pypy/jit/backend/x86/test/test_regloc.py - copied unchanged from r76437, pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_regloc.py pypy/trunk/pypy/jit/backend/x86/test/test_rx86.py - copied unchanged from r76437, pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py pypy/trunk/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py - copied unchanged from r76437, pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py pypy/trunk/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py - copied unchanged from r76437, pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py pypy/trunk/pypy/jit/backend/x86/tool/instruction_encoding.sh - copied unchanged from r76437, pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/tool/instruction_encoding.sh Removed: pypy/trunk/pypy/jit/backend/x86/ri386.py pypy/trunk/pypy/jit/backend/x86/ri386setup.py pypy/trunk/pypy/jit/backend/x86/test/test_ri386.py pypy/trunk/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py Modified: pypy/trunk/ (props changed) pypy/trunk/pypy/config/translationoption.py pypy/trunk/pypy/jit/backend/detect_cpu.py pypy/trunk/pypy/jit/backend/llsupport/regalloc.py pypy/trunk/pypy/jit/backend/test/runner_test.py pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/codebuf.py pypy/trunk/pypy/jit/backend/x86/jump.py pypy/trunk/pypy/jit/backend/x86/regalloc.py pypy/trunk/pypy/jit/backend/x86/runner.py pypy/trunk/pypy/jit/backend/x86/test/conftest.py pypy/trunk/pypy/jit/backend/x86/test/test_assembler.py pypy/trunk/pypy/jit/backend/x86/test/test_basic.py pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py pypy/trunk/pypy/jit/backend/x86/test/test_jump.py pypy/trunk/pypy/jit/backend/x86/test/test_recompilation.py pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py pypy/trunk/pypy/jit/backend/x86/test/test_regalloc2.py pypy/trunk/pypy/jit/backend/x86/test/test_runner.py pypy/trunk/pypy/jit/backend/x86/test/test_symbolic_x86.py pypy/trunk/pypy/jit/backend/x86/test/test_zll_random.py pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py pypy/trunk/pypy/jit/backend/x86/test/test_ztranslation.py pypy/trunk/pypy/jit/backend/x86/tool/viewcode.py pypy/trunk/pypy/module/signal/interp_signal.py Log: merge x86-64-jit-backend to trunk Modified: pypy/trunk/pypy/config/translationoption.py ============================================================================== --- pypy/trunk/pypy/config/translationoption.py (original) +++ pypy/trunk/pypy/config/translationoption.py Mon Aug 2 19:52:15 2010 @@ -342,6 +342,10 @@ 'jit': 'hybrid extraopts jit', } +# For now, 64-bit JIT requires boehm +if IS_64_BITS: + OPT_TABLE['jit'] = OPT_TABLE['jit'].replace('hybrid', 'boehm') + def set_opt_level(config, level): """Apply optimization suggestions on the 'config'. The optimizations depend on the selected level and possibly on the backend. Modified: pypy/trunk/pypy/jit/backend/detect_cpu.py ============================================================================== --- pypy/trunk/pypy/jit/backend/detect_cpu.py (original) +++ pypy/trunk/pypy/jit/backend/detect_cpu.py Mon Aug 2 19:52:15 2010 @@ -56,6 +56,8 @@ return "pypy.jit.backend.x86.runner", "CPU" elif backend_name == 'x86-without-sse2': return "pypy.jit.backend.x86.runner", "CPU386_NO_SSE2" + elif backend_name == 'x86_64': + return "pypy.jit.backend.x86.runner", "CPU_X86_64" elif backend_name == 'cli': return "pypy.jit.backend.cli.runner", "CliCPU" elif backend_name == 'llvm': Modified: pypy/trunk/pypy/jit/backend/llsupport/regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/regalloc.py Mon Aug 2 19:52:15 2010 @@ -22,18 +22,19 @@ def get(self, box): return self.frame_bindings.get(box, None) - def loc(self, box, size): + def loc(self, box): res = self.get(box) if res is not None: return res - newloc = self.frame_pos(self.frame_depth, size) + newloc = self.frame_pos(self.frame_depth, box.type) self.frame_bindings[box] = newloc - self.frame_depth += size + # Objects returned by frame_pos must support frame_size() + self.frame_depth += newloc.frame_size() return newloc # abstract methods that need to be overwritten for specific assemblers @staticmethod - def frame_pos(loc, size): + def frame_pos(loc, type): raise NotImplementedError("Purely abstract") class RegisterManager(object): @@ -43,7 +44,6 @@ all_regs = [] no_lower_byte_regs = [] save_around_call_regs = [] - reg_width = 1 # in terms of stack space eaten def __init__(self, longevity, frame_manager=None, assembler=None): self.free_regs = self.all_regs[:] @@ -148,7 +148,7 @@ loc = self.reg_bindings[v_to_spill] del self.reg_bindings[v_to_spill] if self.frame_manager.get(v_to_spill) is None: - newloc = self.frame_manager.loc(v_to_spill, self.reg_width) + newloc = self.frame_manager.loc(v_to_spill) self.assembler.regalloc_mov(loc, newloc) return loc @@ -204,7 +204,7 @@ try: return self.reg_bindings[box] except KeyError: - return self.frame_manager.loc(box, self.reg_width) + return self.frame_manager.loc(box) def return_constant(self, v, forbidden_vars=[], selected_reg=None, imm_fine=True): @@ -260,7 +260,7 @@ self.reg_bindings[v] = loc self.assembler.regalloc_mov(prev_loc, loc) else: - loc = self.frame_manager.loc(v, self.reg_width) + loc = self.frame_manager.loc(v) self.assembler.regalloc_mov(prev_loc, loc) def force_result_in_reg(self, result_v, v, forbidden_vars=[]): @@ -280,7 +280,7 @@ self.free_regs = [reg for reg in self.free_regs if reg is not loc] return loc if v not in self.reg_bindings: - prev_loc = self.frame_manager.loc(v, self.reg_width) + prev_loc = self.frame_manager.loc(v) loc = self.force_allocate_reg(v, forbidden_vars) self.assembler.regalloc_mov(prev_loc, loc) assert v in self.reg_bindings @@ -300,7 +300,7 @@ def _sync_var(self, v): if not self.frame_manager.get(v): reg = self.reg_bindings[v] - to = self.frame_manager.loc(v, self.reg_width) + to = self.frame_manager.loc(v) self.assembler.regalloc_mov(reg, to) # otherwise it's clean Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/runner_test.py (original) +++ pypy/trunk/pypy/jit/backend/test/runner_test.py Mon Aug 2 19:52:15 2010 @@ -461,6 +461,25 @@ [funcbox] + args, 'float', descr=calldescr) assert abs(res.value - 4.6) < 0.0001 + + def test_call_many_arguments(self): + # Test calling a function with a large number of arguments (more than + # 6, which will force passing some arguments on the stack on 64-bit) + + def func(*args): + assert len(args) == 16 + # Try to sum up args in a way that would probably detect a + # transposed argument + return sum(arg * (2**i) for i, arg in enumerate(args)) + + FUNC = self.FuncType([lltype.Signed]*16, lltype.Signed) + FPTR = self.Ptr(FUNC) + calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + func_ptr = llhelper(FPTR, func) + args = range(16) + funcbox = self.get_funcbox(self.cpu, func_ptr) + res = self.execute_operation(rop.CALL, [funcbox] + map(BoxInt, args), 'int', descr=calldescr) + assert res.value == func(*args) def test_call_stack_alignment(self): # test stack alignment issues, notably for Mac OS/X. @@ -978,6 +997,8 @@ else: assert 0 operations.append(ResOperation(opnum, boxargs, boxres)) + # Unique-ify inputargs + inputargs = list(set(inputargs)) faildescr = BasicFailDescr(1) operations.append(ResOperation(rop.FINISH, [], None, descr=faildescr)) @@ -1050,9 +1071,11 @@ descr=BasicFailDescr(5))] operations[1].fail_args = [] looptoken = LoopToken() - self.cpu.compile_loop(list(testcase), operations, + # Use "set" to unique-ify inputargs + unique_testcase_list = list(set(testcase)) + self.cpu.compile_loop(unique_testcase_list, operations, looptoken) - for i, box in enumerate(testcase): + for i, box in enumerate(unique_testcase_list): self.cpu.set_future_value_float(i, box.value) fail = self.cpu.execute_token(looptoken) if fail.identifier != 5 - (expected_id^expected): @@ -1695,7 +1718,7 @@ def test_assembler_call(self): called = [] def assembler_helper(failindex, virtualizable): - assert self.cpu.get_latest_value_int(0) == 10 + assert self.cpu.get_latest_value_int(0) == 97 called.append(failindex) return 4 + 9 @@ -1708,33 +1731,41 @@ _assembler_helper_ptr) ops = ''' - [i0, i1] - i2 = int_add(i0, i1) - finish(i2)''' + [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] + i10 = int_add(i0, i1) + i11 = int_add(i10, i2) + i12 = int_add(i11, i3) + i13 = int_add(i12, i4) + i14 = int_add(i13, i5) + i15 = int_add(i14, i6) + i16 = int_add(i15, i7) + i17 = int_add(i16, i8) + i18 = int_add(i17, i9) + finish(i18)''' loop = parse(ops) looptoken = LoopToken() looptoken.outermost_jitdriver_sd = FakeJitDriverSD() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) - ARGS = [lltype.Signed, lltype.Signed] + ARGS = [lltype.Signed] * 10 RES = lltype.Signed self.cpu.portal_calldescr = self.cpu.calldescrof( lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES) - self.cpu.set_future_value_int(0, 1) - self.cpu.set_future_value_int(1, 2) + for i in range(10): + self.cpu.set_future_value_int(i, i+1) res = self.cpu.execute_token(looptoken) - assert self.cpu.get_latest_value_int(0) == 3 + assert self.cpu.get_latest_value_int(0) == 55 ops = ''' - [i4, i5] - i6 = int_add(i4, 1) - i3 = call_assembler(i6, i5, descr=looptoken) + [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] + i10 = int_add(i0, 42) + i11 = call_assembler(i10, i1, i2, i3, i4, i5, i6, i7, i8, i9, descr=looptoken) guard_not_forced()[] - finish(i3) + finish(i11) ''' loop = parse(ops, namespace=locals()) othertoken = LoopToken() self.cpu.compile_loop(loop.inputargs, loop.operations, othertoken) - self.cpu.set_future_value_int(0, 4) - self.cpu.set_future_value_int(1, 5) + for i in range(10): + self.cpu.set_future_value_int(i, i+1) res = self.cpu.execute_token(othertoken) assert self.cpu.get_latest_value_int(0) == 13 assert called Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Mon Aug 2 19:52:15 2010 @@ -7,24 +7,34 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.annlowlevel import llhelper from pypy.tool.uid import fixid -from pypy.jit.backend.x86.regalloc import RegAlloc, WORD,\ - X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs, FRAME_FIXED_SIZE,\ - FORCE_INDEX_OFS +from pypy.jit.backend.x86.regalloc import RegAlloc, \ + X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs + +from pypy.jit.backend.x86.arch import FRAME_FIXED_SIZE, FORCE_INDEX_OFS, WORD, IS_X86_32, IS_X86_64 + +from pypy.jit.backend.x86.regloc import (eax, ecx, edx, ebx, + esp, ebp, esi, edi, + xmm0, xmm1, xmm2, xmm3, + xmm4, xmm5, xmm6, xmm7, + r8, r9, r10, r11, + r12, r13, r14, r15, + X86_64_SCRATCH_REG, + X86_64_XMM_SCRATCH_REG, + RegLoc, StackLoc, + ImmedLoc, AddressLoc, imm) + from pypy.rlib.objectmodel import we_are_translated, specialize -from pypy.jit.backend.x86 import codebuf -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86 import rx86, regloc, codebuf from pypy.jit.metainterp.resoperation import rop from pypy.jit.backend.x86.support import values_array from pypy.rlib.debug import debug_print from pypy.rlib import rgc +from pypy.jit.backend.x86.jump import remap_frame_layout from pypy.rlib.streamio import open_file_as_stream -# our calling convention - we pass first 6 args in registers -# and the rest stays on the stack - # darwin requires the stack to be 16 bytes aligned on calls. Same for gcc 4.5.0, # better safe than sorry -CALL_ALIGN = 4 +CALL_ALIGN = 16 // WORD def align_stack_words(words): return (words + CALL_ALIGN - 1) & ~(CALL_ALIGN-1) @@ -32,16 +42,36 @@ class MachineCodeBlockWrapper(object): MC_DEFAULT_SIZE = 1024*1024 - def __init__(self, bigsize, profile_agent=None): + def __init__(self, assembler, bigsize, profile_agent=None): + self.assembler = assembler self.old_mcs = [] # keepalive self.bigsize = bigsize self._mc = self._instantiate_mc() self.function_name = None self.profile_agent = profile_agent + self.reset_reserved_bytes() def _instantiate_mc(self): # hook for testing return codebuf.MachineCodeBlock(self.bigsize) + def ensure_bytes_available(self, num_bytes): + if self.bytes_free() <= (self._reserved_bytes + num_bytes): + self.make_new_mc() + + def reserve_bytes(self, num_bytes): + self.ensure_bytes_available(num_bytes) + self._reserved_bytes += num_bytes + + def reset_reserved_bytes(self): + # XXX er.... pretty random number, just to be sure + # not to write half-instruction + self._reserved_bytes = 64 + + def get_relative_pos(self): + return self._mc.get_relative_pos() + + def overwrite(self, pos, listofchars): + return self._mc.overwrite(pos, listofchars) def bytes_free(self): return self._mc._size - self._mc.get_relative_pos() @@ -63,12 +93,25 @@ def make_new_mc(self): new_mc = self._instantiate_mc() debug_print('[new machine code block at', new_mc.tell(), ']') - self._mc.JMP(rel32(new_mc.tell())) + + if IS_X86_64: + # The scratch register is sometimes used as a temporary + # register, but the JMP below might clobber it. Rather than risk + # subtle bugs, we preserve the scratch register across the jump. + self._mc.PUSH_r(X86_64_SCRATCH_REG.value) + + self._mc.JMP(imm(new_mc.tell())) + + if IS_X86_64: + # Restore scratch reg + new_mc.POP_r(X86_64_SCRATCH_REG.value) if self.function_name is not None: self.end_function(done=False) self.start_pos = new_mc.get_relative_pos() + self.assembler.write_pending_failure_recoveries() + self._mc.done() self.old_mcs.append(self._mc) self._mc = new_mc @@ -82,21 +125,33 @@ def _new_method(name): def method(self, *args): - # XXX er.... pretty random number, just to be sure - # not to write half-instruction - if self.bytes_free() < 64: + if self.bytes_free() < self._reserved_bytes: self.make_new_mc() getattr(self._mc, name)(*args) method.func_name = name return method +for _name in rx86.all_instructions + regloc.all_extra_instructions: + setattr(MachineCodeBlockWrapper, _name, _new_method(_name)) + for name in dir(codebuf.MachineCodeBlock): if name.upper() == name or name == "writechr": setattr(MachineCodeBlockWrapper, name, _new_method(name)) +class GuardToken(object): + def __init__(self, faildescr, failargs, fail_locs, exc, desc_bytes): + self.faildescr = faildescr + self.failargs = failargs + self.fail_locs = fail_locs + self.exc = exc + self.desc_bytes = desc_bytes + + def recovery_stub_size(self): + # XXX: 32 is pulled out of the air + return 32 + len(self.desc_bytes) + class Assembler386(object): mc = None - mc2 = None mc_size = MachineCodeBlockWrapper.MC_DEFAULT_SIZE _float_constants = None _regalloc = None @@ -118,10 +173,11 @@ self.loop_run_counter = values_array(lltype.Signed, 10000) self.loop_names = [] # if we have 10000 loops, we have some other problems I guess - self.loc_float_const_neg = None - self.loc_float_const_abs = None + self.float_const_neg_addr = 0 + self.float_const_abs_addr = 0 self.malloc_fixedsize_slowpath1 = 0 self.malloc_fixedsize_slowpath2 = 0 + self.pending_guard_tokens = None self.setup_failure_recovery() self._loop_counter = 0 self._debug = False @@ -134,7 +190,7 @@ def set_debug(self, v): self._debug = v - def make_sure_mc_exists(self): + def setup(self): if self.mc is None: # the address of the function called by 'new' gc_ll_descr = self.cpu.gc_ll_descr @@ -153,11 +209,7 @@ ll_new_unicode = gc_ll_descr.get_funcptr_for_newunicode() self.malloc_unicode_func_addr = rffi.cast(lltype.Signed, ll_new_unicode) - # done - # we generate the loop body in 'mc' - # 'mc2' is for guard recovery code - self.mc = MachineCodeBlockWrapper(self.mc_size, self.cpu.profile_agent) - self.mc2 = MachineCodeBlockWrapper(self.mc_size) + self.mc = MachineCodeBlockWrapper(self, self.mc_size, self.cpu.profile_agent) self._build_failure_recovery(False) self._build_failure_recovery(True) if self.cpu.supports_floats: @@ -173,6 +225,12 @@ s = s.split(':')[-1] self.set_debug(True) self._output_loop_log = s + ".count" + # Intialize here instead of __init__ to prevent + # pending_guard_tokens from being considered a prebuilt object, + # which sometimes causes memory leaks since the prebuilt list is + # still considered a GC root after we re-assign + # pending_guard_tokens in write_pending_failure_recoveries + self.pending_guard_tokens = [] def finish_once(self): if self._debug: @@ -185,44 +243,48 @@ f.close() def _build_float_constants(self): - # 11 words: 8 words for the data, and up to 3 words for alignment - addr = lltype.malloc(rffi.CArray(lltype.Signed), 11, flavor='raw') + # 44 bytes: 32 bytes for the data, and up to 12 bytes for alignment + addr = lltype.malloc(rffi.CArray(lltype.Char), 44, flavor='raw') if not we_are_translated(): self._keepalive_malloced_float_consts = addr float_constants = rffi.cast(lltype.Signed, addr) float_constants = (float_constants + 15) & ~15 # align to 16 bytes - addr = rffi.cast(rffi.CArrayPtr(lltype.Signed), float_constants) - addr[0] = 0 # \ - addr[1] = -2147483648 # / for neg - addr[2] = 0 # - addr[3] = 0 # - addr[4] = -1 # \ - addr[5] = 2147483647 # / for abs - addr[6] = 0 # - addr[7] = 0 # - self.loc_float_const_neg = heap64(float_constants) - self.loc_float_const_abs = heap64(float_constants + 16) + addr = rffi.cast(rffi.CArrayPtr(lltype.Char), float_constants) + qword_padding = '\x00\x00\x00\x00\x00\x00\x00\x00' + # 0x8000000000000000 + neg_const = '\x00\x00\x00\x00\x00\x00\x00\x80' + # 0x7FFFFFFFFFFFFFFF + abs_const = '\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x7F' + data = neg_const + qword_padding + abs_const + qword_padding + for i in range(len(data)): + addr[i] = data[i] + self.float_const_neg_addr = float_constants + self.float_const_abs_addr = float_constants + 16 def _build_malloc_fixedsize_slowpath(self): - mc = self.mc2._mc # ---------- first helper for the slow path of malloc ---------- - self.malloc_fixedsize_slowpath1 = mc.tell() + self.malloc_fixedsize_slowpath1 = self.mc.tell() if self.cpu.supports_floats: # save the XMM registers in - for i in range(8): # the *caller* frame, from esp+8 - mc.MOVSD(mem64(esp, 8+8*i), xmm_registers[i]) - mc.SUB(edx, eax) # compute the size we want - mc.MOV(mem(esp, 4), edx) # save it as the new argument + for i in range(self.cpu.NUM_REGS):# the *caller* frame, from esp+8 + self.mc.MOVSD_sx((WORD*2)+8*i, i) + self.mc.SUB_rr(edx.value, eax.value) # compute the size we want + if IS_X86_32: + self.mc.MOV_sr(WORD, edx.value) # save it as the new argument + elif IS_X86_64: + # FIXME: We can't just clobber rdi like this, can we? + self.mc.MOV_rr(edi.value, edx.value) + addr = self.cpu.gc_ll_descr.get_malloc_fixedsize_slowpath_addr() - mc.JMP(rel32(addr)) # tail call to the real malloc + self.mc.JMP(imm(addr)) # tail call to the real malloc # ---------- second helper for the slow path of malloc ---------- - self.malloc_fixedsize_slowpath2 = mc.tell() + self.malloc_fixedsize_slowpath2 = self.mc.tell() if self.cpu.supports_floats: # restore the XMM registers - for i in range(8): # from where they were saved - mc.MOVSD(xmm_registers[i], mem64(esp, 8+8*i)) + for i in range(self.cpu.NUM_REGS):# from where they were saved + self.mc.MOVSD_xs(i, (WORD*2)+8*i) nursery_free_adr = self.cpu.gc_ll_descr.get_nursery_free_addr() - mc.MOV(edx, heap(nursery_free_adr)) # load this in EDX - mc.RET() - self.mc2.done() + self.mc.MOV(edx, heap(nursery_free_adr)) # load this in EDX + self.mc.RET() + self.mc.done() def assemble_loop(self, inputargs, operations, looptoken): """adds the following attributes to looptoken: @@ -233,15 +295,17 @@ _x86_param_depth _x86_arglocs """ - self.make_sure_mc_exists() + if not we_are_translated(): + # Arguments should be unique + assert len(set(inputargs)) == len(inputargs) + + self.setup() funcname = self._find_debug_merge_point(operations) + regalloc = RegAlloc(self, self.cpu.translate_support_code) arglocs = regalloc.prepare_loop(inputargs, operations, looptoken) looptoken._x86_arglocs = arglocs - needed_mem = len(arglocs[0]) * 16 + 16 - if needed_mem >= self.mc.bytes_free(): - self.mc.make_new_mc() # profile support name = "Loop # %s: %s" % (looptoken.number, funcname) @@ -256,21 +320,21 @@ self._patch_stackadjust(adr_stackadjust, frame_depth+param_depth) looptoken._x86_frame_depth = frame_depth looptoken._x86_param_depth = param_depth - # we need to make sure here that we don't overload an mc badly. - # a safe estimate is that we need at most 16 bytes per arg - needed_mem = len(arglocs[0]) * 16 + 16 - if needed_mem >= self.mc.bytes_free(): - self.mc.make_new_mc() + looptoken._x86_direct_bootstrap_code = self.mc.tell() self._assemble_bootstrap_direct_call(arglocs, curadr, frame_depth+param_depth) debug_print("Loop #", looptoken.number, "has address", looptoken._x86_loop_code, "to", self.mc.tell()) self.mc.end_function() + self.write_pending_failure_recoveries() - def assemble_bridge(self, faildescr, inputargs, operations): - self.make_sure_mc_exists() + if not we_are_translated(): + # Arguments should be unique + assert len(set(inputargs)) == len(inputargs) + + self.setup() funcname = self._find_debug_merge_point(operations) arglocs = self.rebuild_faillocs_from_descr( @@ -302,6 +366,19 @@ descr_number, "has address", adr_bridge, "to", self.mc.tell()) self.mc.end_function() + self.write_pending_failure_recoveries() + + def write_pending_failure_recoveries(self): + for tok in self.pending_guard_tokens: + # Okay to write to _mc because we've already made sure that + # there's enough space by "reserving" bytes. + addr = self.generate_quick_failure(self.mc._mc, tok.faildescr, tok.failargs, tok.fail_locs, tok.exc, tok.desc_bytes) + tok.faildescr._x86_adr_recovery_stub = addr + self.patch_jump_for_descr(tok.faildescr, addr) + + self.pending_guard_tokens = [] + self.mc.reset_reserved_bytes() + self.mc.done() def _find_debug_merge_point(self, operations): @@ -319,8 +396,21 @@ def patch_jump_for_descr(self, faildescr, adr_new_target): adr_jump_offset = faildescr._x86_adr_jump_offset - mc = codebuf.InMemoryCodeBuilder(adr_jump_offset, adr_jump_offset + 4) - mc.write(packimm32(adr_new_target - adr_jump_offset - 4)) + adr_recovery_stub = faildescr._x86_adr_recovery_stub + offset = adr_new_target - (adr_jump_offset + 4) + # If the new target fits within a rel32 of the jump, just patch + # that. Otherwise, leave the original rel32 to the recovery stub in + # place, but clobber the recovery stub with a jump to the real + # target. + if rx86.fits_in_32bits(offset): + mc = codebuf.InMemoryCodeBuilder(adr_jump_offset, adr_jump_offset + 4) + mc.writeimm32(offset) + else: + # "mov r11, addr; jmp r11" is 13 bytes + mc = codebuf.InMemoryCodeBuilder(adr_recovery_stub, adr_recovery_stub + 13) + mc.MOV_ri(X86_64_SCRATCH_REG.value, adr_new_target) + mc.JMP_r(X86_64_SCRATCH_REG.value) + mc.valgrind_invalidated() mc.done() @@ -337,7 +427,6 @@ self.mc.POP(eax) regalloc.walk_operations(operations) self.mc.done() - self.mc2.done() if we_are_translated() or self.cpu.dont_keepalive_stuff: self._regalloc = None # else keep it around for debugging frame_depth = regalloc.fm.frame_depth @@ -352,7 +441,7 @@ def _patchable_stackadjust(self): # stack adjustment LEA - self.mc.LEA(esp, fixedsize_ebp_ofs(0)) + self.mc.LEA32_rb(esp.value, 0) return self.mc.tell() - 4 def _patch_stackadjust(self, adr_lea, reserved_depth): @@ -361,23 +450,34 @@ # Compute the correct offset for the instruction LEA ESP, [EBP-4*words]. # Given that [EBP] is where we saved EBP, i.e. in the last word # of our fixed frame, then the 'words' value is: - words = (FRAME_FIXED_SIZE - 1) + reserved_depth + words = (self.cpu.FRAME_FIXED_SIZE - 1) + reserved_depth # align, e.g. for Mac OS X aligned_words = align_stack_words(words+2)-2 # 2 = EIP+EBP - mc.write(packimm32(-WORD * aligned_words)) + mc.writeimm32(-WORD * aligned_words) mc.done() def _call_header(self): - self.mc.PUSH(ebp) - self.mc.MOV(ebp, esp) - self.mc.PUSH(ebx) - self.mc.PUSH(esi) - self.mc.PUSH(edi) + self.mc.PUSH_r(ebp.value) + self.mc.MOV_rr(ebp.value, esp.value) + for regloc in self.cpu.CALLEE_SAVE_REGISTERS: + self.mc.PUSH_r(regloc.value) + # NB. the shape of the frame is hard-coded in get_basic_shape() too. # Also, make sure this is consistent with FRAME_FIXED_SIZE. return self._patchable_stackadjust() + def _call_footer(self): + self.mc.LEA_rb(esp.value, -len(self.cpu.CALLEE_SAVE_REGISTERS) * WORD) + + for i in range(len(self.cpu.CALLEE_SAVE_REGISTERS)-1, -1, -1): + self.mc.POP_r(self.cpu.CALLEE_SAVE_REGISTERS[i].value) + + self.mc.POP_r(ebp.value) + self.mc.RET() + def _assemble_bootstrap_direct_call(self, arglocs, jmpadr, stackdepth): + if IS_X86_64: + return self._assemble_bootstrap_direct_call_64(arglocs, jmpadr, stackdepth) # XXX pushing ebx esi and edi is a bit pointless, since we store # all regsiters anyway, for the case of guard_not_forced # XXX this can be improved greatly. Right now it'll behave like @@ -388,23 +488,81 @@ self._patch_stackadjust(adr_stackadjust, stackdepth) for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] - if isinstance(loc, REG): - self.mc.MOV(loc, mem(ebp, (2 + i) * WORD)) + if isinstance(loc, RegLoc): + assert not loc.is_xmm + self.mc.MOV_rb(loc.value, (2 + i) * WORD) loc = floatlocs[i] - if isinstance(loc, XMMREG): - self.mc.MOVSD(loc, mem64(ebp, (1 + i) * 2 * WORD)) + if isinstance(loc, RegLoc): + assert loc.is_xmm + self.mc.MOVSD_xb(loc.value, (1 + i) * 2 * WORD) tmp = eax xmmtmp = xmm0 for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] - if loc is not None and not isinstance(loc, REG): - self.mc.MOV(tmp, mem(ebp, (2 + i) * WORD)) + if loc is not None and not isinstance(loc, RegLoc): + self.mc.MOV_rb(tmp.value, (2 + i) * WORD) self.mc.MOV(loc, tmp) loc = floatlocs[i] - if loc is not None and not isinstance(loc, XMMREG): - self.mc.MOVSD(xmmtmp, mem64(ebp, (1 + i) * 2 * WORD)) - self.mc.MOVSD(loc, xmmtmp) - self.mc.JMP(rel32(jmpadr)) + if loc is not None and not isinstance(loc, RegLoc): + self.mc.MOVSD_xb(xmmtmp.value, (1 + i) * 2 * WORD) + assert isinstance(loc, StackLoc) + self.mc.MOVSD_bx(loc.value, xmmtmp.value) + self.mc.JMP_l(jmpadr) + return adr_stackadjust + + def _assemble_bootstrap_direct_call_64(self, arglocs, jmpadr, stackdepth): + # XXX: Very similar to _emit_call_64 + + src_locs = [] + dst_locs = [] + xmm_src_locs = [] + xmm_dst_locs = [] + get_from_stack = [] + + # In reverse order for use with pop() + unused_gpr = [r9, r8, ecx, edx, esi, edi] + unused_xmm = [xmm7, xmm6, xmm5, xmm4, xmm3, xmm2, xmm1, xmm0] + + nonfloatlocs, floatlocs = arglocs + adr_stackadjust = self._call_header() + self._patch_stackadjust(adr_stackadjust, stackdepth) + + # The lists are padded with Nones + assert len(nonfloatlocs) == len(floatlocs) + + for i in range(len(nonfloatlocs)): + loc = nonfloatlocs[i] + if loc is not None: + if len(unused_gpr) > 0: + src_locs.append(unused_gpr.pop()) + dst_locs.append(loc) + else: + get_from_stack.append((loc, False)) + + floc = floatlocs[i] + if floc is not None: + if len(unused_xmm) > 0: + xmm_src_locs.append(unused_xmm.pop()) + xmm_dst_locs.append(floc) + else: + get_from_stack.append((floc, True)) + + remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG) + remap_frame_layout(self, xmm_src_locs, xmm_dst_locs, X86_64_XMM_SCRATCH_REG) + + for i in range(len(get_from_stack)): + loc, is_xmm = get_from_stack[i] + if is_xmm: + self.mc.MOVSD_xb(X86_64_XMM_SCRATCH_REG.value, (2 + i) * WORD) + self.mc.MOVSD(loc, X86_64_XMM_SCRATCH_REG) + else: + self.mc.MOV_rb(X86_64_SCRATCH_REG.value, (2 + i) * WORD) + # XXX: We're assuming that "loc" won't require regloc to + # clobber the scratch register + self.mc.MOV(loc, X86_64_SCRATCH_REG) + + self.mc.JMP(imm(jmpadr)) + return adr_stackadjust def _assemble_bootstrap_code(self, inputargs, arglocs): @@ -412,11 +570,12 @@ adr_stackadjust = self._call_header() tmp = X86RegisterManager.all_regs[0] xmmtmp = X86XMMRegisterManager.all_regs[0] + self.mc._mc.begin_reuse_scratch_register() for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] if loc is None: continue - if isinstance(loc, REG): + if isinstance(loc, RegLoc): target = loc else: target = tmp @@ -430,17 +589,20 @@ adr = self.fail_boxes_int.get_addr_for_num(i) self.mc.MOV(target, heap(adr)) if target is not loc: - self.mc.MOV(loc, target) + assert isinstance(loc, StackLoc) + self.mc.MOV_br(loc.value, target.value) for i in range(len(floatlocs)): loc = floatlocs[i] if loc is None: continue adr = self.fail_boxes_float.get_addr_for_num(i) - if isinstance(loc, REG): - self.mc.MOVSD(loc, heap64(adr)) + if isinstance(loc, RegLoc): + self.mc.MOVSD(loc, heap(adr)) else: - self.mc.MOVSD(xmmtmp, heap64(adr)) - self.mc.MOVSD(loc, xmmtmp) + self.mc.MOVSD(xmmtmp, heap(adr)) + assert isinstance(loc, StackLoc) + self.mc.MOVSD_bx(loc.value, xmmtmp.value) + self.mc._mc.end_reuse_scratch_register() return adr_stackadjust def dump(self, text): @@ -453,27 +615,10 @@ finally: Box._extended_display = _prev - def _start_block(self): - # Return a 'mc' that can be used to write an "atomic" block, - # i.e. one that will not contain any JMP. - mc = self.mc._mc - if not we_are_translated(): - self._block_started_mc = (self.mc, mc.tell()) - self.mc = "block started" - return mc - - def _stop_block(self): - if not we_are_translated(): - assert self.mc == "block started" - self.mc, orgpos = self._block_started_mc - assert 0 <= self.mc._mc.tell() - orgpos <= 58, ( - "too many bytes in _start_block/_stop_block pair") - del self._block_started_mc - # ------------------------------------------------------------ def mov(self, from_loc, to_loc): - if isinstance(from_loc, XMMREG) or isinstance(to_loc, XMMREG): + if (isinstance(from_loc, RegLoc) and from_loc.is_xmm) or (isinstance(to_loc, RegLoc) and to_loc.is_xmm): self.mc.MOVSD(to_loc, from_loc) else: self.mc.MOV(to_loc, from_loc) @@ -481,24 +626,24 @@ regalloc_mov = mov # legacy interface def regalloc_push(self, loc): - if isinstance(loc, XMMREG): - self.mc.SUB(esp, imm(2*WORD)) - self.mc.MOVSD(mem64(esp, 0), loc) - elif isinstance(loc, MODRM64): + if isinstance(loc, RegLoc) and loc.is_xmm: + self.mc.SUB_ri(esp.value, 2*WORD) + self.mc.MOVSD_sx(0, loc.value) + elif WORD == 4 and isinstance(loc, StackLoc) and loc.width == 8: # XXX evil trick - self.mc.PUSH(mem(ebp, get_ebp_ofs(loc.position))) - self.mc.PUSH(mem(ebp, get_ebp_ofs(loc.position + 1))) + self.mc.PUSH_b(get_ebp_ofs(loc.position)) + self.mc.PUSH_b(get_ebp_ofs(loc.position + 1)) else: self.mc.PUSH(loc) def regalloc_pop(self, loc): - if isinstance(loc, XMMREG): - self.mc.MOVSD(loc, mem64(esp, 0)) - self.mc.ADD(esp, imm(2*WORD)) - elif isinstance(loc, MODRM64): + if isinstance(loc, RegLoc) and loc.is_xmm: + self.mc.MOVSD_xs(loc.value, 0) + self.mc.ADD_ri(esp.value, 2*WORD) + elif WORD == 4 and isinstance(loc, StackLoc) and loc.width == 8: # XXX evil trick - self.mc.POP(mem(ebp, get_ebp_ofs(loc.position + 1))) - self.mc.POP(mem(ebp, get_ebp_ofs(loc.position))) + self.mc.POP_b(get_ebp_ofs(loc.position + 1)) + self.mc.POP_b(get_ebp_ofs(loc.position)) else: self.mc.POP(loc) @@ -515,14 +660,14 @@ faildescr._x86_current_depths = current_depths failargs = guard_op.fail_args guard_opnum = guard_op.opnum - failaddr = self.implement_guard_recovery(guard_opnum, - faildescr, failargs, - faillocs) + guard_token = self.implement_guard_recovery(guard_opnum, + faildescr, failargs, + faillocs) if op is None: dispatch_opnum = guard_opnum else: dispatch_opnum = op.opnum - res = genop_guard_list[dispatch_opnum](self, op, guard_op, failaddr, + res = genop_guard_list[dispatch_opnum](self, op, guard_op, guard_token, arglocs, resloc) faildescr._x86_adr_jump_offset = res @@ -549,103 +694,158 @@ rl = result_loc.lowest8bits() if isinstance(op.args[0], Const): self.mc.CMP(arglocs[1], arglocs[0]) - getattr(self.mc, 'SET' + rev_cond)(rl) + self.mc.SET_ir(rx86.Conditions[rev_cond], rl.value) else: self.mc.CMP(arglocs[0], arglocs[1]) - getattr(self.mc, 'SET' + cond)(rl) - self.mc.MOVZX(result_loc, rl) + self.mc.SET_ir(rx86.Conditions[cond], rl.value) + self.mc.MOVZX8_rr(result_loc.value, rl.value) return genop_cmp def _cmpop_float(cond, is_ne=False): def genop_cmp(self, op, arglocs, result_loc): self.mc.UCOMISD(arglocs[0], arglocs[1]) - rl = result_loc.lowest8bits() - rh = result_loc.higher8bits() - getattr(self.mc, 'SET' + cond)(rl) + tmp1 = result_loc.lowest8bits() + if IS_X86_32: + tmp2 = result_loc.higher8bits() + elif IS_X86_64: + tmp2 = X86_64_SCRATCH_REG.lowest8bits() + + self.mc.SET_ir(rx86.Conditions[cond], tmp1.value) if is_ne: - self.mc.SETP(rh) - self.mc.OR(rl, rh) + self.mc.SET_ir(rx86.Conditions['P'], tmp2.value) + self.mc.OR8_rr(tmp1.value, tmp2.value) else: - self.mc.SETNP(rh) - self.mc.AND(rl, rh) - self.mc.MOVZX(result_loc, rl) + self.mc.SET_ir(rx86.Conditions['NP'], tmp2.value) + self.mc.AND8_rr(tmp1.value, tmp2.value) + self.mc.MOVZX8_rr(result_loc.value, tmp1.value) return genop_cmp def _cmpop_guard(cond, rev_cond, false_cond, false_rev_cond): - def genop_cmp_guard(self, op, guard_op, addr, arglocs, result_loc): + def genop_cmp_guard(self, op, guard_op, guard_token, arglocs, result_loc): guard_opnum = guard_op.opnum if isinstance(op.args[0], Const): self.mc.CMP(arglocs[1], arglocs[0]) if guard_opnum == rop.GUARD_FALSE: - name = 'J' + rev_cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(guard_token, rev_cond) else: - name = 'J' + false_rev_cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(guard_token, false_rev_cond) else: self.mc.CMP(arglocs[0], arglocs[1]) if guard_opnum == rop.GUARD_FALSE: - name = 'J' + cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(guard_token, cond) else: - name = 'J' + false_cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(guard_token, false_cond) return genop_cmp_guard def _cmpop_guard_float(cond, false_cond, need_jp): - def genop_cmp_guard_float(self, op, guard_op, addr, arglocs, + def genop_cmp_guard_float(self, op, guard_op, guard_token, arglocs, result_loc): guard_opnum = guard_op.opnum self.mc.UCOMISD(arglocs[0], arglocs[1]) + # 16 is enough space for the rel8 jumps below and the rel32 + # jump in implement_guard + self.mc.ensure_bytes_available(16 + guard_token.recovery_stub_size()) if guard_opnum == rop.GUARD_FALSE: - mc = self.mc._mc - name = 'J' + cond if need_jp: - mc.JP(rel8(6)) - getattr(mc, name)(rel32(addr)) - return mc.tell() - 4 + self.mc.J_il8(rx86.Conditions['P'], 6) + return self.implement_guard(guard_token, cond) else: if need_jp: - mc = self.mc._mc - mc.JP(rel8(2)) - getattr(mc, 'J' + cond)(rel8(5)) - return self.implement_guard(addr, mc.JMP) - name = 'J' + false_cond - return self.implement_guard(addr, getattr(self.mc, name)) + self.mc.J_il8(rx86.Conditions['P'], 2) + self.mc.J_il8(rx86.Conditions[cond], 5) + return self.implement_guard(guard_token) + return self.implement_guard(guard_token, false_cond) return genop_cmp_guard_float - @specialize.arg(5) - def _emit_call(self, x, arglocs, start=0, tmp=eax, force_mc=False, - mc=None): - if not force_mc: - mc = self.mc + def _emit_call(self, x, arglocs, start=0, tmp=eax): + if IS_X86_64: + return self._emit_call_64(x, arglocs, start) + p = 0 n = len(arglocs) for i in range(start, n): loc = arglocs[i] - if isinstance(loc, REG): - if isinstance(loc, XMMREG): - mc.MOVSD(mem64(esp, p), loc) + if isinstance(loc, RegLoc): + if loc.is_xmm: + self.mc.MOVSD_sx(p, loc.value) else: - mc.MOV(mem(esp, p), loc) + self.mc.MOV_sr(p, loc.value) p += round_up_to_4(loc.width) p = 0 for i in range(start, n): loc = arglocs[i] - if not isinstance(loc, REG): - if isinstance(loc, MODRM64): - mc.MOVSD(xmm0, loc) - mc.MOVSD(mem64(esp, p), xmm0) + if not isinstance(loc, RegLoc): + if loc.width == 8: + self.mc.MOVSD(xmm0, loc) + self.mc.MOVSD_sx(p, xmm0.value) else: - mc.MOV(tmp, loc) - mc.MOV(mem(esp, p), tmp) + self.mc.MOV(tmp, loc) + self.mc.MOV_sr(p, tmp.value) p += round_up_to_4(loc.width) self._regalloc.reserve_param(p//WORD) - mc.CALL(x) + # x is a location + self.mc.CALL(x) self.mark_gc_roots() + + def _emit_call_64(self, x, arglocs, start=0): + src_locs = [] + dst_locs = [] + xmm_src_locs = [] + xmm_dst_locs = [] + pass_on_stack = [] + + # In reverse order for use with pop() + unused_gpr = [r9, r8, ecx, edx, esi, edi] + unused_xmm = [xmm7, xmm6, xmm5, xmm4, xmm3, xmm2, xmm1, xmm0] + + for i in range(start, len(arglocs)): + loc = arglocs[i] + assert isinstance(loc, RegLoc) or isinstance(loc, ImmedLoc) or isinstance(loc, StackLoc) + + # XXX: Should be much simplier to tell whether a location is a float! + if (isinstance(loc, RegLoc) and loc.is_xmm) or (isinstance(loc, StackLoc) and loc.type == FLOAT): + if len(unused_xmm) > 0: + xmm_src_locs.append(loc) + xmm_dst_locs.append(unused_xmm.pop()) + else: + pass_on_stack.append(loc) + else: + if len(unused_gpr) > 0: + src_locs.append(loc) + dst_locs.append(unused_gpr.pop()) + else: + pass_on_stack.append(loc) + # Emit instructions to pass the stack arguments + # XXX: Would be nice to let remap_frame_layout take care of this, but + # we'd need to create something like StackLoc, but relative to esp, + # and I don't know if it's worth it. + for i in range(len(pass_on_stack)): + loc = pass_on_stack[i] + if not isinstance(loc, RegLoc): + if isinstance(loc, StackLoc) and loc.type == FLOAT: + self.mc.MOVSD(X86_64_XMM_SCRATCH_REG, loc) + self.mc.MOVSD_sx(i*WORD, X86_64_XMM_SCRATCH_REG.value) + else: + self.mc.MOV(X86_64_SCRATCH_REG, loc) + self.mc.MOV_sr(i*WORD, X86_64_SCRATCH_REG.value) + else: + # It's a register + if loc.is_xmm: + self.mc.MOVSD_sx(i*WORD, loc.value) + else: + self.mc.MOV_sr(i*WORD, loc.value) + + # Handle register arguments + remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG) + remap_frame_layout(self, xmm_src_locs, xmm_dst_locs, X86_64_XMM_SCRATCH_REG) + + self._regalloc.reserve_param(len(pass_on_stack)) + self.mc.CALL(x) + self.mark_gc_roots() + def call(self, addr, args, res): - self._emit_call(rel32(addr), args) + self._emit_call(imm(addr), args) assert res is eax genop_int_neg = _unaryop("NEG") @@ -656,6 +856,9 @@ genop_int_and = _binaryop("AND", True) genop_int_or = _binaryop("OR", True) genop_int_xor = _binaryop("XOR", True) + genop_int_lshift = _binaryop("SHL") + genop_int_rshift = _binaryop("SAR") + genop_uint_rshift = _binaryop("SHR") genop_float_add = _binaryop("ADDSD", True) genop_float_sub = _binaryop('SUBSD') genop_float_mul = _binaryop('MULSD', True) @@ -702,26 +905,27 @@ genop_guard_float_gt = _cmpop_guard_float("A", "BE", False) genop_guard_float_ge = _cmpop_guard_float("AE", "B", False) - def genop_guard_float_ne(self, op, guard_op, addr, arglocs, result_loc): + def genop_guard_float_ne(self, op, guard_op, guard_token, arglocs, result_loc): guard_opnum = guard_op.opnum self.mc.UCOMISD(arglocs[0], arglocs[1]) - mc = self.mc._mc + # 16 is enough space for the rel8 jumps below and the rel32 + # jump in implement_guard + self.mc.ensure_bytes_available(16 + guard_token.recovery_stub_size()) if guard_opnum == rop.GUARD_TRUE: - mc.JP(rel8(6)) - mc.JE(rel32(addr)) - return mc.tell() - 4 - else: - mc.JP(rel8(2)) - mc.JE(rel8(5)) - return self.implement_guard(addr, mc.JMP) + self.mc.J_il8(rx86.Conditions['P'], 6) + return self.implement_guard(guard_token, 'E') + else: + self.mc.J_il8(rx86.Conditions['P'], 2) + self.mc.J_il8(rx86.Conditions['E'], 5) + return self.implement_guard(guard_token) def genop_float_neg(self, op, arglocs, resloc): # Following what gcc does: res = x ^ 0x8000000000000000 - self.mc.XORPD(arglocs[0], self.loc_float_const_neg) + self.mc.XORPD(arglocs[0], heap(self.float_const_neg_addr)) def genop_float_abs(self, op, arglocs, resloc): # Following what gcc does: res = x & 0x7FFFFFFFFFFFFFFF - self.mc.ANDPD(arglocs[0], self.loc_float_const_abs) + self.mc.ANDPD(arglocs[0], heap(self.float_const_abs_addr)) def genop_cast_float_to_int(self, op, arglocs, resloc): self.mc.CVTTSD2SI(resloc, arglocs[0]) @@ -729,70 +933,56 @@ def genop_cast_int_to_float(self, op, arglocs, resloc): self.mc.CVTSI2SD(resloc, arglocs[0]) - def genop_int_lshift(self, op, arglocs, resloc): - loc, loc2 = arglocs - if loc2 is ecx: - loc2 = cl - self.mc.SHL(loc, loc2) - - def genop_int_rshift(self, op, arglocs, resloc): - loc, loc2 = arglocs - if loc2 is ecx: - loc2 = cl - self.mc.SAR(loc, loc2) - - def genop_uint_rshift(self, op, arglocs, resloc): - loc, loc2 = arglocs - if loc2 is ecx: - loc2 = cl - self.mc.SHR(loc, loc2) - - def genop_guard_int_is_true(self, op, guard_op, addr, arglocs, resloc): + def genop_guard_int_is_true(self, op, guard_op, guard_token, arglocs, resloc): guard_opnum = guard_op.opnum - self.mc.CMP(arglocs[0], imm8(0)) + self.mc.CMP(arglocs[0], imm(0)) if guard_opnum == rop.GUARD_TRUE: - return self.implement_guard(addr, self.mc.JZ) + return self.implement_guard(guard_token, 'Z') else: - return self.implement_guard(addr, self.mc.JNZ) + return self.implement_guard(guard_token, 'NZ') def genop_int_is_true(self, op, arglocs, resloc): - self.mc.CMP(arglocs[0], imm8(0)) + self.mc.CMP(arglocs[0], imm(0)) rl = resloc.lowest8bits() - self.mc.SETNE(rl) - self.mc.MOVZX(resloc, rl) + self.mc.SET_ir(rx86.Conditions['NE'], rl.value) + self.mc.MOVZX8(resloc, rl) - def genop_guard_int_is_zero(self, op, guard_op, addr, arglocs, resloc): + def genop_guard_int_is_zero(self, op, guard_op, guard_token, arglocs, resloc): guard_opnum = guard_op.opnum - self.mc.CMP(arglocs[0], imm8(0)) + self.mc.CMP(arglocs[0], imm(0)) if guard_opnum == rop.GUARD_TRUE: - return self.implement_guard(addr, self.mc.JNZ) + return self.implement_guard(guard_token, 'NZ') else: - return self.implement_guard(addr, self.mc.JZ) + return self.implement_guard(guard_token, 'Z') def genop_int_is_zero(self, op, arglocs, resloc): - self.mc.CMP(arglocs[0], imm8(0)) + self.mc.CMP(arglocs[0], imm(0)) rl = resloc.lowest8bits() - self.mc.SETE(rl) - self.mc.MOVZX(resloc, rl) + self.mc.SET_ir(rx86.Conditions['E'], rl.value) + self.mc.MOVZX8(resloc, rl) def genop_same_as(self, op, arglocs, resloc): self.mov(arglocs[0], resloc) #genop_cast_ptr_to_int = genop_same_as def genop_int_mod(self, op, arglocs, resloc): - self.mc.CDQ() - self.mc.IDIV(ecx) + if IS_X86_32: + self.mc.CDQ() + elif IS_X86_64: + self.mc.CQO() + + self.mc.IDIV_r(ecx.value) genop_int_floordiv = genop_int_mod def genop_uint_floordiv(self, op, arglocs, resloc): - self.mc.XOR(edx, edx) - self.mc.DIV(ecx) + self.mc.XOR_rr(edx.value, edx.value) + self.mc.DIV_r(ecx.value) def genop_new_with_vtable(self, op, arglocs, result_loc): assert result_loc is eax loc_vtable = arglocs[-1] - assert isinstance(loc_vtable, IMM32) + assert isinstance(loc_vtable, ImmedLoc) arglocs = arglocs[:-1] self.call(self.malloc_func_addr, arglocs, eax) # xxx ignore NULL returns for now @@ -800,7 +990,9 @@ def set_vtable(self, loc, loc_vtable): if self.cpu.vtable_offset is not None: - self.mc.MOV(mem(loc, self.cpu.vtable_offset), loc_vtable) + assert isinstance(loc, RegLoc) + assert isinstance(loc_vtable, ImmedLoc) + self.mc.MOV_mi((loc.value, self.cpu.vtable_offset), loc_vtable.value) # XXX genop_new is abused for all varsized mallocs with Boehm, for now # (instead of genop_new_array, genop_newstr, genop_newunicode) @@ -822,16 +1014,19 @@ def genop_getfield_gc(self, op, arglocs, resloc): base_loc, ofs_loc, size_loc = arglocs - assert isinstance(size_loc, IMM32) + assert isinstance(size_loc, ImmedLoc) + assert isinstance(resloc, RegLoc) size = size_loc.value - if size == 1: - self.mc.MOVZX(resloc, addr8_add(base_loc, ofs_loc)) + + source_addr = AddressLoc(base_loc, ofs_loc) + if resloc.is_xmm: + self.mc.MOVSD(resloc, source_addr) + elif size == 1: + self.mc.MOVZX8(resloc, source_addr) elif size == 2: - self.mc.MOVZX(resloc, addr_add(base_loc, ofs_loc)) + self.mc.MOVZX16(resloc, source_addr) elif size == WORD: - self.mc.MOV(resloc, addr_add(base_loc, ofs_loc)) - elif size == 8: - self.mc.MOVSD(resloc, addr64_add(base_loc, ofs_loc)) + self.mc.MOV(resloc, source_addr) else: raise NotImplementedError("getfield size = %d" % size) @@ -841,16 +1036,16 @@ def genop_getarrayitem_gc(self, op, arglocs, resloc): base_loc, ofs_loc, scale, ofs = arglocs - assert isinstance(ofs, IMM32) - assert isinstance(scale, IMM32) + assert isinstance(ofs, ImmedLoc) + assert isinstance(scale, ImmedLoc) if op.result.type == FLOAT: - self.mc.MOVSD(resloc, addr64_add(base_loc, ofs_loc, ofs.value, + self.mc.MOVSD(resloc, addr_add(base_loc, ofs_loc, ofs.value, scale.value)) else: if scale.value == 0: - self.mc.MOVZX(resloc, addr8_add(base_loc, ofs_loc, ofs.value, + self.mc.MOVZX8(resloc, addr_add(base_loc, ofs_loc, ofs.value, scale.value)) - elif scale.value == 2: + elif (1 << scale.value) == WORD: self.mc.MOV(resloc, addr_add(base_loc, ofs_loc, ofs.value, scale.value)) else: @@ -862,34 +1057,33 @@ def genop_discard_setfield_gc(self, op, arglocs): base_loc, ofs_loc, size_loc, value_loc = arglocs - assert isinstance(size_loc, IMM32) + assert isinstance(size_loc, ImmedLoc) size = size_loc.value - if size == WORD * 2: - self.mc.MOVSD(addr64_add(base_loc, ofs_loc), value_loc) + dest_addr = AddressLoc(base_loc, ofs_loc) + if isinstance(value_loc, RegLoc) and value_loc.is_xmm: + self.mc.MOVSD(dest_addr, value_loc) elif size == WORD: - self.mc.MOV(addr_add(base_loc, ofs_loc), value_loc) + self.mc.MOV(dest_addr, value_loc) elif size == 2: - self.mc.MOV16(addr_add(base_loc, ofs_loc), value_loc) + self.mc.MOV16(dest_addr, value_loc) elif size == 1: - self.mc.MOV(addr8_add(base_loc, ofs_loc), value_loc.lowest8bits()) + self.mc.MOV8(dest_addr, value_loc.lowest8bits()) else: print "[asmgen]setfield addr size %d" % size raise NotImplementedError("Addr size %d" % size) def genop_discard_setarrayitem_gc(self, op, arglocs): base_loc, ofs_loc, value_loc, scale_loc, baseofs = arglocs - assert isinstance(baseofs, IMM32) - assert isinstance(scale_loc, IMM32) + assert isinstance(baseofs, ImmedLoc) + assert isinstance(scale_loc, ImmedLoc) + dest_addr = AddressLoc(base_loc, ofs_loc, scale_loc.value, baseofs.value) if op.args[2].type == FLOAT: - self.mc.MOVSD(addr64_add(base_loc, ofs_loc, baseofs.value, - scale_loc.value), value_loc) + self.mc.MOVSD(dest_addr, value_loc) else: - if scale_loc.value == 2: - self.mc.MOV(addr_add(base_loc, ofs_loc, baseofs.value, - scale_loc.value), value_loc) + if (1 << scale_loc.value) == WORD: + self.mc.MOV(dest_addr, value_loc) elif scale_loc.value == 0: - self.mc.MOV(addr8_add(base_loc, ofs_loc, baseofs.value, - scale_loc.value), value_loc.lowest8bits()) + self.mc.MOV8(dest_addr, value_loc.lowest8bits()) else: raise NotImplementedError("scale = %d" % scale_loc.value) @@ -898,17 +1092,17 @@ basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) assert itemsize == 1 - self.mc.MOV(addr8_add(base_loc, ofs_loc, basesize), - val_loc.lowest8bits()) + dest_addr = AddressLoc(base_loc, ofs_loc, 0, basesize) + self.mc.MOV8(dest_addr, val_loc.lowest8bits()) def genop_discard_unicodesetitem(self, op, arglocs): base_loc, ofs_loc, val_loc = arglocs basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) if itemsize == 4: - self.mc.MOV(addr_add(base_loc, ofs_loc, basesize, 2), val_loc) + self.mc.MOV32(AddressLoc(base_loc, ofs_loc, 2, basesize), val_loc) elif itemsize == 2: - self.mc.MOV16(addr_add(base_loc, ofs_loc, basesize, 1), val_loc) + self.mc.MOV16(AddressLoc(base_loc, ofs_loc, 1, basesize), val_loc) else: assert 0, itemsize @@ -929,7 +1123,7 @@ def genop_arraylen_gc(self, op, arglocs, resloc): base_loc, ofs_loc = arglocs - assert isinstance(ofs_loc, IMM32) + assert isinstance(ofs_loc, ImmedLoc) self.mc.MOV(resloc, addr_add_const(base_loc, ofs_loc.value)) def genop_strgetitem(self, op, arglocs, resloc): @@ -937,83 +1131,83 @@ basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) assert itemsize == 1 - self.mc.MOVZX(resloc, addr8_add(base_loc, ofs_loc, basesize)) + self.mc.MOVZX8(resloc, AddressLoc(base_loc, ofs_loc, 0, basesize)) def genop_unicodegetitem(self, op, arglocs, resloc): base_loc, ofs_loc = arglocs basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) if itemsize == 4: - self.mc.MOV(resloc, addr_add(base_loc, ofs_loc, basesize, 2)) + self.mc.MOV32(resloc, AddressLoc(base_loc, ofs_loc, 2, basesize)) elif itemsize == 2: - self.mc.MOVZX(resloc, addr_add(base_loc, ofs_loc, basesize, 1)) + self.mc.MOVZX16(resloc, AddressLoc(base_loc, ofs_loc, 1, basesize)) else: assert 0, itemsize - def genop_guard_guard_true(self, ign_1, guard_op, addr, locs, ign_2): + def genop_guard_guard_true(self, ign_1, guard_op, guard_token, locs, ign_2): loc = locs[0] self.mc.TEST(loc, loc) - return self.implement_guard(addr, self.mc.JZ) + return self.implement_guard(guard_token, 'Z') genop_guard_guard_nonnull = genop_guard_guard_true - def genop_guard_guard_no_exception(self, ign_1, guard_op, addr, + def genop_guard_guard_no_exception(self, ign_1, guard_op, guard_token, locs, ign_2): self.mc.CMP(heap(self.cpu.pos_exception()), imm(0)) - return self.implement_guard(addr, self.mc.JNZ) + return self.implement_guard(guard_token, 'NZ') - def genop_guard_guard_exception(self, ign_1, guard_op, addr, + def genop_guard_guard_exception(self, ign_1, guard_op, guard_token, locs, resloc): loc = locs[0] loc1 = locs[1] self.mc.MOV(loc1, heap(self.cpu.pos_exception())) self.mc.CMP(loc1, loc) - addr = self.implement_guard(addr, self.mc.JNE) + addr = self.implement_guard(guard_token, 'NE') if resloc is not None: self.mc.MOV(resloc, heap(self.cpu.pos_exc_value())) self.mc.MOV(heap(self.cpu.pos_exception()), imm(0)) self.mc.MOV(heap(self.cpu.pos_exc_value()), imm(0)) return addr - def _gen_guard_overflow(self, guard_op, addr): + def _gen_guard_overflow(self, guard_op, guard_token): guard_opnum = guard_op.opnum if guard_opnum == rop.GUARD_NO_OVERFLOW: - return self.implement_guard(addr, self.mc.JO) + return self.implement_guard(guard_token, 'O') elif guard_opnum == rop.GUARD_OVERFLOW: - return self.implement_guard(addr, self.mc.JNO) + return self.implement_guard(guard_token, 'NO') else: print "int_xxx_ovf followed by", guard_op.getopname() raise AssertionError - def genop_guard_int_add_ovf(self, op, guard_op, addr, arglocs, result_loc): + def genop_guard_int_add_ovf(self, op, guard_op, guard_token, arglocs, result_loc): self.genop_int_add(op, arglocs, result_loc) - return self._gen_guard_overflow(guard_op, addr) + return self._gen_guard_overflow(guard_op, guard_token) - def genop_guard_int_sub_ovf(self, op, guard_op, addr, arglocs, result_loc): + def genop_guard_int_sub_ovf(self, op, guard_op, guard_token, arglocs, result_loc): self.genop_int_sub(op, arglocs, result_loc) - return self._gen_guard_overflow(guard_op, addr) + return self._gen_guard_overflow(guard_op, guard_token) - def genop_guard_int_mul_ovf(self, op, guard_op, addr, arglocs, result_loc): + def genop_guard_int_mul_ovf(self, op, guard_op, guard_token, arglocs, result_loc): self.genop_int_mul(op, arglocs, result_loc) - return self._gen_guard_overflow(guard_op, addr) + return self._gen_guard_overflow(guard_op, guard_token) - def genop_guard_guard_false(self, ign_1, guard_op, addr, locs, ign_2): + def genop_guard_guard_false(self, ign_1, guard_op, guard_token, locs, ign_2): loc = locs[0] self.mc.TEST(loc, loc) - return self.implement_guard(addr, self.mc.JNZ) + return self.implement_guard(guard_token, 'NZ') genop_guard_guard_isnull = genop_guard_guard_false - def genop_guard_guard_value(self, ign_1, guard_op, addr, locs, ign_2): + def genop_guard_guard_value(self, ign_1, guard_op, guard_token, locs, ign_2): if guard_op.args[0].type == FLOAT: assert guard_op.args[1].type == FLOAT self.mc.UCOMISD(locs[0], locs[1]) else: self.mc.CMP(locs[0], locs[1]) - return self.implement_guard(addr, self.mc.JNE) + return self.implement_guard(guard_token, 'NE') - def _cmp_guard_class(self, mc, locs): + def _cmp_guard_class(self, locs): offset = self.cpu.vtable_offset if offset is not None: - mc.CMP(mem(locs[0], offset), locs[1]) + self.mc.CMP(mem(locs[0], offset), locs[1]) else: # XXX hard-coded assumption: to go from an object to its class # we use the following algorithm: @@ -1022,7 +1216,7 @@ # - multiply by 4 and use it as an offset in type_info_group # - add 16 bytes, to go past the TYPE_INFO structure loc = locs[1] - assert isinstance(loc, IMM32) + assert isinstance(loc, ImmedLoc) classptr = loc.value # here, we have to go back from 'classptr' to the value expected # from reading the 16 bits in the object header @@ -1031,37 +1225,37 @@ type_info_group = llop.gc_get_type_info_group(llmemory.Address) type_info_group = rffi.cast(lltype.Signed, type_info_group) expected_typeid = (classptr - sizeof_ti - type_info_group) >> 2 - mc.CMP16(mem(locs[0], 0), imm32(expected_typeid)) + self.mc.CMP16(mem(locs[0], 0), ImmedLoc(expected_typeid)) - def genop_guard_guard_class(self, ign_1, guard_op, addr, locs, ign_2): - mc = self._start_block() - self._cmp_guard_class(mc, locs) - self._stop_block() - return self.implement_guard(addr, self.mc.JNE) + def genop_guard_guard_class(self, ign_1, guard_op, guard_token, locs, ign_2): + self.mc.ensure_bytes_available(256) + self._cmp_guard_class(locs) + return self.implement_guard(guard_token, 'NE') def genop_guard_guard_nonnull_class(self, ign_1, guard_op, - addr, locs, ign_2): - mc = self._start_block() - mc.CMP(locs[0], imm8(1)) - mc.JB(rel8_patched_later) - jb_location = mc.get_relative_pos() - self._cmp_guard_class(mc, locs) + guard_token, locs, ign_2): + self.mc.ensure_bytes_available(256) + self.mc.CMP(locs[0], imm(1)) + # Patched below + self.mc.J_il8(rx86.Conditions['B'], 0) + jb_location = self.mc.get_relative_pos() + self._cmp_guard_class(locs) # patch the JB above - offset = mc.get_relative_pos() - jb_location + offset = self.mc.get_relative_pos() - jb_location assert 0 < offset <= 127 - mc.overwrite(jb_location-1, [chr(offset)]) - self._stop_block() + self.mc.overwrite(jb_location-1, [chr(offset)]) # - return self.implement_guard(addr, self.mc.JNE) + return self.implement_guard(guard_token, 'NE') def implement_guard_recovery(self, guard_opnum, faildescr, failargs, fail_locs): exc = (guard_opnum == rop.GUARD_EXCEPTION or guard_opnum == rop.GUARD_NO_EXCEPTION or guard_opnum == rop.GUARD_NOT_FORCED) - return self.generate_quick_failure(faildescr, failargs, fail_locs, exc) + desc_bytes = self.failure_recovery_description(failargs, fail_locs) + return GuardToken(faildescr, failargs, fail_locs, exc, desc_bytes) - def generate_quick_failure(self, faildescr, failargs, fail_locs, exc): + def generate_quick_failure(self, mc, faildescr, failargs, fail_locs, exc, desc_bytes): """Generate the initial code for handling a failure. We try to keep it as compact as possible. The idea is that this code is executed at most once (and very often, zero times); when @@ -1069,38 +1263,43 @@ really handle recovery from this particular failure. """ fail_index = self.cpu.get_fail_descr_number(faildescr) - bytes_needed = 20 + 5 * len(failargs) # conservative estimate - if self.mc2.bytes_free() < bytes_needed: - self.mc2.make_new_mc() - mc = self.mc2._mc addr = mc.tell() withfloats = False for box in failargs: if box is not None and box.type == FLOAT: withfloats = True break - mc.CALL(rel32(self.failure_recovery_code[exc + 2 * withfloats])) + mc.CALL(imm(self.failure_recovery_code[exc + 2 * withfloats])) # write tight data that describes the failure recovery faildescr._x86_failure_recovery_bytecode = mc.tell() - self.write_failure_recovery_description(mc, failargs, fail_locs) + for byte in desc_bytes: + mc.writechr(ord(byte)) # write the fail_index too - mc.write(packimm32(fail_index)) + mc.writeimm32(fail_index) # for testing the decoding, write a final byte 0xCC if not we_are_translated(): mc.writechr(0xCC) faildescr._x86_debug_faillocs = [loc for loc in fail_locs if loc is not None] + + # Make sure the recovery stub is at least 16 bytes long (for the + # case where we overwrite the recovery stub with a 64-bit absolute + # jump) + while mc.tell() - addr < 16: + mc.writechr(0x00) return addr DESCR_REF = 0x00 DESCR_INT = 0x01 DESCR_FLOAT = 0x02 DESCR_SPECIAL = 0x03 - CODE_FROMSTACK = 4*8 + # XXX: 4*8 works on i386, should we optimize for that case? + CODE_FROMSTACK = 4*16 CODE_STOP = 0 | DESCR_SPECIAL CODE_HOLE = 4 | DESCR_SPECIAL - def write_failure_recovery_description(self, mc, failargs, locs): + def failure_recovery_description(self, failargs, locs): + desc_bytes = [] for i in range(len(failargs)): arg = failargs[i] if arg is not None: @@ -1113,24 +1312,30 @@ else: raise AssertionError("bogus kind") loc = locs[i] - if isinstance(loc, MODRM): + if isinstance(loc, StackLoc): n = self.CODE_FROMSTACK//4 + loc.position else: - assert isinstance(loc, REG) - n = loc.op + assert isinstance(loc, RegLoc) + n = loc.value n = kind + 4*n while n > 0x7F: - mc.writechr((n & 0x7F) | 0x80) + desc_bytes.append(chr((n & 0x7F) | 0x80)) n >>= 7 else: n = self.CODE_HOLE - mc.writechr(n) - mc.writechr(self.CODE_STOP) + desc_bytes.append(chr(n)) + desc_bytes.append(chr(self.CODE_STOP)) # assert that the fail_boxes lists are big enough assert len(failargs) <= self.fail_boxes_int.SIZE + return desc_bytes + + def write_failure_recovery_description(self, mc, failargs, locs): + for byte in self.failure_recovery_description(failargs, locs): + mc.writechr(ord(byte)) def rebuild_faillocs_from_descr(self, bytecode): from pypy.jit.backend.x86.regalloc import X86FrameManager + descr_to_box_type = [REF, INT, FLOAT] bytecode = rffi.cast(rffi.UCHARP, bytecode) arglocs = [] while 1: @@ -1155,7 +1360,7 @@ size = 2 else: size = 1 - loc = X86FrameManager.frame_pos(code, size) + loc = X86FrameManager.frame_pos(code, descr_to_box_type[kind]) elif code == self.CODE_STOP: break elif code == self.CODE_HOLE: @@ -1165,16 +1370,16 @@ kind = code & 3 code >>= 2 if kind == self.DESCR_FLOAT: - loc = xmm_registers[code] + loc = regloc.XMMREGLOCS[code] else: - loc = registers[code] + loc = regloc.REGLOCS[code] arglocs.append(loc) return arglocs[:] @rgc.no_collect def grab_frame_values(self, bytecode, frame_addr, allregisters): # no malloc allowed here!! - self.fail_ebp = allregisters[16 + ebp.op] + self.fail_ebp = allregisters[16 + ebp.value] num = 0 value_hi = 0 while 1: @@ -1197,7 +1402,7 @@ code = (code - self.CODE_FROMSTACK) >> 2 stackloc = frame_addr + get_ebp_ofs(code) value = rffi.cast(rffi.LONGP, stackloc)[0] - if kind == self.DESCR_FLOAT: + if kind == self.DESCR_FLOAT and WORD == 4: value_hi = value value = rffi.cast(rffi.LONGP, stackloc - 4)[0] else: @@ -1211,8 +1416,11 @@ break code >>= 2 if kind == self.DESCR_FLOAT: - value = allregisters[2*code] - value_hi = allregisters[2*code + 1] + if WORD == 4: + value = allregisters[2*code] + value_hi = allregisters[2*code + 1] + else: + value = allregisters[code] else: value = allregisters[16 + code] @@ -1223,7 +1431,8 @@ tgt = self.fail_boxes_ptr.get_addr_for_num(num) elif kind == self.DESCR_FLOAT: tgt = self.fail_boxes_float.get_addr_for_num(num) - rffi.cast(rffi.LONGP, tgt)[1] = value_hi + if WORD == 4: + rffi.cast(rffi.LONGP, tgt)[1] = value_hi else: assert 0, "bogus kind" rffi.cast(rffi.LONGP, tgt)[0] = value @@ -1232,7 +1441,8 @@ if not we_are_translated(): assert bytecode[4] == 0xCC self.fail_boxes_count = num - fail_index = rffi.cast(rffi.LONGP, bytecode)[0] + fail_index = rffi.cast(rffi.INTP, bytecode)[0] + fail_index = rffi.cast(lltype.Signed, fail_index) return fail_index def setup_failure_recovery(self): @@ -1243,8 +1453,8 @@ # original value of the registers, optionally the original # value of XMM registers, and finally a reference to the # recovery bytecode. See _build_failure_recovery() for details. - stack_at_ebp = registers[ebp.op] - bytecode = rffi.cast(rffi.UCHARP, registers[8]) + stack_at_ebp = registers[ebp.value] + bytecode = rffi.cast(rffi.UCHARP, registers[self.cpu.NUM_REGS]) allregisters = rffi.ptradd(registers, -16) return self.grab_frame_values(bytecode, stack_at_ebp, allregisters) @@ -1259,23 +1469,23 @@ self.failure_recovery_func) failure_recovery_func = rffi.cast(lltype.Signed, failure_recovery_func) - mc = self.mc2._mc + mc = self.mc._mc # Assume that we are called at the beginning, when there is no risk # that 'mc' runs out of space. Checked by asserts in mc.write(). recovery_addr = mc.tell() - mc.PUSH(edi) - mc.PUSH(esi) - mc.PUSH(ebp) - mc.PUSH(esp) # <-- not really used, but needed to take up the space - mc.PUSH(ebx) - mc.PUSH(edx) - mc.PUSH(ecx) - mc.PUSH(eax) - mc.MOV(esi, esp) + + # Push all general purpose registers + for gpr in range(self.cpu.NUM_REGS-1, -1, -1): + mc.PUSH_r(gpr) + + # ebx/rbx is callee-save in both i386 and x86-64 + mc.MOV_rr(ebx.value, esp.value) + if withfloats: - mc.SUB(esp, imm(8*8)) - for i in range(8): - mc.MOVSD(mem64(esp, 8*i), xmm_registers[i]) + # Push all float registers + mc.SUB_ri(esp.value, self.cpu.NUM_REGS*8) + for i in range(self.cpu.NUM_REGS): + mc.MOVSD_sx(8*i, i) # we call a provided function that will # - call our on_leave_jitted_hook which will mark @@ -1283,7 +1493,7 @@ # avoid unwarranted freeing # - optionally save exception depending on the flag addr = self.cpu.get_on_leave_jitted_int(save_exception=exc) - mc.CALL(rel32(addr)) + mc.CALL(imm(addr)) # the following call saves all values from the stack and from # registers to the right 'fail_boxes_' location. @@ -1293,50 +1503,58 @@ # bytecode, pushed just before by the CALL instruction written by # generate_quick_failure(). XXX misaligned stack in the call, but # it's ok because failure_recovery_func is not calling anything more - mc.PUSH(esi) - mc.CALL(rel32(failure_recovery_func)) + + # XXX + if IS_X86_32: + mc.PUSH_r(ebx.value) + elif IS_X86_64: + mc.MOV_rr(edi.value, ebx.value) + # XXX: Correct to only align the stack on 64-bit? + mc.AND_ri(esp.value, -16) + else: + raise AssertionError("Shouldn't happen") + + mc.CALL(imm(failure_recovery_func)) # returns in eax the fail_index # now we return from the complete frame, which starts from - # _assemble_bootstrap_code(). The LEA below throws away most - # of the frame, including all the PUSHes that we did just above. - mc.LEA(esp, addr_add(ebp, imm(-3 * WORD))) - mc.POP(edi) # [ebp-12] - mc.POP(esi) # [ebp-8] - mc.POP(ebx) # [ebp-4] - mc.POP(ebp) # [ebp] - mc.RET() - self.mc2.done() + # _assemble_bootstrap_code(). The LEA in _call_footer below throws + # away most of the frame, including all the PUSHes that we did just + # above. + + self._call_footer() + self.mc.done() self.failure_recovery_code[exc + 2 * withfloats] = recovery_addr def generate_failure(self, fail_index, locs, exc, locs_are_ref): - mc = self.mc + self.mc._mc.begin_reuse_scratch_register() for i in range(len(locs)): loc = locs[i] - if isinstance(loc, REG): - if loc.width == 8: + if isinstance(loc, RegLoc): + if loc.is_xmm: adr = self.fail_boxes_float.get_addr_for_num(i) - mc.MOVSD(heap64(adr), loc) + self.mc.MOVSD(heap(adr), loc) else: if locs_are_ref[i]: adr = self.fail_boxes_ptr.get_addr_for_num(i) else: adr = self.fail_boxes_int.get_addr_for_num(i) - mc.MOV(heap(adr), loc) + self.mc.MOV(heap(adr), loc) for i in range(len(locs)): loc = locs[i] - if not isinstance(loc, REG): - if loc.width == 8: - mc.MOVSD(xmm0, loc) + if not isinstance(loc, RegLoc): + if isinstance(loc, StackLoc) and loc.type == FLOAT: + self.mc.MOVSD_xb(xmm0.value, loc.value) adr = self.fail_boxes_float.get_addr_for_num(i) - mc.MOVSD(heap64(adr), xmm0) + self.mc.MOVSD(heap(adr), xmm0) else: if locs_are_ref[i]: adr = self.fail_boxes_ptr.get_addr_for_num(i) else: adr = self.fail_boxes_int.get_addr_for_num(i) - mc.MOV(eax, loc) - mc.MOV(heap(adr), eax) + self.mc.MOV(eax, loc) + self.mc.MOV(heap(adr), eax) + self.mc._mc.end_reuse_scratch_register() # we call a provided function that will # - call our on_leave_jitted_hook which will mark @@ -1344,28 +1562,31 @@ # avoid unwarranted freeing # - optionally save exception depending on the flag addr = self.cpu.get_on_leave_jitted_int(save_exception=exc) - mc.CALL(rel32(addr)) + self.mc.CALL(imm(addr)) + + self.mc.MOV_ri(eax.value, fail_index) - mc.LEA(esp, addr_add(ebp, imm(-3 * WORD))) - mc.MOV(eax, imm(fail_index)) - mc.POP(edi) # [ebp-12] - mc.POP(esi) # [ebp-8] - mc.POP(ebx) # [ebp-4] - mc.POP(ebp) # [ebp] - mc.RET() - - @specialize.arg(2) - def implement_guard(self, addr, emit_jump): - emit_jump(rel32(addr)) + # exit function + self._call_footer() + + def implement_guard(self, guard_token, condition=None): + self.mc.reserve_bytes(guard_token.recovery_stub_size()) + self.pending_guard_tokens.append(guard_token) + # XXX: These jumps are patched later, the self.mc.tell() are just + # dummy values + if condition: + self.mc.J_il(rx86.Conditions[condition], self.mc.tell()) + else: + self.mc.JMP_l(self.mc.tell()) return self.mc.tell() - 4 def genop_call(self, op, arglocs, resloc): sizeloc = arglocs[0] - assert isinstance(sizeloc, IMM32) + assert isinstance(sizeloc, ImmedLoc) size = sizeloc.value if isinstance(op.args[0], Const): - x = rel32(op.args[0].getint()) + x = imm(op.args[0].getint()) else: x = arglocs[1] if x is eax: @@ -1375,35 +1596,35 @@ self._emit_call(x, arglocs, 2, tmp=tmp) - if isinstance(resloc, MODRM64): - self.mc.FSTP(resloc) + if isinstance(resloc, StackLoc) and resloc.width == 8 and IS_X86_32: + self.mc.FSTP_b(resloc.value) elif size == 1: - self.mc.AND(eax, imm(0xff)) + self.mc.AND_ri(eax.value, 0xff) elif size == 2: - self.mc.AND(eax, imm(0xffff)) + self.mc.AND_ri(eax.value, 0xffff) - def genop_guard_call_may_force(self, op, guard_op, addr, + def genop_guard_call_may_force(self, op, guard_op, guard_token, arglocs, result_loc): faildescr = guard_op.descr fail_index = self.cpu.get_fail_descr_number(faildescr) - self.mc.MOV(mem(ebp, FORCE_INDEX_OFS), imm(fail_index)) + self.mc.MOV_bi(FORCE_INDEX_OFS, fail_index) self.genop_call(op, arglocs, result_loc) - self.mc.CMP(mem(ebp, FORCE_INDEX_OFS), imm(0)) - return self.implement_guard(addr, self.mc.JL) + self.mc.CMP_bi(FORCE_INDEX_OFS, 0) + return self.implement_guard(guard_token, 'L') - def genop_guard_call_assembler(self, op, guard_op, addr, + def genop_guard_call_assembler(self, op, guard_op, guard_token, arglocs, result_loc): faildescr = guard_op.descr fail_index = self.cpu.get_fail_descr_number(faildescr) - self.mc.MOV(mem(ebp, FORCE_INDEX_OFS), imm(fail_index)) + self.mc.MOV_bi(FORCE_INDEX_OFS, fail_index) descr = op.descr assert isinstance(descr, LoopToken) assert len(arglocs) - 2 == len(descr._x86_arglocs[0]) # # Write a call to the direct_bootstrap_code of the target assembler - self._emit_call(rel32(descr._x86_direct_bootstrap_code), arglocs, 2, + self._emit_call(imm(descr._x86_direct_bootstrap_code), arglocs, 2, tmp=eax) - mc = self._start_block() + self.mc.ensure_bytes_available(256) if op.result is None: assert result_loc is None value = self.cpu.done_with_this_frame_void_v @@ -1419,26 +1640,27 @@ value = self.cpu.done_with_this_frame_float_v else: raise AssertionError(kind) - mc.CMP(eax, imm(value)) - mc.JE(rel8_patched_later) # goto B if we get 'done_with_this_frame' - je_location = mc.get_relative_pos() + self.mc.CMP_ri(eax.value, value) + # patched later + self.mc.J_il8(rx86.Conditions['E'], 0) # goto B if we get 'done_with_this_frame' + je_location = self.mc.get_relative_pos() # # Path A: use assembler_helper_adr jd = descr.outermost_jitdriver_sd assert jd is not None asm_helper_adr = self.cpu.cast_adr_to_int(jd.assembler_helper_adr) - self._emit_call(rel32(asm_helper_adr), [eax, arglocs[1]], 0, - tmp=ecx, force_mc=True, mc=mc) - if isinstance(result_loc, MODRM64): - mc.FSTP(result_loc) + self._emit_call(imm(asm_helper_adr), [eax, arglocs[1]], 0, + tmp=ecx) + if IS_X86_32 and isinstance(result_loc, StackLoc) and result_loc.type == FLOAT: + self.mc.FSTP_b(result_loc.value) #else: result_loc is already either eax or None, checked below - mc.JMP(rel8_patched_later) # done - jmp_location = mc.get_relative_pos() + self.mc.JMP_l8(0) # jump to done, patched later + jmp_location = self.mc.get_relative_pos() # # Path B: fast path. Must load the return value, and reset the token offset = jmp_location - je_location assert 0 < offset <= 127 - mc.overwrite(je_location - 1, [chr(offset)]) + self.mc.overwrite(je_location - 1, [chr(offset)]) # # Reset the vable token --- XXX really too much special logic here:-( if jd.index_of_virtualizable >= 0: @@ -1446,8 +1668,8 @@ fielddescr = jd.vable_token_descr assert isinstance(fielddescr, BaseFieldDescr) ofs = fielddescr.offset - mc.MOV(eax, arglocs[1]) - mc.MOV(addr_add(eax, imm(ofs)), imm(0)) + self.mc.MOV(eax, arglocs[1]) + self.mc.MOV_mi((eax.value, ofs), 0) # in the line above, TOKEN_NONE = 0 # if op.result is not None: @@ -1456,27 +1678,26 @@ if kind == FLOAT: xmmtmp = X86XMMRegisterManager.all_regs[0] adr = self.fail_boxes_float.get_addr_for_num(0) - mc.MOVSD(xmmtmp, heap64(adr)) - mc.MOVSD(result_loc, xmmtmp) + self.mc.MOVSD(xmmtmp, heap(adr)) + self.mc.MOVSD(result_loc, xmmtmp) else: assert result_loc is eax if kind == INT: adr = self.fail_boxes_int.get_addr_for_num(0) - mc.MOV(eax, heap(adr)) + self.mc.MOV(eax, heap(adr)) elif kind == REF: adr = self.fail_boxes_ptr.get_addr_for_num(0) - mc.XOR(eax, eax) - mc.XCHG(eax, heap(adr)) + self.mc.XOR_rr(eax.value, eax.value) + self.mc.XCHG(eax, heap(adr)) else: raise AssertionError(kind) # # Here we join Path A and Path B again - offset = mc.get_relative_pos() - jmp_location + offset = self.mc.get_relative_pos() - jmp_location assert 0 <= offset <= 127 - mc.overwrite(jmp_location - 1, [chr(offset)]) - self._stop_block() - self.mc.CMP(mem(ebp, FORCE_INDEX_OFS), imm(0)) - return self.implement_guard(addr, self.mc.JL) + self.mc.overwrite(jmp_location - 1, [chr(offset)]) + self.mc.CMP_bi(FORCE_INDEX_OFS, 0) + return self.implement_guard(guard_token, 'L') def genop_discard_cond_call_gc_wb(self, op, arglocs): # use 'mc._mc' directly instead of 'mc', to avoid @@ -1486,31 +1707,41 @@ cls = self.cpu.gc_ll_descr.has_write_barrier_class() assert cls is not None and isinstance(descr, cls) loc_base = arglocs[0] - mc = self._start_block() - mc.TEST(mem8(loc_base, descr.jit_wb_if_flag_byteofs), - imm8(descr.jit_wb_if_flag_singlebyte)) - mc.JZ(rel8_patched_later) - jz_location = mc.get_relative_pos() + self.mc.ensure_bytes_available(256) + self.mc.TEST8_mi((loc_base.value, descr.jit_wb_if_flag_byteofs), + descr.jit_wb_if_flag_singlebyte) + self.mc.J_il8(rx86.Conditions['Z'], 0) # patched later + jz_location = self.mc.get_relative_pos() # the following is supposed to be the slow path, so whenever possible # we choose the most compact encoding over the most efficient one. for i in range(len(arglocs)-1, -1, -1): - mc.PUSH(arglocs[i]) + self.mc.PUSH(arglocs[i]) + + if IS_X86_64: + # We clobber these registers to pass the arguments, but that's + # okay, because consider_cond_call_gc_wb makes sure that any + # caller-save registers with values in them are present in arglocs, + # so they are saved on the stack above and restored below + self.mc.MOV_rs(edi.value, 0) + self.mc.MOV_rs(esi.value, 8) + # misaligned stack in the call, but it's ok because the write barrier # is not going to call anything more. Also, this assumes that the # write barrier does not touch the xmm registers. - mc.CALL(rel32(descr.get_write_barrier_fn(self.cpu))) + self.mc.CALL(imm(descr.get_write_barrier_fn(self.cpu))) for i in range(len(arglocs)): loc = arglocs[i] - assert isinstance(loc, REG) - mc.POP(loc) + assert isinstance(loc, RegLoc) + self.mc.POP(loc) # patch the JZ above - offset = mc.get_relative_pos() - jz_location + offset = self.mc.get_relative_pos() - jz_location assert 0 < offset <= 127 - mc.overwrite(jz_location-1, [chr(offset)]) - self._stop_block() + self.mc.overwrite(jz_location-1, [chr(offset)]) def genop_force_token(self, op, arglocs, resloc): - self.mc.LEA(resloc, mem(ebp, FORCE_INDEX_OFS)) + # RegAlloc.consider_force_token ensures this: + assert isinstance(resloc, RegLoc) + self.mc.LEA_rb(resloc.value, FORCE_INDEX_OFS) def not_implemented_op_discard(self, op, arglocs): msg = "not implemented operation: %s" % op.getopname() @@ -1538,17 +1769,16 @@ return loop_token._x86_arglocs def closing_jump(self, loop_token): - self.mc.JMP(rel32(loop_token._x86_loop_code)) + self.mc.JMP(imm(loop_token._x86_loop_code)) def malloc_cond_fixedsize(self, nursery_free_adr, nursery_top_adr, size, tid): - # don't use self.mc - mc = self._start_block() - mc.MOV(eax, heap(nursery_free_adr)) - mc.LEA(edx, addr_add(eax, imm(size))) - mc.CMP(edx, heap(nursery_top_adr)) - mc.JNA(rel8_patched_later) - jmp_adr = mc.get_relative_pos() + self.mc.ensure_bytes_available(256) + self.mc.MOV(eax, heap(nursery_free_adr)) + self.mc.LEA_rm(edx.value, (eax.value, size)) + self.mc.CMP(edx, heap(nursery_top_adr)) + self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later + jmp_adr = self.mc.get_relative_pos() # See comments in _build_malloc_fixedsize_slowpath for the # details of the two helper functions that we are calling below. @@ -1564,17 +1794,16 @@ # reserve room for the argument to the real malloc and the # 8 saved XMM regs self._regalloc.reserve_param(1+16) - mc.CALL(rel32(slowpath_addr1)) + self.mc.CALL(imm(slowpath_addr1)) self.mark_gc_roots() slowpath_addr2 = self.malloc_fixedsize_slowpath2 - mc.CALL(rel32(slowpath_addr2)) + self.mc.CALL(imm(slowpath_addr2)) - offset = mc.get_relative_pos() - jmp_adr + offset = self.mc.get_relative_pos() - jmp_adr assert 0 < offset <= 127 - mc.overwrite(jmp_adr-1, [chr(offset)]) - mc.MOV(addr_add(eax, imm(0)), imm(tid)) - mc.MOV(heap(nursery_free_adr), edx) - self._stop_block() + self.mc.overwrite(jmp_adr-1, [chr(offset)]) + self.mc.MOV_mi((eax.value, 0), tid) + self.mc.MOV(heap(nursery_free_adr), edx) genop_discard_list = [Assembler386.not_implemented_op_discard] * rop._LAST genop_list = [Assembler386.not_implemented_op] * rop._LAST @@ -1594,32 +1823,20 @@ num = getattr(rop, opname.upper()) genop_list[num] = value -def new_addr_add(heap, mem, memsib): - def addr_add(reg_or_imm1, reg_or_imm2, offset=0, scale=0): - if isinstance(reg_or_imm1, IMM32): - if isinstance(reg_or_imm2, IMM32): - return heap(reg_or_imm1.value + offset + - (reg_or_imm2.value << scale)) - else: - return memsib(None, reg_or_imm2, scale, reg_or_imm1.value + offset) - else: - if isinstance(reg_or_imm2, IMM32): - return mem(reg_or_imm1, offset + (reg_or_imm2.value << scale)) - else: - return memsib(reg_or_imm1, reg_or_imm2, scale, offset) - return addr_add - -addr8_add = new_addr_add(heap8, mem8, memSIB8) -addr_add = new_addr_add(heap, mem, memSIB) -addr64_add = new_addr_add(heap64, mem64, memSIB64) - -def addr_add_const(reg_or_imm1, offset): - if isinstance(reg_or_imm1, IMM32): - return heap(reg_or_imm1.value + offset) - else: - return mem(reg_or_imm1, offset) - def round_up_to_4(size): if size < 4: return 4 return size + +# XXX: ri386 migration shims: +def addr_add(reg_or_imm1, reg_or_imm2, offset=0, scale=0): + return AddressLoc(reg_or_imm1, reg_or_imm2, scale, offset) + +def addr_add_const(reg_or_imm1, offset): + return AddressLoc(reg_or_imm1, ImmedLoc(0), 0, offset) + +def mem(loc, offset): + return AddressLoc(loc, ImmedLoc(0), 0, offset) + +def heap(addr): + return AddressLoc(ImmedLoc(addr), ImmedLoc(0), 0, 0) Modified: pypy/trunk/pypy/jit/backend/x86/codebuf.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/codebuf.py (original) +++ pypy/trunk/pypy/jit/backend/x86/codebuf.py Mon Aug 2 19:52:15 2010 @@ -2,12 +2,20 @@ import os, sys from pypy.rpython.lltypesystem import lltype, rffi from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.jit.backend.x86.ri386 import I386CodeBuilder +from pypy.jit.backend.x86.rx86 import X86_32_CodeBuilder, X86_64_CodeBuilder +from pypy.jit.backend.x86.regloc import LocationCodeBuilder from pypy.rlib.rmmap import PTR, alloc, free from pypy.rlib.debug import make_sure_not_resized +from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 +from pypy.rlib.objectmodel import we_are_translated +# XXX: Seems nasty to change the superclass of InMemoryCodeBuilder like this +if IS_X86_32: + codebuilder_cls = X86_32_CodeBuilder +elif IS_X86_64: + codebuilder_cls = X86_64_CodeBuilder -class InMemoryCodeBuilder(I386CodeBuilder): +class InMemoryCodeBuilder(codebuilder_cls, LocationCodeBuilder): _last_dump_start = 0 def __init__(self, start, end): @@ -31,13 +39,15 @@ def write(self, listofchars): self._pos = self.overwrite(self._pos, listofchars) - def writechr(self, n): - # purely for performance: don't make the one-element list [chr(n)] + def writechar(self, char): pos = self._pos assert pos + 1 <= self._size - self._data[pos] = chr(n) + self._data[pos] = char self._pos = pos + 1 + def writechr(self, n): + self.writechar(chr(n)) + def get_relative_pos(self): return self._pos @@ -50,11 +60,6 @@ self._pos = pos self._last_dump_start = pos - def execute(self, arg1, arg2): - # XXX old testing stuff - fnptr = rffi.cast(lltype.Ptr(BINARYFN), self._data) - return fnptr(arg1, arg2) - def done(self): # normally, no special action is needed here if machine_code_dumper.enabled: @@ -77,9 +82,6 @@ valgrind.discard_translations(self._data, self._size) -BINARYFN = lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed) - - class MachineCodeDumper: enabled = True log_fd = -1 @@ -107,7 +109,10 @@ return False # log the executable name from pypy.jit.backend.hlinfo import highleveljitinfo - os.write(self.log_fd, 'BACKEND i386\n') + if IS_X86_32: + os.write(self.log_fd, 'BACKEND x86\n') + elif IS_X86_64: + os.write(self.log_fd, 'BACKEND x86_64\n') if highleveljitinfo.sys_executable: os.write(self.log_fd, 'SYS_EXECUTABLE %s\n' % ( highleveljitinfo.sys_executable,)) @@ -137,6 +142,12 @@ def __init__(self, map_size): data = alloc(map_size) + if IS_X86_64 and not we_are_translated(): + # Hack to make sure that mcs are not within 32-bits of one + # another for testing purposes + from pypy.rlib.rmmap import hint + hint.pos += 0xFFFFFFFF + self._init(data, map_size) def __del__(self): Modified: pypy/trunk/pypy/jit/backend/x86/jump.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/jump.py (original) +++ pypy/trunk/pypy/jit/backend/x86/jump.py Mon Aug 2 19:52:15 2010 @@ -1,23 +1,6 @@ import sys from pypy.tool.pairtype import extendabletype -from pypy.jit.backend.x86.ri386 import * - -class __extend__(OPERAND): - __metaclass__ = extendabletype - def _getregkey(self): - raise AssertionError("should only happen to registers and frame " - "positions") - -class __extend__(REG): - __metaclass__ = extendabletype - def _getregkey(self): - return ~self.op - -class __extend__(MODRM): - __metaclass__ = extendabletype - def _getregkey(self): - return self.position - +from pypy.jit.backend.x86.regloc import ImmedLoc, StackLoc def remap_frame_layout(assembler, src_locations, dst_locations, tmpreg): pending_dests = len(dst_locations) @@ -27,7 +10,7 @@ srccount[dst._getregkey()] = 0 for i in range(len(dst_locations)): src = src_locations[i] - if isinstance(src, IMM32): + if isinstance(src, ImmedLoc): continue key = src._getregkey() if key in srccount: @@ -46,7 +29,7 @@ srccount[key] = -1 # means "it's done" pending_dests -= 1 src = src_locations[i] - if not isinstance(src, IMM32): + if not isinstance(src, ImmedLoc): key = src._getregkey() if key in srccount: srccount[key] -= 1 @@ -80,7 +63,7 @@ assert pending_dests == 0 def _move(assembler, src, dst, tmpreg): - if isinstance(dst, MODRM) and isinstance(src, MODRM): + if dst.is_memory_reference() and src.is_memory_reference(): assembler.regalloc_mov(src, tmpreg) src = tmpreg assembler.regalloc_mov(src, dst) Modified: pypy/trunk/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regalloc.py Mon Aug 2 19:52:15 2010 @@ -5,7 +5,7 @@ from pypy.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr, ResOperation, BoxPtr, LoopToken, INT, REF, FLOAT) -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.regloc import * from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi, rstr from pypy.rlib.objectmodel import we_are_translated from pypy.rlib import rgc @@ -17,16 +17,7 @@ from pypy.jit.backend.llsupport.descr import BaseCallDescr, BaseSizeDescr from pypy.jit.backend.llsupport.regalloc import FrameManager, RegisterManager,\ TempBox - -WORD = 4 -FRAME_FIXED_SIZE = 5 # ebp + ebx + esi + edi + force_index = 5 words -FORCE_INDEX_OFS = -4*WORD - -width_of_type = { - INT : 1, - REF : 1, - FLOAT : 2, - } +from pypy.jit.backend.x86.arch import WORD, FRAME_FIXED_SIZE, IS_X86_32, IS_X86_64 class X86RegisterManager(RegisterManager): @@ -50,6 +41,12 @@ print "convert_to_imm: got a %s" % c raise AssertionError +class X86_64_RegisterManager(X86RegisterManager): + # r11 omitted because it's used as scratch + all_regs = [eax, ecx, edx, ebx, esi, edi, r8, r9, r10, r12, r13, r14, r15] + no_lower_byte_regs = [] + save_around_call_regs = [eax, ecx, edx, esi, edi, r8, r9, r10] + class FloatConstants(object): BASE_CONSTANT_SIZE = 1000 @@ -80,7 +77,6 @@ all_regs = [xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7] # we never need lower byte I hope save_around_call_regs = all_regs - reg_width = 2 def __init__(self, longevity, frame_manager=None, assembler=None): RegisterManager.__init__(self, longevity, frame_manager=frame_manager, @@ -94,27 +90,35 @@ def convert_to_imm(self, c): adr = self.float_constants.record_float(c.getfloat()) - return heap64(adr) + return AddressLoc(ImmedLoc(adr), ImmedLoc(0), 0, 0) def after_call(self, v): # the result is stored in st0, but we don't have this around, # so genop_call will move it to some frame location immediately # after the call - return self.frame_manager.loc(v, 2) + return self.frame_manager.loc(v) -class X86FrameManager(FrameManager): +class X86_64_XMMRegisterManager(X86XMMRegisterManager): + # xmm15 reserved for scratch use + all_regs = [xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14] + save_around_call_regs = all_regs + + def call_result_location(self, v): + return xmm0 + + def after_call(self, v): + # We use RegisterManager's implementation, since X86XMMRegisterManager + # places the result on the stack, which we don't need to do when the + # calling convention places the result in xmm0 + return RegisterManager.after_call(self, v) +class X86FrameManager(FrameManager): @staticmethod - def frame_pos(i, size): - if size == 1: - res = mem(ebp, get_ebp_ofs(i)) - elif size == 2: - res = mem64(ebp, get_ebp_ofs(i + 1)) - else: - print "Unimplemented size %d" % i - raise NotImplementedError("unimplemented size %d" % i) - res.position = i - return res + def frame_pos(i, box_type): + if IS_X86_32 and box_type == FLOAT: + return StackLoc(i, get_ebp_ofs(i+1), 2, box_type) + else: + return StackLoc(i, get_ebp_ofs(i), 1, box_type) class RegAlloc(object): exc = False @@ -135,11 +139,21 @@ # compute longevity of variables longevity = self._compute_vars_longevity(inputargs, operations) self.longevity = longevity - self.rm = X86RegisterManager(longevity, - frame_manager = self.fm, - assembler = self.assembler) - self.xrm = X86XMMRegisterManager(longevity, frame_manager = self.fm, - assembler = self.assembler) + # XXX + if cpu.WORD == 4: + gpr_reg_mgr_cls = X86RegisterManager + xmm_reg_mgr_cls = X86XMMRegisterManager + elif cpu.WORD == 8: + gpr_reg_mgr_cls = X86_64_RegisterManager + xmm_reg_mgr_cls = X86_64_XMMRegisterManager + else: + raise AssertionError("Word size should be 4 or 8") + + self.rm = gpr_reg_mgr_cls(longevity, + frame_manager = self.fm, + assembler = self.assembler) + self.xrm = xmm_reg_mgr_cls(longevity, frame_manager = self.fm, + assembler = self.assembler) def prepare_loop(self, inputargs, operations, looptoken): self._prepare(inputargs, operations) @@ -184,7 +198,7 @@ if reg: loc = reg else: - loc = self.fm.loc(arg, width_of_type[arg.type]) + loc = self.fm.loc(arg) if arg.type == FLOAT: floatlocs[i] = loc else: @@ -252,23 +266,23 @@ arg = inputargs[i] i += 1 if arg.type == FLOAT: - if isinstance(loc, REG): + if isinstance(loc, RegLoc): self.xrm.reg_bindings[arg] = loc used[loc] = None else: self.fm.frame_bindings[arg] = loc else: - if isinstance(loc, REG): + if isinstance(loc, RegLoc): self.rm.reg_bindings[arg] = loc used[loc] = None else: self.fm.frame_bindings[arg] = loc self.rm.free_regs = [] - for reg in X86RegisterManager.all_regs: + for reg in self.rm.all_regs: if reg not in used: self.rm.free_regs.append(reg) self.xrm.free_regs = [] - for reg in X86XMMRegisterManager.all_regs: + for reg in self.xrm.all_regs: if reg not in used: self.xrm.free_regs.append(reg) # note: we need to make a copy of inputargs because possibly_free_vars @@ -646,7 +660,7 @@ vable_index = jd.index_of_virtualizable if vable_index >= 0: self.rm._sync_var(op.args[vable_index]) - vable = self.fm.loc(op.args[vable_index], 1) + vable = self.fm.loc(op.args[vable_index]) else: vable = imm(0) self._call(op, [imm(size), vable] + @@ -670,7 +684,7 @@ # function, a GC write barrier, is known not to touch them. # See remember_young_pointer() in rpython/memory/gc/generation.py. for v, reg in self.rm.reg_bindings.items(): - if ((reg is eax or reg is ecx or reg is edx) + if (reg in self.rm.save_around_call_regs and self.rm.stays_alive(v)): arglocs.append(reg) self.PerformDiscard(op, arglocs) @@ -809,7 +823,7 @@ def consider_setfield_gc(self, op): ofs_loc, size_loc, ptr = self._unpack_fielddescr(op.descr) - assert isinstance(size_loc, IMM32) + assert isinstance(size_loc, ImmedLoc) if size_loc.value == 1: need_lower_byte = True else: @@ -950,7 +964,7 @@ shape = gcrootmap.get_basic_shape() for v, val in self.fm.frame_bindings.items(): if (isinstance(v, BoxPtr) and self.rm.stays_alive(v)): - assert isinstance(val, MODRM) + assert isinstance(val, StackLoc) gcrootmap.add_ebp_offset(shape, get_ebp_ofs(val.position)) for v, reg in self.rm.reg_bindings.items(): if reg is eax: Modified: pypy/trunk/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/runner.py (original) +++ pypy/trunk/pypy/jit/backend/x86/runner.py Mon Aug 2 19:52:15 2010 @@ -4,11 +4,13 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.jit.metainterp import history, compile from pypy.jit.backend.x86.assembler import Assembler386 -from pypy.jit.backend.x86.regalloc import FORCE_INDEX_OFS +from pypy.jit.backend.x86.arch import FORCE_INDEX_OFS from pypy.jit.backend.x86.profagent import ProfileAgent from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU +from pypy.jit.backend.x86 import regloc +import sys -class CPU386(AbstractLLCPU): +class AbstractX86CPU(AbstractLLCPU): debug = True supports_floats = True @@ -132,10 +134,28 @@ assert fail_index == fail_index_2 return faildescr +class CPU386(AbstractX86CPU): + WORD = 4 + NUM_REGS = 8 + CALLEE_SAVE_REGISTERS = [regloc.ebx, regloc.esi, regloc.edi] + FRAME_FIXED_SIZE = len(CALLEE_SAVE_REGISTERS) + 2 + + def __init__(self, *args, **kwargs): + assert sys.maxint == (2**31 - 1) + super(CPU386, self).__init__(*args, **kwargs) class CPU386_NO_SSE2(CPU386): supports_floats = False +class CPU_X86_64(AbstractX86CPU): + WORD = 8 + NUM_REGS = 16 + CALLEE_SAVE_REGISTERS = [regloc.ebx, regloc.r12, regloc.r13, regloc.r14, regloc.r15] + FRAME_FIXED_SIZE = len(CALLEE_SAVE_REGISTERS) + 2 + + def __init__(self, *args, **kwargs): + assert sys.maxint == (2**63 - 1) + super(CPU_X86_64, self).__init__(*args, **kwargs) CPU = CPU386 Modified: pypy/trunk/pypy/jit/backend/x86/test/conftest.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/conftest.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/conftest.py Mon Aug 2 19:52:15 2010 @@ -3,6 +3,5 @@ cpu = detect_cpu.autodetect() def pytest_runtest_setup(item): - if cpu != 'x86': - py.test.skip("x86 directory skipped: cpu is %r" % (cpu,)) - + if cpu not in ('x86', 'x86_64'): + py.test.skip("x86/x86_64 tests skipped: cpu is %r" % (cpu,)) Modified: pypy/trunk/pypy/jit/backend/x86/test/test_assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_assembler.py Mon Aug 2 19:52:15 2010 @@ -1,14 +1,19 @@ -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.regloc import * from pypy.jit.backend.x86.assembler import Assembler386, MachineCodeBlockWrapper from pypy.jit.backend.x86.regalloc import X86FrameManager, get_ebp_ofs -from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat +from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, INT, REF, FLOAT from pypy.rlib.rarithmetic import intmask from pypy.rpython.lltypesystem import lltype, llmemory, rffi +from pypy.jit.backend.x86.arch import WORD, IS_X86_32, IS_X86_64 +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.regalloc import X86RegisterManager, X86_64_RegisterManager, X86XMMRegisterManager, X86_64_XMMRegisterManager +ACTUAL_CPU = getcpuclass() class FakeCPU: rtyper = None supports_floats = True + NUM_REGS = ACTUAL_CPU.NUM_REGS class FakeMC: def __init__(self, base_address=0): @@ -25,7 +30,14 @@ self.content.append(("JMP", args)) def done(self): pass + def PUSH_r(self, reg): + pass + def POP_r(self, reg): + pass +class FakeAssembler: + def write_pending_failure_recoveries(self): + pass def test_write_failure_recovery_description(): assembler = Assembler386(FakeCPU()) @@ -33,12 +45,12 @@ failargs = [BoxInt(), BoxPtr(), BoxFloat()] * 3 failargs.insert(6, None) failargs.insert(7, None) - locs = [X86FrameManager.frame_pos(0, 1), - X86FrameManager.frame_pos(1, 1), - X86FrameManager.frame_pos(10, 2), - X86FrameManager.frame_pos(100, 1), - X86FrameManager.frame_pos(101, 1), - X86FrameManager.frame_pos(110, 2), + locs = [X86FrameManager.frame_pos(0, INT), + X86FrameManager.frame_pos(1, REF), + X86FrameManager.frame_pos(10, FLOAT), + X86FrameManager.frame_pos(100, INT), + X86FrameManager.frame_pos(101, REF), + X86FrameManager.frame_pos(110, FLOAT), None, None, ebx, @@ -46,17 +58,17 @@ xmm2] assert len(failargs) == len(locs) assembler.write_failure_recovery_description(mc, failargs, locs) - nums = [Assembler386.DESCR_INT + 4*(8+0), - Assembler386.DESCR_REF + 4*(8+1), - Assembler386.DESCR_FLOAT + 4*(8+10), - Assembler386.DESCR_INT + 4*(8+100), - Assembler386.DESCR_REF + 4*(8+101), - Assembler386.DESCR_FLOAT + 4*(8+110), + nums = [Assembler386.DESCR_INT + 4*(16+0), + Assembler386.DESCR_REF + 4*(16+1), + Assembler386.DESCR_FLOAT + 4*(16+10), + Assembler386.DESCR_INT + 4*(16+100), + Assembler386.DESCR_REF + 4*(16+101), + Assembler386.DESCR_FLOAT + 4*(16+110), Assembler386.CODE_HOLE, Assembler386.CODE_HOLE, - Assembler386.DESCR_INT + 4*ebx.op, - Assembler386.DESCR_REF + 4*esi.op, - Assembler386.DESCR_FLOAT + 4*xmm2.op] + Assembler386.DESCR_INT + 4*ebx.value, + Assembler386.DESCR_REF + 4*esi.value, + Assembler386.DESCR_FLOAT + 4*xmm2.value] double_byte_nums = [] for num in nums[3:6]: double_byte_nums.append((num & 0x7F) | 0x80) @@ -94,6 +106,9 @@ return lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S)) def get_random_float(): + # Returns , , + # NB: on 64-bit, will be the entire float and + # will be random garbage from malloc! assert withfloats value = random.random() - 0.5 # make sure it fits into 64 bits @@ -101,9 +116,16 @@ rffi.cast(rffi.DOUBLEP, tmp)[0] = value return rffi.cast(rffi.DOUBLEP, tmp)[0], tmp[0], tmp[1] + if IS_X86_32: + main_registers = X86RegisterManager.all_regs + xmm_registers = X86XMMRegisterManager.all_regs + elif IS_X86_64: + main_registers = X86_64_RegisterManager.all_regs + xmm_registers = X86_64_XMMRegisterManager.all_regs + # memory locations: 26 integers, 26 pointers, 26 floats # main registers: half of them as signed and the other half as ptrs - # xmm registers: all floats, from xmm0 to xmm7 + # xmm registers: all floats, from xmm0 to xmm(7|15) # holes: 8 locations = [] baseloc = 4 @@ -117,18 +139,17 @@ content = ([('int', locations.pop()) for _ in range(26)] + [('ptr', locations.pop()) for _ in range(26)] + [(['int', 'ptr'][random.randrange(0, 2)], reg) - for reg in [eax, ecx, edx, ebx, esi, edi]]) + for reg in main_registers]) if withfloats: content += ([('float', locations.pop()) for _ in range(26)] + - [('float', reg) for reg in [xmm0, xmm1, xmm2, xmm3, - xmm4, xmm5, xmm6, xmm7]]) + [('float', reg) for reg in xmm_registers]) for i in range(8): content.append(('hole', None)) random.shuffle(content) # prepare the expected target arrays, the descr_bytecode, # the 'registers' and the 'stack' arrays according to 'content' - xmmregisters = lltype.malloc(rffi.LONGP.TO, 16+9, flavor='raw') + xmmregisters = lltype.malloc(rffi.LONGP.TO, 16+ACTUAL_CPU.NUM_REGS+1, flavor='raw') registers = rffi.ptradd(xmmregisters, 16) stacklen = baseloc + 10 stack = lltype.malloc(rffi.LONGP.TO, stacklen, flavor='raw') @@ -140,8 +161,8 @@ assert loc >= 0 ofs = get_ebp_ofs(loc) assert ofs < 0 - assert (ofs % 4) == 0 - stack[stacklen + ofs//4] = value + assert (ofs % WORD) == 0 + stack[stacklen + ofs//WORD] = value descr_bytecode = [] for i, (kind, loc) in enumerate(content): @@ -152,12 +173,18 @@ value, lo, hi = get_random_float() expected_floats[i] = value kind = Assembler386.DESCR_FLOAT - if isinstance(loc, REG): - xmmregisters[2*loc.op] = lo - xmmregisters[2*loc.op+1] = hi + if isinstance(loc, RegLoc): + if WORD == 4: + xmmregisters[2*loc.value] = lo + xmmregisters[2*loc.value+1] = hi + elif WORD == 8: + xmmregisters[loc.value] = lo else: - write_in_stack(loc, hi) - write_in_stack(loc+1, lo) + if WORD == 4: + write_in_stack(loc, hi) + write_in_stack(loc+1, lo) + elif WORD == 8: + write_in_stack(loc, lo) else: if kind == 'int': value = get_random_int() @@ -170,15 +197,15 @@ value = rffi.cast(rffi.LONG, value) else: assert 0, kind - if isinstance(loc, REG): - registers[loc.op] = value + if isinstance(loc, RegLoc): + registers[loc.value] = value else: write_in_stack(loc, value) - if isinstance(loc, REG): - num = kind + 4*loc.op + if isinstance(loc, RegLoc): + num = kind + 4*loc.value else: - num = kind + 4*(8+loc) + num = kind + Assembler386.CODE_FROMSTACK + (4*loc) while num >= 0x80: descr_bytecode.append((num & 0x7F) | 0x80) num >>= 7 @@ -195,8 +222,8 @@ for i in range(len(descr_bytecode)): assert 0 <= descr_bytecode[i] <= 255 descr_bytes[i] = rffi.cast(rffi.UCHAR, descr_bytecode[i]) - registers[8] = rffi.cast(rffi.LONG, descr_bytes) - registers[ebp.op] = rffi.cast(rffi.LONG, stack) + 4*stacklen + registers[ACTUAL_CPU.NUM_REGS] = rffi.cast(rffi.LONG, descr_bytes) + registers[ebp.value] = rffi.cast(rffi.LONG, stack) + WORD*stacklen # run! assembler = Assembler386(FakeCPU()) @@ -237,7 +264,8 @@ def test_mc_wrapper_profile_agent(): agent = FakeProfileAgent() - mc = FakeMCWrapper(100, agent) + assembler = FakeAssembler() + mc = FakeMCWrapper(assembler, 100, agent) mc.start_function("abc") mc.writechr("x") mc.writechr("x") Modified: pypy/trunk/pypy/jit/backend/x86/test/test_basic.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_basic.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_basic.py Mon Aug 2 19:52:15 2010 @@ -1,5 +1,5 @@ import py -from pypy.jit.backend.x86.runner import CPU386 +from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.metainterp.warmspot import ll_meta_interp from pypy.jit.metainterp.test import test_basic from pypy.jit.codewriter.policy import StopAtXPolicy @@ -7,7 +7,7 @@ class Jit386Mixin(test_basic.LLJitMixin): type_system = 'lltype' - CPUClass = CPU386 + CPUClass = getcpuclass() def check_jumps(self, maxcount): pass Modified: pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py Mon Aug 2 19:52:15 2010 @@ -9,13 +9,13 @@ from pypy.jit.codewriter import heaptracker from pypy.jit.backend.llsupport.descr import GcCache from pypy.jit.backend.llsupport.gc import GcLLDescription -from pypy.jit.backend.x86.runner import CPU -from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, FRAME_FIXED_SIZE +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.regalloc import RegAlloc +from pypy.jit.backend.x86.arch import WORD, FRAME_FIXED_SIZE from pypy.jit.metainterp.test.oparser import parse from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.annlowlevel import llhelper from pypy.rpython.lltypesystem import rclass, rstr -from pypy.jit.backend.x86.ri386 import * from pypy.jit.backend.llsupport.gc import GcLLDescr_framework, GcRefList, GcPtrFieldDescr from pypy.jit.backend.x86.test.test_regalloc import MockAssembler @@ -23,6 +23,8 @@ from pypy.jit.backend.x86.regalloc import X86RegisterManager, X86FrameManager,\ X86XMMRegisterManager +CPU = getcpuclass() + class MockGcRootMap(object): def get_basic_shape(self): return ['shape'] @@ -84,7 +86,7 @@ mark = regalloc.get_mark_gc_roots(cpu.gc_ll_descr.gcrootmap) assert mark[0] == 'compressed' base = -WORD * FRAME_FIXED_SIZE - expected = ['ebx', 'esi', 'edi', base, base-4, base-8] + expected = ['ebx', 'esi', 'edi', base, base-WORD, base-WORD*2] assert dict.fromkeys(mark[1:]) == dict.fromkeys(expected) class TestRegallocGcIntegration(BaseTestRegalloc): @@ -175,7 +177,7 @@ self.addrs[1] = self.addrs[0] + 64 # 64 bytes def malloc_slowpath(size): - assert size == 8 + assert size == WORD*2 nadr = rffi.cast(lltype.Signed, self.nursery) self.addrs[0] = nadr + size return nadr @@ -199,7 +201,7 @@ return rffi.cast(lltype.Signed, self.addrs) def get_nursery_top_addr(self): - return rffi.cast(lltype.Signed, self.addrs) + 4 + return rffi.cast(lltype.Signed, self.addrs) + WORD def get_malloc_fixedsize_slowpath_addr(self): fptr = llhelper(lltype.Ptr(self.MALLOC_SLOWPATH), self.malloc_slowpath) @@ -213,7 +215,7 @@ def setup_method(self, method): cpu = CPU(None, None) - cpu.vtable_offset = 4 + cpu.vtable_offset = WORD cpu.gc_ll_descr = GCDescrFastpathMalloc() NODE = lltype.Struct('node', ('tid', lltype.Signed), @@ -249,7 +251,7 @@ assert gc_ll_descr.nursery[0] == self.nodedescr.tid assert gc_ll_descr.nursery[1] == 42 nurs_adr = rffi.cast(lltype.Signed, gc_ll_descr.nursery) - assert gc_ll_descr.addrs[0] == nurs_adr + 8 + assert gc_ll_descr.addrs[0] == nurs_adr + (WORD*2) def test_malloc_slowpath(self): ops = ''' @@ -269,7 +271,7 @@ # this should call slow path once gc_ll_descr = self.cpu.gc_ll_descr nadr = rffi.cast(lltype.Signed, gc_ll_descr.nursery) - assert gc_ll_descr.addrs[0] == nadr + 8 + assert gc_ll_descr.addrs[0] == nadr + (WORD*2) def test_new_with_vtable(self): ops = ''' @@ -284,4 +286,4 @@ assert gc_ll_descr.nursery[0] == self.descrsize.tid assert gc_ll_descr.nursery[1] == self.vtable_int nurs_adr = rffi.cast(lltype.Signed, gc_ll_descr.nursery) - assert gc_ll_descr.addrs[0] == nurs_adr + 12 + assert gc_ll_descr.addrs[0] == nurs_adr + (WORD*3) Modified: pypy/trunk/pypy/jit/backend/x86/test/test_jump.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_jump.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_jump.py Mon Aug 2 19:52:15 2010 @@ -1,6 +1,7 @@ -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.regloc import * from pypy.jit.backend.x86.regalloc import X86FrameManager from pypy.jit.backend.x86.jump import remap_frame_layout +from pypy.jit.metainterp.history import INT frame_pos = X86FrameManager.frame_pos @@ -25,7 +26,7 @@ continue assert len(op1) == len(op2) for x, y in zip(op1, op2): - if isinstance(x, MODRM) and isinstance(y, MODRM): + if isinstance(x, StackLoc) and isinstance(y, MODRM): assert x.byte == y.byte assert x.extradata == y.extradata else: @@ -41,9 +42,9 @@ remap_frame_layout(assembler, [eax, ebx, ecx, edx, esi, edi], [eax, ebx, ecx, edx, esi, edi], '?') assert assembler.ops == [] - s8 = frame_pos(1, 1) - s12 = frame_pos(31, 1) - s20 = frame_pos(6, 1) + s8 = frame_pos(1, INT) + s12 = frame_pos(31, INT) + s20 = frame_pos(6, INT) remap_frame_layout(assembler, [eax, ebx, ecx, s20, s8, edx, s12, esi, edi], [eax, ebx, ecx, s20, s8, edx, s12, esi, edi], '?') @@ -58,10 +59,10 @@ def test_simple_framelocs(): assembler = MockAssembler() - s8 = frame_pos(0, 1) - s12 = frame_pos(13, 1) - s20 = frame_pos(20, 1) - s24 = frame_pos(221, 1) + s8 = frame_pos(0, INT) + s12 = frame_pos(13, INT) + s20 = frame_pos(20, INT) + s24 = frame_pos(221, INT) remap_frame_layout(assembler, [s8, eax, s12], [s20, s24, edi], edx) assert assembler.ops == [('mov', s8, edx), ('mov', edx, s20), @@ -70,10 +71,10 @@ def test_reordering(): assembler = MockAssembler() - s8 = frame_pos(8, 1) - s12 = frame_pos(12, 1) - s20 = frame_pos(19, 1) - s24 = frame_pos(1, 1) + s8 = frame_pos(8, INT) + s12 = frame_pos(12, INT) + s20 = frame_pos(19, INT) + s24 = frame_pos(1, INT) remap_frame_layout(assembler, [eax, s8, s20, ebx], [s8, ebx, eax, edi], '?') assert assembler.got([('mov', ebx, edi), @@ -83,10 +84,10 @@ def test_cycle(): assembler = MockAssembler() - s8 = frame_pos(8, 1) - s12 = frame_pos(12, 1) - s20 = frame_pos(19, 1) - s24 = frame_pos(1, 1) + s8 = frame_pos(8, INT) + s12 = frame_pos(12, INT) + s20 = frame_pos(19, INT) + s24 = frame_pos(1, INT) remap_frame_layout(assembler, [eax, s8, s20, ebx], [s8, ebx, eax, s20], '?') assert assembler.got([('push', s8), @@ -97,12 +98,12 @@ def test_cycle_2(): assembler = MockAssembler() - s8 = frame_pos(8, 1) - s12 = frame_pos(12, 1) - s20 = frame_pos(19, 1) - s24 = frame_pos(1, 1) - s2 = frame_pos(2, 1) - s3 = frame_pos(3, 1) + s8 = frame_pos(8, INT) + s12 = frame_pos(12, INT) + s20 = frame_pos(19, INT) + s24 = frame_pos(1, INT) + s2 = frame_pos(2, INT) + s3 = frame_pos(3, INT) remap_frame_layout(assembler, [eax, s8, edi, s20, eax, s20, s24, esi, s2, s3], [s8, s20, edi, eax, edx, s24, ebx, s12, s3, s2], @@ -127,14 +128,14 @@ remap_frame_layout(assembler, [c3], [eax], '?') assert assembler.ops == [('mov', c3, eax)] assembler = MockAssembler() - s12 = frame_pos(12, 1) + s12 = frame_pos(12, INT) remap_frame_layout(assembler, [c3], [s12], '?') assert assembler.ops == [('mov', c3, s12)] def test_constants_and_cycle(): assembler = MockAssembler() c3 = imm(3) - s12 = frame_pos(13, 1) + s12 = frame_pos(13, INT) remap_frame_layout(assembler, [ebx, c3, s12], [s12, eax, ebx], edi) assert assembler.ops == [('mov', c3, eax), Modified: pypy/trunk/pypy/jit/backend/x86/test/test_recompilation.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_recompilation.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_recompilation.py Mon Aug 2 19:52:15 2010 @@ -1,6 +1,5 @@ - -from pypy.jit.backend.x86.runner import CPU from pypy.jit.backend.x86.test.test_regalloc import BaseTestRegalloc +from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 class TestRecompilation(BaseTestRegalloc): def test_compile_bridge_not_deeper(self): @@ -51,7 +50,9 @@ descr = loop.operations[2].descr new = descr._x86_bridge_frame_depth assert descr._x86_bridge_param_depth == 0 - assert new > previous + # XXX: Maybe add enough ops to force stack on 64-bit as well? + if IS_X86_32: + assert new > previous self.cpu.set_future_value_int(0, 0) fail = self.run(loop) assert fail.identifier == 2 @@ -111,7 +112,9 @@ guard_op = loop.operations[5] loop_frame_depth = loop.token._x86_frame_depth assert loop.token._x86_param_depth == 0 - assert guard_op.descr._x86_bridge_frame_depth > loop_frame_depth + # XXX: Maybe add enough ops to force stack on 64-bit as well? + if IS_X86_32: + assert guard_op.descr._x86_bridge_frame_depth > loop_frame_depth assert guard_op.descr._x86_bridge_param_depth == 0 self.cpu.set_future_value_int(0, 0) self.cpu.set_future_value_int(1, 0) Modified: pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py Mon Aug 2 19:52:15 2010 @@ -7,15 +7,17 @@ BoxPtr, ConstPtr, LoopToken, BasicFailDescr from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.backend.llsupport.descr import GcCache -from pypy.jit.backend.x86.runner import CPU -from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, X86RegisterManager,\ +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.regalloc import RegAlloc, X86RegisterManager,\ FloatConstants +from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 from pypy.jit.metainterp.test.oparser import parse from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.annlowlevel import llhelper from pypy.rpython.lltypesystem import rclass, rstr -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.rx86 import * +CPU = getcpuclass() class MockGcDescr(GcCache): def get_funcptr_for_new(self): return 123 @@ -92,13 +94,20 @@ def f2(x, y): return x*y + def f10(*args): + assert len(args) == 10 + return sum(args) + F1PTR = lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Signed)) F2PTR = lltype.Ptr(lltype.FuncType([lltype.Signed]*2, lltype.Signed)) + F10PTR = lltype.Ptr(lltype.FuncType([lltype.Signed]*10, lltype.Signed)) f1ptr = llhelper(F1PTR, f1) f2ptr = llhelper(F2PTR, f2) + f10ptr = llhelper(F10PTR, f10) f1_calldescr = cpu.calldescrof(F1PTR.TO, F1PTR.TO.ARGS, F1PTR.TO.RESULT) f2_calldescr = cpu.calldescrof(F2PTR.TO, F2PTR.TO.ARGS, F2PTR.TO.RESULT) + f10_calldescr = cpu.calldescrof(F10PTR.TO, F10PTR.TO.ARGS, F10PTR.TO.RESULT) namespace = locals().copy() type_system = 'lltype' @@ -541,6 +550,12 @@ assert self.getints(9) == [0, 1, 1, 1, 1, 1, 1, 1, 1] class TestRegAllocCallAndStackDepth(BaseTestRegalloc): + def expected_param_depth(self, num_args): + # Assumes the arguments are all non-float + if IS_X86_32: + return num_args + elif IS_X86_64: + return max(num_args - 6, 0) def test_one_call(self): ops = ''' @@ -550,7 +565,7 @@ ''' loop = self.interpret(ops, [4, 7, 9, 9 ,9, 9, 9, 9, 9, 9, 9]) assert self.getints(11) == [5, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9] - assert loop.token._x86_param_depth == 1 + assert loop.token._x86_param_depth == self.expected_param_depth(1) def test_two_calls(self): ops = ''' @@ -561,8 +576,21 @@ ''' loop = self.interpret(ops, [4, 7, 9, 9 ,9, 9, 9, 9, 9, 9, 9]) assert self.getints(11) == [5*7, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9] - assert loop.token._x86_param_depth == 2 - + assert loop.token._x86_param_depth == self.expected_param_depth(2) + + def test_call_many_arguments(self): + # NB: The first and last arguments in the call are constants. This + # is primarily for x86-64, to ensure that loading a constant to an + # argument register or to the stack works correctly + ops = ''' + [i0, i1, i2, i3, i4, i5, i6, i7] + i8 = call(ConstClass(f10ptr), 1, i0, i1, i2, i3, i4, i5, i6, i7, 10, descr=f10_calldescr) + finish(i8) + ''' + loop = self.interpret(ops, [2, 3, 4, 5, 6, 7, 8, 9]) + assert self.getint(0) == 55 + assert loop.token._x86_param_depth == self.expected_param_depth(10) + def test_bridge_calls_1(self): ops = ''' [i0, i1] @@ -579,7 +607,7 @@ ''' bridge = self.attach_bridge(ops, loop, -2) - assert loop.operations[-2].descr._x86_bridge_param_depth == 2 + assert loop.operations[-2].descr._x86_bridge_param_depth == self.expected_param_depth(2) self.cpu.set_future_value_int(0, 4) self.cpu.set_future_value_int(1, 7) @@ -602,7 +630,7 @@ ''' bridge = self.attach_bridge(ops, loop, -2) - assert loop.operations[-2].descr._x86_bridge_param_depth == 2 + assert loop.operations[-2].descr._x86_bridge_param_depth == self.expected_param_depth(2) self.cpu.set_future_value_int(0, 4) self.cpu.set_future_value_int(1, 7) Modified: pypy/trunk/pypy/jit/backend/x86/test/test_regalloc2.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_regalloc2.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_regalloc2.py Mon Aug 2 19:52:15 2010 @@ -2,7 +2,9 @@ from pypy.jit.metainterp.history import ResOperation, BoxInt, ConstInt,\ BoxPtr, ConstPtr, BasicFailDescr, LoopToken from pypy.jit.metainterp.resoperation import rop -from pypy.jit.backend.x86.runner import CPU +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.arch import WORD +CPU = getcpuclass() def test_bug_rshift(): v1 = BoxInt() @@ -281,5 +283,8 @@ assert cpu.get_latest_value_int(16) == -57344 assert cpu.get_latest_value_int(17) == 1 assert cpu.get_latest_value_int(18) == -1 - assert cpu.get_latest_value_int(19) == -2147483648 + if WORD == 4: + assert cpu.get_latest_value_int(19) == -2147483648 + elif WORD == 8: + assert cpu.get_latest_value_int(19) == 19327352832 assert cpu.get_latest_value_int(20) == -49 Modified: pypy/trunk/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_runner.py Mon Aug 2 19:52:15 2010 @@ -2,9 +2,9 @@ from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rstr, rclass from pypy.jit.metainterp.history import ResOperation, LoopToken from pypy.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstPtr, - Box, BasicFailDescr) -from pypy.jit.backend.x86.runner import CPU -from pypy.jit.backend.x86.regalloc import WORD + Box, BoxFloat, BasicFailDescr) +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.arch import WORD from pypy.jit.backend.llsupport import symbolic from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.executor import execute @@ -15,6 +15,8 @@ import sys import os +CPU = getcpuclass() + class FakeStats(object): pass @@ -59,7 +61,7 @@ assert u.chars[3] == u'd' @staticmethod - def _resbuf(res, item_tp=ctypes.c_int): + def _resbuf(res, item_tp=ctypes.c_long): return ctypes.cast(res.value._obj.intval, ctypes.POINTER(item_tp)) def test_allocations(self): @@ -74,8 +76,11 @@ return ctypes.cast(buf, ctypes.c_void_p).value func = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int)(f) addr = ctypes.cast(func, ctypes.c_void_p).value + # ctypes produces an unsigned value. We need it to be signed for, eg, + # relative addressing to work properly. + addr = rffi.cast(lltype.Signed, addr) - self.cpu.assembler.make_sure_mc_exists() + self.cpu.assembler.setup() self.cpu.assembler.malloc_func_addr = addr ofs = symbolic.get_field_token(rstr.STR, 'chars', False)[0] @@ -360,7 +365,9 @@ self.cpu.compile_bridge(faildescr1, [i1b], bridge) name, address, size = agent.functions[1] assert name == "Bridge # 0: bye" - assert address == loopaddress + loopsize + # Would be exactly ==, but there are some guard failure recovery + # stubs in-between + assert address >= loopaddress + loopsize assert size >= 10 # randomish number self.cpu.set_future_value_int(0, 2) @@ -386,7 +393,7 @@ ops.append(ResOperation(rop.FINISH, [v], None, descr=BasicFailDescr())) looptoken = LoopToken() - self.cpu.assembler.make_sure_mc_exists() + self.cpu.assembler.setup() old_mc_mc = self.cpu.assembler.mc._mc self.cpu.compile_loop([base_v], ops, looptoken) assert self.cpu.assembler.mc._mc != old_mc_mc # overflowed @@ -394,6 +401,63 @@ self.cpu.execute_token(looptoken) assert self.cpu.get_latest_value_int(0) == 1024 + def test_overflow_guard_float_cmp(self): + # The float comparisons on x86 tend to use small relative jumps, + # which may run into trouble if they fall on the edge of a + # MachineCodeBlock change. + a = BoxFloat(1.0) + b = BoxFloat(2.0) + failed = BoxInt(41) + finished = BoxInt(42) + + # We select guards that will always succeed, so that execution will + # continue through the entire set of comparisions + ops_to_test = ( + (rop.FLOAT_LT, [a, b], rop.GUARD_TRUE), + (rop.FLOAT_LT, [b, a], rop.GUARD_FALSE), + + (rop.FLOAT_LE, [a, a], rop.GUARD_TRUE), + (rop.FLOAT_LE, [a, b], rop.GUARD_TRUE), + (rop.FLOAT_LE, [b, a], rop.GUARD_FALSE), + + (rop.FLOAT_EQ, [a, a], rop.GUARD_TRUE), + (rop.FLOAT_EQ, [a, b], rop.GUARD_FALSE), + + (rop.FLOAT_NE, [a, b], rop.GUARD_TRUE), + (rop.FLOAT_NE, [a, a], rop.GUARD_FALSE), + + (rop.FLOAT_GT, [b, a], rop.GUARD_TRUE), + (rop.FLOAT_GT, [a, b], rop.GUARD_FALSE), + + (rop.FLOAT_GE, [a, a], rop.GUARD_TRUE), + (rop.FLOAT_GE, [b, a], rop.GUARD_TRUE), + (rop.FLOAT_GE, [a, b], rop.GUARD_FALSE), + ) + + for float_op, args, guard_op in ops_to_test: + ops = [] + + for i in range(200): + cmp_result = BoxInt() + ops.append(ResOperation(float_op, args, cmp_result)) + ops.append(ResOperation(guard_op, [cmp_result], None, descr=BasicFailDescr())) + ops[-1].fail_args = [failed] + + ops.append(ResOperation(rop.FINISH, [finished], None, descr=BasicFailDescr())) + + looptoken = LoopToken() + self.cpu.compile_loop([a, b, failed, finished], ops, looptoken) + self.cpu.set_future_value_float(0, a.value) + self.cpu.set_future_value_float(1, b.value) + self.cpu.set_future_value_int(2, failed.value) + self.cpu.set_future_value_int(3, finished.value) + self.cpu.execute_token(looptoken) + + # Really just a sanity check. We're actually interested in + # whether the test segfaults. + assert self.cpu.get_latest_value_int(0) == finished.value + + class TestDebuggingAssembler(object): def setup_method(self, meth): self.pypylog = os.environ.get('PYPYLOG', None) Modified: pypy/trunk/pypy/jit/backend/x86/test/test_symbolic_x86.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_symbolic_x86.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_symbolic_x86.py Mon Aug 2 19:52:15 2010 @@ -1,6 +1,7 @@ import py from pypy.jit.backend.llsupport.symbolic import * from pypy.rpython.lltypesystem import lltype, rffi +from pypy.jit.backend.x86.arch import WORD # This test file is here and not in llsupport/test/ because it checks # that we get correct numbers for a 32-bit machine. @@ -19,32 +20,32 @@ ofs_z, size_z = get_field_token(S, 'z', False) # ofs_x might be 0 or not, depending on how we count the headers # but the rest should be as expected for a 386 machine - assert size_x == size_y == size_z == 4 + assert size_x == size_y == size_z == WORD assert ofs_x >= 0 - assert ofs_y == ofs_x + 4 - assert ofs_z == ofs_x + 8 + assert ofs_y == ofs_x + WORD + assert ofs_z == ofs_x + (WORD*2) def test_struct_size(): ofs_z, size_z = get_field_token(S, 'z', False) totalsize = get_size(S, False) - assert totalsize == ofs_z + 4 + assert totalsize == ofs_z + WORD def test_primitive_size(): - assert get_size(lltype.Signed, False) == 4 + assert get_size(lltype.Signed, False) == WORD assert get_size(lltype.Char, False) == 1 - assert get_size(lltype.Ptr(S), False) == 4 + assert get_size(lltype.Ptr(S), False) == WORD def test_array_token(): A = lltype.GcArray(lltype.Char) basesize, itemsize, ofs_length = get_array_token(A, False) - assert basesize >= 4 # at least the 'length', maybe some gc headers + assert basesize >= WORD # at least the 'length', maybe some gc headers assert itemsize == 1 - assert ofs_length == basesize - 4 + assert ofs_length == basesize - WORD A = lltype.GcArray(lltype.Signed) basesize, itemsize, ofs_length = get_array_token(A, False) - assert basesize >= 4 # at least the 'length', maybe some gc headers - assert itemsize == 4 - assert ofs_length == basesize - 4 + assert basesize >= WORD # at least the 'length', maybe some gc headers + assert itemsize == WORD + assert ofs_length == basesize - WORD def test_varsized_struct_size(): S1 = lltype.GcStruct('S1', ('parent', S), @@ -54,9 +55,9 @@ ofs_extra, size_extra = get_field_token(S1, 'extra', False) basesize, itemsize, ofs_length = get_array_token(S1, False) assert size_parent == ofs_extra - assert size_extra == 4 - assert ofs_length == ofs_extra + 4 - assert basesize == ofs_length + 4 + assert size_extra == WORD + assert ofs_length == ofs_extra + WORD + assert basesize == ofs_length + WORD assert itemsize == 1 def test_string(): Modified: pypy/trunk/pypy/jit/backend/x86/test/test_zll_random.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_zll_random.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_zll_random.py Mon Aug 2 19:52:15 2010 @@ -1,9 +1,11 @@ from pypy.jit.backend.test.test_random import check_random_function, Random from pypy.jit.backend.test.test_ll_random import LLtypeOperationBuilder -from pypy.jit.backend.x86.runner import CPU386 +from pypy.jit.backend.detect_cpu import getcpuclass + +CPU = getcpuclass() def test_stress(): - cpu = CPU386(None, None) + cpu = CPU(None, None) r = Random() for i in range(1000): check_random_function(cpu, LLtypeOperationBuilder, r, i, 1000) Modified: pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py Mon Aug 2 19:52:15 2010 @@ -17,6 +17,8 @@ from pypy.jit.backend.llsupport.gc import GcRefList, GcRootMap_asmgcc from pypy.jit.backend.llsupport.gc import GcLLDescr_framework from pypy.tool.udir import udir +from pypy.jit.backend.x86.arch import IS_X86_64 +import py.test class X(object): def __init__(self, x=0): @@ -126,6 +128,10 @@ class TestCompileHybrid(object): def setup_class(cls): + if IS_X86_64: + # No hybrid GC on 64-bit for the time being + py.test.skip() + funcs = [] name_to_func = {} for fullname in dir(cls): Modified: pypy/trunk/pypy/jit/backend/x86/test/test_ztranslation.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_ztranslation.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_ztranslation.py Mon Aug 2 19:52:15 2010 @@ -3,13 +3,14 @@ from pypy.rlib.jit import JitDriver, OPTIMIZER_FULL, unroll_parameters from pypy.rlib.jit import PARAMETERS, dont_look_inside from pypy.jit.metainterp.jitprof import Profiler -from pypy.jit.backend.x86.runner import CPU386 +from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.backend.test.support import CCompiledMixin from pypy.jit.codewriter.policy import StopAtXPolicy from pypy.translator.translator import TranslationContext +from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 class TestTranslationX86(CCompiledMixin): - CPUClass = CPU386 + CPUClass = getcpuclass() def _check_cbuilder(self, cbuilder): # We assume here that we have sse2. If not, the CPUClass @@ -114,7 +115,7 @@ class TestTranslationRemoveTypePtrX86(CCompiledMixin): - CPUClass = CPU386 + CPUClass = getcpuclass() def _get_TranslationContext(self): t = TranslationContext() @@ -125,6 +126,10 @@ return t def test_external_exception_handling_translates(self): + # FIXME + if IS_X86_64: + import py.test; py.test.skip() + jitdriver = JitDriver(greens = [], reds = ['n', 'total']) class ImDone(Exception): Modified: pypy/trunk/pypy/jit/backend/x86/tool/viewcode.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/tool/viewcode.py (original) +++ pypy/trunk/pypy/jit/backend/x86/tool/viewcode.py Mon Aug 2 19:52:15 2010 @@ -31,16 +31,22 @@ if sys.platform == "win32": XXX # lots more in Psyco -def machine_code_dump(data, originaddr): - # the disassembler to use. 'objdump' writes GNU-style instructions. - # 'ndisasm' would use Intel syntax, but you need to fix the output parsing. - objdump = ('objdump -M intel -b binary -m i386 ' +def machine_code_dump(data, originaddr, backend_name): + objdump_backend_option = { + 'x86': 'i386', + 'x86_64': 'x86-64', + } + objdump = ('objdump -M intel,%(backend)s -b binary -m i386 ' '--adjust-vma=%(origin)d -D %(file)s') # f = open(tmpfile, 'wb') f.write(data) f.close() - g = os.popen(objdump % {'file': tmpfile, 'origin': originaddr}, 'r') + g = os.popen(objdump % { + 'file': tmpfile, + 'origin': originaddr, + 'backend': objdump_backend_option[backend_name], + }, 'r') result = g.readlines() g.close() return result[6:] # drop some objdump cruft @@ -126,7 +132,7 @@ def disassemble(self): if not hasattr(self, 'text'): - lines = machine_code_dump(self.data, self.addr) + lines = machine_code_dump(self.data, self.addr, self.world.backend_name) # instead of adding symbol names in the dumps we could # also make the 0xNNNNNNNN addresses be red and show the # symbol name when the mouse is over them @@ -171,10 +177,13 @@ self.jumps = {} self.symbols = {} self.logentries = {} + self.backend_name = None def parse(self, f, textonly=True): for line in f: - if line.startswith('CODE_DUMP '): + if line.startswith('BACKEND '): + self.backend_name = line.split(' ')[1].strip() + elif line.startswith('CODE_DUMP '): pieces = line.split() assert pieces[1].startswith('@') assert pieces[2].startswith('+') Modified: pypy/trunk/pypy/module/signal/interp_signal.py ============================================================================== --- pypy/trunk/pypy/module/signal/interp_signal.py (original) +++ pypy/trunk/pypy/module/signal/interp_signal.py Mon Aug 2 19:52:15 2010 @@ -7,6 +7,7 @@ from pypy.translator.tool.cbuild import ExternalCompilationInfo import py from pypy.tool import autopath +from pypy.rlib import jit def setup(): for key, value in cpy_signal.__dict__.items(): @@ -159,10 +160,12 @@ return space.wrap(SIG_DFL) getsignal.unwrap_spec = [ObjSpace, int] + at jit.dont_look_inside def alarm(space, timeout): return space.wrap(c_alarm(timeout)) alarm.unwrap_spec = [ObjSpace, int] + at jit.dont_look_inside def pause(space): c_pause() return space.w_None @@ -173,6 +176,7 @@ raise OperationError(space.w_ValueError, space.wrap("signal number out of range")) + at jit.dont_look_inside def signal(space, signum, w_handler): """ signal(sig, action) -> action From fijal at codespeak.net Mon Aug 2 20:42:14 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 2 Aug 2010 20:42:14 +0200 (CEST) Subject: [pypy-svn] r76439 - in pypy/branch/improved-asm-logging/pypy/jit/backend/x86: . test Message-ID: <20100802184214.71599282BDB@codespeak.net> Author: fijal Date: Mon Aug 2 20:42:10 2010 New Revision: 76439 Modified: pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py pypy/branch/improved-asm-logging/pypy/jit/backend/x86/test/test_runner.py Log: hopefully fix carl's concerns and have an array per-loop. Modified: pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/improved-asm-logging/pypy/jit/backend/x86/assembler.py Mon Aug 2 20:42:10 2010 @@ -13,11 +13,12 @@ from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.jit.backend.x86 import codebuf from pypy.jit.backend.x86.ri386 import * -from pypy.jit.metainterp.resoperation import rop +from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.backend.x86.support import values_array from pypy.rlib.debug import debug_print from pypy.rlib import rgc from pypy.rlib.streamio import open_file_as_stream +from pypy.jit.metainterp.history import ConstInt, BoxInt # our calling convention - we pass first 6 args in registers # and the rest stays on the stack @@ -94,6 +95,8 @@ if name.upper() == name or name == "writechr": setattr(MachineCodeBlockWrapper, name, _new_method(name)) +DEBUG_COUNTER = rffi.CArray(lltype.Signed) + class Assembler386(object): mc = None mc2 = None @@ -115,16 +118,15 @@ self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) self.fail_boxes_float = values_array(lltype.Float, failargs_limit) self.fail_ebp = 0 - self.loop_run_counter = values_array(lltype.Signed, 10000) - self.loop_names = [] + self.loop_run_counters = [] # if we have 10000 loops, we have some other problems I guess self.loc_float_const_neg = None self.loc_float_const_abs = None self.malloc_fixedsize_slowpath1 = 0 self.malloc_fixedsize_slowpath2 = 0 self.setup_failure_recovery() - self._loop_counter = 0 self._debug = False + self.debug_counter_descr = cpu.arraydescrof(DEBUG_COUNTER) def leave_jitted_hook(self): ptrs = self.fail_boxes_ptr.ar @@ -179,9 +181,9 @@ output_log = self._output_loop_log assert output_log is not None f = open_file_as_stream(output_log, "w") - for i in range(self._loop_counter): - f.write(self.loop_names[i] + ":" + - str(self.loop_run_counter.getitem(i)) + "\n") + for i in range(len(self.loop_run_counters)): + name, arr = self.loop_run_counters[i] + f.write(name + ":" + str(arr[0]) + "\n") f.close() def _build_float_constants(self): @@ -237,6 +239,7 @@ funcname = self._find_debug_merge_point(operations) regalloc = RegAlloc(self, self.cpu.translate_support_code) + operations = self._inject_debugging_code(operations) arglocs = regalloc.prepare_loop(inputargs, operations, looptoken) looptoken._x86_arglocs = arglocs needed_mem = len(arglocs[0]) * 16 + 16 @@ -279,6 +282,7 @@ assert ([loc.assembler() for loc in arglocs] == [loc.assembler() for loc in faildescr._x86_debug_faillocs]) regalloc = RegAlloc(self, self.cpu.translate_support_code) + operations = self._inject_debugging_code(operations) fail_depths = faildescr._x86_current_depths regalloc.prepare_bridge(fail_depths, inputargs, arglocs, operations) @@ -310,11 +314,12 @@ funcname = op.args[0]._get_str() break else: - funcname = "" % self._loop_counter + funcname = "" % len(self.loop_run_counters) # invent the counter, so we don't get too confused if self._debug: - self.loop_names.append(funcname) - self._loop_counter += 1 + arr = lltype.malloc(DEBUG_COUNTER, 1, flavor='raw') + arr[0] = 0 + self.loop_run_counters.append((funcname, arr)) return funcname def patch_jump_for_descr(self, faildescr, adr_new_target): @@ -324,17 +329,31 @@ mc.valgrind_invalidated() mc.done() - def _assemble(self, regalloc, operations): - self._regalloc = regalloc + def _inject_debugging_code(self, operations): if self._debug: # before doing anything, let's increase a counter - # we need one register free (a bit of a hack, but whatever) - self.mc.PUSH(eax) - adr = self.loop_run_counter.get_addr_for_num(self._loop_counter - 1) - self.mc.MOV(eax, heap(adr)) - self.mc.ADD(eax, imm(1)) - self.mc.MOV(heap(adr), eax) - self.mc.POP(eax) + c_adr = ConstInt(rffi.cast(lltype.Signed, + self.loop_run_counters[-1][1])) + box = BoxInt() + box2 = BoxInt() + ops = [ResOperation(rop.GETARRAYITEM_RAW, [c_adr, ConstInt(0)], + box, descr=self.debug_counter_descr), + ResOperation(rop.INT_ADD, [box, ConstInt(1)], box2), + ResOperation(rop.SETARRAYITEM_RAW, [c_adr, ConstInt(0), + box2], + None, descr=self.debug_counter_descr)] + operations = ops + operations + # # we need one register free (a bit of a hack, but whatever) + # self.mc.PUSH(eax) + # adr = rffi.cast(lltype.Signed, self.loop_run_counters[-1][1]) + # self.mc.MOV(eax, heap(adr)) + # self.mc.ADD(eax, imm(1)) + # self.mc.MOV(heap(adr), eax) + # self.mc.POP(eax) + return operations + + def _assemble(self, regalloc, operations): + self._regalloc = regalloc regalloc.walk_operations(operations) self.mc.done() self.mc2.done() Modified: pypy/branch/improved-asm-logging/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/branch/improved-asm-logging/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/branch/improved-asm-logging/pypy/jit/backend/x86/test/test_runner.py Mon Aug 2 20:42:10 2010 @@ -420,8 +420,9 @@ self.cpu.set_future_value_int(0, 0) self.cpu.execute_token(ops.token) # check debugging info - assert self.cpu.assembler.loop_names == ["xyz"] - assert self.cpu.assembler.loop_run_counter.getitem(0) == 10 + name, arr = self.cpu.assembler.loop_run_counters[0] + assert name == 'xyz' + assert arr[0] == 10 self.cpu.finish_once() lines = py.path.local(self.logfile + ".count").readlines() assert lines[0] == 'xyz:10\n' From jcreigh at codespeak.net Mon Aug 2 20:58:58 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Mon, 2 Aug 2010 20:58:58 +0200 (CEST) Subject: [pypy-svn] r76440 - in pypy/branch/asmgcc-64: . lib-python lib-python/modified-2.5.2 lib-python/modified-2.5.2/ctypes lib_pypy lib_pypy/_ctypes lib_pypy/pypy_test pypy/annotation pypy/annotation/test pypy/config pypy/interpreter pypy/interpreter/astcompiler pypy/interpreter/astcompiler/tools pypy/interpreter/test pypy/jit/backend pypy/jit/backend/llgraph pypy/jit/backend/llsupport pypy/jit/backend/llsupport/test pypy/jit/backend/test pypy/jit/backend/x86 pypy/jit/backend/x86/test pypy/jit/backend/x86/tool pypy/jit/codewriter pypy/jit/codewriter/test pypy/jit/metainterp pypy/jit/metainterp/test pypy/jit/tool pypy/module/__builtin__ pypy/module/_ast pypy/module/_ast/test pypy/module/_file pypy/module/_file/test pypy/module/_rawffi/test pypy/module/cpyext pypy/module/posix pypy/module/posix/test pypy/module/signal pypy/module/test_lib_pypy/ctypes_tests pypy/objspace/std pypy/objspace/std/test pypy/rlib pypy/rlib/test pypy/rpython pypy/rpython/lltypesystem pypy/rpython/module pypy/rpython/module/test pypy/rpython/test pypy/translator/c/test pypy/translator/goal pypy/translator/goal/test2 pypy/translator/platform Message-ID: <20100802185858.25739282BDB@codespeak.net> Author: jcreigh Date: Mon Aug 2 20:58:49 2010 New Revision: 76440 Added: pypy/branch/asmgcc-64/pypy/jit/backend/x86/arch.py - copied unchanged from r76439, pypy/trunk/pypy/jit/backend/x86/arch.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/regloc.py - copied unchanged from r76439, pypy/trunk/pypy/jit/backend/x86/regloc.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/rx86.py - copied unchanged from r76439, pypy/trunk/pypy/jit/backend/x86/rx86.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_regloc.py - copied unchanged from r76439, pypy/trunk/pypy/jit/backend/x86/test/test_regloc.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_rx86.py - copied unchanged from r76439, pypy/trunk/pypy/jit/backend/x86/test/test_rx86.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py - copied unchanged from r76439, pypy/trunk/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py - copied unchanged from r76439, pypy/trunk/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/tool/instruction_encoding.sh - copied unchanged from r76439, pypy/trunk/pypy/jit/backend/x86/tool/instruction_encoding.sh pypy/branch/asmgcc-64/pypy/jit/tool/gen-trace-mode-keywords.py - copied unchanged from r76439, pypy/trunk/pypy/jit/tool/gen-trace-mode-keywords.py pypy/branch/asmgcc-64/pypy/jit/tool/pypytrace-mode.el - copied unchanged from r76439, pypy/trunk/pypy/jit/tool/pypytrace-mode.el pypy/branch/asmgcc-64/pypy/rlib/test/test_rposix.py - copied unchanged from r76439, pypy/trunk/pypy/rlib/test/test_rposix.py pypy/branch/asmgcc-64/pypy/rpython/module/ll_win32file.py - copied unchanged from r76439, pypy/trunk/pypy/rpython/module/ll_win32file.py Removed: pypy/branch/asmgcc-64/pypy/jit/backend/x86/ri386.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/ri386setup.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_ri386.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py Modified: pypy/branch/asmgcc-64/ (props changed) pypy/branch/asmgcc-64/lib-python/ (props changed) pypy/branch/asmgcc-64/lib-python/conftest.py pypy/branch/asmgcc-64/lib-python/modified-2.5.2/ctypes/__init__.py pypy/branch/asmgcc-64/lib-python/modified-2.5.2/site.py pypy/branch/asmgcc-64/lib_pypy/ (props changed) pypy/branch/asmgcc-64/lib_pypy/__init__.py pypy/branch/asmgcc-64/lib_pypy/_ctypes/__init__.py pypy/branch/asmgcc-64/lib_pypy/_ctypes/array.py pypy/branch/asmgcc-64/lib_pypy/_ctypes/builtin.py pypy/branch/asmgcc-64/lib_pypy/_ctypes/primitive.py pypy/branch/asmgcc-64/lib_pypy/datetime.py pypy/branch/asmgcc-64/lib_pypy/dbm.py (props changed) pypy/branch/asmgcc-64/lib_pypy/pypy_test/test_ctypes_support.py pypy/branch/asmgcc-64/lib_pypy/pypy_test/test_datetime.py pypy/branch/asmgcc-64/lib_pypy/pypy_test/test_functools.py (props changed) pypy/branch/asmgcc-64/pypy/annotation/bookkeeper.py pypy/branch/asmgcc-64/pypy/annotation/model.py pypy/branch/asmgcc-64/pypy/annotation/test/test_model.py pypy/branch/asmgcc-64/pypy/config/translationoption.py pypy/branch/asmgcc-64/pypy/interpreter/astcompiler/assemble.py pypy/branch/asmgcc-64/pypy/interpreter/astcompiler/ast.py pypy/branch/asmgcc-64/pypy/interpreter/astcompiler/codegen.py pypy/branch/asmgcc-64/pypy/interpreter/astcompiler/consts.py pypy/branch/asmgcc-64/pypy/interpreter/astcompiler/symtable.py pypy/branch/asmgcc-64/pypy/interpreter/astcompiler/tools/asdl_py.py pypy/branch/asmgcc-64/pypy/interpreter/baseobjspace.py pypy/branch/asmgcc-64/pypy/interpreter/error.py pypy/branch/asmgcc-64/pypy/interpreter/gateway.py pypy/branch/asmgcc-64/pypy/interpreter/pyopcode.py pypy/branch/asmgcc-64/pypy/interpreter/test/test_compiler.py pypy/branch/asmgcc-64/pypy/interpreter/test/test_gateway.py pypy/branch/asmgcc-64/pypy/jit/backend/detect_cpu.py pypy/branch/asmgcc-64/pypy/jit/backend/llgraph/llimpl.py pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/descr.py pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/gc.py pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/regalloc.py pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/support.py pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/test/test_descr.py pypy/branch/asmgcc-64/pypy/jit/backend/test/runner_test.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/assembler.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/codebuf.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/jump.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/regalloc.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/runner.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/conftest.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_assembler.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_basic.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_gc_integration.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_jump.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_recompilation.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_regalloc.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_regalloc2.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_runner.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_symbolic_x86.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_zll_random.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_zrpy_gc.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_ztranslation.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/tool/viewcode.py pypy/branch/asmgcc-64/pypy/jit/codewriter/codewriter.py pypy/branch/asmgcc-64/pypy/jit/codewriter/jtransform.py pypy/branch/asmgcc-64/pypy/jit/codewriter/test/test_flatten.py pypy/branch/asmgcc-64/pypy/jit/metainterp/optimizeopt.py pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_loop.py pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_loop_spec.py pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_recursive.py pypy/branch/asmgcc-64/pypy/module/__builtin__/compiling.py pypy/branch/asmgcc-64/pypy/module/_ast/__init__.py pypy/branch/asmgcc-64/pypy/module/_ast/test/test_ast.py pypy/branch/asmgcc-64/pypy/module/_file/interp_file.py pypy/branch/asmgcc-64/pypy/module/_file/test/test_file.py pypy/branch/asmgcc-64/pypy/module/_rawffi/test/test__rawffi.py pypy/branch/asmgcc-64/pypy/module/cpyext/methodobject.py pypy/branch/asmgcc-64/pypy/module/posix/interp_posix.py pypy/branch/asmgcc-64/pypy/module/posix/test/test_posix2.py pypy/branch/asmgcc-64/pypy/module/signal/interp_signal.py pypy/branch/asmgcc-64/pypy/module/test_lib_pypy/ctypes_tests/ (props changed) pypy/branch/asmgcc-64/pypy/objspace/std/callmethod.py pypy/branch/asmgcc-64/pypy/objspace/std/test/test_callmethod.py pypy/branch/asmgcc-64/pypy/rlib/objectmodel.py pypy/branch/asmgcc-64/pypy/rlib/rarithmetic.py pypy/branch/asmgcc-64/pypy/rlib/rdynload.py pypy/branch/asmgcc-64/pypy/rlib/rposix.py pypy/branch/asmgcc-64/pypy/rlib/rwin32.py pypy/branch/asmgcc-64/pypy/rlib/streamio.py pypy/branch/asmgcc-64/pypy/rlib/test/test_objectmodel.py pypy/branch/asmgcc-64/pypy/rlib/test/test_rarithmetic.py pypy/branch/asmgcc-64/pypy/rpython/extfunc.py pypy/branch/asmgcc-64/pypy/rpython/lltypesystem/rffi.py pypy/branch/asmgcc-64/pypy/rpython/lltypesystem/rstr.py pypy/branch/asmgcc-64/pypy/rpython/module/ll_os.py pypy/branch/asmgcc-64/pypy/rpython/module/ll_os_stat.py pypy/branch/asmgcc-64/pypy/rpython/module/test/test_ll_os_stat.py pypy/branch/asmgcc-64/pypy/rpython/rstr.py pypy/branch/asmgcc-64/pypy/rpython/test/test_extfunc.py pypy/branch/asmgcc-64/pypy/rpython/test/test_rstr.py pypy/branch/asmgcc-64/pypy/translator/c/test/test_typed.py pypy/branch/asmgcc-64/pypy/translator/goal/app_main.py pypy/branch/asmgcc-64/pypy/translator/goal/test2/test_app_main.py pypy/branch/asmgcc-64/pypy/translator/platform/__init__.py pypy/branch/asmgcc-64/pypy/translator/platform/darwin.py pypy/branch/asmgcc-64/pypy/translator/platform/posix.py pypy/branch/asmgcc-64/pypy/translator/platform/windows.py Log: merged changes from trunk through r76439 Modified: pypy/branch/asmgcc-64/lib-python/conftest.py ============================================================================== --- pypy/branch/asmgcc-64/lib-python/conftest.py (original) +++ pypy/branch/asmgcc-64/lib-python/conftest.py Mon Aug 2 20:58:49 2010 @@ -464,11 +464,7 @@ RegrTest('test_coding.py'), RegrTest('test_complex_args.py'), RegrTest('test_contextlib.py', usemodules="thread"), - # we skip test ctypes, since we adapted it massively in order - # to test what we want to support. There are real failures, - # but it's about missing features that we don't want to support - # now - RegrTest('test_ctypes.py', skip="we have a replacement"), + RegrTest('test_ctypes.py', usemodules="_rawffi"), RegrTest('test_defaultdict.py'), RegrTest('test_email_renamed.py'), RegrTest('test_exception_variations.py'), Modified: pypy/branch/asmgcc-64/lib-python/modified-2.5.2/ctypes/__init__.py ============================================================================== --- pypy/branch/asmgcc-64/lib-python/modified-2.5.2/ctypes/__init__.py (original) +++ pypy/branch/asmgcc-64/lib-python/modified-2.5.2/ctypes/__init__.py Mon Aug 2 20:58:49 2010 @@ -471,7 +471,7 @@ # functions -from _ctypes import _memmove_addr, _memset_addr, _string_at_addr, _cast_addr +from _ctypes import _memmove_addr, _memset_addr, _cast_addr ## void *memmove(void *, const void *, size_t); memmove = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)(_memmove_addr) @@ -490,24 +490,34 @@ def cast(obj, typ): return _cast(obj, obj, typ) -_string_at = CFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr) +try: + from _ctypes import _string_at_addr +except ImportError: + from _ctypes import _string_at +else: + _string_at = CFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr) + def string_at(ptr, size=-1): """string_at(addr[, size]) -> string Return the string at addr.""" return _string_at(ptr, size) +def wstring_at(ptr, size=-1): + """wstring_at(addr[, size]) -> string + + Return the string at addr.""" + return _wstring_at(ptr, size) + try: from _ctypes import _wstring_at_addr except ImportError: - pass + try: + from _ctypes import _wstring_at + except ImportError: + del wstring_at else: _wstring_at = CFUNCTYPE(py_object, c_void_p, c_int)(_wstring_at_addr) - def wstring_at(ptr, size=-1): - """wstring_at(addr[, size]) -> string - - Return the string at addr.""" - return _wstring_at(ptr, size) if _os.name in ("nt", "ce"): # COM stuff Modified: pypy/branch/asmgcc-64/lib-python/modified-2.5.2/site.py ============================================================================== --- pypy/branch/asmgcc-64/lib-python/modified-2.5.2/site.py (original) +++ pypy/branch/asmgcc-64/lib-python/modified-2.5.2/site.py Mon Aug 2 20:58:49 2010 @@ -175,7 +175,7 @@ def addsitepackages(known_paths): """Add site-packages to sys.path, in a PyPy-specific way.""" - if hasattr(sys, 'pypy_version_info'): + if hasattr(sys, 'pypy_version_info') and hasattr(sys, 'prefix'): from distutils.sysconfig import get_python_lib sitedir = get_python_lib(standard_lib=False) if os.path.isdir(sitedir): Modified: pypy/branch/asmgcc-64/lib_pypy/__init__.py ============================================================================== --- pypy/branch/asmgcc-64/lib_pypy/__init__.py (original) +++ pypy/branch/asmgcc-64/lib_pypy/__init__.py Mon Aug 2 20:58:49 2010 @@ -3,6 +3,5 @@ # Without this check, you would be able to do 'import __init__' from a pypy # prompt -import sys; print sys.path if __name__ != 'lib_pypy': raise ImportError, '__init__' Modified: pypy/branch/asmgcc-64/lib_pypy/_ctypes/__init__.py ============================================================================== --- pypy/branch/asmgcc-64/lib_pypy/_ctypes/__init__.py (original) +++ pypy/branch/asmgcc-64/lib_pypy/_ctypes/__init__.py Mon Aug 2 20:58:49 2010 @@ -7,8 +7,8 @@ from _ctypes.dll import dlopen from _ctypes.structure import Structure from _ctypes.array import Array -from _ctypes.builtin import _memmove_addr, _string_at_addr, _memset_addr,\ - set_conversion_mode, _wstring_at_addr +from _ctypes.builtin import _memmove_addr, _string_at, _memset_addr,\ + set_conversion_mode, _wstring_at from _ctypes.union import Union import os as _os Modified: pypy/branch/asmgcc-64/lib_pypy/_ctypes/array.py ============================================================================== --- pypy/branch/asmgcc-64/lib_pypy/_ctypes/array.py (original) +++ pypy/branch/asmgcc-64/lib_pypy/_ctypes/array.py Mon Aug 2 20:58:49 2010 @@ -4,7 +4,6 @@ from _ctypes.basics import _CData, cdata_from_address, _CDataMeta, sizeof from _ctypes.basics import keepalive_key, store_reference, ensure_objects from _ctypes.basics import CArgObject -from _ctypes.builtin import _string_at_addr, _wstring_at_addr def _create_unicode(buffer, maxlength): res = [] Modified: pypy/branch/asmgcc-64/lib_pypy/_ctypes/builtin.py ============================================================================== --- pypy/branch/asmgcc-64/lib_pypy/_ctypes/builtin.py (original) +++ pypy/branch/asmgcc-64/lib_pypy/_ctypes/builtin.py Mon Aug 2 20:58:49 2010 @@ -8,10 +8,10 @@ _memmove_addr = _rawffi.get_libc().getaddressindll('memmove') _memset_addr = _rawffi.get_libc().getaddressindll('memset') -def _string_at_addr(addr, lgt): +def _string_at(addr, lgt): # address here can be almost anything import ctypes - arg = ctypes.c_char_p._CData_value(addr) + arg = ctypes.c_void_p._CData_value(addr) return _rawffi.charp2rawstring(arg, lgt) def set_conversion_mode(encoding, errors): @@ -20,9 +20,9 @@ ConvMode.encoding = encoding return old_cm -def _wstring_at_addr(addr, lgt): +def _wstring_at(addr, lgt): import ctypes - arg = ctypes.c_wchar_p._CData_value(addr) + arg = ctypes.c_void_p._CData_value(addr) # XXX purely applevel if lgt == -1: lgt = sys.maxint Modified: pypy/branch/asmgcc-64/lib_pypy/_ctypes/primitive.py ============================================================================== --- pypy/branch/asmgcc-64/lib_pypy/_ctypes/primitive.py (original) +++ pypy/branch/asmgcc-64/lib_pypy/_ctypes/primitive.py Mon Aug 2 20:58:49 2010 @@ -86,6 +86,8 @@ return value if isinstance(value, _Pointer): return cls.from_address(value._buffer.buffer) + if isinstance(value, (int, long)): + return cls(value) FROM_PARAM_BY_TYPE = { 'z': from_param_char_p, @@ -141,13 +143,13 @@ result.value = property(_getvalue, _setvalue) elif tp == 'Z': # c_wchar_p - from _ctypes import Array, _Pointer, _wstring_at_addr + from _ctypes import Array, _Pointer, _wstring_at def _getvalue(self): addr = self._buffer[0] if addr == 0: return None else: - return _wstring_at_addr(addr, -1) + return _wstring_at(addr, -1) def _setvalue(self, value): if isinstance(value, basestring): @@ -216,14 +218,14 @@ SysAllocStringLen = windll.oleaut32.SysAllocStringLen SysStringLen = windll.oleaut32.SysStringLen SysFreeString = windll.oleaut32.SysFreeString - from _ctypes import _wstring_at_addr + from _ctypes import _wstring_at def _getvalue(self): addr = self._buffer[0] if addr == 0: return None else: size = SysStringLen(addr) - return _wstring_at_addr(addr, size) + return _wstring_at(addr, size) def _setvalue(self, value): if isinstance(value, basestring): @@ -254,18 +256,21 @@ from_address = cdata_from_address def from_param(self, value): + if isinstance(value, self): + return value + from_param_f = FROM_PARAM_BY_TYPE.get(self._type_) if from_param_f: res = from_param_f(self, value) if res is not None: return res - - if isinstance(value, self): - return value - try: - return self(value) - except (TypeError, ValueError): - return super(SimpleType, self).from_param(value) + else: + try: + return self(value) + except (TypeError, ValueError): + pass + + return super(SimpleType, self).from_param(value) def _CData_output(self, resbuffer, base=None, index=-1): output = super(SimpleType, self)._CData_output(resbuffer, base, index) Modified: pypy/branch/asmgcc-64/lib_pypy/datetime.py ============================================================================== --- pypy/branch/asmgcc-64/lib_pypy/datetime.py (original) +++ pypy/branch/asmgcc-64/lib_pypy/datetime.py Mon Aug 2 20:58:49 2010 @@ -1412,7 +1412,7 @@ def utcfromtimestamp(cls, t): "Construct a UTC datetime from a POSIX timestamp (like time.time())." - if 1 - (t % 1.0) < 0.000001: + if 1 - (t % 1.0) < 0.0000005: t = float(int(t)) + 1 if t < 0: t -= 1 Modified: pypy/branch/asmgcc-64/lib_pypy/pypy_test/test_ctypes_support.py ============================================================================== --- pypy/branch/asmgcc-64/lib_pypy/pypy_test/test_ctypes_support.py (original) +++ pypy/branch/asmgcc-64/lib_pypy/pypy_test/test_ctypes_support.py Mon Aug 2 20:58:49 2010 @@ -20,3 +20,14 @@ assert get_errno() != 0 set_errno(0) assert get_errno() == 0 + +def test_argument_conversion_and_checks(): + import ctypes + libc = ctypes.cdll.LoadLibrary("libc.so.6") + libc.strlen.argtypes = ctypes.c_char_p, + libc.strlen.restype = ctypes.c_size_t + assert libc.strlen("eggs") == 4 + + # Should raise ArgumentError, not segfault + py.test.raises(ctypes.ArgumentError, libc.strlen, False) + Modified: pypy/branch/asmgcc-64/lib_pypy/pypy_test/test_datetime.py ============================================================================== --- pypy/branch/asmgcc-64/lib_pypy/pypy_test/test_datetime.py (original) +++ pypy/branch/asmgcc-64/lib_pypy/pypy_test/test_datetime.py Mon Aug 2 20:58:49 2010 @@ -15,4 +15,18 @@ expected = datetime.datetime(*(time.strptime(string, format)[0:6])) got = datetime.datetime.strptime(string, format) assert expected == got + +def test_datetime_rounding(): + b = 0.0000001 + a = 0.9999994 + + assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 + assert datetime.datetime.utcfromtimestamp(a).second == 0 + a += b + assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 + assert datetime.datetime.utcfromtimestamp(a).second == 0 + a += b + assert datetime.datetime.utcfromtimestamp(a).microsecond == 0 + assert datetime.datetime.utcfromtimestamp(a).second == 1 + Modified: pypy/branch/asmgcc-64/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/annotation/bookkeeper.py (original) +++ pypy/branch/asmgcc-64/pypy/annotation/bookkeeper.py Mon Aug 2 20:58:49 2010 @@ -739,7 +739,7 @@ """ w_tuple = SomeTuple def newtuple(self, items_s): - if items_s == [Ellipsis]: + if len(items_s) == 1 and items_s[0] is Ellipsis: res = SomeObject() # hack to get a SomeObject as the *arg res.from_ellipsis = True return res Modified: pypy/branch/asmgcc-64/pypy/annotation/model.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/annotation/model.py (original) +++ pypy/branch/asmgcc-64/pypy/annotation/model.py Mon Aug 2 20:58:49 2010 @@ -34,7 +34,7 @@ from pypy.tool.pairtype import pair, extendabletype from pypy.tool.tls import tlsobject from pypy.rlib.rarithmetic import r_uint, r_ulonglong, base_int -from pypy.rlib.rarithmetic import r_singlefloat +from pypy.rlib.rarithmetic import r_singlefloat, isnan import inspect, weakref DEBUG = False # set to False to disable recording of debugging information @@ -162,6 +162,13 @@ # pretend it's a float. immutable = True + def __eq__(self, other): + # NaN unpleasantness. + if (self.is_constant() and other.is_constant() and + isnan(self.const) and isnan(other.const)): + return True + return super(SomeFloat, self).__eq__(other) + def can_be_none(self): return False Modified: pypy/branch/asmgcc-64/pypy/annotation/test/test_model.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/annotation/test/test_model.py (original) +++ pypy/branch/asmgcc-64/pypy/annotation/test/test_model.py Mon Aug 2 20:58:49 2010 @@ -200,6 +200,15 @@ s2.const=cls assert s1.contains(s2) +def test_nan(): + f1 = SomeFloat() + f1.const = float("nan") + f2 = SomeFloat() + f2.const = float("nan") + assert f1.contains(f1) + assert f2.contains(f1) + assert f1.contains(f2) + if __name__ == '__main__': for name, value in globals().items(): if name.startswith('test_'): Modified: pypy/branch/asmgcc-64/pypy/config/translationoption.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/config/translationoption.py (original) +++ pypy/branch/asmgcc-64/pypy/config/translationoption.py Mon Aug 2 20:58:49 2010 @@ -342,6 +342,10 @@ 'jit': 'hybrid extraopts jit', } +# For now, 64-bit JIT requires boehm +if IS_64_BITS: + OPT_TABLE['jit'] = OPT_TABLE['jit'].replace('hybrid', 'boehm') + def set_opt_level(config, level): """Apply optimization suggestions on the 'config'. The optimizations depend on the selected level and possibly on the backend. Modified: pypy/branch/asmgcc-64/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/asmgcc-64/pypy/interpreter/astcompiler/assemble.py Mon Aug 2 20:58:49 2010 @@ -566,22 +566,22 @@ return (oparg % 256) + 2 * (oparg / 256) def _compute_CALL_FUNCTION(arg): - return _num_args(arg) + return -_num_args(arg) def _compute_CALL_FUNCTION_VAR(arg): - return _num_args(arg) - 1 + return -_num_args(arg) - 1 def _compute_CALL_FUNCTION_KW(arg): - return _num_args(arg) - 1 + return -_num_args(arg) - 1 def _compute_CALL_FUNCTION_VAR_KW(arg): - return _num_args(arg) - 2 + return -_num_args(arg) - 2 def _compute_CALL_LIKELY_BUILTIN(arg): return -(arg & 0xFF) + 1 def _compute_CALL_METHOD(arg): - return -arg - 1 + return -_num_args(arg) - 1 _stack_effect_computers = {} Modified: pypy/branch/asmgcc-64/pypy/interpreter/astcompiler/ast.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/interpreter/astcompiler/ast.py (original) +++ pypy/branch/asmgcc-64/pypy/interpreter/astcompiler/ast.py Mon Aug 2 20:58:49 2010 @@ -230,6 +230,7 @@ visitor.visit_FunctionDef(self) def mutate_over(self, visitor): + self.args = self.args.mutate_over(visitor) if self.body: visitor._mutate_sequence(self.body) if self.decorators: @@ -784,6 +785,8 @@ def mutate_over(self, visitor): if self.body: visitor._mutate_sequence(self.body) + if self.handlers: + visitor._mutate_sequence(self.handlers) if self.orelse: visitor._mutate_sequence(self.orelse) return visitor.visit_TryExcept(self) @@ -927,6 +930,8 @@ visitor.visit_Import(self) def mutate_over(self, visitor): + if self.names: + visitor._mutate_sequence(self.names) return visitor.visit_Import(self) def sync_app_attrs(self, space): @@ -965,6 +970,8 @@ visitor.visit_ImportFrom(self) def mutate_over(self, visitor): + if self.names: + visitor._mutate_sequence(self.names) return visitor.visit_ImportFrom(self) def sync_app_attrs(self, space): @@ -1282,6 +1289,7 @@ visitor.visit_Lambda(self) def mutate_over(self, visitor): + self.args = self.args.mutate_over(visitor) self.body = self.body.mutate_over(visitor) return visitor.visit_Lambda(self) @@ -1398,6 +1406,8 @@ def mutate_over(self, visitor): self.elt = self.elt.mutate_over(visitor) + if self.generators: + visitor._mutate_sequence(self.generators) return visitor.visit_ListComp(self) def sync_app_attrs(self, space): @@ -1437,6 +1447,8 @@ def mutate_over(self, visitor): self.elt = self.elt.mutate_over(visitor) + if self.generators: + visitor._mutate_sequence(self.generators) return visitor.visit_GeneratorExp(self) def sync_app_attrs(self, space): @@ -1562,6 +1574,8 @@ self.func = self.func.mutate_over(visitor) if self.args: visitor._mutate_sequence(self.args) + if self.keywords: + visitor._mutate_sequence(self.keywords) if self.starargs: self.starargs = self.starargs.mutate_over(visitor) if self.kwargs: @@ -2293,6 +2307,13 @@ self.w_ifs = None self.initialization_state = 7 + def mutate_over(self, visitor): + self.target = self.target.mutate_over(visitor) + self.iter = self.iter.mutate_over(visitor) + if self.ifs: + visitor._mutate_sequence(self.ifs) + return visitor.visit_comprehension(self) + def walkabout(self, visitor): visitor.visit_comprehension(self) @@ -2327,6 +2348,15 @@ self.col_offset = col_offset self.initialization_state = 31 + def mutate_over(self, visitor): + if self.type: + self.type = self.type.mutate_over(visitor) + if self.name: + self.name = self.name.mutate_over(visitor) + if self.body: + visitor._mutate_sequence(self.body) + return visitor.visit_excepthandler(self) + def walkabout(self, visitor): visitor.visit_excepthandler(self) @@ -2366,6 +2396,13 @@ self.w_defaults = None self.initialization_state = 15 + def mutate_over(self, visitor): + if self.args: + visitor._mutate_sequence(self.args) + if self.defaults: + visitor._mutate_sequence(self.defaults) + return visitor.visit_arguments(self) + def walkabout(self, visitor): visitor.visit_arguments(self) @@ -2407,6 +2444,10 @@ self.value = value self.initialization_state = 3 + def mutate_over(self, visitor): + self.value = self.value.mutate_over(visitor) + return visitor.visit_keyword(self) + def walkabout(self, visitor): visitor.visit_keyword(self) @@ -2426,6 +2467,9 @@ self.asname = asname self.initialization_state = 3 + def mutate_over(self, visitor): + return visitor.visit_alias(self) + def walkabout(self, visitor): visitor.visit_alias(self) Modified: pypy/branch/asmgcc-64/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/asmgcc-64/pypy/interpreter/astcompiler/codegen.py Mon Aug 2 20:58:49 2010 @@ -258,9 +258,11 @@ # Load decorators first, but apply them after the function is created. if func.decorators: self.visit_sequence(func.decorators) - if func.args.defaults: - self.visit_sequence(func.args.defaults) - num_defaults = len(func.args.defaults) + args = func.args + assert isinstance(args, ast.arguments) + if args.defaults: + self.visit_sequence(args.defaults) + num_defaults = len(args.defaults) else: num_defaults = 0 code = self.sub_scope(FunctionCodeGenerator, func.name, func, @@ -274,9 +276,11 @@ def visit_Lambda(self, lam): self.update_position(lam.lineno) - if lam.args.defaults: - self.visit_sequence(lam.args.defaults) - default_count = len(lam.args.defaults) + args = lam.args + assert isinstance(args, ast.arguments) + if args.defaults: + self.visit_sequence(args.defaults) + default_count = len(args.defaults) else: default_count = 0 code = self.sub_scope(LambdaCodeGenerator, "", lam, lam.lineno) @@ -956,9 +960,12 @@ elif call_type == 3: op = ops.CALL_FUNCTION_VAR_KW self.emit_op_arg(op, arg) + + def _call_has_no_star_args(self, call): + return not call.starargs and not call.kwargs def _call_has_simple_args(self, call): - return not call.starargs and not call.kwargs and not call.keywords + return self._call_has_no_star_args(call) and not call.keywords def _optimize_builtin_call(self, call): if not self.space.config.objspace.opcodes.CALL_LIKELY_BUILTIN or \ @@ -984,7 +991,7 @@ def _optimize_method_call(self, call): if not self.space.config.objspace.opcodes.CALL_METHOD or \ - not self._call_has_simple_args(call) or \ + not self._call_has_no_star_args(call) or \ not isinstance(call.func, ast.Attribute): return False attr_lookup = call.func @@ -996,7 +1003,12 @@ arg_count = len(call.args) else: arg_count = 0 - self.emit_op_arg(ops.CALL_METHOD, arg_count) + if call.keywords: + self.visit_sequence(call.keywords) + kwarg_count = len(call.keywords) + else: + kwarg_count = 0 + self.emit_op_arg(ops.CALL_METHOD, (kwarg_count << 8) | arg_count) return True def _listcomp_generator(self, list_name, gens, gen_index, elt): @@ -1275,9 +1287,11 @@ else: self.add_const(self.space.w_None) start = 0 - if func.args.args: - self._handle_nested_args(func.args.args) - self.argcount = len(func.args.args) + args = func.args + assert isinstance(args, ast.arguments) + if args.args: + self._handle_nested_args(args.args) + self.argcount = len(args.args) for i in range(start, len(func.body)): func.body[i].walkabout(self) @@ -1286,9 +1300,11 @@ def _compile(self, lam): assert isinstance(lam, ast.Lambda) - if lam.args.args: - self._handle_nested_args(lam.args.args) - self.argcount = len(lam.args.args) + args = lam.args + assert isinstance(args, ast.arguments) + if args.args: + self._handle_nested_args(args.args) + self.argcount = len(args.args) # Prevent a string from being the first constant and thus a docstring. self.add_const(self.space.w_None) lam.body.walkabout(self) Modified: pypy/branch/asmgcc-64/pypy/interpreter/astcompiler/consts.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/interpreter/astcompiler/consts.py (original) +++ pypy/branch/asmgcc-64/pypy/interpreter/astcompiler/consts.py Mon Aug 2 20:58:49 2010 @@ -18,4 +18,4 @@ PyCF_SOURCE_IS_UTF8 = 0x0100 PyCF_DONT_IMPLY_DEDENT = 0x0200 -PyCF_AST_ONLY = 0x0400 +PyCF_ONLY_AST = 0x0400 Modified: pypy/branch/asmgcc-64/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/asmgcc-64/pypy/interpreter/astcompiler/symtable.py Mon Aug 2 20:58:49 2010 @@ -353,8 +353,10 @@ def visit_FunctionDef(self, func): self.note_symbol(func.name, SYM_ASSIGNED) # Function defaults and decorators happen in the outer scope. - if func.args.defaults: - self.visit_sequence(func.args.defaults) + args = func.args + assert isinstance(args, ast.arguments) + if args.defaults: + self.visit_sequence(args.defaults) if func.decorators: self.visit_sequence(func.decorators) new_scope = FunctionScope(func.name, func.lineno, func.col_offset) @@ -420,8 +422,10 @@ self.note_symbol(name, SYM_GLOBAL) def visit_Lambda(self, lamb): - if lamb.args.defaults: - self.visit_sequence(lamb.args.defaults) + args = lamb.args + assert isinstance(args, ast.arguments) + if args.defaults: + self.visit_sequence(args.defaults) new_scope = FunctionScope("lambda", lamb.lineno, lamb.col_offset) self.push_scope(new_scope, lamb) lamb.args.walkabout(self) Modified: pypy/branch/asmgcc-64/pypy/interpreter/astcompiler/tools/asdl_py.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/interpreter/astcompiler/tools/asdl_py.py (original) +++ pypy/branch/asmgcc-64/pypy/interpreter/astcompiler/tools/asdl_py.py Mon Aug 2 20:58:49 2010 @@ -100,6 +100,7 @@ self.emit("") self.make_constructor(product.fields, product) self.emit("") + self.make_mutate_over(product, name) self.emit("def walkabout(self, visitor):", 1) self.emit("visitor.visit_%s(self)" % (name,), 2) self.emit("") @@ -183,6 +184,26 @@ have_everything = self.data.required_masks[node] | \ self.data.optional_masks[node] self.emit("self.initialization_state = %i" % (have_everything,), 2) + + def make_mutate_over(self, cons, name): + self.emit("def mutate_over(self, visitor):", 1) + for field in cons.fields: + if (field.type.value not in asdl.builtin_types and + field.type.value not in self.data.simple_types): + if field.opt or field.seq: + level = 3 + self.emit("if self.%s:" % (field.name,), 2) + else: + level = 2 + if field.seq: + sub = (field.name,) + self.emit("visitor._mutate_sequence(self.%s)" % sub, level) + else: + sub = (field.name, field.name) + self.emit("self.%s = self.%s.mutate_over(visitor)" % sub, + level) + self.emit("return visitor.visit_%s(self)" % (name,), 2) + self.emit("") def visitConstructor(self, cons, base, extra_attributes): self.emit("class %s(%s):" % (cons.name, base)) @@ -199,24 +220,7 @@ self.emit("def walkabout(self, visitor):", 1) self.emit("visitor.visit_%s(self)" % (cons.name,), 2) self.emit("") - self.emit("def mutate_over(self, visitor):", 1) - for field in cons.fields: - if field.type.value not in asdl.builtin_types and \ - field.type.value not in self.data.prod_simple: - if field.opt or field.seq: - level = 3 - self.emit("if self.%s:" % (field.name,), 2) - else: - level = 2 - if field.seq: - sub = (field.name,) - self.emit("visitor._mutate_sequence(self.%s)" % sub, level) - else: - sub = (field.name, field.name) - self.emit("self.%s = self.%s.mutate_over(visitor)" % sub, - level) - self.emit("return visitor.visit_%s(self)" % (cons.name,), 2) - self.emit("") + self.make_mutate_over(cons, cons.name) self.make_var_syncer(cons.fields + self.data.cons_attributes[cons], cons, cons.name) Modified: pypy/branch/asmgcc-64/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/asmgcc-64/pypy/interpreter/baseobjspace.py Mon Aug 2 20:58:49 2010 @@ -1102,17 +1102,6 @@ self.wrap('argument must be a unicode')) return self.unicode_w(w_obj) - def path_w(self, w_obj): - """ Like str_w, but if the object is unicode, encode it using - filesystemencoding - """ - filesystemencoding = self.sys.filesystemencoding - if (filesystemencoding and - self.is_true(self.isinstance(w_obj, self.w_unicode))): - w_obj = self.call_method(w_obj, "encode", - self.wrap(filesystemencoding)) - return self.str_w(w_obj) - def bool_w(self, w_obj): # Unwraps a bool, also accepting an int for compatibility. # This is here mostly just for gateway.int_unwrapping_space_method(). Modified: pypy/branch/asmgcc-64/pypy/interpreter/error.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/interpreter/error.py (original) +++ pypy/branch/asmgcc-64/pypy/interpreter/error.py Mon Aug 2 20:58:49 2010 @@ -344,7 +344,7 @@ else: _WINDOWS = True - def wrap_windowserror(space, e, filename=None): + def wrap_windowserror(space, e, w_filename=None): from pypy.rlib import rwin32 winerror = e.winerror @@ -353,19 +353,19 @@ except ValueError: msg = 'Windows Error %d' % winerror exc = space.w_WindowsError - if filename is not None: + if w_filename is not None: w_error = space.call_function(exc, space.wrap(winerror), - space.wrap(msg), space.wrap(filename)) + space.wrap(msg), w_filename) else: w_error = space.call_function(exc, space.wrap(winerror), space.wrap(msg)) return OperationError(exc, w_error) -def wrap_oserror(space, e, filename=None, exception_name='w_OSError'): +def wrap_oserror2(space, e, w_filename=None, exception_name='w_OSError'): assert isinstance(e, OSError) if _WINDOWS and isinstance(e, WindowsError): - return wrap_windowserror(space, e, filename) + return wrap_windowserror(space, e, w_filename) errno = e.errno try: @@ -373,10 +373,21 @@ except ValueError: msg = 'error %d' % errno exc = getattr(space, exception_name) - if filename is not None: + if w_filename is not None: w_error = space.call_function(exc, space.wrap(errno), - space.wrap(msg), space.wrap(filename)) + space.wrap(msg), w_filename) else: - w_error = space.call_function(exc, space.wrap(errno), space.wrap(msg)) + w_error = space.call_function(exc, space.wrap(errno), + space.wrap(msg)) return OperationError(exc, w_error) +wrap_oserror2._annspecialcase_ = 'specialize:arg(3)' + +def wrap_oserror(space, e, filename=None, exception_name='w_OSError'): + if filename is not None: + return wrap_oserror2(space, e, space.wrap(filename), + exception_name=exception_name) + else: + return wrap_oserror2(space, e, None, + exception_name=exception_name) wrap_oserror._annspecialcase_ = 'specialize:arg(3)' + Modified: pypy/branch/asmgcc-64/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/interpreter/gateway.py (original) +++ pypy/branch/asmgcc-64/pypy/interpreter/gateway.py Mon Aug 2 20:58:49 2010 @@ -137,9 +137,6 @@ def visit_c_nonnegint(self, el, app_sig): self.checked_space_method(el, app_sig) - def visit_path(self, el, app_sig): - self.checked_space_method(el, app_sig) - def visit__Wrappable(self, el, app_sig): name = el.__name__ argname = self.orig_arg() @@ -241,9 +238,6 @@ def visit_bufferstr(self, typ): self.run_args.append("space.bufferstr_w(%s)" % (self.scopenext(),)) - def visit_path(self, typ): - self.run_args.append("space.path_w(%s)" % (self.scopenext(),)) - def visit_nonnegint(self, typ): self.run_args.append("space.nonnegint_w(%s)" % (self.scopenext(),)) @@ -371,9 +365,6 @@ def visit_bufferstr(self, typ): self.unwrap.append("space.bufferstr_w(%s)" % (self.nextarg(),)) - def visit_path(self, typ): - self.unwrap.append("space.path_w(%s)" % (self.nextarg(),)) - def visit_nonnegint(self, typ): self.unwrap.append("space.nonnegint_w(%s)" % (self.nextarg(),)) Modified: pypy/branch/asmgcc-64/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/asmgcc-64/pypy/interpreter/pyopcode.py Mon Aug 2 20:58:49 2010 @@ -211,10 +211,6 @@ next_instr = block.handle(self, unroller) return next_instr # now inside a 'finally' block - if opcode == self.opcodedesc.YIELD_VALUE.index: - #self.last_instr = intmask(next_instr - 1) XXX clean up! - raise Yield - if opcode == self.opcodedesc.END_FINALLY.index: unroller = self.end_finally() if isinstance(unroller, SuspendedUnroller): @@ -239,7 +235,7 @@ if not opdesc.is_enabled(space): continue if opdesc.methodname in ( - 'EXTENDED_ARG', 'RETURN_VALUE', 'YIELD_VALUE', + 'EXTENDED_ARG', 'RETURN_VALUE', 'END_FINALLY', 'JUMP_ABSOLUTE'): continue # opcodes implemented above @@ -814,6 +810,9 @@ self.space.str_w(w_name)) self.pushvalue(w_obj) + def YIELD_VALUE(self, oparg, next_instr): + raise Yield + def jump_absolute(self, jumpto, next_instr, ec): return jumpto Modified: pypy/branch/asmgcc-64/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/interpreter/test/test_compiler.py (original) +++ pypy/branch/asmgcc-64/pypy/interpreter/test/test_compiler.py Mon Aug 2 20:58:49 2010 @@ -4,6 +4,7 @@ from pypy.interpreter.pycode import PyCode from pypy.interpreter.error import OperationError from pypy.interpreter.argument import Arguments +from pypy.conftest import gettestobjspace class BaseTestCompiler: def setup_method(self, method): @@ -838,6 +839,47 @@ sys.stdout = save_stdout output = s.getvalue() assert "STOP_CODE" not in output + + def test_optimize_list_comp(self): + source = """def _f(a): + return [x for x in a if None] + """ + exec source + code = _f.func_code + + import StringIO, sys, dis + s = StringIO.StringIO() + out = sys.stdout + sys.stdout = s + try: + dis.dis(code) + finally: + sys.stdout = out + output = s.getvalue() + assert "LOAD_GLOBAL" not in output + +class AppTestCallMethod(object): + def setup_class(cls): + cls.space = gettestobjspace(**{'objspace.opcodes.CALL_METHOD': True}) + + def test_call_method_kwargs(self): + source = """def _f(a): + return a.f(a=a) + """ + exec source + code = _f.func_code + + import StringIO, sys, dis + s = StringIO.StringIO() + out = sys.stdout + sys.stdout = s + try: + dis.dis(code) + finally: + sys.stdout = out + output = s.getvalue() + assert "CALL_METHOD" in output + class AppTestExceptions: def test_indentation_error(self): Modified: pypy/branch/asmgcc-64/pypy/interpreter/test/test_gateway.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/interpreter/test/test_gateway.py (original) +++ pypy/branch/asmgcc-64/pypy/interpreter/test/test_gateway.py Mon Aug 2 20:58:49 2010 @@ -454,16 +454,6 @@ assert len(l) == 1 assert space.eq_w(l[0], w("foo")) - def test_interp2app_unwrap_spec_path(self, monkeypatch): - space = self.space - def g(space, p): - return p - - app_g = gateway.interp2app(g, unwrap_spec=[gateway.ObjSpace, 'path']) - w_app_g = space.wrap(app_g) - monkeypatch.setattr(space.sys, "filesystemencoding", "utf-8") - w_res = space.call_function(w_app_g, space.wrap(u"?")) - def test_interp2app_classmethod(self): space = self.space w = space.wrap Modified: pypy/branch/asmgcc-64/pypy/jit/backend/detect_cpu.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/detect_cpu.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/detect_cpu.py Mon Aug 2 20:58:49 2010 @@ -56,6 +56,8 @@ return "pypy.jit.backend.x86.runner", "CPU" elif backend_name == 'x86-without-sse2': return "pypy.jit.backend.x86.runner", "CPU386_NO_SSE2" + elif backend_name == 'x86_64': + return "pypy.jit.backend.x86.runner", "CPU_X86_64" elif backend_name == 'cli': return "pypy.jit.backend.cli.runner", "CliCPU" elif backend_name == 'llvm': Modified: pypy/branch/asmgcc-64/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/llgraph/llimpl.py Mon Aug 2 20:58:49 2010 @@ -421,11 +421,9 @@ global _last_exception assert _last_exception is None, "exception left behind" verbose = True - operations = self.loop.operations - opindex = 0 + self.opindex = 0 while True: - self.opindex = opindex - op = operations[opindex] + op = self.loop.operations[self.opindex] args = [self.getenv(v) for v in op.args] if not op.is_final(): try: @@ -439,8 +437,8 @@ args = [self.getenv(v) for v in op.fail_args if v] assert len(op.jump_target.inputargs) == len(args) self.env = dict(zip(op.jump_target.inputargs, args)) - operations = op.jump_target.operations - opindex = 0 + self.loop = op.jump_target + self.opindex = 0 continue else: self._populate_fail_args(op) @@ -465,14 +463,13 @@ raise Exception("op.result.concretetype is %r" % (RESTYPE,)) self.env[op.result] = x - opindex += 1 + self.opindex += 1 continue if op.opnum == rop.JUMP: assert len(op.jump_target.inputargs) == len(args) self.env = dict(zip(op.jump_target.inputargs, args)) self.loop = op.jump_target - operations = self.loop.operations - opindex = 0 + self.opindex = 0 _stats.exec_jumps += 1 elif op.opnum == rop.FINISH: if self.verbose: @@ -1549,6 +1546,8 @@ setannotation(do_getarrayitem_gc_int, annmodel.SomeInteger()) setannotation(do_getarrayitem_gc_ptr, annmodel.SomePtr(llmemory.GCREF)) setannotation(do_getarrayitem_gc_float, annmodel.SomeFloat()) +setannotation(do_getarrayitem_raw_int, annmodel.SomeInteger()) +setannotation(do_getarrayitem_raw_float, annmodel.SomeFloat()) setannotation(do_getfield_gc_int, annmodel.SomeInteger()) setannotation(do_getfield_gc_ptr, annmodel.SomePtr(llmemory.GCREF)) setannotation(do_getfield_gc_float, annmodel.SomeFloat()) @@ -1560,6 +1559,8 @@ setannotation(do_setarrayitem_gc_int, annmodel.s_None) setannotation(do_setarrayitem_gc_ptr, annmodel.s_None) setannotation(do_setarrayitem_gc_float, annmodel.s_None) +setannotation(do_setarrayitem_raw_int, annmodel.s_None) +setannotation(do_setarrayitem_raw_float, annmodel.s_None) setannotation(do_setfield_gc_int, annmodel.s_None) setannotation(do_setfield_gc_ptr, annmodel.s_None) setannotation(do_setfield_gc_float, annmodel.s_None) Modified: pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/descr.py Mon Aug 2 20:58:49 2010 @@ -23,10 +23,10 @@ self._cache_call = {} def init_size_descr(self, STRUCT, sizedescr): - pass + assert isinstance(STRUCT, lltype.GcStruct) def init_array_descr(self, ARRAY, arraydescr): - pass + assert isinstance(ARRAY, lltype.GcArray) # ____________________________________________________________ @@ -67,9 +67,11 @@ class BaseFieldDescr(AbstractDescr): offset = 0 # help translation + name = '' _clsname = '' - def __init__(self, offset): + def __init__(self, name, offset): + self.name = name self.offset = offset def sort_key(self): @@ -88,7 +90,7 @@ return self._is_float_field def repr_of_descr(self): - return '<%s %s>' % (self._clsname, self.offset) + return '<%s %s %s>' % (self._clsname, self.name, self.offset) class NonGcPtrFieldDescr(BaseFieldDescr): @@ -113,7 +115,8 @@ offset, _ = symbolic.get_field_token(STRUCT, fieldname, gccache.translate_support_code) FIELDTYPE = getattr(STRUCT, fieldname) - fielddescr = getFieldDescrClass(FIELDTYPE)(offset) + name = '%s.%s' % (STRUCT._name, fieldname) + fielddescr = getFieldDescrClass(FIELDTYPE)(name, offset) cachedict = cache.setdefault(STRUCT, {}) cachedict[fieldname] = fielddescr return fielddescr @@ -205,7 +208,8 @@ assert basesize == arraydescr.get_base_size(False) assert itemsize == arraydescr.get_item_size(False) assert ofslength == arraydescr.get_ofs_length(False) - gccache.init_array_descr(ARRAY, arraydescr) + if isinstance(ARRAY, lltype.GcArray): + gccache.init_array_descr(ARRAY, arraydescr) cache[ARRAY] = arraydescr return arraydescr Modified: pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/gc.py Mon Aug 2 20:58:49 2010 @@ -351,7 +351,7 @@ gcrootmap = cls() self.gcrootmap = gcrootmap self.gcrefs = GcRefList() - self.single_gcref_descr = GcPtrFieldDescr(0) + self.single_gcref_descr = GcPtrFieldDescr('', 0) # make a TransformerLayoutBuilder and save it on the translator # where it can be fished and reused by the FrameworkGCTransformer Modified: pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/regalloc.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/regalloc.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/regalloc.py Mon Aug 2 20:58:49 2010 @@ -22,18 +22,19 @@ def get(self, box): return self.frame_bindings.get(box, None) - def loc(self, box, size): + def loc(self, box): res = self.get(box) if res is not None: return res - newloc = self.frame_pos(self.frame_depth, size) + newloc = self.frame_pos(self.frame_depth, box.type) self.frame_bindings[box] = newloc - self.frame_depth += size + # Objects returned by frame_pos must support frame_size() + self.frame_depth += newloc.frame_size() return newloc # abstract methods that need to be overwritten for specific assemblers @staticmethod - def frame_pos(loc, size): + def frame_pos(loc, type): raise NotImplementedError("Purely abstract") class RegisterManager(object): @@ -43,7 +44,6 @@ all_regs = [] no_lower_byte_regs = [] save_around_call_regs = [] - reg_width = 1 # in terms of stack space eaten def __init__(self, longevity, frame_manager=None, assembler=None): self.free_regs = self.all_regs[:] @@ -148,7 +148,7 @@ loc = self.reg_bindings[v_to_spill] del self.reg_bindings[v_to_spill] if self.frame_manager.get(v_to_spill) is None: - newloc = self.frame_manager.loc(v_to_spill, self.reg_width) + newloc = self.frame_manager.loc(v_to_spill) self.assembler.regalloc_mov(loc, newloc) return loc @@ -204,7 +204,7 @@ try: return self.reg_bindings[box] except KeyError: - return self.frame_manager.loc(box, self.reg_width) + return self.frame_manager.loc(box) def return_constant(self, v, forbidden_vars=[], selected_reg=None, imm_fine=True): @@ -260,7 +260,7 @@ self.reg_bindings[v] = loc self.assembler.regalloc_mov(prev_loc, loc) else: - loc = self.frame_manager.loc(v, self.reg_width) + loc = self.frame_manager.loc(v) self.assembler.regalloc_mov(prev_loc, loc) def force_result_in_reg(self, result_v, v, forbidden_vars=[]): @@ -280,7 +280,7 @@ self.free_regs = [reg for reg in self.free_regs if reg is not loc] return loc if v not in self.reg_bindings: - prev_loc = self.frame_manager.loc(v, self.reg_width) + prev_loc = self.frame_manager.loc(v) loc = self.force_allocate_reg(v, forbidden_vars) self.assembler.regalloc_mov(prev_loc, loc) assert v in self.reg_bindings @@ -300,7 +300,7 @@ def _sync_var(self, v): if not self.frame_manager.get(v): reg = self.reg_bindings[v] - to = self.frame_manager.loc(v, self.reg_width) + to = self.frame_manager.loc(v) self.assembler.regalloc_mov(reg, to) # otherwise it's clean Modified: pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/support.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/support.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/support.py Mon Aug 2 20:58:49 2010 @@ -30,7 +30,13 @@ funcobj = get_funcobj(fnptr) if hasattr(funcobj, 'graph'): - llinterp = LLInterpreter(rtyper) #, exc_data_ptr=exc_data_ptr) + # cache the llinterp; otherwise the remember_malloc/remember_free + # done on the LLInterpreter don't match + try: + llinterp = rtyper._on_top_of_llinterp_llinterp + except AttributeError: + llinterp = LLInterpreter(rtyper) #, exc_data_ptr=exc_data_ptr) + rtyper._on_top_of_llinterp_llinterp = llinterp def on_top_of_llinterp(*args): real_args = process_args(args) return llinterp.eval_graph(funcobj.graph, real_args) Modified: pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/test/test_descr.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/test/test_descr.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/test/test_descr.py Mon Aug 2 20:58:49 2010 @@ -53,6 +53,10 @@ assert descr_y.__class__ is GcPtrFieldDescr assert descr_z.__class__ is NonGcPtrFieldDescr assert descr_f.__class__ is clsf + assert descr_x.name == 'S.x' + assert descr_y.name == 'S.y' + assert descr_z.name == 'S.z' + assert descr_f.name == 'S.f' if not tsc: assert descr_x.offset < descr_y.offset < descr_z.offset assert descr_x.sort_key() < descr_y.sort_key() < descr_z.sort_key() @@ -228,11 +232,11 @@ # descr2 = get_field_descr(c0, S, 'y') o, _ = symbolic.get_field_token(S, 'y', False) - assert descr2.repr_of_descr() == '' % o + assert descr2.repr_of_descr() == '' % o # descr2i = get_field_descr(c0, S, 'x') o, _ = symbolic.get_field_token(S, 'x', False) - assert descr2i.repr_of_descr() == '' % o + assert descr2i.repr_of_descr() == '' % o # descr3 = get_array_descr(c0, lltype.GcArray(lltype.Ptr(S))) assert descr3.repr_of_descr() == '' Modified: pypy/branch/asmgcc-64/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/test/runner_test.py Mon Aug 2 20:58:49 2010 @@ -461,6 +461,25 @@ [funcbox] + args, 'float', descr=calldescr) assert abs(res.value - 4.6) < 0.0001 + + def test_call_many_arguments(self): + # Test calling a function with a large number of arguments (more than + # 6, which will force passing some arguments on the stack on 64-bit) + + def func(*args): + assert len(args) == 16 + # Try to sum up args in a way that would probably detect a + # transposed argument + return sum(arg * (2**i) for i, arg in enumerate(args)) + + FUNC = self.FuncType([lltype.Signed]*16, lltype.Signed) + FPTR = self.Ptr(FUNC) + calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + func_ptr = llhelper(FPTR, func) + args = range(16) + funcbox = self.get_funcbox(self.cpu, func_ptr) + res = self.execute_operation(rop.CALL, [funcbox] + map(BoxInt, args), 'int', descr=calldescr) + assert res.value == func(*args) def test_call_stack_alignment(self): # test stack alignment issues, notably for Mac OS/X. @@ -978,6 +997,8 @@ else: assert 0 operations.append(ResOperation(opnum, boxargs, boxres)) + # Unique-ify inputargs + inputargs = list(set(inputargs)) faildescr = BasicFailDescr(1) operations.append(ResOperation(rop.FINISH, [], None, descr=faildescr)) @@ -1050,9 +1071,11 @@ descr=BasicFailDescr(5))] operations[1].fail_args = [] looptoken = LoopToken() - self.cpu.compile_loop(list(testcase), operations, + # Use "set" to unique-ify inputargs + unique_testcase_list = list(set(testcase)) + self.cpu.compile_loop(unique_testcase_list, operations, looptoken) - for i, box in enumerate(testcase): + for i, box in enumerate(unique_testcase_list): self.cpu.set_future_value_float(i, box.value) fail = self.cpu.execute_token(looptoken) if fail.identifier != 5 - (expected_id^expected): @@ -1695,7 +1718,7 @@ def test_assembler_call(self): called = [] def assembler_helper(failindex, virtualizable): - assert self.cpu.get_latest_value_int(0) == 10 + assert self.cpu.get_latest_value_int(0) == 97 called.append(failindex) return 4 + 9 @@ -1708,33 +1731,41 @@ _assembler_helper_ptr) ops = ''' - [i0, i1] - i2 = int_add(i0, i1) - finish(i2)''' + [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] + i10 = int_add(i0, i1) + i11 = int_add(i10, i2) + i12 = int_add(i11, i3) + i13 = int_add(i12, i4) + i14 = int_add(i13, i5) + i15 = int_add(i14, i6) + i16 = int_add(i15, i7) + i17 = int_add(i16, i8) + i18 = int_add(i17, i9) + finish(i18)''' loop = parse(ops) looptoken = LoopToken() looptoken.outermost_jitdriver_sd = FakeJitDriverSD() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) - ARGS = [lltype.Signed, lltype.Signed] + ARGS = [lltype.Signed] * 10 RES = lltype.Signed self.cpu.portal_calldescr = self.cpu.calldescrof( lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES) - self.cpu.set_future_value_int(0, 1) - self.cpu.set_future_value_int(1, 2) + for i in range(10): + self.cpu.set_future_value_int(i, i+1) res = self.cpu.execute_token(looptoken) - assert self.cpu.get_latest_value_int(0) == 3 + assert self.cpu.get_latest_value_int(0) == 55 ops = ''' - [i4, i5] - i6 = int_add(i4, 1) - i3 = call_assembler(i6, i5, descr=looptoken) + [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] + i10 = int_add(i0, 42) + i11 = call_assembler(i10, i1, i2, i3, i4, i5, i6, i7, i8, i9, descr=looptoken) guard_not_forced()[] - finish(i3) + finish(i11) ''' loop = parse(ops, namespace=locals()) othertoken = LoopToken() self.cpu.compile_loop(loop.inputargs, loop.operations, othertoken) - self.cpu.set_future_value_int(0, 4) - self.cpu.set_future_value_int(1, 5) + for i in range(10): + self.cpu.set_future_value_int(i, i+1) res = self.cpu.execute_token(othertoken) assert self.cpu.get_latest_value_int(0) == 13 assert called Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/assembler.py Mon Aug 2 20:58:49 2010 @@ -1,4 +1,4 @@ -import sys +import sys, os from pypy.jit.backend.llsupport import symbolic from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxFloat from pypy.jit.metainterp.history import AbstractFailDescr, INT, REF, FLOAT,\ @@ -7,23 +7,34 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.annlowlevel import llhelper from pypy.tool.uid import fixid -from pypy.jit.backend.x86.regalloc import RegAlloc, WORD,\ - X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs, FRAME_FIXED_SIZE,\ - FORCE_INDEX_OFS +from pypy.jit.backend.x86.regalloc import RegAlloc, \ + X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs + +from pypy.jit.backend.x86.arch import FRAME_FIXED_SIZE, FORCE_INDEX_OFS, WORD, IS_X86_32, IS_X86_64 + +from pypy.jit.backend.x86.regloc import (eax, ecx, edx, ebx, + esp, ebp, esi, edi, + xmm0, xmm1, xmm2, xmm3, + xmm4, xmm5, xmm6, xmm7, + r8, r9, r10, r11, + r12, r13, r14, r15, + X86_64_SCRATCH_REG, + X86_64_XMM_SCRATCH_REG, + RegLoc, StackLoc, + ImmedLoc, AddressLoc, imm) + from pypy.rlib.objectmodel import we_are_translated, specialize -from pypy.jit.backend.x86 import codebuf -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86 import rx86, regloc, codebuf from pypy.jit.metainterp.resoperation import rop from pypy.jit.backend.x86.support import values_array from pypy.rlib.debug import debug_print from pypy.rlib import rgc - -# our calling convention - we pass first 6 args in registers -# and the rest stays on the stack +from pypy.jit.backend.x86.jump import remap_frame_layout +from pypy.rlib.streamio import open_file_as_stream # darwin requires the stack to be 16 bytes aligned on calls. Same for gcc 4.5.0, # better safe than sorry -CALL_ALIGN = 4 +CALL_ALIGN = 16 // WORD def align_stack_words(words): return (words + CALL_ALIGN - 1) & ~(CALL_ALIGN-1) @@ -31,16 +42,36 @@ class MachineCodeBlockWrapper(object): MC_DEFAULT_SIZE = 1024*1024 - def __init__(self, bigsize, profile_agent=None): + def __init__(self, assembler, bigsize, profile_agent=None): + self.assembler = assembler self.old_mcs = [] # keepalive self.bigsize = bigsize self._mc = self._instantiate_mc() self.function_name = None self.profile_agent = profile_agent + self.reset_reserved_bytes() def _instantiate_mc(self): # hook for testing return codebuf.MachineCodeBlock(self.bigsize) + def ensure_bytes_available(self, num_bytes): + if self.bytes_free() <= (self._reserved_bytes + num_bytes): + self.make_new_mc() + + def reserve_bytes(self, num_bytes): + self.ensure_bytes_available(num_bytes) + self._reserved_bytes += num_bytes + + def reset_reserved_bytes(self): + # XXX er.... pretty random number, just to be sure + # not to write half-instruction + self._reserved_bytes = 64 + + def get_relative_pos(self): + return self._mc.get_relative_pos() + + def overwrite(self, pos, listofchars): + return self._mc.overwrite(pos, listofchars) def bytes_free(self): return self._mc._size - self._mc.get_relative_pos() @@ -62,12 +93,25 @@ def make_new_mc(self): new_mc = self._instantiate_mc() debug_print('[new machine code block at', new_mc.tell(), ']') - self._mc.JMP(rel32(new_mc.tell())) + + if IS_X86_64: + # The scratch register is sometimes used as a temporary + # register, but the JMP below might clobber it. Rather than risk + # subtle bugs, we preserve the scratch register across the jump. + self._mc.PUSH_r(X86_64_SCRATCH_REG.value) + + self._mc.JMP(imm(new_mc.tell())) + + if IS_X86_64: + # Restore scratch reg + new_mc.POP_r(X86_64_SCRATCH_REG.value) if self.function_name is not None: self.end_function(done=False) self.start_pos = new_mc.get_relative_pos() + self.assembler.write_pending_failure_recoveries() + self._mc.done() self.old_mcs.append(self._mc) self._mc = new_mc @@ -81,24 +125,37 @@ def _new_method(name): def method(self, *args): - # XXX er.... pretty random number, just to be sure - # not to write half-instruction - if self.bytes_free() < 64: + if self.bytes_free() < self._reserved_bytes: self.make_new_mc() getattr(self._mc, name)(*args) method.func_name = name return method +for _name in rx86.all_instructions + regloc.all_extra_instructions: + setattr(MachineCodeBlockWrapper, _name, _new_method(_name)) + for name in dir(codebuf.MachineCodeBlock): if name.upper() == name or name == "writechr": setattr(MachineCodeBlockWrapper, name, _new_method(name)) +class GuardToken(object): + def __init__(self, faildescr, failargs, fail_locs, exc, desc_bytes): + self.faildescr = faildescr + self.failargs = failargs + self.fail_locs = fail_locs + self.exc = exc + self.desc_bytes = desc_bytes + + def recovery_stub_size(self): + # XXX: 32 is pulled out of the air + return 32 + len(self.desc_bytes) + class Assembler386(object): mc = None - mc2 = None mc_size = MachineCodeBlockWrapper.MC_DEFAULT_SIZE _float_constants = None _regalloc = None + _output_loop_log = None def __init__(self, cpu, translate_support_code=False, failargs_limit=1000): @@ -113,18 +170,27 @@ self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) self.fail_boxes_float = values_array(lltype.Float, failargs_limit) self.fail_ebp = 0 - self.loc_float_const_neg = None - self.loc_float_const_abs = None + self.loop_run_counter = values_array(lltype.Signed, 10000) + self.loop_names = [] + # if we have 10000 loops, we have some other problems I guess + self.float_const_neg_addr = 0 + self.float_const_abs_addr = 0 self.malloc_fixedsize_slowpath1 = 0 self.malloc_fixedsize_slowpath2 = 0 + self.pending_guard_tokens = None self.setup_failure_recovery() + self._loop_counter = 0 + self._debug = False def leave_jitted_hook(self): ptrs = self.fail_boxes_ptr.ar llop.gc_assume_young_pointers(lltype.Void, llmemory.cast_ptr_to_adr(ptrs)) - def make_sure_mc_exists(self): + def set_debug(self, v): + self._debug = v + + def setup(self): if self.mc is None: # the address of the function called by 'new' gc_ll_descr = self.cpu.gc_ll_descr @@ -143,11 +209,7 @@ ll_new_unicode = gc_ll_descr.get_funcptr_for_newunicode() self.malloc_unicode_func_addr = rffi.cast(lltype.Signed, ll_new_unicode) - # done - # we generate the loop body in 'mc' - # 'mc2' is for guard recovery code - self.mc = MachineCodeBlockWrapper(self.mc_size, self.cpu.profile_agent) - self.mc2 = MachineCodeBlockWrapper(self.mc_size) + self.mc = MachineCodeBlockWrapper(self, self.mc_size, self.cpu.profile_agent) self._build_failure_recovery(False) self._build_failure_recovery(True) if self.cpu.supports_floats: @@ -157,46 +219,72 @@ self._build_float_constants() if hasattr(gc_ll_descr, 'get_malloc_fixedsize_slowpath_addr'): self._build_malloc_fixedsize_slowpath() + s = os.environ.get('PYPYLOG') + if s: + if s.find(':') != -1: + s = s.split(':')[-1] + self.set_debug(True) + self._output_loop_log = s + ".count" + # Intialize here instead of __init__ to prevent + # pending_guard_tokens from being considered a prebuilt object, + # which sometimes causes memory leaks since the prebuilt list is + # still considered a GC root after we re-assign + # pending_guard_tokens in write_pending_failure_recoveries + self.pending_guard_tokens = [] + + def finish_once(self): + if self._debug: + output_log = self._output_loop_log + assert output_log is not None + f = open_file_as_stream(output_log, "w") + for i in range(self._loop_counter): + f.write(self.loop_names[i] + ":" + + str(self.loop_run_counter.getitem(i)) + "\n") + f.close() def _build_float_constants(self): - # 11 words: 8 words for the data, and up to 3 words for alignment - addr = lltype.malloc(rffi.CArray(lltype.Signed), 11, flavor='raw') + # 44 bytes: 32 bytes for the data, and up to 12 bytes for alignment + addr = lltype.malloc(rffi.CArray(lltype.Char), 44, flavor='raw') if not we_are_translated(): self._keepalive_malloced_float_consts = addr float_constants = rffi.cast(lltype.Signed, addr) float_constants = (float_constants + 15) & ~15 # align to 16 bytes - addr = rffi.cast(rffi.CArrayPtr(lltype.Signed), float_constants) - addr[0] = 0 # \ - addr[1] = -2147483648 # / for neg - addr[2] = 0 # - addr[3] = 0 # - addr[4] = -1 # \ - addr[5] = 2147483647 # / for abs - addr[6] = 0 # - addr[7] = 0 # - self.loc_float_const_neg = heap64(float_constants) - self.loc_float_const_abs = heap64(float_constants + 16) + addr = rffi.cast(rffi.CArrayPtr(lltype.Char), float_constants) + qword_padding = '\x00\x00\x00\x00\x00\x00\x00\x00' + # 0x8000000000000000 + neg_const = '\x00\x00\x00\x00\x00\x00\x00\x80' + # 0x7FFFFFFFFFFFFFFF + abs_const = '\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x7F' + data = neg_const + qword_padding + abs_const + qword_padding + for i in range(len(data)): + addr[i] = data[i] + self.float_const_neg_addr = float_constants + self.float_const_abs_addr = float_constants + 16 def _build_malloc_fixedsize_slowpath(self): - mc = self.mc2._mc # ---------- first helper for the slow path of malloc ---------- - self.malloc_fixedsize_slowpath1 = mc.tell() + self.malloc_fixedsize_slowpath1 = self.mc.tell() if self.cpu.supports_floats: # save the XMM registers in - for i in range(8): # the *caller* frame, from esp+8 - mc.MOVSD(mem64(esp, 8+8*i), xmm_registers[i]) - mc.SUB(edx, eax) # compute the size we want - mc.MOV(mem(esp, 4), edx) # save it as the new argument + for i in range(self.cpu.NUM_REGS):# the *caller* frame, from esp+8 + self.mc.MOVSD_sx((WORD*2)+8*i, i) + self.mc.SUB_rr(edx.value, eax.value) # compute the size we want + if IS_X86_32: + self.mc.MOV_sr(WORD, edx.value) # save it as the new argument + elif IS_X86_64: + # FIXME: We can't just clobber rdi like this, can we? + self.mc.MOV_rr(edi.value, edx.value) + addr = self.cpu.gc_ll_descr.get_malloc_fixedsize_slowpath_addr() - mc.JMP(rel32(addr)) # tail call to the real malloc + self.mc.JMP(imm(addr)) # tail call to the real malloc # ---------- second helper for the slow path of malloc ---------- - self.malloc_fixedsize_slowpath2 = mc.tell() + self.malloc_fixedsize_slowpath2 = self.mc.tell() if self.cpu.supports_floats: # restore the XMM registers - for i in range(8): # from where they were saved - mc.MOVSD(xmm_registers[i], mem64(esp, 8+8*i)) + for i in range(self.cpu.NUM_REGS):# from where they were saved + self.mc.MOVSD_xs(i, (WORD*2)+8*i) nursery_free_adr = self.cpu.gc_ll_descr.get_nursery_free_addr() - mc.MOV(edx, heap(nursery_free_adr)) # load this in EDX - mc.RET() - self.mc2.done() + self.mc.MOV(edx, heap(nursery_free_adr)) # load this in EDX + self.mc.RET() + self.mc.done() def assemble_loop(self, inputargs, operations, looptoken): """adds the following attributes to looptoken: @@ -207,15 +295,17 @@ _x86_param_depth _x86_arglocs """ + if not we_are_translated(): + # Arguments should be unique + assert len(set(inputargs)) == len(inputargs) + + self.setup() funcname = self._find_debug_merge_point(operations) - self.make_sure_mc_exists() + regalloc = RegAlloc(self, self.cpu.translate_support_code) arglocs = regalloc.prepare_loop(inputargs, operations, looptoken) looptoken._x86_arglocs = arglocs - needed_mem = len(arglocs[0]) * 16 + 16 - if needed_mem >= self.mc.bytes_free(): - self.mc.make_new_mc() # profile support name = "Loop # %s: %s" % (looptoken.number, funcname) @@ -230,23 +320,23 @@ self._patch_stackadjust(adr_stackadjust, frame_depth+param_depth) looptoken._x86_frame_depth = frame_depth looptoken._x86_param_depth = param_depth - # we need to make sure here that we don't overload an mc badly. - # a safe estimate is that we need at most 16 bytes per arg - needed_mem = len(arglocs[0]) * 16 + 16 - if needed_mem >= self.mc.bytes_free(): - self.mc.make_new_mc() + looptoken._x86_direct_bootstrap_code = self.mc.tell() self._assemble_bootstrap_direct_call(arglocs, curadr, frame_depth+param_depth) debug_print("Loop #", looptoken.number, "has address", looptoken._x86_loop_code, "to", self.mc.tell()) self.mc.end_function() + self.write_pending_failure_recoveries() - def assemble_bridge(self, faildescr, inputargs, operations): + if not we_are_translated(): + # Arguments should be unique + assert len(set(inputargs)) == len(inputargs) + + self.setup() funcname = self._find_debug_merge_point(operations) - self.make_sure_mc_exists() arglocs = self.rebuild_faillocs_from_descr( faildescr._x86_failure_recovery_bytecode) if not we_are_translated(): @@ -276,25 +366,67 @@ descr_number, "has address", adr_bridge, "to", self.mc.tell()) self.mc.end_function() + self.write_pending_failure_recoveries() + + def write_pending_failure_recoveries(self): + for tok in self.pending_guard_tokens: + # Okay to write to _mc because we've already made sure that + # there's enough space by "reserving" bytes. + addr = self.generate_quick_failure(self.mc._mc, tok.faildescr, tok.failargs, tok.fail_locs, tok.exc, tok.desc_bytes) + tok.faildescr._x86_adr_recovery_stub = addr + self.patch_jump_for_descr(tok.faildescr, addr) + + self.pending_guard_tokens = [] + self.mc.reset_reserved_bytes() + self.mc.done() def _find_debug_merge_point(self, operations): + for op in operations: if op.opnum == rop.DEBUG_MERGE_POINT: - return op.args[0]._get_str() - return "" + funcname = op.args[0]._get_str() + break + else: + funcname = "" % self._loop_counter + # invent the counter, so we don't get too confused + if self._debug: + self.loop_names.append(funcname) + self._loop_counter += 1 + return funcname def patch_jump_for_descr(self, faildescr, adr_new_target): adr_jump_offset = faildescr._x86_adr_jump_offset - mc = codebuf.InMemoryCodeBuilder(adr_jump_offset, adr_jump_offset + 4) - mc.write(packimm32(adr_new_target - adr_jump_offset - 4)) + adr_recovery_stub = faildescr._x86_adr_recovery_stub + offset = adr_new_target - (adr_jump_offset + 4) + # If the new target fits within a rel32 of the jump, just patch + # that. Otherwise, leave the original rel32 to the recovery stub in + # place, but clobber the recovery stub with a jump to the real + # target. + if rx86.fits_in_32bits(offset): + mc = codebuf.InMemoryCodeBuilder(adr_jump_offset, adr_jump_offset + 4) + mc.writeimm32(offset) + else: + # "mov r11, addr; jmp r11" is 13 bytes + mc = codebuf.InMemoryCodeBuilder(adr_recovery_stub, adr_recovery_stub + 13) + mc.MOV_ri(X86_64_SCRATCH_REG.value, adr_new_target) + mc.JMP_r(X86_64_SCRATCH_REG.value) + mc.valgrind_invalidated() mc.done() def _assemble(self, regalloc, operations): self._regalloc = regalloc + if self._debug: + # before doing anything, let's increase a counter + # we need one register free (a bit of a hack, but whatever) + self.mc.PUSH(eax) + adr = self.loop_run_counter.get_addr_for_num(self._loop_counter - 1) + self.mc.MOV(eax, heap(adr)) + self.mc.ADD(eax, imm(1)) + self.mc.MOV(heap(adr), eax) + self.mc.POP(eax) regalloc.walk_operations(operations) self.mc.done() - self.mc2.done() if we_are_translated() or self.cpu.dont_keepalive_stuff: self._regalloc = None # else keep it around for debugging frame_depth = regalloc.fm.frame_depth @@ -309,7 +441,7 @@ def _patchable_stackadjust(self): # stack adjustment LEA - self.mc.LEA(esp, fixedsize_ebp_ofs(0)) + self.mc.LEA32_rb(esp.value, 0) return self.mc.tell() - 4 def _patch_stackadjust(self, adr_lea, reserved_depth): @@ -318,23 +450,34 @@ # Compute the correct offset for the instruction LEA ESP, [EBP-4*words]. # Given that [EBP] is where we saved EBP, i.e. in the last word # of our fixed frame, then the 'words' value is: - words = (FRAME_FIXED_SIZE - 1) + reserved_depth + words = (self.cpu.FRAME_FIXED_SIZE - 1) + reserved_depth # align, e.g. for Mac OS X aligned_words = align_stack_words(words+2)-2 # 2 = EIP+EBP - mc.write(packimm32(-WORD * aligned_words)) + mc.writeimm32(-WORD * aligned_words) mc.done() def _call_header(self): - self.mc.PUSH(ebp) - self.mc.MOV(ebp, esp) - self.mc.PUSH(ebx) - self.mc.PUSH(esi) - self.mc.PUSH(edi) + self.mc.PUSH_r(ebp.value) + self.mc.MOV_rr(ebp.value, esp.value) + for regloc in self.cpu.CALLEE_SAVE_REGISTERS: + self.mc.PUSH_r(regloc.value) + # NB. the shape of the frame is hard-coded in get_basic_shape() too. # Also, make sure this is consistent with FRAME_FIXED_SIZE. return self._patchable_stackadjust() + def _call_footer(self): + self.mc.LEA_rb(esp.value, -len(self.cpu.CALLEE_SAVE_REGISTERS) * WORD) + + for i in range(len(self.cpu.CALLEE_SAVE_REGISTERS)-1, -1, -1): + self.mc.POP_r(self.cpu.CALLEE_SAVE_REGISTERS[i].value) + + self.mc.POP_r(ebp.value) + self.mc.RET() + def _assemble_bootstrap_direct_call(self, arglocs, jmpadr, stackdepth): + if IS_X86_64: + return self._assemble_bootstrap_direct_call_64(arglocs, jmpadr, stackdepth) # XXX pushing ebx esi and edi is a bit pointless, since we store # all regsiters anyway, for the case of guard_not_forced # XXX this can be improved greatly. Right now it'll behave like @@ -345,23 +488,81 @@ self._patch_stackadjust(adr_stackadjust, stackdepth) for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] - if isinstance(loc, REG): - self.mc.MOV(loc, mem(ebp, (2 + i) * WORD)) + if isinstance(loc, RegLoc): + assert not loc.is_xmm + self.mc.MOV_rb(loc.value, (2 + i) * WORD) loc = floatlocs[i] - if isinstance(loc, XMMREG): - self.mc.MOVSD(loc, mem64(ebp, (1 + i) * 2 * WORD)) + if isinstance(loc, RegLoc): + assert loc.is_xmm + self.mc.MOVSD_xb(loc.value, (1 + i) * 2 * WORD) tmp = eax xmmtmp = xmm0 for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] - if loc is not None and not isinstance(loc, REG): - self.mc.MOV(tmp, mem(ebp, (2 + i) * WORD)) + if loc is not None and not isinstance(loc, RegLoc): + self.mc.MOV_rb(tmp.value, (2 + i) * WORD) self.mc.MOV(loc, tmp) loc = floatlocs[i] - if loc is not None and not isinstance(loc, XMMREG): - self.mc.MOVSD(xmmtmp, mem64(ebp, (1 + i) * 2 * WORD)) - self.mc.MOVSD(loc, xmmtmp) - self.mc.JMP(rel32(jmpadr)) + if loc is not None and not isinstance(loc, RegLoc): + self.mc.MOVSD_xb(xmmtmp.value, (1 + i) * 2 * WORD) + assert isinstance(loc, StackLoc) + self.mc.MOVSD_bx(loc.value, xmmtmp.value) + self.mc.JMP_l(jmpadr) + return adr_stackadjust + + def _assemble_bootstrap_direct_call_64(self, arglocs, jmpadr, stackdepth): + # XXX: Very similar to _emit_call_64 + + src_locs = [] + dst_locs = [] + xmm_src_locs = [] + xmm_dst_locs = [] + get_from_stack = [] + + # In reverse order for use with pop() + unused_gpr = [r9, r8, ecx, edx, esi, edi] + unused_xmm = [xmm7, xmm6, xmm5, xmm4, xmm3, xmm2, xmm1, xmm0] + + nonfloatlocs, floatlocs = arglocs + adr_stackadjust = self._call_header() + self._patch_stackadjust(adr_stackadjust, stackdepth) + + # The lists are padded with Nones + assert len(nonfloatlocs) == len(floatlocs) + + for i in range(len(nonfloatlocs)): + loc = nonfloatlocs[i] + if loc is not None: + if len(unused_gpr) > 0: + src_locs.append(unused_gpr.pop()) + dst_locs.append(loc) + else: + get_from_stack.append((loc, False)) + + floc = floatlocs[i] + if floc is not None: + if len(unused_xmm) > 0: + xmm_src_locs.append(unused_xmm.pop()) + xmm_dst_locs.append(floc) + else: + get_from_stack.append((floc, True)) + + remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG) + remap_frame_layout(self, xmm_src_locs, xmm_dst_locs, X86_64_XMM_SCRATCH_REG) + + for i in range(len(get_from_stack)): + loc, is_xmm = get_from_stack[i] + if is_xmm: + self.mc.MOVSD_xb(X86_64_XMM_SCRATCH_REG.value, (2 + i) * WORD) + self.mc.MOVSD(loc, X86_64_XMM_SCRATCH_REG) + else: + self.mc.MOV_rb(X86_64_SCRATCH_REG.value, (2 + i) * WORD) + # XXX: We're assuming that "loc" won't require regloc to + # clobber the scratch register + self.mc.MOV(loc, X86_64_SCRATCH_REG) + + self.mc.JMP(imm(jmpadr)) + return adr_stackadjust def _assemble_bootstrap_code(self, inputargs, arglocs): @@ -369,11 +570,12 @@ adr_stackadjust = self._call_header() tmp = X86RegisterManager.all_regs[0] xmmtmp = X86XMMRegisterManager.all_regs[0] + self.mc._mc.begin_reuse_scratch_register() for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] if loc is None: continue - if isinstance(loc, REG): + if isinstance(loc, RegLoc): target = loc else: target = tmp @@ -387,17 +589,20 @@ adr = self.fail_boxes_int.get_addr_for_num(i) self.mc.MOV(target, heap(adr)) if target is not loc: - self.mc.MOV(loc, target) + assert isinstance(loc, StackLoc) + self.mc.MOV_br(loc.value, target.value) for i in range(len(floatlocs)): loc = floatlocs[i] if loc is None: continue adr = self.fail_boxes_float.get_addr_for_num(i) - if isinstance(loc, REG): - self.mc.MOVSD(loc, heap64(adr)) + if isinstance(loc, RegLoc): + self.mc.MOVSD(loc, heap(adr)) else: - self.mc.MOVSD(xmmtmp, heap64(adr)) - self.mc.MOVSD(loc, xmmtmp) + self.mc.MOVSD(xmmtmp, heap(adr)) + assert isinstance(loc, StackLoc) + self.mc.MOVSD_bx(loc.value, xmmtmp.value) + self.mc._mc.end_reuse_scratch_register() return adr_stackadjust def dump(self, text): @@ -410,27 +615,10 @@ finally: Box._extended_display = _prev - def _start_block(self): - # Return a 'mc' that can be used to write an "atomic" block, - # i.e. one that will not contain any JMP. - mc = self.mc._mc - if not we_are_translated(): - self._block_started_mc = (self.mc, mc.tell()) - self.mc = "block started" - return mc - - def _stop_block(self): - if not we_are_translated(): - assert self.mc == "block started" - self.mc, orgpos = self._block_started_mc - assert 0 <= self.mc._mc.tell() - orgpos <= 58, ( - "too many bytes in _start_block/_stop_block pair") - del self._block_started_mc - # ------------------------------------------------------------ def mov(self, from_loc, to_loc): - if isinstance(from_loc, XMMREG) or isinstance(to_loc, XMMREG): + if (isinstance(from_loc, RegLoc) and from_loc.is_xmm) or (isinstance(to_loc, RegLoc) and to_loc.is_xmm): self.mc.MOVSD(to_loc, from_loc) else: self.mc.MOV(to_loc, from_loc) @@ -438,24 +626,24 @@ regalloc_mov = mov # legacy interface def regalloc_push(self, loc): - if isinstance(loc, XMMREG): - self.mc.SUB(esp, imm(2*WORD)) - self.mc.MOVSD(mem64(esp, 0), loc) - elif isinstance(loc, MODRM64): + if isinstance(loc, RegLoc) and loc.is_xmm: + self.mc.SUB_ri(esp.value, 2*WORD) + self.mc.MOVSD_sx(0, loc.value) + elif WORD == 4 and isinstance(loc, StackLoc) and loc.width == 8: # XXX evil trick - self.mc.PUSH(mem(ebp, get_ebp_ofs(loc.position))) - self.mc.PUSH(mem(ebp, get_ebp_ofs(loc.position + 1))) + self.mc.PUSH_b(get_ebp_ofs(loc.position)) + self.mc.PUSH_b(get_ebp_ofs(loc.position + 1)) else: self.mc.PUSH(loc) def regalloc_pop(self, loc): - if isinstance(loc, XMMREG): - self.mc.MOVSD(loc, mem64(esp, 0)) - self.mc.ADD(esp, imm(2*WORD)) - elif isinstance(loc, MODRM64): + if isinstance(loc, RegLoc) and loc.is_xmm: + self.mc.MOVSD_xs(loc.value, 0) + self.mc.ADD_ri(esp.value, 2*WORD) + elif WORD == 4 and isinstance(loc, StackLoc) and loc.width == 8: # XXX evil trick - self.mc.POP(mem(ebp, get_ebp_ofs(loc.position + 1))) - self.mc.POP(mem(ebp, get_ebp_ofs(loc.position))) + self.mc.POP_b(get_ebp_ofs(loc.position + 1)) + self.mc.POP_b(get_ebp_ofs(loc.position)) else: self.mc.POP(loc) @@ -472,14 +660,14 @@ faildescr._x86_current_depths = current_depths failargs = guard_op.fail_args guard_opnum = guard_op.opnum - failaddr = self.implement_guard_recovery(guard_opnum, - faildescr, failargs, - faillocs) + guard_token = self.implement_guard_recovery(guard_opnum, + faildescr, failargs, + faillocs) if op is None: dispatch_opnum = guard_opnum else: dispatch_opnum = op.opnum - res = genop_guard_list[dispatch_opnum](self, op, guard_op, failaddr, + res = genop_guard_list[dispatch_opnum](self, op, guard_op, guard_token, arglocs, resloc) faildescr._x86_adr_jump_offset = res @@ -506,103 +694,158 @@ rl = result_loc.lowest8bits() if isinstance(op.args[0], Const): self.mc.CMP(arglocs[1], arglocs[0]) - getattr(self.mc, 'SET' + rev_cond)(rl) + self.mc.SET_ir(rx86.Conditions[rev_cond], rl.value) else: self.mc.CMP(arglocs[0], arglocs[1]) - getattr(self.mc, 'SET' + cond)(rl) - self.mc.MOVZX(result_loc, rl) + self.mc.SET_ir(rx86.Conditions[cond], rl.value) + self.mc.MOVZX8_rr(result_loc.value, rl.value) return genop_cmp def _cmpop_float(cond, is_ne=False): def genop_cmp(self, op, arglocs, result_loc): self.mc.UCOMISD(arglocs[0], arglocs[1]) - rl = result_loc.lowest8bits() - rh = result_loc.higher8bits() - getattr(self.mc, 'SET' + cond)(rl) + tmp1 = result_loc.lowest8bits() + if IS_X86_32: + tmp2 = result_loc.higher8bits() + elif IS_X86_64: + tmp2 = X86_64_SCRATCH_REG.lowest8bits() + + self.mc.SET_ir(rx86.Conditions[cond], tmp1.value) if is_ne: - self.mc.SETP(rh) - self.mc.OR(rl, rh) + self.mc.SET_ir(rx86.Conditions['P'], tmp2.value) + self.mc.OR8_rr(tmp1.value, tmp2.value) else: - self.mc.SETNP(rh) - self.mc.AND(rl, rh) - self.mc.MOVZX(result_loc, rl) + self.mc.SET_ir(rx86.Conditions['NP'], tmp2.value) + self.mc.AND8_rr(tmp1.value, tmp2.value) + self.mc.MOVZX8_rr(result_loc.value, tmp1.value) return genop_cmp def _cmpop_guard(cond, rev_cond, false_cond, false_rev_cond): - def genop_cmp_guard(self, op, guard_op, addr, arglocs, result_loc): + def genop_cmp_guard(self, op, guard_op, guard_token, arglocs, result_loc): guard_opnum = guard_op.opnum if isinstance(op.args[0], Const): self.mc.CMP(arglocs[1], arglocs[0]) if guard_opnum == rop.GUARD_FALSE: - name = 'J' + rev_cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(guard_token, rev_cond) else: - name = 'J' + false_rev_cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(guard_token, false_rev_cond) else: self.mc.CMP(arglocs[0], arglocs[1]) if guard_opnum == rop.GUARD_FALSE: - name = 'J' + cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(guard_token, cond) else: - name = 'J' + false_cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(guard_token, false_cond) return genop_cmp_guard def _cmpop_guard_float(cond, false_cond, need_jp): - def genop_cmp_guard_float(self, op, guard_op, addr, arglocs, + def genop_cmp_guard_float(self, op, guard_op, guard_token, arglocs, result_loc): guard_opnum = guard_op.opnum self.mc.UCOMISD(arglocs[0], arglocs[1]) + # 16 is enough space for the rel8 jumps below and the rel32 + # jump in implement_guard + self.mc.ensure_bytes_available(16 + guard_token.recovery_stub_size()) if guard_opnum == rop.GUARD_FALSE: - mc = self.mc._mc - name = 'J' + cond if need_jp: - mc.JP(rel8(6)) - getattr(mc, name)(rel32(addr)) - return mc.tell() - 4 + self.mc.J_il8(rx86.Conditions['P'], 6) + return self.implement_guard(guard_token, cond) else: if need_jp: - mc = self.mc._mc - mc.JP(rel8(2)) - getattr(mc, 'J' + cond)(rel8(5)) - return self.implement_guard(addr, mc.JMP) - name = 'J' + false_cond - return self.implement_guard(addr, getattr(self.mc, name)) + self.mc.J_il8(rx86.Conditions['P'], 2) + self.mc.J_il8(rx86.Conditions[cond], 5) + return self.implement_guard(guard_token) + return self.implement_guard(guard_token, false_cond) return genop_cmp_guard_float - @specialize.arg(5) - def _emit_call(self, x, arglocs, start=0, tmp=eax, force_mc=False, - mc=None): - if not force_mc: - mc = self.mc + def _emit_call(self, x, arglocs, start=0, tmp=eax): + if IS_X86_64: + return self._emit_call_64(x, arglocs, start) + p = 0 n = len(arglocs) for i in range(start, n): loc = arglocs[i] - if isinstance(loc, REG): - if isinstance(loc, XMMREG): - mc.MOVSD(mem64(esp, p), loc) + if isinstance(loc, RegLoc): + if loc.is_xmm: + self.mc.MOVSD_sx(p, loc.value) else: - mc.MOV(mem(esp, p), loc) + self.mc.MOV_sr(p, loc.value) p += round_up_to_4(loc.width) p = 0 for i in range(start, n): loc = arglocs[i] - if not isinstance(loc, REG): - if isinstance(loc, MODRM64): - mc.MOVSD(xmm0, loc) - mc.MOVSD(mem64(esp, p), xmm0) + if not isinstance(loc, RegLoc): + if loc.width == 8: + self.mc.MOVSD(xmm0, loc) + self.mc.MOVSD_sx(p, xmm0.value) else: - mc.MOV(tmp, loc) - mc.MOV(mem(esp, p), tmp) + self.mc.MOV(tmp, loc) + self.mc.MOV_sr(p, tmp.value) p += round_up_to_4(loc.width) self._regalloc.reserve_param(p//WORD) - mc.CALL(x) + # x is a location + self.mc.CALL(x) self.mark_gc_roots() + + def _emit_call_64(self, x, arglocs, start=0): + src_locs = [] + dst_locs = [] + xmm_src_locs = [] + xmm_dst_locs = [] + pass_on_stack = [] + + # In reverse order for use with pop() + unused_gpr = [r9, r8, ecx, edx, esi, edi] + unused_xmm = [xmm7, xmm6, xmm5, xmm4, xmm3, xmm2, xmm1, xmm0] + + for i in range(start, len(arglocs)): + loc = arglocs[i] + assert isinstance(loc, RegLoc) or isinstance(loc, ImmedLoc) or isinstance(loc, StackLoc) + + # XXX: Should be much simplier to tell whether a location is a float! + if (isinstance(loc, RegLoc) and loc.is_xmm) or (isinstance(loc, StackLoc) and loc.type == FLOAT): + if len(unused_xmm) > 0: + xmm_src_locs.append(loc) + xmm_dst_locs.append(unused_xmm.pop()) + else: + pass_on_stack.append(loc) + else: + if len(unused_gpr) > 0: + src_locs.append(loc) + dst_locs.append(unused_gpr.pop()) + else: + pass_on_stack.append(loc) + # Emit instructions to pass the stack arguments + # XXX: Would be nice to let remap_frame_layout take care of this, but + # we'd need to create something like StackLoc, but relative to esp, + # and I don't know if it's worth it. + for i in range(len(pass_on_stack)): + loc = pass_on_stack[i] + if not isinstance(loc, RegLoc): + if isinstance(loc, StackLoc) and loc.type == FLOAT: + self.mc.MOVSD(X86_64_XMM_SCRATCH_REG, loc) + self.mc.MOVSD_sx(i*WORD, X86_64_XMM_SCRATCH_REG.value) + else: + self.mc.MOV(X86_64_SCRATCH_REG, loc) + self.mc.MOV_sr(i*WORD, X86_64_SCRATCH_REG.value) + else: + # It's a register + if loc.is_xmm: + self.mc.MOVSD_sx(i*WORD, loc.value) + else: + self.mc.MOV_sr(i*WORD, loc.value) + + # Handle register arguments + remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG) + remap_frame_layout(self, xmm_src_locs, xmm_dst_locs, X86_64_XMM_SCRATCH_REG) + + self._regalloc.reserve_param(len(pass_on_stack)) + self.mc.CALL(x) + self.mark_gc_roots() + def call(self, addr, args, res): - self._emit_call(rel32(addr), args) + self._emit_call(imm(addr), args) assert res is eax genop_int_neg = _unaryop("NEG") @@ -613,6 +856,9 @@ genop_int_and = _binaryop("AND", True) genop_int_or = _binaryop("OR", True) genop_int_xor = _binaryop("XOR", True) + genop_int_lshift = _binaryop("SHL") + genop_int_rshift = _binaryop("SAR") + genop_uint_rshift = _binaryop("SHR") genop_float_add = _binaryop("ADDSD", True) genop_float_sub = _binaryop('SUBSD') genop_float_mul = _binaryop('MULSD', True) @@ -659,26 +905,27 @@ genop_guard_float_gt = _cmpop_guard_float("A", "BE", False) genop_guard_float_ge = _cmpop_guard_float("AE", "B", False) - def genop_guard_float_ne(self, op, guard_op, addr, arglocs, result_loc): + def genop_guard_float_ne(self, op, guard_op, guard_token, arglocs, result_loc): guard_opnum = guard_op.opnum self.mc.UCOMISD(arglocs[0], arglocs[1]) - mc = self.mc._mc + # 16 is enough space for the rel8 jumps below and the rel32 + # jump in implement_guard + self.mc.ensure_bytes_available(16 + guard_token.recovery_stub_size()) if guard_opnum == rop.GUARD_TRUE: - mc.JP(rel8(6)) - mc.JE(rel32(addr)) - return mc.tell() - 4 - else: - mc.JP(rel8(2)) - mc.JE(rel8(5)) - return self.implement_guard(addr, mc.JMP) + self.mc.J_il8(rx86.Conditions['P'], 6) + return self.implement_guard(guard_token, 'E') + else: + self.mc.J_il8(rx86.Conditions['P'], 2) + self.mc.J_il8(rx86.Conditions['E'], 5) + return self.implement_guard(guard_token) def genop_float_neg(self, op, arglocs, resloc): # Following what gcc does: res = x ^ 0x8000000000000000 - self.mc.XORPD(arglocs[0], self.loc_float_const_neg) + self.mc.XORPD(arglocs[0], heap(self.float_const_neg_addr)) def genop_float_abs(self, op, arglocs, resloc): # Following what gcc does: res = x & 0x7FFFFFFFFFFFFFFF - self.mc.ANDPD(arglocs[0], self.loc_float_const_abs) + self.mc.ANDPD(arglocs[0], heap(self.float_const_abs_addr)) def genop_cast_float_to_int(self, op, arglocs, resloc): self.mc.CVTTSD2SI(resloc, arglocs[0]) @@ -686,70 +933,56 @@ def genop_cast_int_to_float(self, op, arglocs, resloc): self.mc.CVTSI2SD(resloc, arglocs[0]) - def genop_int_lshift(self, op, arglocs, resloc): - loc, loc2 = arglocs - if loc2 is ecx: - loc2 = cl - self.mc.SHL(loc, loc2) - - def genop_int_rshift(self, op, arglocs, resloc): - loc, loc2 = arglocs - if loc2 is ecx: - loc2 = cl - self.mc.SAR(loc, loc2) - - def genop_uint_rshift(self, op, arglocs, resloc): - loc, loc2 = arglocs - if loc2 is ecx: - loc2 = cl - self.mc.SHR(loc, loc2) - - def genop_guard_int_is_true(self, op, guard_op, addr, arglocs, resloc): + def genop_guard_int_is_true(self, op, guard_op, guard_token, arglocs, resloc): guard_opnum = guard_op.opnum - self.mc.CMP(arglocs[0], imm8(0)) + self.mc.CMP(arglocs[0], imm(0)) if guard_opnum == rop.GUARD_TRUE: - return self.implement_guard(addr, self.mc.JZ) + return self.implement_guard(guard_token, 'Z') else: - return self.implement_guard(addr, self.mc.JNZ) + return self.implement_guard(guard_token, 'NZ') def genop_int_is_true(self, op, arglocs, resloc): - self.mc.CMP(arglocs[0], imm8(0)) + self.mc.CMP(arglocs[0], imm(0)) rl = resloc.lowest8bits() - self.mc.SETNE(rl) - self.mc.MOVZX(resloc, rl) + self.mc.SET_ir(rx86.Conditions['NE'], rl.value) + self.mc.MOVZX8(resloc, rl) - def genop_guard_int_is_zero(self, op, guard_op, addr, arglocs, resloc): + def genop_guard_int_is_zero(self, op, guard_op, guard_token, arglocs, resloc): guard_opnum = guard_op.opnum - self.mc.CMP(arglocs[0], imm8(0)) + self.mc.CMP(arglocs[0], imm(0)) if guard_opnum == rop.GUARD_TRUE: - return self.implement_guard(addr, self.mc.JNZ) + return self.implement_guard(guard_token, 'NZ') else: - return self.implement_guard(addr, self.mc.JZ) + return self.implement_guard(guard_token, 'Z') def genop_int_is_zero(self, op, arglocs, resloc): - self.mc.CMP(arglocs[0], imm8(0)) + self.mc.CMP(arglocs[0], imm(0)) rl = resloc.lowest8bits() - self.mc.SETE(rl) - self.mc.MOVZX(resloc, rl) + self.mc.SET_ir(rx86.Conditions['E'], rl.value) + self.mc.MOVZX8(resloc, rl) def genop_same_as(self, op, arglocs, resloc): self.mov(arglocs[0], resloc) #genop_cast_ptr_to_int = genop_same_as def genop_int_mod(self, op, arglocs, resloc): - self.mc.CDQ() - self.mc.IDIV(ecx) + if IS_X86_32: + self.mc.CDQ() + elif IS_X86_64: + self.mc.CQO() + + self.mc.IDIV_r(ecx.value) genop_int_floordiv = genop_int_mod def genop_uint_floordiv(self, op, arglocs, resloc): - self.mc.XOR(edx, edx) - self.mc.DIV(ecx) + self.mc.XOR_rr(edx.value, edx.value) + self.mc.DIV_r(ecx.value) def genop_new_with_vtable(self, op, arglocs, result_loc): assert result_loc is eax loc_vtable = arglocs[-1] - assert isinstance(loc_vtable, IMM32) + assert isinstance(loc_vtable, ImmedLoc) arglocs = arglocs[:-1] self.call(self.malloc_func_addr, arglocs, eax) # xxx ignore NULL returns for now @@ -757,7 +990,9 @@ def set_vtable(self, loc, loc_vtable): if self.cpu.vtable_offset is not None: - self.mc.MOV(mem(loc, self.cpu.vtable_offset), loc_vtable) + assert isinstance(loc, RegLoc) + assert isinstance(loc_vtable, ImmedLoc) + self.mc.MOV_mi((loc.value, self.cpu.vtable_offset), loc_vtable.value) # XXX genop_new is abused for all varsized mallocs with Boehm, for now # (instead of genop_new_array, genop_newstr, genop_newunicode) @@ -779,16 +1014,19 @@ def genop_getfield_gc(self, op, arglocs, resloc): base_loc, ofs_loc, size_loc = arglocs - assert isinstance(size_loc, IMM32) + assert isinstance(size_loc, ImmedLoc) + assert isinstance(resloc, RegLoc) size = size_loc.value - if size == 1: - self.mc.MOVZX(resloc, addr8_add(base_loc, ofs_loc)) + + source_addr = AddressLoc(base_loc, ofs_loc) + if resloc.is_xmm: + self.mc.MOVSD(resloc, source_addr) + elif size == 1: + self.mc.MOVZX8(resloc, source_addr) elif size == 2: - self.mc.MOVZX(resloc, addr_add(base_loc, ofs_loc)) + self.mc.MOVZX16(resloc, source_addr) elif size == WORD: - self.mc.MOV(resloc, addr_add(base_loc, ofs_loc)) - elif size == 8: - self.mc.MOVSD(resloc, addr64_add(base_loc, ofs_loc)) + self.mc.MOV(resloc, source_addr) else: raise NotImplementedError("getfield size = %d" % size) @@ -798,16 +1036,16 @@ def genop_getarrayitem_gc(self, op, arglocs, resloc): base_loc, ofs_loc, scale, ofs = arglocs - assert isinstance(ofs, IMM32) - assert isinstance(scale, IMM32) + assert isinstance(ofs, ImmedLoc) + assert isinstance(scale, ImmedLoc) if op.result.type == FLOAT: - self.mc.MOVSD(resloc, addr64_add(base_loc, ofs_loc, ofs.value, + self.mc.MOVSD(resloc, addr_add(base_loc, ofs_loc, ofs.value, scale.value)) else: if scale.value == 0: - self.mc.MOVZX(resloc, addr8_add(base_loc, ofs_loc, ofs.value, + self.mc.MOVZX8(resloc, addr_add(base_loc, ofs_loc, ofs.value, scale.value)) - elif scale.value == 2: + elif (1 << scale.value) == WORD: self.mc.MOV(resloc, addr_add(base_loc, ofs_loc, ofs.value, scale.value)) else: @@ -819,34 +1057,33 @@ def genop_discard_setfield_gc(self, op, arglocs): base_loc, ofs_loc, size_loc, value_loc = arglocs - assert isinstance(size_loc, IMM32) + assert isinstance(size_loc, ImmedLoc) size = size_loc.value - if size == WORD * 2: - self.mc.MOVSD(addr64_add(base_loc, ofs_loc), value_loc) + dest_addr = AddressLoc(base_loc, ofs_loc) + if isinstance(value_loc, RegLoc) and value_loc.is_xmm: + self.mc.MOVSD(dest_addr, value_loc) elif size == WORD: - self.mc.MOV(addr_add(base_loc, ofs_loc), value_loc) + self.mc.MOV(dest_addr, value_loc) elif size == 2: - self.mc.MOV16(addr_add(base_loc, ofs_loc), value_loc) + self.mc.MOV16(dest_addr, value_loc) elif size == 1: - self.mc.MOV(addr8_add(base_loc, ofs_loc), value_loc.lowest8bits()) + self.mc.MOV8(dest_addr, value_loc.lowest8bits()) else: print "[asmgen]setfield addr size %d" % size raise NotImplementedError("Addr size %d" % size) def genop_discard_setarrayitem_gc(self, op, arglocs): base_loc, ofs_loc, value_loc, scale_loc, baseofs = arglocs - assert isinstance(baseofs, IMM32) - assert isinstance(scale_loc, IMM32) + assert isinstance(baseofs, ImmedLoc) + assert isinstance(scale_loc, ImmedLoc) + dest_addr = AddressLoc(base_loc, ofs_loc, scale_loc.value, baseofs.value) if op.args[2].type == FLOAT: - self.mc.MOVSD(addr64_add(base_loc, ofs_loc, baseofs.value, - scale_loc.value), value_loc) + self.mc.MOVSD(dest_addr, value_loc) else: - if scale_loc.value == 2: - self.mc.MOV(addr_add(base_loc, ofs_loc, baseofs.value, - scale_loc.value), value_loc) + if (1 << scale_loc.value) == WORD: + self.mc.MOV(dest_addr, value_loc) elif scale_loc.value == 0: - self.mc.MOV(addr8_add(base_loc, ofs_loc, baseofs.value, - scale_loc.value), value_loc.lowest8bits()) + self.mc.MOV8(dest_addr, value_loc.lowest8bits()) else: raise NotImplementedError("scale = %d" % scale_loc.value) @@ -855,17 +1092,17 @@ basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) assert itemsize == 1 - self.mc.MOV(addr8_add(base_loc, ofs_loc, basesize), - val_loc.lowest8bits()) + dest_addr = AddressLoc(base_loc, ofs_loc, 0, basesize) + self.mc.MOV8(dest_addr, val_loc.lowest8bits()) def genop_discard_unicodesetitem(self, op, arglocs): base_loc, ofs_loc, val_loc = arglocs basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) if itemsize == 4: - self.mc.MOV(addr_add(base_loc, ofs_loc, basesize, 2), val_loc) + self.mc.MOV32(AddressLoc(base_loc, ofs_loc, 2, basesize), val_loc) elif itemsize == 2: - self.mc.MOV16(addr_add(base_loc, ofs_loc, basesize, 1), val_loc) + self.mc.MOV16(AddressLoc(base_loc, ofs_loc, 1, basesize), val_loc) else: assert 0, itemsize @@ -886,7 +1123,7 @@ def genop_arraylen_gc(self, op, arglocs, resloc): base_loc, ofs_loc = arglocs - assert isinstance(ofs_loc, IMM32) + assert isinstance(ofs_loc, ImmedLoc) self.mc.MOV(resloc, addr_add_const(base_loc, ofs_loc.value)) def genop_strgetitem(self, op, arglocs, resloc): @@ -894,83 +1131,83 @@ basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) assert itemsize == 1 - self.mc.MOVZX(resloc, addr8_add(base_loc, ofs_loc, basesize)) + self.mc.MOVZX8(resloc, AddressLoc(base_loc, ofs_loc, 0, basesize)) def genop_unicodegetitem(self, op, arglocs, resloc): base_loc, ofs_loc = arglocs basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) if itemsize == 4: - self.mc.MOV(resloc, addr_add(base_loc, ofs_loc, basesize, 2)) + self.mc.MOV32(resloc, AddressLoc(base_loc, ofs_loc, 2, basesize)) elif itemsize == 2: - self.mc.MOVZX(resloc, addr_add(base_loc, ofs_loc, basesize, 1)) + self.mc.MOVZX16(resloc, AddressLoc(base_loc, ofs_loc, 1, basesize)) else: assert 0, itemsize - def genop_guard_guard_true(self, ign_1, guard_op, addr, locs, ign_2): + def genop_guard_guard_true(self, ign_1, guard_op, guard_token, locs, ign_2): loc = locs[0] self.mc.TEST(loc, loc) - return self.implement_guard(addr, self.mc.JZ) + return self.implement_guard(guard_token, 'Z') genop_guard_guard_nonnull = genop_guard_guard_true - def genop_guard_guard_no_exception(self, ign_1, guard_op, addr, + def genop_guard_guard_no_exception(self, ign_1, guard_op, guard_token, locs, ign_2): self.mc.CMP(heap(self.cpu.pos_exception()), imm(0)) - return self.implement_guard(addr, self.mc.JNZ) + return self.implement_guard(guard_token, 'NZ') - def genop_guard_guard_exception(self, ign_1, guard_op, addr, + def genop_guard_guard_exception(self, ign_1, guard_op, guard_token, locs, resloc): loc = locs[0] loc1 = locs[1] self.mc.MOV(loc1, heap(self.cpu.pos_exception())) self.mc.CMP(loc1, loc) - addr = self.implement_guard(addr, self.mc.JNE) + addr = self.implement_guard(guard_token, 'NE') if resloc is not None: self.mc.MOV(resloc, heap(self.cpu.pos_exc_value())) self.mc.MOV(heap(self.cpu.pos_exception()), imm(0)) self.mc.MOV(heap(self.cpu.pos_exc_value()), imm(0)) return addr - def _gen_guard_overflow(self, guard_op, addr): + def _gen_guard_overflow(self, guard_op, guard_token): guard_opnum = guard_op.opnum if guard_opnum == rop.GUARD_NO_OVERFLOW: - return self.implement_guard(addr, self.mc.JO) + return self.implement_guard(guard_token, 'O') elif guard_opnum == rop.GUARD_OVERFLOW: - return self.implement_guard(addr, self.mc.JNO) + return self.implement_guard(guard_token, 'NO') else: print "int_xxx_ovf followed by", guard_op.getopname() raise AssertionError - def genop_guard_int_add_ovf(self, op, guard_op, addr, arglocs, result_loc): + def genop_guard_int_add_ovf(self, op, guard_op, guard_token, arglocs, result_loc): self.genop_int_add(op, arglocs, result_loc) - return self._gen_guard_overflow(guard_op, addr) + return self._gen_guard_overflow(guard_op, guard_token) - def genop_guard_int_sub_ovf(self, op, guard_op, addr, arglocs, result_loc): + def genop_guard_int_sub_ovf(self, op, guard_op, guard_token, arglocs, result_loc): self.genop_int_sub(op, arglocs, result_loc) - return self._gen_guard_overflow(guard_op, addr) + return self._gen_guard_overflow(guard_op, guard_token) - def genop_guard_int_mul_ovf(self, op, guard_op, addr, arglocs, result_loc): + def genop_guard_int_mul_ovf(self, op, guard_op, guard_token, arglocs, result_loc): self.genop_int_mul(op, arglocs, result_loc) - return self._gen_guard_overflow(guard_op, addr) + return self._gen_guard_overflow(guard_op, guard_token) - def genop_guard_guard_false(self, ign_1, guard_op, addr, locs, ign_2): + def genop_guard_guard_false(self, ign_1, guard_op, guard_token, locs, ign_2): loc = locs[0] self.mc.TEST(loc, loc) - return self.implement_guard(addr, self.mc.JNZ) + return self.implement_guard(guard_token, 'NZ') genop_guard_guard_isnull = genop_guard_guard_false - def genop_guard_guard_value(self, ign_1, guard_op, addr, locs, ign_2): + def genop_guard_guard_value(self, ign_1, guard_op, guard_token, locs, ign_2): if guard_op.args[0].type == FLOAT: assert guard_op.args[1].type == FLOAT self.mc.UCOMISD(locs[0], locs[1]) else: self.mc.CMP(locs[0], locs[1]) - return self.implement_guard(addr, self.mc.JNE) + return self.implement_guard(guard_token, 'NE') - def _cmp_guard_class(self, mc, locs): + def _cmp_guard_class(self, locs): offset = self.cpu.vtable_offset if offset is not None: - mc.CMP(mem(locs[0], offset), locs[1]) + self.mc.CMP(mem(locs[0], offset), locs[1]) else: # XXX hard-coded assumption: to go from an object to its class # we use the following algorithm: @@ -979,7 +1216,7 @@ # - multiply by 4 and use it as an offset in type_info_group # - add 16 bytes, to go past the TYPE_INFO structure loc = locs[1] - assert isinstance(loc, IMM32) + assert isinstance(loc, ImmedLoc) classptr = loc.value # here, we have to go back from 'classptr' to the value expected # from reading the 16 bits in the object header @@ -988,37 +1225,37 @@ type_info_group = llop.gc_get_type_info_group(llmemory.Address) type_info_group = rffi.cast(lltype.Signed, type_info_group) expected_typeid = (classptr - sizeof_ti - type_info_group) >> 2 - mc.CMP16(mem(locs[0], 0), imm32(expected_typeid)) + self.mc.CMP16(mem(locs[0], 0), ImmedLoc(expected_typeid)) - def genop_guard_guard_class(self, ign_1, guard_op, addr, locs, ign_2): - mc = self._start_block() - self._cmp_guard_class(mc, locs) - self._stop_block() - return self.implement_guard(addr, self.mc.JNE) + def genop_guard_guard_class(self, ign_1, guard_op, guard_token, locs, ign_2): + self.mc.ensure_bytes_available(256) + self._cmp_guard_class(locs) + return self.implement_guard(guard_token, 'NE') def genop_guard_guard_nonnull_class(self, ign_1, guard_op, - addr, locs, ign_2): - mc = self._start_block() - mc.CMP(locs[0], imm8(1)) - mc.JB(rel8_patched_later) - jb_location = mc.get_relative_pos() - self._cmp_guard_class(mc, locs) + guard_token, locs, ign_2): + self.mc.ensure_bytes_available(256) + self.mc.CMP(locs[0], imm(1)) + # Patched below + self.mc.J_il8(rx86.Conditions['B'], 0) + jb_location = self.mc.get_relative_pos() + self._cmp_guard_class(locs) # patch the JB above - offset = mc.get_relative_pos() - jb_location + offset = self.mc.get_relative_pos() - jb_location assert 0 < offset <= 127 - mc.overwrite(jb_location-1, [chr(offset)]) - self._stop_block() + self.mc.overwrite(jb_location-1, [chr(offset)]) # - return self.implement_guard(addr, self.mc.JNE) + return self.implement_guard(guard_token, 'NE') def implement_guard_recovery(self, guard_opnum, faildescr, failargs, fail_locs): exc = (guard_opnum == rop.GUARD_EXCEPTION or guard_opnum == rop.GUARD_NO_EXCEPTION or guard_opnum == rop.GUARD_NOT_FORCED) - return self.generate_quick_failure(faildescr, failargs, fail_locs, exc) + desc_bytes = self.failure_recovery_description(failargs, fail_locs) + return GuardToken(faildescr, failargs, fail_locs, exc, desc_bytes) - def generate_quick_failure(self, faildescr, failargs, fail_locs, exc): + def generate_quick_failure(self, mc, faildescr, failargs, fail_locs, exc, desc_bytes): """Generate the initial code for handling a failure. We try to keep it as compact as possible. The idea is that this code is executed at most once (and very often, zero times); when @@ -1026,38 +1263,43 @@ really handle recovery from this particular failure. """ fail_index = self.cpu.get_fail_descr_number(faildescr) - bytes_needed = 20 + 5 * len(failargs) # conservative estimate - if self.mc2.bytes_free() < bytes_needed: - self.mc2.make_new_mc() - mc = self.mc2._mc addr = mc.tell() withfloats = False for box in failargs: if box is not None and box.type == FLOAT: withfloats = True break - mc.CALL(rel32(self.failure_recovery_code[exc + 2 * withfloats])) + mc.CALL(imm(self.failure_recovery_code[exc + 2 * withfloats])) # write tight data that describes the failure recovery faildescr._x86_failure_recovery_bytecode = mc.tell() - self.write_failure_recovery_description(mc, failargs, fail_locs) + for byte in desc_bytes: + mc.writechr(ord(byte)) # write the fail_index too - mc.write(packimm32(fail_index)) + mc.writeimm32(fail_index) # for testing the decoding, write a final byte 0xCC if not we_are_translated(): mc.writechr(0xCC) faildescr._x86_debug_faillocs = [loc for loc in fail_locs if loc is not None] + + # Make sure the recovery stub is at least 16 bytes long (for the + # case where we overwrite the recovery stub with a 64-bit absolute + # jump) + while mc.tell() - addr < 16: + mc.writechr(0x00) return addr DESCR_REF = 0x00 DESCR_INT = 0x01 DESCR_FLOAT = 0x02 DESCR_SPECIAL = 0x03 - CODE_FROMSTACK = 4*8 + # XXX: 4*8 works on i386, should we optimize for that case? + CODE_FROMSTACK = 4*16 CODE_STOP = 0 | DESCR_SPECIAL CODE_HOLE = 4 | DESCR_SPECIAL - def write_failure_recovery_description(self, mc, failargs, locs): + def failure_recovery_description(self, failargs, locs): + desc_bytes = [] for i in range(len(failargs)): arg = failargs[i] if arg is not None: @@ -1070,24 +1312,30 @@ else: raise AssertionError("bogus kind") loc = locs[i] - if isinstance(loc, MODRM): + if isinstance(loc, StackLoc): n = self.CODE_FROMSTACK//4 + loc.position else: - assert isinstance(loc, REG) - n = loc.op + assert isinstance(loc, RegLoc) + n = loc.value n = kind + 4*n while n > 0x7F: - mc.writechr((n & 0x7F) | 0x80) + desc_bytes.append(chr((n & 0x7F) | 0x80)) n >>= 7 else: n = self.CODE_HOLE - mc.writechr(n) - mc.writechr(self.CODE_STOP) + desc_bytes.append(chr(n)) + desc_bytes.append(chr(self.CODE_STOP)) # assert that the fail_boxes lists are big enough assert len(failargs) <= self.fail_boxes_int.SIZE + return desc_bytes + + def write_failure_recovery_description(self, mc, failargs, locs): + for byte in self.failure_recovery_description(failargs, locs): + mc.writechr(ord(byte)) def rebuild_faillocs_from_descr(self, bytecode): from pypy.jit.backend.x86.regalloc import X86FrameManager + descr_to_box_type = [REF, INT, FLOAT] bytecode = rffi.cast(rffi.UCHARP, bytecode) arglocs = [] while 1: @@ -1112,7 +1360,7 @@ size = 2 else: size = 1 - loc = X86FrameManager.frame_pos(code, size) + loc = X86FrameManager.frame_pos(code, descr_to_box_type[kind]) elif code == self.CODE_STOP: break elif code == self.CODE_HOLE: @@ -1122,16 +1370,16 @@ kind = code & 3 code >>= 2 if kind == self.DESCR_FLOAT: - loc = xmm_registers[code] + loc = regloc.XMMREGLOCS[code] else: - loc = registers[code] + loc = regloc.REGLOCS[code] arglocs.append(loc) return arglocs[:] @rgc.no_collect def grab_frame_values(self, bytecode, frame_addr, allregisters): # no malloc allowed here!! - self.fail_ebp = allregisters[16 + ebp.op] + self.fail_ebp = allregisters[16 + ebp.value] num = 0 value_hi = 0 while 1: @@ -1154,7 +1402,7 @@ code = (code - self.CODE_FROMSTACK) >> 2 stackloc = frame_addr + get_ebp_ofs(code) value = rffi.cast(rffi.LONGP, stackloc)[0] - if kind == self.DESCR_FLOAT: + if kind == self.DESCR_FLOAT and WORD == 4: value_hi = value value = rffi.cast(rffi.LONGP, stackloc - 4)[0] else: @@ -1168,8 +1416,11 @@ break code >>= 2 if kind == self.DESCR_FLOAT: - value = allregisters[2*code] - value_hi = allregisters[2*code + 1] + if WORD == 4: + value = allregisters[2*code] + value_hi = allregisters[2*code + 1] + else: + value = allregisters[code] else: value = allregisters[16 + code] @@ -1180,7 +1431,8 @@ tgt = self.fail_boxes_ptr.get_addr_for_num(num) elif kind == self.DESCR_FLOAT: tgt = self.fail_boxes_float.get_addr_for_num(num) - rffi.cast(rffi.LONGP, tgt)[1] = value_hi + if WORD == 4: + rffi.cast(rffi.LONGP, tgt)[1] = value_hi else: assert 0, "bogus kind" rffi.cast(rffi.LONGP, tgt)[0] = value @@ -1189,7 +1441,8 @@ if not we_are_translated(): assert bytecode[4] == 0xCC self.fail_boxes_count = num - fail_index = rffi.cast(rffi.LONGP, bytecode)[0] + fail_index = rffi.cast(rffi.INTP, bytecode)[0] + fail_index = rffi.cast(lltype.Signed, fail_index) return fail_index def setup_failure_recovery(self): @@ -1200,8 +1453,8 @@ # original value of the registers, optionally the original # value of XMM registers, and finally a reference to the # recovery bytecode. See _build_failure_recovery() for details. - stack_at_ebp = registers[ebp.op] - bytecode = rffi.cast(rffi.UCHARP, registers[8]) + stack_at_ebp = registers[ebp.value] + bytecode = rffi.cast(rffi.UCHARP, registers[self.cpu.NUM_REGS]) allregisters = rffi.ptradd(registers, -16) return self.grab_frame_values(bytecode, stack_at_ebp, allregisters) @@ -1216,23 +1469,23 @@ self.failure_recovery_func) failure_recovery_func = rffi.cast(lltype.Signed, failure_recovery_func) - mc = self.mc2._mc + mc = self.mc._mc # Assume that we are called at the beginning, when there is no risk # that 'mc' runs out of space. Checked by asserts in mc.write(). recovery_addr = mc.tell() - mc.PUSH(edi) - mc.PUSH(esi) - mc.PUSH(ebp) - mc.PUSH(esp) # <-- not really used, but needed to take up the space - mc.PUSH(ebx) - mc.PUSH(edx) - mc.PUSH(ecx) - mc.PUSH(eax) - mc.MOV(esi, esp) + + # Push all general purpose registers + for gpr in range(self.cpu.NUM_REGS-1, -1, -1): + mc.PUSH_r(gpr) + + # ebx/rbx is callee-save in both i386 and x86-64 + mc.MOV_rr(ebx.value, esp.value) + if withfloats: - mc.SUB(esp, imm(8*8)) - for i in range(8): - mc.MOVSD(mem64(esp, 8*i), xmm_registers[i]) + # Push all float registers + mc.SUB_ri(esp.value, self.cpu.NUM_REGS*8) + for i in range(self.cpu.NUM_REGS): + mc.MOVSD_sx(8*i, i) # we call a provided function that will # - call our on_leave_jitted_hook which will mark @@ -1240,7 +1493,7 @@ # avoid unwarranted freeing # - optionally save exception depending on the flag addr = self.cpu.get_on_leave_jitted_int(save_exception=exc) - mc.CALL(rel32(addr)) + mc.CALL(imm(addr)) # the following call saves all values from the stack and from # registers to the right 'fail_boxes_' location. @@ -1250,50 +1503,58 @@ # bytecode, pushed just before by the CALL instruction written by # generate_quick_failure(). XXX misaligned stack in the call, but # it's ok because failure_recovery_func is not calling anything more - mc.PUSH(esi) - mc.CALL(rel32(failure_recovery_func)) + + # XXX + if IS_X86_32: + mc.PUSH_r(ebx.value) + elif IS_X86_64: + mc.MOV_rr(edi.value, ebx.value) + # XXX: Correct to only align the stack on 64-bit? + mc.AND_ri(esp.value, -16) + else: + raise AssertionError("Shouldn't happen") + + mc.CALL(imm(failure_recovery_func)) # returns in eax the fail_index # now we return from the complete frame, which starts from - # _assemble_bootstrap_code(). The LEA below throws away most - # of the frame, including all the PUSHes that we did just above. - mc.LEA(esp, addr_add(ebp, imm(-3 * WORD))) - mc.POP(edi) # [ebp-12] - mc.POP(esi) # [ebp-8] - mc.POP(ebx) # [ebp-4] - mc.POP(ebp) # [ebp] - mc.RET() - self.mc2.done() + # _assemble_bootstrap_code(). The LEA in _call_footer below throws + # away most of the frame, including all the PUSHes that we did just + # above. + + self._call_footer() + self.mc.done() self.failure_recovery_code[exc + 2 * withfloats] = recovery_addr def generate_failure(self, fail_index, locs, exc, locs_are_ref): - mc = self.mc + self.mc._mc.begin_reuse_scratch_register() for i in range(len(locs)): loc = locs[i] - if isinstance(loc, REG): - if loc.width == 8: + if isinstance(loc, RegLoc): + if loc.is_xmm: adr = self.fail_boxes_float.get_addr_for_num(i) - mc.MOVSD(heap64(adr), loc) + self.mc.MOVSD(heap(adr), loc) else: if locs_are_ref[i]: adr = self.fail_boxes_ptr.get_addr_for_num(i) else: adr = self.fail_boxes_int.get_addr_for_num(i) - mc.MOV(heap(adr), loc) + self.mc.MOV(heap(adr), loc) for i in range(len(locs)): loc = locs[i] - if not isinstance(loc, REG): - if loc.width == 8: - mc.MOVSD(xmm0, loc) + if not isinstance(loc, RegLoc): + if isinstance(loc, StackLoc) and loc.type == FLOAT: + self.mc.MOVSD_xb(xmm0.value, loc.value) adr = self.fail_boxes_float.get_addr_for_num(i) - mc.MOVSD(heap64(adr), xmm0) + self.mc.MOVSD(heap(adr), xmm0) else: if locs_are_ref[i]: adr = self.fail_boxes_ptr.get_addr_for_num(i) else: adr = self.fail_boxes_int.get_addr_for_num(i) - mc.MOV(eax, loc) - mc.MOV(heap(adr), eax) + self.mc.MOV(eax, loc) + self.mc.MOV(heap(adr), eax) + self.mc._mc.end_reuse_scratch_register() # we call a provided function that will # - call our on_leave_jitted_hook which will mark @@ -1301,28 +1562,31 @@ # avoid unwarranted freeing # - optionally save exception depending on the flag addr = self.cpu.get_on_leave_jitted_int(save_exception=exc) - mc.CALL(rel32(addr)) + self.mc.CALL(imm(addr)) - mc.LEA(esp, addr_add(ebp, imm(-3 * WORD))) - mc.MOV(eax, imm(fail_index)) - mc.POP(edi) # [ebp-12] - mc.POP(esi) # [ebp-8] - mc.POP(ebx) # [ebp-4] - mc.POP(ebp) # [ebp] - mc.RET() - - @specialize.arg(2) - def implement_guard(self, addr, emit_jump): - emit_jump(rel32(addr)) + self.mc.MOV_ri(eax.value, fail_index) + + # exit function + self._call_footer() + + def implement_guard(self, guard_token, condition=None): + self.mc.reserve_bytes(guard_token.recovery_stub_size()) + self.pending_guard_tokens.append(guard_token) + # XXX: These jumps are patched later, the self.mc.tell() are just + # dummy values + if condition: + self.mc.J_il(rx86.Conditions[condition], self.mc.tell()) + else: + self.mc.JMP_l(self.mc.tell()) return self.mc.tell() - 4 def genop_call(self, op, arglocs, resloc): sizeloc = arglocs[0] - assert isinstance(sizeloc, IMM32) + assert isinstance(sizeloc, ImmedLoc) size = sizeloc.value if isinstance(op.args[0], Const): - x = rel32(op.args[0].getint()) + x = imm(op.args[0].getint()) else: x = arglocs[1] if x is eax: @@ -1332,35 +1596,35 @@ self._emit_call(x, arglocs, 2, tmp=tmp) - if isinstance(resloc, MODRM64): - self.mc.FSTP(resloc) + if isinstance(resloc, StackLoc) and resloc.width == 8 and IS_X86_32: + self.mc.FSTP_b(resloc.value) elif size == 1: - self.mc.AND(eax, imm(0xff)) + self.mc.AND_ri(eax.value, 0xff) elif size == 2: - self.mc.AND(eax, imm(0xffff)) + self.mc.AND_ri(eax.value, 0xffff) - def genop_guard_call_may_force(self, op, guard_op, addr, + def genop_guard_call_may_force(self, op, guard_op, guard_token, arglocs, result_loc): faildescr = guard_op.descr fail_index = self.cpu.get_fail_descr_number(faildescr) - self.mc.MOV(mem(ebp, FORCE_INDEX_OFS), imm(fail_index)) + self.mc.MOV_bi(FORCE_INDEX_OFS, fail_index) self.genop_call(op, arglocs, result_loc) - self.mc.CMP(mem(ebp, FORCE_INDEX_OFS), imm(0)) - return self.implement_guard(addr, self.mc.JL) + self.mc.CMP_bi(FORCE_INDEX_OFS, 0) + return self.implement_guard(guard_token, 'L') - def genop_guard_call_assembler(self, op, guard_op, addr, + def genop_guard_call_assembler(self, op, guard_op, guard_token, arglocs, result_loc): faildescr = guard_op.descr fail_index = self.cpu.get_fail_descr_number(faildescr) - self.mc.MOV(mem(ebp, FORCE_INDEX_OFS), imm(fail_index)) + self.mc.MOV_bi(FORCE_INDEX_OFS, fail_index) descr = op.descr assert isinstance(descr, LoopToken) assert len(arglocs) - 2 == len(descr._x86_arglocs[0]) # # Write a call to the direct_bootstrap_code of the target assembler - self._emit_call(rel32(descr._x86_direct_bootstrap_code), arglocs, 2, + self._emit_call(imm(descr._x86_direct_bootstrap_code), arglocs, 2, tmp=eax) - mc = self._start_block() + self.mc.ensure_bytes_available(256) if op.result is None: assert result_loc is None value = self.cpu.done_with_this_frame_void_v @@ -1376,26 +1640,27 @@ value = self.cpu.done_with_this_frame_float_v else: raise AssertionError(kind) - mc.CMP(eax, imm(value)) - mc.JE(rel8_patched_later) # goto B if we get 'done_with_this_frame' - je_location = mc.get_relative_pos() + self.mc.CMP_ri(eax.value, value) + # patched later + self.mc.J_il8(rx86.Conditions['E'], 0) # goto B if we get 'done_with_this_frame' + je_location = self.mc.get_relative_pos() # # Path A: use assembler_helper_adr jd = descr.outermost_jitdriver_sd assert jd is not None asm_helper_adr = self.cpu.cast_adr_to_int(jd.assembler_helper_adr) - self._emit_call(rel32(asm_helper_adr), [eax, arglocs[1]], 0, - tmp=ecx, force_mc=True, mc=mc) - if isinstance(result_loc, MODRM64): - mc.FSTP(result_loc) + self._emit_call(imm(asm_helper_adr), [eax, arglocs[1]], 0, + tmp=ecx) + if IS_X86_32 and isinstance(result_loc, StackLoc) and result_loc.type == FLOAT: + self.mc.FSTP_b(result_loc.value) #else: result_loc is already either eax or None, checked below - mc.JMP(rel8_patched_later) # done - jmp_location = mc.get_relative_pos() + self.mc.JMP_l8(0) # jump to done, patched later + jmp_location = self.mc.get_relative_pos() # # Path B: fast path. Must load the return value, and reset the token offset = jmp_location - je_location assert 0 < offset <= 127 - mc.overwrite(je_location - 1, [chr(offset)]) + self.mc.overwrite(je_location - 1, [chr(offset)]) # # Reset the vable token --- XXX really too much special logic here:-( if jd.index_of_virtualizable >= 0: @@ -1403,8 +1668,8 @@ fielddescr = jd.vable_token_descr assert isinstance(fielddescr, BaseFieldDescr) ofs = fielddescr.offset - mc.MOV(eax, arglocs[1]) - mc.MOV(addr_add(eax, imm(ofs)), imm(0)) + self.mc.MOV(eax, arglocs[1]) + self.mc.MOV_mi((eax.value, ofs), 0) # in the line above, TOKEN_NONE = 0 # if op.result is not None: @@ -1413,27 +1678,26 @@ if kind == FLOAT: xmmtmp = X86XMMRegisterManager.all_regs[0] adr = self.fail_boxes_float.get_addr_for_num(0) - mc.MOVSD(xmmtmp, heap64(adr)) - mc.MOVSD(result_loc, xmmtmp) + self.mc.MOVSD(xmmtmp, heap(adr)) + self.mc.MOVSD(result_loc, xmmtmp) else: assert result_loc is eax if kind == INT: adr = self.fail_boxes_int.get_addr_for_num(0) - mc.MOV(eax, heap(adr)) + self.mc.MOV(eax, heap(adr)) elif kind == REF: adr = self.fail_boxes_ptr.get_addr_for_num(0) - mc.XOR(eax, eax) - mc.XCHG(eax, heap(adr)) + self.mc.XOR_rr(eax.value, eax.value) + self.mc.XCHG(eax, heap(adr)) else: raise AssertionError(kind) # # Here we join Path A and Path B again - offset = mc.get_relative_pos() - jmp_location + offset = self.mc.get_relative_pos() - jmp_location assert 0 <= offset <= 127 - mc.overwrite(jmp_location - 1, [chr(offset)]) - self._stop_block() - self.mc.CMP(mem(ebp, FORCE_INDEX_OFS), imm(0)) - return self.implement_guard(addr, self.mc.JL) + self.mc.overwrite(jmp_location - 1, [chr(offset)]) + self.mc.CMP_bi(FORCE_INDEX_OFS, 0) + return self.implement_guard(guard_token, 'L') def genop_discard_cond_call_gc_wb(self, op, arglocs): # use 'mc._mc' directly instead of 'mc', to avoid @@ -1443,31 +1707,41 @@ cls = self.cpu.gc_ll_descr.has_write_barrier_class() assert cls is not None and isinstance(descr, cls) loc_base = arglocs[0] - mc = self._start_block() - mc.TEST(mem8(loc_base, descr.jit_wb_if_flag_byteofs), - imm8(descr.jit_wb_if_flag_singlebyte)) - mc.JZ(rel8_patched_later) - jz_location = mc.get_relative_pos() + self.mc.ensure_bytes_available(256) + self.mc.TEST8_mi((loc_base.value, descr.jit_wb_if_flag_byteofs), + descr.jit_wb_if_flag_singlebyte) + self.mc.J_il8(rx86.Conditions['Z'], 0) # patched later + jz_location = self.mc.get_relative_pos() # the following is supposed to be the slow path, so whenever possible # we choose the most compact encoding over the most efficient one. for i in range(len(arglocs)-1, -1, -1): - mc.PUSH(arglocs[i]) + self.mc.PUSH(arglocs[i]) + + if IS_X86_64: + # We clobber these registers to pass the arguments, but that's + # okay, because consider_cond_call_gc_wb makes sure that any + # caller-save registers with values in them are present in arglocs, + # so they are saved on the stack above and restored below + self.mc.MOV_rs(edi.value, 0) + self.mc.MOV_rs(esi.value, 8) + # misaligned stack in the call, but it's ok because the write barrier # is not going to call anything more. Also, this assumes that the # write barrier does not touch the xmm registers. - mc.CALL(rel32(descr.get_write_barrier_fn(self.cpu))) + self.mc.CALL(imm(descr.get_write_barrier_fn(self.cpu))) for i in range(len(arglocs)): loc = arglocs[i] - assert isinstance(loc, REG) - mc.POP(loc) + assert isinstance(loc, RegLoc) + self.mc.POP(loc) # patch the JZ above - offset = mc.get_relative_pos() - jz_location + offset = self.mc.get_relative_pos() - jz_location assert 0 < offset <= 127 - mc.overwrite(jz_location-1, [chr(offset)]) - self._stop_block() + self.mc.overwrite(jz_location-1, [chr(offset)]) def genop_force_token(self, op, arglocs, resloc): - self.mc.LEA(resloc, mem(ebp, FORCE_INDEX_OFS)) + # RegAlloc.consider_force_token ensures this: + assert isinstance(resloc, RegLoc) + self.mc.LEA_rb(resloc.value, FORCE_INDEX_OFS) def not_implemented_op_discard(self, op, arglocs): msg = "not implemented operation: %s" % op.getopname() @@ -1495,17 +1769,16 @@ return loop_token._x86_arglocs def closing_jump(self, loop_token): - self.mc.JMP(rel32(loop_token._x86_loop_code)) + self.mc.JMP(imm(loop_token._x86_loop_code)) def malloc_cond_fixedsize(self, nursery_free_adr, nursery_top_adr, size, tid): - # don't use self.mc - mc = self._start_block() - mc.MOV(eax, heap(nursery_free_adr)) - mc.LEA(edx, addr_add(eax, imm(size))) - mc.CMP(edx, heap(nursery_top_adr)) - mc.JNA(rel8_patched_later) - jmp_adr = mc.get_relative_pos() + self.mc.ensure_bytes_available(256) + self.mc.MOV(eax, heap(nursery_free_adr)) + self.mc.LEA_rm(edx.value, (eax.value, size)) + self.mc.CMP(edx, heap(nursery_top_adr)) + self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later + jmp_adr = self.mc.get_relative_pos() # See comments in _build_malloc_fixedsize_slowpath for the # details of the two helper functions that we are calling below. @@ -1521,17 +1794,16 @@ # reserve room for the argument to the real malloc and the # 8 saved XMM regs self._regalloc.reserve_param(1+16) - mc.CALL(rel32(slowpath_addr1)) + self.mc.CALL(imm(slowpath_addr1)) self.mark_gc_roots() slowpath_addr2 = self.malloc_fixedsize_slowpath2 - mc.CALL(rel32(slowpath_addr2)) + self.mc.CALL(imm(slowpath_addr2)) - offset = mc.get_relative_pos() - jmp_adr + offset = self.mc.get_relative_pos() - jmp_adr assert 0 < offset <= 127 - mc.overwrite(jmp_adr-1, [chr(offset)]) - mc.MOV(addr_add(eax, imm(0)), imm(tid)) - mc.MOV(heap(nursery_free_adr), edx) - self._stop_block() + self.mc.overwrite(jmp_adr-1, [chr(offset)]) + self.mc.MOV_mi((eax.value, 0), tid) + self.mc.MOV(heap(nursery_free_adr), edx) genop_discard_list = [Assembler386.not_implemented_op_discard] * rop._LAST genop_list = [Assembler386.not_implemented_op] * rop._LAST @@ -1551,32 +1823,20 @@ num = getattr(rop, opname.upper()) genop_list[num] = value -def new_addr_add(heap, mem, memsib): - def addr_add(reg_or_imm1, reg_or_imm2, offset=0, scale=0): - if isinstance(reg_or_imm1, IMM32): - if isinstance(reg_or_imm2, IMM32): - return heap(reg_or_imm1.value + offset + - (reg_or_imm2.value << scale)) - else: - return memsib(None, reg_or_imm2, scale, reg_or_imm1.value + offset) - else: - if isinstance(reg_or_imm2, IMM32): - return mem(reg_or_imm1, offset + (reg_or_imm2.value << scale)) - else: - return memsib(reg_or_imm1, reg_or_imm2, scale, offset) - return addr_add - -addr8_add = new_addr_add(heap8, mem8, memSIB8) -addr_add = new_addr_add(heap, mem, memSIB) -addr64_add = new_addr_add(heap64, mem64, memSIB64) - -def addr_add_const(reg_or_imm1, offset): - if isinstance(reg_or_imm1, IMM32): - return heap(reg_or_imm1.value + offset) - else: - return mem(reg_or_imm1, offset) - def round_up_to_4(size): if size < 4: return 4 return size + +# XXX: ri386 migration shims: +def addr_add(reg_or_imm1, reg_or_imm2, offset=0, scale=0): + return AddressLoc(reg_or_imm1, reg_or_imm2, scale, offset) + +def addr_add_const(reg_or_imm1, offset): + return AddressLoc(reg_or_imm1, ImmedLoc(0), 0, offset) + +def mem(loc, offset): + return AddressLoc(loc, ImmedLoc(0), 0, offset) + +def heap(addr): + return AddressLoc(ImmedLoc(addr), ImmedLoc(0), 0, 0) Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/codebuf.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/codebuf.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/codebuf.py Mon Aug 2 20:58:49 2010 @@ -2,12 +2,20 @@ import os, sys from pypy.rpython.lltypesystem import lltype, rffi from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.jit.backend.x86.ri386 import I386CodeBuilder +from pypy.jit.backend.x86.rx86 import X86_32_CodeBuilder, X86_64_CodeBuilder +from pypy.jit.backend.x86.regloc import LocationCodeBuilder from pypy.rlib.rmmap import PTR, alloc, free from pypy.rlib.debug import make_sure_not_resized +from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 +from pypy.rlib.objectmodel import we_are_translated +# XXX: Seems nasty to change the superclass of InMemoryCodeBuilder like this +if IS_X86_32: + codebuilder_cls = X86_32_CodeBuilder +elif IS_X86_64: + codebuilder_cls = X86_64_CodeBuilder -class InMemoryCodeBuilder(I386CodeBuilder): +class InMemoryCodeBuilder(codebuilder_cls, LocationCodeBuilder): _last_dump_start = 0 def __init__(self, start, end): @@ -31,13 +39,15 @@ def write(self, listofchars): self._pos = self.overwrite(self._pos, listofchars) - def writechr(self, n): - # purely for performance: don't make the one-element list [chr(n)] + def writechar(self, char): pos = self._pos assert pos + 1 <= self._size - self._data[pos] = chr(n) + self._data[pos] = char self._pos = pos + 1 + def writechr(self, n): + self.writechar(chr(n)) + def get_relative_pos(self): return self._pos @@ -50,11 +60,6 @@ self._pos = pos self._last_dump_start = pos - def execute(self, arg1, arg2): - # XXX old testing stuff - fnptr = rffi.cast(lltype.Ptr(BINARYFN), self._data) - return fnptr(arg1, arg2) - def done(self): # normally, no special action is needed here if machine_code_dumper.enabled: @@ -77,9 +82,6 @@ valgrind.discard_translations(self._data, self._size) -BINARYFN = lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed) - - class MachineCodeDumper: enabled = True log_fd = -1 @@ -107,7 +109,10 @@ return False # log the executable name from pypy.jit.backend.hlinfo import highleveljitinfo - os.write(self.log_fd, 'BACKEND i386\n') + if IS_X86_32: + os.write(self.log_fd, 'BACKEND x86\n') + elif IS_X86_64: + os.write(self.log_fd, 'BACKEND x86_64\n') if highleveljitinfo.sys_executable: os.write(self.log_fd, 'SYS_EXECUTABLE %s\n' % ( highleveljitinfo.sys_executable,)) @@ -137,6 +142,12 @@ def __init__(self, map_size): data = alloc(map_size) + if IS_X86_64 and not we_are_translated(): + # Hack to make sure that mcs are not within 32-bits of one + # another for testing purposes + from pypy.rlib.rmmap import hint + hint.pos += 0xFFFFFFFF + self._init(data, map_size) def __del__(self): Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/jump.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/jump.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/jump.py Mon Aug 2 20:58:49 2010 @@ -1,23 +1,6 @@ import sys from pypy.tool.pairtype import extendabletype -from pypy.jit.backend.x86.ri386 import * - -class __extend__(OPERAND): - __metaclass__ = extendabletype - def _getregkey(self): - raise AssertionError("should only happen to registers and frame " - "positions") - -class __extend__(REG): - __metaclass__ = extendabletype - def _getregkey(self): - return ~self.op - -class __extend__(MODRM): - __metaclass__ = extendabletype - def _getregkey(self): - return self.position - +from pypy.jit.backend.x86.regloc import ImmedLoc, StackLoc def remap_frame_layout(assembler, src_locations, dst_locations, tmpreg): pending_dests = len(dst_locations) @@ -27,7 +10,7 @@ srccount[dst._getregkey()] = 0 for i in range(len(dst_locations)): src = src_locations[i] - if isinstance(src, IMM32): + if isinstance(src, ImmedLoc): continue key = src._getregkey() if key in srccount: @@ -46,7 +29,7 @@ srccount[key] = -1 # means "it's done" pending_dests -= 1 src = src_locations[i] - if not isinstance(src, IMM32): + if not isinstance(src, ImmedLoc): key = src._getregkey() if key in srccount: srccount[key] -= 1 @@ -80,7 +63,7 @@ assert pending_dests == 0 def _move(assembler, src, dst, tmpreg): - if isinstance(dst, MODRM) and isinstance(src, MODRM): + if dst.is_memory_reference() and src.is_memory_reference(): assembler.regalloc_mov(src, tmpreg) src = tmpreg assembler.regalloc_mov(src, dst) Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/regalloc.py Mon Aug 2 20:58:49 2010 @@ -5,7 +5,7 @@ from pypy.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr, ResOperation, BoxPtr, LoopToken, INT, REF, FLOAT) -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.regloc import * from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi, rstr from pypy.rlib.objectmodel import we_are_translated from pypy.rlib import rgc @@ -17,16 +17,7 @@ from pypy.jit.backend.llsupport.descr import BaseCallDescr, BaseSizeDescr from pypy.jit.backend.llsupport.regalloc import FrameManager, RegisterManager,\ TempBox - -WORD = 4 -FRAME_FIXED_SIZE = 5 # ebp + ebx + esi + edi + force_index = 5 words -FORCE_INDEX_OFS = -4*WORD - -width_of_type = { - INT : 1, - REF : 1, - FLOAT : 2, - } +from pypy.jit.backend.x86.arch import WORD, FRAME_FIXED_SIZE, IS_X86_32, IS_X86_64 class X86RegisterManager(RegisterManager): @@ -50,6 +41,12 @@ print "convert_to_imm: got a %s" % c raise AssertionError +class X86_64_RegisterManager(X86RegisterManager): + # r11 omitted because it's used as scratch + all_regs = [eax, ecx, edx, ebx, esi, edi, r8, r9, r10, r12, r13, r14, r15] + no_lower_byte_regs = [] + save_around_call_regs = [eax, ecx, edx, esi, edi, r8, r9, r10] + class FloatConstants(object): BASE_CONSTANT_SIZE = 1000 @@ -80,7 +77,6 @@ all_regs = [xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7] # we never need lower byte I hope save_around_call_regs = all_regs - reg_width = 2 def __init__(self, longevity, frame_manager=None, assembler=None): RegisterManager.__init__(self, longevity, frame_manager=frame_manager, @@ -94,27 +90,35 @@ def convert_to_imm(self, c): adr = self.float_constants.record_float(c.getfloat()) - return heap64(adr) + return AddressLoc(ImmedLoc(adr), ImmedLoc(0), 0, 0) def after_call(self, v): # the result is stored in st0, but we don't have this around, # so genop_call will move it to some frame location immediately # after the call - return self.frame_manager.loc(v, 2) + return self.frame_manager.loc(v) -class X86FrameManager(FrameManager): +class X86_64_XMMRegisterManager(X86XMMRegisterManager): + # xmm15 reserved for scratch use + all_regs = [xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14] + save_around_call_regs = all_regs + + def call_result_location(self, v): + return xmm0 + + def after_call(self, v): + # We use RegisterManager's implementation, since X86XMMRegisterManager + # places the result on the stack, which we don't need to do when the + # calling convention places the result in xmm0 + return RegisterManager.after_call(self, v) +class X86FrameManager(FrameManager): @staticmethod - def frame_pos(i, size): - if size == 1: - res = mem(ebp, get_ebp_ofs(i)) - elif size == 2: - res = mem64(ebp, get_ebp_ofs(i + 1)) - else: - print "Unimplemented size %d" % i - raise NotImplementedError("unimplemented size %d" % i) - res.position = i - return res + def frame_pos(i, box_type): + if IS_X86_32 and box_type == FLOAT: + return StackLoc(i, get_ebp_ofs(i+1), 2, box_type) + else: + return StackLoc(i, get_ebp_ofs(i), 1, box_type) class RegAlloc(object): exc = False @@ -135,11 +139,21 @@ # compute longevity of variables longevity = self._compute_vars_longevity(inputargs, operations) self.longevity = longevity - self.rm = X86RegisterManager(longevity, - frame_manager = self.fm, - assembler = self.assembler) - self.xrm = X86XMMRegisterManager(longevity, frame_manager = self.fm, - assembler = self.assembler) + # XXX + if cpu.WORD == 4: + gpr_reg_mgr_cls = X86RegisterManager + xmm_reg_mgr_cls = X86XMMRegisterManager + elif cpu.WORD == 8: + gpr_reg_mgr_cls = X86_64_RegisterManager + xmm_reg_mgr_cls = X86_64_XMMRegisterManager + else: + raise AssertionError("Word size should be 4 or 8") + + self.rm = gpr_reg_mgr_cls(longevity, + frame_manager = self.fm, + assembler = self.assembler) + self.xrm = xmm_reg_mgr_cls(longevity, frame_manager = self.fm, + assembler = self.assembler) def prepare_loop(self, inputargs, operations, looptoken): self._prepare(inputargs, operations) @@ -184,7 +198,7 @@ if reg: loc = reg else: - loc = self.fm.loc(arg, width_of_type[arg.type]) + loc = self.fm.loc(arg) if arg.type == FLOAT: floatlocs[i] = loc else: @@ -252,23 +266,23 @@ arg = inputargs[i] i += 1 if arg.type == FLOAT: - if isinstance(loc, REG): + if isinstance(loc, RegLoc): self.xrm.reg_bindings[arg] = loc used[loc] = None else: self.fm.frame_bindings[arg] = loc else: - if isinstance(loc, REG): + if isinstance(loc, RegLoc): self.rm.reg_bindings[arg] = loc used[loc] = None else: self.fm.frame_bindings[arg] = loc self.rm.free_regs = [] - for reg in X86RegisterManager.all_regs: + for reg in self.rm.all_regs: if reg not in used: self.rm.free_regs.append(reg) self.xrm.free_regs = [] - for reg in X86XMMRegisterManager.all_regs: + for reg in self.xrm.all_regs: if reg not in used: self.xrm.free_regs.append(reg) # note: we need to make a copy of inputargs because possibly_free_vars @@ -646,7 +660,7 @@ vable_index = jd.index_of_virtualizable if vable_index >= 0: self.rm._sync_var(op.args[vable_index]) - vable = self.fm.loc(op.args[vable_index], 1) + vable = self.fm.loc(op.args[vable_index]) else: vable = imm(0) self._call(op, [imm(size), vable] + @@ -670,7 +684,7 @@ # function, a GC write barrier, is known not to touch them. # See remember_young_pointer() in rpython/memory/gc/generation.py. for v, reg in self.rm.reg_bindings.items(): - if ((reg is eax or reg is ecx or reg is edx) + if (reg in self.rm.save_around_call_regs and self.rm.stays_alive(v)): arglocs.append(reg) self.PerformDiscard(op, arglocs) @@ -809,7 +823,7 @@ def consider_setfield_gc(self, op): ofs_loc, size_loc, ptr = self._unpack_fielddescr(op.descr) - assert isinstance(size_loc, IMM32) + assert isinstance(size_loc, ImmedLoc) if size_loc.value == 1: need_lower_byte = True else: @@ -950,7 +964,7 @@ shape = gcrootmap.get_basic_shape() for v, val in self.fm.frame_bindings.items(): if (isinstance(v, BoxPtr) and self.rm.stays_alive(v)): - assert isinstance(val, MODRM) + assert isinstance(val, StackLoc) gcrootmap.add_ebp_offset(shape, get_ebp_ofs(val.position)) for v, reg in self.rm.reg_bindings.items(): if reg is eax: Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/runner.py Mon Aug 2 20:58:49 2010 @@ -4,11 +4,13 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.jit.metainterp import history, compile from pypy.jit.backend.x86.assembler import Assembler386 -from pypy.jit.backend.x86.regalloc import FORCE_INDEX_OFS +from pypy.jit.backend.x86.arch import FORCE_INDEX_OFS from pypy.jit.backend.x86.profagent import ProfileAgent from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU +from pypy.jit.backend.x86 import regloc +import sys -class CPU386(AbstractLLCPU): +class AbstractX86CPU(AbstractLLCPU): debug = True supports_floats = True @@ -44,6 +46,7 @@ self.profile_agent.startup() def finish_once(self): + self.assembler.finish_once() self.profile_agent.shutdown() def compile_loop(self, inputargs, operations, looptoken): @@ -131,10 +134,28 @@ assert fail_index == fail_index_2 return faildescr +class CPU386(AbstractX86CPU): + WORD = 4 + NUM_REGS = 8 + CALLEE_SAVE_REGISTERS = [regloc.ebx, regloc.esi, regloc.edi] + FRAME_FIXED_SIZE = len(CALLEE_SAVE_REGISTERS) + 2 + + def __init__(self, *args, **kwargs): + assert sys.maxint == (2**31 - 1) + super(CPU386, self).__init__(*args, **kwargs) class CPU386_NO_SSE2(CPU386): supports_floats = False +class CPU_X86_64(AbstractX86CPU): + WORD = 8 + NUM_REGS = 16 + CALLEE_SAVE_REGISTERS = [regloc.ebx, regloc.r12, regloc.r13, regloc.r14, regloc.r15] + FRAME_FIXED_SIZE = len(CALLEE_SAVE_REGISTERS) + 2 + + def __init__(self, *args, **kwargs): + assert sys.maxint == (2**63 - 1) + super(CPU_X86_64, self).__init__(*args, **kwargs) CPU = CPU386 Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/conftest.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/conftest.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/conftest.py Mon Aug 2 20:58:49 2010 @@ -3,6 +3,5 @@ cpu = detect_cpu.autodetect() def pytest_runtest_setup(item): - if cpu != 'x86': - py.test.skip("x86 directory skipped: cpu is %r" % (cpu,)) - + if cpu not in ('x86', 'x86_64'): + py.test.skip("x86/x86_64 tests skipped: cpu is %r" % (cpu,)) Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_assembler.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_assembler.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_assembler.py Mon Aug 2 20:58:49 2010 @@ -1,14 +1,19 @@ -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.regloc import * from pypy.jit.backend.x86.assembler import Assembler386, MachineCodeBlockWrapper from pypy.jit.backend.x86.regalloc import X86FrameManager, get_ebp_ofs -from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat +from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, INT, REF, FLOAT from pypy.rlib.rarithmetic import intmask from pypy.rpython.lltypesystem import lltype, llmemory, rffi +from pypy.jit.backend.x86.arch import WORD, IS_X86_32, IS_X86_64 +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.regalloc import X86RegisterManager, X86_64_RegisterManager, X86XMMRegisterManager, X86_64_XMMRegisterManager +ACTUAL_CPU = getcpuclass() class FakeCPU: rtyper = None supports_floats = True + NUM_REGS = ACTUAL_CPU.NUM_REGS class FakeMC: def __init__(self, base_address=0): @@ -25,7 +30,14 @@ self.content.append(("JMP", args)) def done(self): pass + def PUSH_r(self, reg): + pass + def POP_r(self, reg): + pass +class FakeAssembler: + def write_pending_failure_recoveries(self): + pass def test_write_failure_recovery_description(): assembler = Assembler386(FakeCPU()) @@ -33,12 +45,12 @@ failargs = [BoxInt(), BoxPtr(), BoxFloat()] * 3 failargs.insert(6, None) failargs.insert(7, None) - locs = [X86FrameManager.frame_pos(0, 1), - X86FrameManager.frame_pos(1, 1), - X86FrameManager.frame_pos(10, 2), - X86FrameManager.frame_pos(100, 1), - X86FrameManager.frame_pos(101, 1), - X86FrameManager.frame_pos(110, 2), + locs = [X86FrameManager.frame_pos(0, INT), + X86FrameManager.frame_pos(1, REF), + X86FrameManager.frame_pos(10, FLOAT), + X86FrameManager.frame_pos(100, INT), + X86FrameManager.frame_pos(101, REF), + X86FrameManager.frame_pos(110, FLOAT), None, None, ebx, @@ -46,17 +58,17 @@ xmm2] assert len(failargs) == len(locs) assembler.write_failure_recovery_description(mc, failargs, locs) - nums = [Assembler386.DESCR_INT + 4*(8+0), - Assembler386.DESCR_REF + 4*(8+1), - Assembler386.DESCR_FLOAT + 4*(8+10), - Assembler386.DESCR_INT + 4*(8+100), - Assembler386.DESCR_REF + 4*(8+101), - Assembler386.DESCR_FLOAT + 4*(8+110), + nums = [Assembler386.DESCR_INT + 4*(16+0), + Assembler386.DESCR_REF + 4*(16+1), + Assembler386.DESCR_FLOAT + 4*(16+10), + Assembler386.DESCR_INT + 4*(16+100), + Assembler386.DESCR_REF + 4*(16+101), + Assembler386.DESCR_FLOAT + 4*(16+110), Assembler386.CODE_HOLE, Assembler386.CODE_HOLE, - Assembler386.DESCR_INT + 4*ebx.op, - Assembler386.DESCR_REF + 4*esi.op, - Assembler386.DESCR_FLOAT + 4*xmm2.op] + Assembler386.DESCR_INT + 4*ebx.value, + Assembler386.DESCR_REF + 4*esi.value, + Assembler386.DESCR_FLOAT + 4*xmm2.value] double_byte_nums = [] for num in nums[3:6]: double_byte_nums.append((num & 0x7F) | 0x80) @@ -94,6 +106,9 @@ return lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S)) def get_random_float(): + # Returns , , + # NB: on 64-bit, will be the entire float and + # will be random garbage from malloc! assert withfloats value = random.random() - 0.5 # make sure it fits into 64 bits @@ -101,9 +116,16 @@ rffi.cast(rffi.DOUBLEP, tmp)[0] = value return rffi.cast(rffi.DOUBLEP, tmp)[0], tmp[0], tmp[1] + if IS_X86_32: + main_registers = X86RegisterManager.all_regs + xmm_registers = X86XMMRegisterManager.all_regs + elif IS_X86_64: + main_registers = X86_64_RegisterManager.all_regs + xmm_registers = X86_64_XMMRegisterManager.all_regs + # memory locations: 26 integers, 26 pointers, 26 floats # main registers: half of them as signed and the other half as ptrs - # xmm registers: all floats, from xmm0 to xmm7 + # xmm registers: all floats, from xmm0 to xmm(7|15) # holes: 8 locations = [] baseloc = 4 @@ -117,18 +139,17 @@ content = ([('int', locations.pop()) for _ in range(26)] + [('ptr', locations.pop()) for _ in range(26)] + [(['int', 'ptr'][random.randrange(0, 2)], reg) - for reg in [eax, ecx, edx, ebx, esi, edi]]) + for reg in main_registers]) if withfloats: content += ([('float', locations.pop()) for _ in range(26)] + - [('float', reg) for reg in [xmm0, xmm1, xmm2, xmm3, - xmm4, xmm5, xmm6, xmm7]]) + [('float', reg) for reg in xmm_registers]) for i in range(8): content.append(('hole', None)) random.shuffle(content) # prepare the expected target arrays, the descr_bytecode, # the 'registers' and the 'stack' arrays according to 'content' - xmmregisters = lltype.malloc(rffi.LONGP.TO, 16+9, flavor='raw') + xmmregisters = lltype.malloc(rffi.LONGP.TO, 16+ACTUAL_CPU.NUM_REGS+1, flavor='raw') registers = rffi.ptradd(xmmregisters, 16) stacklen = baseloc + 10 stack = lltype.malloc(rffi.LONGP.TO, stacklen, flavor='raw') @@ -140,8 +161,8 @@ assert loc >= 0 ofs = get_ebp_ofs(loc) assert ofs < 0 - assert (ofs % 4) == 0 - stack[stacklen + ofs//4] = value + assert (ofs % WORD) == 0 + stack[stacklen + ofs//WORD] = value descr_bytecode = [] for i, (kind, loc) in enumerate(content): @@ -152,12 +173,18 @@ value, lo, hi = get_random_float() expected_floats[i] = value kind = Assembler386.DESCR_FLOAT - if isinstance(loc, REG): - xmmregisters[2*loc.op] = lo - xmmregisters[2*loc.op+1] = hi + if isinstance(loc, RegLoc): + if WORD == 4: + xmmregisters[2*loc.value] = lo + xmmregisters[2*loc.value+1] = hi + elif WORD == 8: + xmmregisters[loc.value] = lo else: - write_in_stack(loc, hi) - write_in_stack(loc+1, lo) + if WORD == 4: + write_in_stack(loc, hi) + write_in_stack(loc+1, lo) + elif WORD == 8: + write_in_stack(loc, lo) else: if kind == 'int': value = get_random_int() @@ -170,15 +197,15 @@ value = rffi.cast(rffi.LONG, value) else: assert 0, kind - if isinstance(loc, REG): - registers[loc.op] = value + if isinstance(loc, RegLoc): + registers[loc.value] = value else: write_in_stack(loc, value) - if isinstance(loc, REG): - num = kind + 4*loc.op + if isinstance(loc, RegLoc): + num = kind + 4*loc.value else: - num = kind + 4*(8+loc) + num = kind + Assembler386.CODE_FROMSTACK + (4*loc) while num >= 0x80: descr_bytecode.append((num & 0x7F) | 0x80) num >>= 7 @@ -195,8 +222,8 @@ for i in range(len(descr_bytecode)): assert 0 <= descr_bytecode[i] <= 255 descr_bytes[i] = rffi.cast(rffi.UCHAR, descr_bytecode[i]) - registers[8] = rffi.cast(rffi.LONG, descr_bytes) - registers[ebp.op] = rffi.cast(rffi.LONG, stack) + 4*stacklen + registers[ACTUAL_CPU.NUM_REGS] = rffi.cast(rffi.LONG, descr_bytes) + registers[ebp.value] = rffi.cast(rffi.LONG, stack) + WORD*stacklen # run! assembler = Assembler386(FakeCPU()) @@ -237,7 +264,8 @@ def test_mc_wrapper_profile_agent(): agent = FakeProfileAgent() - mc = FakeMCWrapper(100, agent) + assembler = FakeAssembler() + mc = FakeMCWrapper(assembler, 100, agent) mc.start_function("abc") mc.writechr("x") mc.writechr("x") Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_basic.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_basic.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_basic.py Mon Aug 2 20:58:49 2010 @@ -1,5 +1,5 @@ import py -from pypy.jit.backend.x86.runner import CPU386 +from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.metainterp.warmspot import ll_meta_interp from pypy.jit.metainterp.test import test_basic from pypy.jit.codewriter.policy import StopAtXPolicy @@ -7,7 +7,7 @@ class Jit386Mixin(test_basic.LLJitMixin): type_system = 'lltype' - CPUClass = CPU386 + CPUClass = getcpuclass() def check_jumps(self, maxcount): pass Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_gc_integration.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_gc_integration.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_gc_integration.py Mon Aug 2 20:58:49 2010 @@ -9,13 +9,13 @@ from pypy.jit.codewriter import heaptracker from pypy.jit.backend.llsupport.descr import GcCache from pypy.jit.backend.llsupport.gc import GcLLDescription -from pypy.jit.backend.x86.runner import CPU -from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, FRAME_FIXED_SIZE +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.regalloc import RegAlloc +from pypy.jit.backend.x86.arch import WORD, FRAME_FIXED_SIZE from pypy.jit.metainterp.test.oparser import parse from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.annlowlevel import llhelper from pypy.rpython.lltypesystem import rclass, rstr -from pypy.jit.backend.x86.ri386 import * from pypy.jit.backend.llsupport.gc import GcLLDescr_framework, GcRefList, GcPtrFieldDescr from pypy.jit.backend.x86.test.test_regalloc import MockAssembler @@ -23,6 +23,8 @@ from pypy.jit.backend.x86.regalloc import X86RegisterManager, X86FrameManager,\ X86XMMRegisterManager +CPU = getcpuclass() + class MockGcRootMap(object): def get_basic_shape(self): return ['shape'] @@ -51,7 +53,7 @@ def initialize(self): self.gcrefs = GcRefList() self.gcrefs.initialize() - self.single_gcref_descr = GcPtrFieldDescr(0) + self.single_gcref_descr = GcPtrFieldDescr('', 0) rewrite_assembler = GcLLDescr_framework.rewrite_assembler.im_func @@ -84,7 +86,7 @@ mark = regalloc.get_mark_gc_roots(cpu.gc_ll_descr.gcrootmap) assert mark[0] == 'compressed' base = -WORD * FRAME_FIXED_SIZE - expected = ['ebx', 'esi', 'edi', base, base-4, base-8] + expected = ['ebx', 'esi', 'edi', base, base-WORD, base-WORD*2] assert dict.fromkeys(mark[1:]) == dict.fromkeys(expected) class TestRegallocGcIntegration(BaseTestRegalloc): @@ -175,7 +177,7 @@ self.addrs[1] = self.addrs[0] + 64 # 64 bytes def malloc_slowpath(size): - assert size == 8 + assert size == WORD*2 nadr = rffi.cast(lltype.Signed, self.nursery) self.addrs[0] = nadr + size return nadr @@ -199,7 +201,7 @@ return rffi.cast(lltype.Signed, self.addrs) def get_nursery_top_addr(self): - return rffi.cast(lltype.Signed, self.addrs) + 4 + return rffi.cast(lltype.Signed, self.addrs) + WORD def get_malloc_fixedsize_slowpath_addr(self): fptr = llhelper(lltype.Ptr(self.MALLOC_SLOWPATH), self.malloc_slowpath) @@ -213,7 +215,7 @@ def setup_method(self, method): cpu = CPU(None, None) - cpu.vtable_offset = 4 + cpu.vtable_offset = WORD cpu.gc_ll_descr = GCDescrFastpathMalloc() NODE = lltype.Struct('node', ('tid', lltype.Signed), @@ -249,7 +251,7 @@ assert gc_ll_descr.nursery[0] == self.nodedescr.tid assert gc_ll_descr.nursery[1] == 42 nurs_adr = rffi.cast(lltype.Signed, gc_ll_descr.nursery) - assert gc_ll_descr.addrs[0] == nurs_adr + 8 + assert gc_ll_descr.addrs[0] == nurs_adr + (WORD*2) def test_malloc_slowpath(self): ops = ''' @@ -269,7 +271,7 @@ # this should call slow path once gc_ll_descr = self.cpu.gc_ll_descr nadr = rffi.cast(lltype.Signed, gc_ll_descr.nursery) - assert gc_ll_descr.addrs[0] == nadr + 8 + assert gc_ll_descr.addrs[0] == nadr + (WORD*2) def test_new_with_vtable(self): ops = ''' @@ -284,4 +286,4 @@ assert gc_ll_descr.nursery[0] == self.descrsize.tid assert gc_ll_descr.nursery[1] == self.vtable_int nurs_adr = rffi.cast(lltype.Signed, gc_ll_descr.nursery) - assert gc_ll_descr.addrs[0] == nurs_adr + 12 + assert gc_ll_descr.addrs[0] == nurs_adr + (WORD*3) Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_jump.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_jump.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_jump.py Mon Aug 2 20:58:49 2010 @@ -1,6 +1,7 @@ -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.regloc import * from pypy.jit.backend.x86.regalloc import X86FrameManager from pypy.jit.backend.x86.jump import remap_frame_layout +from pypy.jit.metainterp.history import INT frame_pos = X86FrameManager.frame_pos @@ -25,7 +26,7 @@ continue assert len(op1) == len(op2) for x, y in zip(op1, op2): - if isinstance(x, MODRM) and isinstance(y, MODRM): + if isinstance(x, StackLoc) and isinstance(y, MODRM): assert x.byte == y.byte assert x.extradata == y.extradata else: @@ -41,9 +42,9 @@ remap_frame_layout(assembler, [eax, ebx, ecx, edx, esi, edi], [eax, ebx, ecx, edx, esi, edi], '?') assert assembler.ops == [] - s8 = frame_pos(1, 1) - s12 = frame_pos(31, 1) - s20 = frame_pos(6, 1) + s8 = frame_pos(1, INT) + s12 = frame_pos(31, INT) + s20 = frame_pos(6, INT) remap_frame_layout(assembler, [eax, ebx, ecx, s20, s8, edx, s12, esi, edi], [eax, ebx, ecx, s20, s8, edx, s12, esi, edi], '?') @@ -58,10 +59,10 @@ def test_simple_framelocs(): assembler = MockAssembler() - s8 = frame_pos(0, 1) - s12 = frame_pos(13, 1) - s20 = frame_pos(20, 1) - s24 = frame_pos(221, 1) + s8 = frame_pos(0, INT) + s12 = frame_pos(13, INT) + s20 = frame_pos(20, INT) + s24 = frame_pos(221, INT) remap_frame_layout(assembler, [s8, eax, s12], [s20, s24, edi], edx) assert assembler.ops == [('mov', s8, edx), ('mov', edx, s20), @@ -70,10 +71,10 @@ def test_reordering(): assembler = MockAssembler() - s8 = frame_pos(8, 1) - s12 = frame_pos(12, 1) - s20 = frame_pos(19, 1) - s24 = frame_pos(1, 1) + s8 = frame_pos(8, INT) + s12 = frame_pos(12, INT) + s20 = frame_pos(19, INT) + s24 = frame_pos(1, INT) remap_frame_layout(assembler, [eax, s8, s20, ebx], [s8, ebx, eax, edi], '?') assert assembler.got([('mov', ebx, edi), @@ -83,10 +84,10 @@ def test_cycle(): assembler = MockAssembler() - s8 = frame_pos(8, 1) - s12 = frame_pos(12, 1) - s20 = frame_pos(19, 1) - s24 = frame_pos(1, 1) + s8 = frame_pos(8, INT) + s12 = frame_pos(12, INT) + s20 = frame_pos(19, INT) + s24 = frame_pos(1, INT) remap_frame_layout(assembler, [eax, s8, s20, ebx], [s8, ebx, eax, s20], '?') assert assembler.got([('push', s8), @@ -97,12 +98,12 @@ def test_cycle_2(): assembler = MockAssembler() - s8 = frame_pos(8, 1) - s12 = frame_pos(12, 1) - s20 = frame_pos(19, 1) - s24 = frame_pos(1, 1) - s2 = frame_pos(2, 1) - s3 = frame_pos(3, 1) + s8 = frame_pos(8, INT) + s12 = frame_pos(12, INT) + s20 = frame_pos(19, INT) + s24 = frame_pos(1, INT) + s2 = frame_pos(2, INT) + s3 = frame_pos(3, INT) remap_frame_layout(assembler, [eax, s8, edi, s20, eax, s20, s24, esi, s2, s3], [s8, s20, edi, eax, edx, s24, ebx, s12, s3, s2], @@ -127,14 +128,14 @@ remap_frame_layout(assembler, [c3], [eax], '?') assert assembler.ops == [('mov', c3, eax)] assembler = MockAssembler() - s12 = frame_pos(12, 1) + s12 = frame_pos(12, INT) remap_frame_layout(assembler, [c3], [s12], '?') assert assembler.ops == [('mov', c3, s12)] def test_constants_and_cycle(): assembler = MockAssembler() c3 = imm(3) - s12 = frame_pos(13, 1) + s12 = frame_pos(13, INT) remap_frame_layout(assembler, [ebx, c3, s12], [s12, eax, ebx], edi) assert assembler.ops == [('mov', c3, eax), Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_recompilation.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_recompilation.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_recompilation.py Mon Aug 2 20:58:49 2010 @@ -1,6 +1,5 @@ - -from pypy.jit.backend.x86.runner import CPU from pypy.jit.backend.x86.test.test_regalloc import BaseTestRegalloc +from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 class TestRecompilation(BaseTestRegalloc): def test_compile_bridge_not_deeper(self): @@ -51,7 +50,9 @@ descr = loop.operations[2].descr new = descr._x86_bridge_frame_depth assert descr._x86_bridge_param_depth == 0 - assert new > previous + # XXX: Maybe add enough ops to force stack on 64-bit as well? + if IS_X86_32: + assert new > previous self.cpu.set_future_value_int(0, 0) fail = self.run(loop) assert fail.identifier == 2 @@ -111,7 +112,9 @@ guard_op = loop.operations[5] loop_frame_depth = loop.token._x86_frame_depth assert loop.token._x86_param_depth == 0 - assert guard_op.descr._x86_bridge_frame_depth > loop_frame_depth + # XXX: Maybe add enough ops to force stack on 64-bit as well? + if IS_X86_32: + assert guard_op.descr._x86_bridge_frame_depth > loop_frame_depth assert guard_op.descr._x86_bridge_param_depth == 0 self.cpu.set_future_value_int(0, 0) self.cpu.set_future_value_int(1, 0) Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_regalloc.py Mon Aug 2 20:58:49 2010 @@ -7,15 +7,17 @@ BoxPtr, ConstPtr, LoopToken, BasicFailDescr from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.backend.llsupport.descr import GcCache -from pypy.jit.backend.x86.runner import CPU -from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, X86RegisterManager,\ +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.regalloc import RegAlloc, X86RegisterManager,\ FloatConstants +from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 from pypy.jit.metainterp.test.oparser import parse from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.annlowlevel import llhelper from pypy.rpython.lltypesystem import rclass, rstr -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.rx86 import * +CPU = getcpuclass() class MockGcDescr(GcCache): def get_funcptr_for_new(self): return 123 @@ -92,13 +94,20 @@ def f2(x, y): return x*y + def f10(*args): + assert len(args) == 10 + return sum(args) + F1PTR = lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Signed)) F2PTR = lltype.Ptr(lltype.FuncType([lltype.Signed]*2, lltype.Signed)) + F10PTR = lltype.Ptr(lltype.FuncType([lltype.Signed]*10, lltype.Signed)) f1ptr = llhelper(F1PTR, f1) f2ptr = llhelper(F2PTR, f2) + f10ptr = llhelper(F10PTR, f10) f1_calldescr = cpu.calldescrof(F1PTR.TO, F1PTR.TO.ARGS, F1PTR.TO.RESULT) f2_calldescr = cpu.calldescrof(F2PTR.TO, F2PTR.TO.ARGS, F2PTR.TO.RESULT) + f10_calldescr = cpu.calldescrof(F10PTR.TO, F10PTR.TO.ARGS, F10PTR.TO.RESULT) namespace = locals().copy() type_system = 'lltype' @@ -541,6 +550,12 @@ assert self.getints(9) == [0, 1, 1, 1, 1, 1, 1, 1, 1] class TestRegAllocCallAndStackDepth(BaseTestRegalloc): + def expected_param_depth(self, num_args): + # Assumes the arguments are all non-float + if IS_X86_32: + return num_args + elif IS_X86_64: + return max(num_args - 6, 0) def test_one_call(self): ops = ''' @@ -550,7 +565,7 @@ ''' loop = self.interpret(ops, [4, 7, 9, 9 ,9, 9, 9, 9, 9, 9, 9]) assert self.getints(11) == [5, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9] - assert loop.token._x86_param_depth == 1 + assert loop.token._x86_param_depth == self.expected_param_depth(1) def test_two_calls(self): ops = ''' @@ -561,8 +576,21 @@ ''' loop = self.interpret(ops, [4, 7, 9, 9 ,9, 9, 9, 9, 9, 9, 9]) assert self.getints(11) == [5*7, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9] - assert loop.token._x86_param_depth == 2 - + assert loop.token._x86_param_depth == self.expected_param_depth(2) + + def test_call_many_arguments(self): + # NB: The first and last arguments in the call are constants. This + # is primarily for x86-64, to ensure that loading a constant to an + # argument register or to the stack works correctly + ops = ''' + [i0, i1, i2, i3, i4, i5, i6, i7] + i8 = call(ConstClass(f10ptr), 1, i0, i1, i2, i3, i4, i5, i6, i7, 10, descr=f10_calldescr) + finish(i8) + ''' + loop = self.interpret(ops, [2, 3, 4, 5, 6, 7, 8, 9]) + assert self.getint(0) == 55 + assert loop.token._x86_param_depth == self.expected_param_depth(10) + def test_bridge_calls_1(self): ops = ''' [i0, i1] @@ -579,7 +607,7 @@ ''' bridge = self.attach_bridge(ops, loop, -2) - assert loop.operations[-2].descr._x86_bridge_param_depth == 2 + assert loop.operations[-2].descr._x86_bridge_param_depth == self.expected_param_depth(2) self.cpu.set_future_value_int(0, 4) self.cpu.set_future_value_int(1, 7) @@ -602,7 +630,7 @@ ''' bridge = self.attach_bridge(ops, loop, -2) - assert loop.operations[-2].descr._x86_bridge_param_depth == 2 + assert loop.operations[-2].descr._x86_bridge_param_depth == self.expected_param_depth(2) self.cpu.set_future_value_int(0, 4) self.cpu.set_future_value_int(1, 7) Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_regalloc2.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_regalloc2.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_regalloc2.py Mon Aug 2 20:58:49 2010 @@ -2,7 +2,9 @@ from pypy.jit.metainterp.history import ResOperation, BoxInt, ConstInt,\ BoxPtr, ConstPtr, BasicFailDescr, LoopToken from pypy.jit.metainterp.resoperation import rop -from pypy.jit.backend.x86.runner import CPU +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.arch import WORD +CPU = getcpuclass() def test_bug_rshift(): v1 = BoxInt() @@ -281,5 +283,8 @@ assert cpu.get_latest_value_int(16) == -57344 assert cpu.get_latest_value_int(17) == 1 assert cpu.get_latest_value_int(18) == -1 - assert cpu.get_latest_value_int(19) == -2147483648 + if WORD == 4: + assert cpu.get_latest_value_int(19) == -2147483648 + elif WORD == 8: + assert cpu.get_latest_value_int(19) == 19327352832 assert cpu.get_latest_value_int(20) == -49 Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_runner.py Mon Aug 2 20:58:49 2010 @@ -2,15 +2,20 @@ from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rstr, rclass from pypy.jit.metainterp.history import ResOperation, LoopToken from pypy.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstPtr, - Box, BasicFailDescr) -from pypy.jit.backend.x86.runner import CPU -from pypy.jit.backend.x86.regalloc import WORD + Box, BoxFloat, BasicFailDescr) +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.arch import WORD from pypy.jit.backend.llsupport import symbolic from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.executor import execute from pypy.jit.backend.test.runner_test import LLtypeBackendTest +from pypy.jit.metainterp.test.oparser import parse +from pypy.tool.udir import udir import ctypes import sys +import os + +CPU = getcpuclass() class FakeStats(object): pass @@ -56,7 +61,7 @@ assert u.chars[3] == u'd' @staticmethod - def _resbuf(res, item_tp=ctypes.c_int): + def _resbuf(res, item_tp=ctypes.c_long): return ctypes.cast(res.value._obj.intval, ctypes.POINTER(item_tp)) def test_allocations(self): @@ -71,8 +76,11 @@ return ctypes.cast(buf, ctypes.c_void_p).value func = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int)(f) addr = ctypes.cast(func, ctypes.c_void_p).value + # ctypes produces an unsigned value. We need it to be signed for, eg, + # relative addressing to work properly. + addr = rffi.cast(lltype.Signed, addr) - self.cpu.assembler.make_sure_mc_exists() + self.cpu.assembler.setup() self.cpu.assembler.malloc_func_addr = addr ofs = symbolic.get_field_token(rstr.STR, 'chars', False)[0] @@ -357,7 +365,9 @@ self.cpu.compile_bridge(faildescr1, [i1b], bridge) name, address, size = agent.functions[1] assert name == "Bridge # 0: bye" - assert address == loopaddress + loopsize + # Would be exactly ==, but there are some guard failure recovery + # stubs in-between + assert address >= loopaddress + loopsize assert size >= 10 # randomish number self.cpu.set_future_value_int(0, 2) @@ -383,10 +393,99 @@ ops.append(ResOperation(rop.FINISH, [v], None, descr=BasicFailDescr())) looptoken = LoopToken() - self.cpu.assembler.make_sure_mc_exists() + self.cpu.assembler.setup() old_mc_mc = self.cpu.assembler.mc._mc self.cpu.compile_loop([base_v], ops, looptoken) assert self.cpu.assembler.mc._mc != old_mc_mc # overflowed self.cpu.set_future_value_int(0, base_v.value) self.cpu.execute_token(looptoken) assert self.cpu.get_latest_value_int(0) == 1024 + + def test_overflow_guard_float_cmp(self): + # The float comparisons on x86 tend to use small relative jumps, + # which may run into trouble if they fall on the edge of a + # MachineCodeBlock change. + a = BoxFloat(1.0) + b = BoxFloat(2.0) + failed = BoxInt(41) + finished = BoxInt(42) + + # We select guards that will always succeed, so that execution will + # continue through the entire set of comparisions + ops_to_test = ( + (rop.FLOAT_LT, [a, b], rop.GUARD_TRUE), + (rop.FLOAT_LT, [b, a], rop.GUARD_FALSE), + + (rop.FLOAT_LE, [a, a], rop.GUARD_TRUE), + (rop.FLOAT_LE, [a, b], rop.GUARD_TRUE), + (rop.FLOAT_LE, [b, a], rop.GUARD_FALSE), + + (rop.FLOAT_EQ, [a, a], rop.GUARD_TRUE), + (rop.FLOAT_EQ, [a, b], rop.GUARD_FALSE), + + (rop.FLOAT_NE, [a, b], rop.GUARD_TRUE), + (rop.FLOAT_NE, [a, a], rop.GUARD_FALSE), + + (rop.FLOAT_GT, [b, a], rop.GUARD_TRUE), + (rop.FLOAT_GT, [a, b], rop.GUARD_FALSE), + + (rop.FLOAT_GE, [a, a], rop.GUARD_TRUE), + (rop.FLOAT_GE, [b, a], rop.GUARD_TRUE), + (rop.FLOAT_GE, [a, b], rop.GUARD_FALSE), + ) + + for float_op, args, guard_op in ops_to_test: + ops = [] + + for i in range(200): + cmp_result = BoxInt() + ops.append(ResOperation(float_op, args, cmp_result)) + ops.append(ResOperation(guard_op, [cmp_result], None, descr=BasicFailDescr())) + ops[-1].fail_args = [failed] + + ops.append(ResOperation(rop.FINISH, [finished], None, descr=BasicFailDescr())) + + looptoken = LoopToken() + self.cpu.compile_loop([a, b, failed, finished], ops, looptoken) + self.cpu.set_future_value_float(0, a.value) + self.cpu.set_future_value_float(1, b.value) + self.cpu.set_future_value_int(2, failed.value) + self.cpu.set_future_value_int(3, finished.value) + self.cpu.execute_token(looptoken) + + # Really just a sanity check. We're actually interested in + # whether the test segfaults. + assert self.cpu.get_latest_value_int(0) == finished.value + + +class TestDebuggingAssembler(object): + def setup_method(self, meth): + self.pypylog = os.environ.get('PYPYLOG', None) + self.logfile = str(udir.join('x86_runner.log')) + os.environ['PYPYLOG'] = "mumble:" + self.logfile + self.cpu = CPU(rtyper=None, stats=FakeStats()) + + def teardown_method(self, meth): + if self.pypylog is not None: + os.environ['PYPYLOG'] = self.pypylog + + def test_debugger_on(self): + loop = """ + [i0] + debug_merge_point('xyz') + i1 = int_add(i0, 1) + i2 = int_ge(i1, 10) + guard_false(i2) [] + jump(i1) + """ + ops = parse(loop) + self.cpu.assembler.set_debug(True) + self.cpu.compile_loop(ops.inputargs, ops.operations, ops.token) + self.cpu.set_future_value_int(0, 0) + self.cpu.execute_token(ops.token) + # check debugging info + assert self.cpu.assembler.loop_names == ["xyz"] + assert self.cpu.assembler.loop_run_counter.getitem(0) == 10 + self.cpu.finish_once() + lines = py.path.local(self.logfile + ".count").readlines() + assert lines[0] == 'xyz:10\n' Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_symbolic_x86.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_symbolic_x86.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_symbolic_x86.py Mon Aug 2 20:58:49 2010 @@ -1,6 +1,7 @@ import py from pypy.jit.backend.llsupport.symbolic import * from pypy.rpython.lltypesystem import lltype, rffi +from pypy.jit.backend.x86.arch import WORD # This test file is here and not in llsupport/test/ because it checks # that we get correct numbers for a 32-bit machine. @@ -19,32 +20,32 @@ ofs_z, size_z = get_field_token(S, 'z', False) # ofs_x might be 0 or not, depending on how we count the headers # but the rest should be as expected for a 386 machine - assert size_x == size_y == size_z == 4 + assert size_x == size_y == size_z == WORD assert ofs_x >= 0 - assert ofs_y == ofs_x + 4 - assert ofs_z == ofs_x + 8 + assert ofs_y == ofs_x + WORD + assert ofs_z == ofs_x + (WORD*2) def test_struct_size(): ofs_z, size_z = get_field_token(S, 'z', False) totalsize = get_size(S, False) - assert totalsize == ofs_z + 4 + assert totalsize == ofs_z + WORD def test_primitive_size(): - assert get_size(lltype.Signed, False) == 4 + assert get_size(lltype.Signed, False) == WORD assert get_size(lltype.Char, False) == 1 - assert get_size(lltype.Ptr(S), False) == 4 + assert get_size(lltype.Ptr(S), False) == WORD def test_array_token(): A = lltype.GcArray(lltype.Char) basesize, itemsize, ofs_length = get_array_token(A, False) - assert basesize >= 4 # at least the 'length', maybe some gc headers + assert basesize >= WORD # at least the 'length', maybe some gc headers assert itemsize == 1 - assert ofs_length == basesize - 4 + assert ofs_length == basesize - WORD A = lltype.GcArray(lltype.Signed) basesize, itemsize, ofs_length = get_array_token(A, False) - assert basesize >= 4 # at least the 'length', maybe some gc headers - assert itemsize == 4 - assert ofs_length == basesize - 4 + assert basesize >= WORD # at least the 'length', maybe some gc headers + assert itemsize == WORD + assert ofs_length == basesize - WORD def test_varsized_struct_size(): S1 = lltype.GcStruct('S1', ('parent', S), @@ -54,9 +55,9 @@ ofs_extra, size_extra = get_field_token(S1, 'extra', False) basesize, itemsize, ofs_length = get_array_token(S1, False) assert size_parent == ofs_extra - assert size_extra == 4 - assert ofs_length == ofs_extra + 4 - assert basesize == ofs_length + 4 + assert size_extra == WORD + assert ofs_length == ofs_extra + WORD + assert basesize == ofs_length + WORD assert itemsize == 1 def test_string(): Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_zll_random.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_zll_random.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_zll_random.py Mon Aug 2 20:58:49 2010 @@ -1,9 +1,11 @@ from pypy.jit.backend.test.test_random import check_random_function, Random from pypy.jit.backend.test.test_ll_random import LLtypeOperationBuilder -from pypy.jit.backend.x86.runner import CPU386 +from pypy.jit.backend.detect_cpu import getcpuclass + +CPU = getcpuclass() def test_stress(): - cpu = CPU386(None, None) + cpu = CPU(None, None) r = Random() for i in range(1000): check_random_function(cpu, LLtypeOperationBuilder, r, i, 1000) Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_zrpy_gc.py Mon Aug 2 20:58:49 2010 @@ -17,6 +17,8 @@ from pypy.jit.backend.llsupport.gc import GcRefList, GcRootMap_asmgcc from pypy.jit.backend.llsupport.gc import GcLLDescr_framework from pypy.tool.udir import udir +from pypy.jit.backend.x86.arch import IS_X86_64 +import py.test class X(object): def __init__(self, x=0): @@ -126,6 +128,10 @@ class TestCompileHybrid(object): def setup_class(cls): + if IS_X86_64: + # No hybrid GC on 64-bit for the time being + py.test.skip() + funcs = [] name_to_func = {} for fullname in dir(cls): Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_ztranslation.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_ztranslation.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_ztranslation.py Mon Aug 2 20:58:49 2010 @@ -3,13 +3,14 @@ from pypy.rlib.jit import JitDriver, OPTIMIZER_FULL, unroll_parameters from pypy.rlib.jit import PARAMETERS, dont_look_inside from pypy.jit.metainterp.jitprof import Profiler -from pypy.jit.backend.x86.runner import CPU386 +from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.backend.test.support import CCompiledMixin from pypy.jit.codewriter.policy import StopAtXPolicy from pypy.translator.translator import TranslationContext +from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 class TestTranslationX86(CCompiledMixin): - CPUClass = CPU386 + CPUClass = getcpuclass() def _check_cbuilder(self, cbuilder): # We assume here that we have sse2. If not, the CPUClass @@ -114,7 +115,7 @@ class TestTranslationRemoveTypePtrX86(CCompiledMixin): - CPUClass = CPU386 + CPUClass = getcpuclass() def _get_TranslationContext(self): t = TranslationContext() @@ -125,6 +126,10 @@ return t def test_external_exception_handling_translates(self): + # FIXME + if IS_X86_64: + import py.test; py.test.skip() + jitdriver = JitDriver(greens = [], reds = ['n', 'total']) class ImDone(Exception): Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/tool/viewcode.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/tool/viewcode.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/tool/viewcode.py Mon Aug 2 20:58:49 2010 @@ -31,16 +31,22 @@ if sys.platform == "win32": XXX # lots more in Psyco -def machine_code_dump(data, originaddr): - # the disassembler to use. 'objdump' writes GNU-style instructions. - # 'ndisasm' would use Intel syntax, but you need to fix the output parsing. - objdump = ('objdump -M intel -b binary -m i386 ' +def machine_code_dump(data, originaddr, backend_name): + objdump_backend_option = { + 'x86': 'i386', + 'x86_64': 'x86-64', + } + objdump = ('objdump -M intel,%(backend)s -b binary -m i386 ' '--adjust-vma=%(origin)d -D %(file)s') # f = open(tmpfile, 'wb') f.write(data) f.close() - g = os.popen(objdump % {'file': tmpfile, 'origin': originaddr}, 'r') + g = os.popen(objdump % { + 'file': tmpfile, + 'origin': originaddr, + 'backend': objdump_backend_option[backend_name], + }, 'r') result = g.readlines() g.close() return result[6:] # drop some objdump cruft @@ -126,7 +132,7 @@ def disassemble(self): if not hasattr(self, 'text'): - lines = machine_code_dump(self.data, self.addr) + lines = machine_code_dump(self.data, self.addr, self.world.backend_name) # instead of adding symbol names in the dumps we could # also make the 0xNNNNNNNN addresses be red and show the # symbol name when the mouse is over them @@ -171,10 +177,13 @@ self.jumps = {} self.symbols = {} self.logentries = {} + self.backend_name = None def parse(self, f, textonly=True): for line in f: - if line.startswith('CODE_DUMP '): + if line.startswith('BACKEND '): + self.backend_name = line.split(' ')[1].strip() + elif line.startswith('CODE_DUMP '): pieces = line.split() assert pieces[1].startswith('@') assert pieces[2].startswith('+') Modified: pypy/branch/asmgcc-64/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/codewriter/codewriter.py Mon Aug 2 20:58:49 2010 @@ -14,11 +14,12 @@ class CodeWriter(object): callcontrol = None # for tests - def __init__(self, cpu=None, jitdrivers_sd=[]): + def __init__(self, cpu=None, jitdrivers_sd=[], debug=False): self.cpu = cpu self.assembler = Assembler() self.callcontrol = CallControl(cpu, jitdrivers_sd) self._seen_files = set() + self.debug = debug def transform_func_to_jitcode(self, func, values, type_system='lltype'): """For testing.""" @@ -60,7 +61,8 @@ self.assembler.assemble(ssarepr, jitcode) # # print the resulting assembler - self.print_ssa_repr(ssarepr, portal_jd, verbose) + if self.debug: + self.print_ssa_repr(ssarepr, portal_jd, verbose) def make_jitcodes(self, verbose=False): log.info("making JitCodes...") Modified: pypy/branch/asmgcc-64/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/codewriter/jtransform.py Mon Aug 2 20:58:49 2010 @@ -667,7 +667,7 @@ return self._rewrite_symmetric(op) def _is_gc(self, v): - return v.concretetype.TO._gckind == 'gc' + return getattr(getattr(v.concretetype, "TO", None), "_gckind", "?") == 'gc' def _rewrite_cmp_ptrs(self, op): if self._is_gc(op.args[0]): @@ -701,6 +701,42 @@ #return op raise NotImplementedError("cast_ptr_to_int") + def rewrite_op_force_cast(self, op): + from pypy.rpython.lltypesystem.rffi import size_and_sign, sizeof + from pypy.rlib.rarithmetic import intmask + assert not self._is_gc(op.args[0]) + size1, unsigned1 = size_and_sign(op.args[0].concretetype) + size2, unsigned2 = size_and_sign(op.result.concretetype) + if size2 >= sizeof(lltype.Signed): + return # the target type is LONG or ULONG + # + def bounds(size, unsigned): + if unsigned: + return 0, 1<<(8*size) + else: + return -(1<<(8*size-1)), 1<<(8*size-1) + min1, max1 = bounds(size1, unsigned1) + min2, max2 = bounds(size2, unsigned2) + if min2 <= min1 <= max1 <= max2: + return # the target type includes the source range + # + result = [] + v1 = op.args[0] + if min2: + c_min2 = Constant(min2, lltype.Signed) + v2 = Variable(); v2.concretetype = lltype.Signed + result.append(SpaceOperation('int_sub', [v1, c_min2], v2)) + else: + v2 = v1 + c_mask = Constant(int((1<<(8*size2))-1), lltype.Signed) + v3 = Variable(); v3.concretetype = lltype.Signed + result.append(SpaceOperation('int_and', [v2, c_mask], v3)) + if min2: + result.append(SpaceOperation('int_add', [v3, c_min2], op.result)) + else: + result[-1].result = op.result + return result + # ---------- # Renames, from the _old opname to the _new one. # The new operation is optionally further processed by rewrite_operation(). Modified: pypy/branch/asmgcc-64/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/codewriter/test/test_flatten.py Mon Aug 2 20:58:49 2010 @@ -729,3 +729,108 @@ int_between %i0, %i1, %i2 -> %i3 int_return %i3 """, transform=True) + + def test_force_cast(self): + from pypy.rpython.lltypesystem import rffi + + for FROM, TO, expected in [ + (rffi.SIGNEDCHAR, rffi.SIGNEDCHAR, ""), + (rffi.SIGNEDCHAR, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.SIGNEDCHAR, rffi.SHORT, ""), + (rffi.SIGNEDCHAR, rffi.USHORT, "int_and %i0, $65535 -> %i1"), + (rffi.SIGNEDCHAR, rffi.LONG, ""), + (rffi.SIGNEDCHAR, rffi.ULONG, ""), + + (rffi.UCHAR, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.UCHAR, rffi.UCHAR, ""), + (rffi.UCHAR, rffi.SHORT, ""), + (rffi.UCHAR, rffi.USHORT, ""), + (rffi.UCHAR, rffi.LONG, ""), + (rffi.UCHAR, rffi.ULONG, ""), + + (rffi.SHORT, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.SHORT, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.SHORT, rffi.SHORT, ""), + (rffi.SHORT, rffi.USHORT, "int_and %i0, $65535 -> %i1"), + (rffi.SHORT, rffi.LONG, ""), + (rffi.SHORT, rffi.ULONG, ""), + + (rffi.USHORT, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.USHORT, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.USHORT, rffi.SHORT, """int_sub %i0, $-32768 -> %i1 + int_and %i1, $65535 -> %i2 + int_add %i2, $-32768 -> %i3"""), + (rffi.USHORT, rffi.USHORT, ""), + (rffi.USHORT, rffi.LONG, ""), + (rffi.USHORT, rffi.ULONG, ""), + + (rffi.LONG, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.LONG, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.LONG, rffi.SHORT, """int_sub %i0, $-32768 -> %i1 + int_and %i1, $65535 -> %i2 + int_add %i2, $-32768 -> %i3"""), + (rffi.LONG, rffi.USHORT, "int_and %i0, $65535 -> %i1"), + (rffi.LONG, rffi.LONG, ""), + (rffi.LONG, rffi.ULONG, ""), + + (rffi.ULONG, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.ULONG, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.ULONG, rffi.SHORT, """int_sub %i0, $-32768 -> %i1 + int_and %i1, $65535 -> %i2 + int_add %i2, $-32768 -> %i3"""), + (rffi.ULONG, rffi.USHORT, "int_and %i0, $65535 -> %i1"), + (rffi.ULONG, rffi.LONG, ""), + (rffi.ULONG, rffi.ULONG, ""), + ]: + expected = [s.strip() for s in expected.splitlines()] + check_force_cast(FROM, TO, expected, 42) + check_force_cast(FROM, TO, expected, -42) + expected.append('int_return %i' + str(len(expected))) + expected = '\n'.join(expected) + # + def f(n): + return rffi.cast(TO, n) + self.encoding_test(f, [rffi.cast(FROM, 42)], expected, + transform=True) + + def test_force_cast_pointer(self): + from pypy.rpython.lltypesystem import rffi + def h(p): + return rffi.cast(rffi.VOIDP, p) + self.encoding_test(h, [lltype.nullptr(rffi.CCHARP.TO)], """ + int_return %i0 + """, transform=True) + + +def check_force_cast(FROM, TO, operations, value): + """Check that the test is correctly written...""" + from pypy.rpython.lltypesystem import rffi + import re + r = re.compile('(\w+) \%i\d, \$(-?\d+)') + # + value = rffi.cast(FROM, value) + value = rffi.cast(lltype.Signed, value) + # + expected_value = rffi.cast(TO, value) + expected_value = rffi.cast(lltype.Signed, expected_value) + # + for op in operations: + match = r.match(op) + assert match, "line %r does not match regexp" % (op,) + opname = match.group(1) + if opname == 'int_add': value += int(match.group(2)) + elif opname == 'int_sub': value -= int(match.group(2)) + elif opname == 'int_and': value &= int(match.group(2)) + else: assert 0, opname + # + assert rffi.cast(lltype.Signed, value) == expected_value Modified: pypy/branch/asmgcc-64/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/metainterp/optimizeopt.py Mon Aug 2 20:58:49 2010 @@ -18,6 +18,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.jit.metainterp.history import AbstractDescr, make_hashable_int + def optimize_loop_1(metainterp_sd, loop): """Optimize loop.operations to make it match the input of loop.specnodes and to remove internal overheadish operations. Note that loop.specnodes @@ -992,6 +993,25 @@ self.make_equal_to(op.result, v1) else: self.optimize_default(op) + + def optimize_INT_SUB(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + if v2.is_constant() and v2.box.getint() == 0: + self.make_equal_to(op.result, v1) + else: + return self.optimize_default(op) + + def optimize_INT_ADD(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + # If one side of the op is 0 the result is the other side. + if v1.is_constant() and v1.box.getint() == 0: + self.make_equal_to(op.result, v2) + elif v2.is_constant() and v2.box.getint() == 0: + self.make_equal_to(op.result, v1) + else: + self.optimize_default(op) optimize_ops = _findall(Optimizer, 'optimize_') Modified: pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_loop.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_loop.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_loop.py Mon Aug 2 20:58:49 2010 @@ -9,6 +9,10 @@ class LoopTest(object): optimizer = OPTIMIZER_SIMPLE + automatic_promotion_result = { + 'int_add' : 6, 'int_gt' : 1, 'guard_false' : 1, 'jump' : 1, + 'guard_value' : 3 + } def meta_interp(self, f, args, policy=None): return ll_meta_interp(f, args, optimizer=self.optimizer, @@ -477,9 +481,9 @@ res = self.meta_interp(main_interpreter_loop, [1]) assert res == main_interpreter_loop(1) self.check_loop_count(1) - # XXX maybe later optimize guard_value away - self.check_loops({'int_add' : 6, 'int_gt' : 1, - 'guard_false' : 1, 'jump' : 1, 'guard_value' : 3}) + # These loops do different numbers of ops based on which optimizer we + # are testing with. + self.check_loops(self.automatic_promotion_result) def test_can_enter_jit_outside_main_loop(self): myjitdriver = JitDriver(greens=[], reds=['i', 'j', 'a']) Modified: pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_loop_spec.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_loop_spec.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_loop_spec.py Mon Aug 2 20:58:49 2010 @@ -5,6 +5,10 @@ class LoopSpecTest(test_loop.LoopTest): optimizer = OPTIMIZER_FULL + automatic_promotion_result = { + 'int_add' : 3, 'int_gt' : 1, 'guard_false' : 1, 'jump' : 1, + 'guard_value' : 1 + } # ====> test_loop.py Modified: pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_optimizeopt.py Mon Aug 2 20:58:49 2010 @@ -2051,8 +2051,41 @@ jump(i1, i0) """ self.optimize_loop(ops, 'Not, Not', expected) - - + + def test_fold_partially_constant_ops(self): + ops = """ + [i0] + i1 = int_sub(i0, 0) + jump(i1) + """ + expected = """ + [i0] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + ops = """ + [i0] + i1 = int_add(i0, 0) + jump(i1) + """ + expected = """ + [i0] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + ops = """ + [i0] + i1 = int_add(0, i0) + jump(i1) + """ + expected = """ + [i0] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + # ---------- def make_fail_descr(self): Modified: pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_recursive.py Mon Aug 2 20:58:49 2010 @@ -523,7 +523,7 @@ def test_trace_from_start(self): def p(pc, code): code = hlstr(code) - return "%s %d %s" % (code, pc, code[pc]) + return "'%s' at %d: %s" % (code, pc, code[pc]) def c(pc, code): return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], @@ -537,9 +537,9 @@ op = code[pc] if op == "+": n += 7 - if op == "-": + elif op == "-": n -= 1 - if op == "c": + elif op == "c": n = f('---', n) elif op == "l": if n > 0: @@ -556,6 +556,7 @@ result = 0 for i in range(m): result += f('+-cl--', i) + g(50) self.meta_interp(g, [50], backendopt=True) self.check_tree_loop_count(3) self.check_history(int_add=1) Modified: pypy/branch/asmgcc-64/pypy/module/__builtin__/compiling.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/__builtin__/compiling.py (original) +++ pypy/branch/asmgcc-64/pypy/module/__builtin__/compiling.py Mon Aug 2 20:58:49 2010 @@ -38,7 +38,7 @@ str_ = space.str_w(w_source) ec = space.getexecutioncontext() - if flags & ~(ec.compiler.compiler_flags | consts.PyCF_AST_ONLY | + if flags & ~(ec.compiler.compiler_flags | consts.PyCF_ONLY_AST | consts.PyCF_DONT_IMPLY_DEDENT | consts.PyCF_SOURCE_IS_UTF8): raise OperationError(space.w_ValueError, space.wrap("compile() unrecognized flags")) @@ -53,7 +53,7 @@ "or 'eval' or 'single'")) if ast_node is None: - if flags & consts.PyCF_AST_ONLY: + if flags & consts.PyCF_ONLY_AST: mod = ec.compiler.compile_to_ast(str_, filename, mode, flags) return space.wrap(mod) else: Modified: pypy/branch/asmgcc-64/pypy/module/_ast/__init__.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/_ast/__init__.py (original) +++ pypy/branch/asmgcc-64/pypy/module/_ast/__init__.py Mon Aug 2 20:58:49 2010 @@ -5,7 +5,7 @@ class Module(MixedModule): interpleveldefs = { - "PyCF_AST_ONLY" : "space.wrap(%s)" % consts.PyCF_AST_ONLY + "PyCF_ONLY_AST" : "space.wrap(%s)" % consts.PyCF_ONLY_AST } appleveldefs = {} Modified: pypy/branch/asmgcc-64/pypy/module/_ast/test/test_ast.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/_ast/test/test_ast.py (original) +++ pypy/branch/asmgcc-64/pypy/module/_ast/test/test_ast.py Mon Aug 2 20:58:49 2010 @@ -10,7 +10,7 @@ cls.w_get_ast = cls.space.appexec([], """(): def get_ast(source, mode="exec"): import _ast as ast - mod = compile(source, "", mode, ast.PyCF_AST_ONLY) + mod = compile(source, "", mode, ast.PyCF_ONLY_AST) assert isinstance(mod, ast.mod) return mod return get_ast""") Modified: pypy/branch/asmgcc-64/pypy/module/_file/interp_file.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/_file/interp_file.py (original) +++ pypy/branch/asmgcc-64/pypy/module/_file/interp_file.py Mon Aug 2 20:58:49 2010 @@ -4,6 +4,7 @@ from pypy.rlib.rarithmetic import r_longlong from pypy.module._file.interp_stream import W_AbstractStream from pypy.module._file.interp_stream import StreamErrors, wrap_streamerror +from pypy.module.posix.interp_posix import dispatch_filename from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import ObjSpace, W_Root, Arguments from pypy.interpreter.typedef import TypeDef, GetSetProperty @@ -81,11 +82,11 @@ # file lock. They don't convert StreamErrors to OperationErrors, too. def direct___init__(self, w_name, mode='r', buffering=-1): - name = self.space.str_w(w_name) self.direct_close() self.w_name = w_name self.check_mode_ok(mode) - stream = streamio.open_file_as_stream(name, mode, buffering) + stream = dispatch_filename(streamio.open_file_as_stream)( + self.space, w_name, mode, buffering) fd = stream.try_to_find_file_descriptor() self.fdopenstream(stream, fd, mode) Modified: pypy/branch/asmgcc-64/pypy/module/_file/test/test_file.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/_file/test/test_file.py (original) +++ pypy/branch/asmgcc-64/pypy/module/_file/test/test_file.py Mon Aug 2 20:58:49 2010 @@ -125,6 +125,15 @@ assert type(res) is str f.close() + def test_unicode_filename(self): + import sys + try: + u'\xe9'.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + skip("encoding not good enough") + f = self.file(self.temppath + u'\xe9', "w") + f.close() + def test_oserror_has_filename(self): try: f = self.file("file that is clearly not there") Modified: pypy/branch/asmgcc-64/pypy/module/_rawffi/test/test__rawffi.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/_rawffi/test/test__rawffi.py (original) +++ pypy/branch/asmgcc-64/pypy/module/_rawffi/test/test__rawffi.py Mon Aug 2 20:58:49 2010 @@ -677,7 +677,12 @@ a = A(1) a[0] = -1234 a.free() - + + def test_long_with_fromaddress(self): + import _rawffi + addr = -1 + raises(ValueError, _rawffi.Array('u').fromaddress, addr, 100) + def test_passing_raw_pointers(self): import _rawffi lib = _rawffi.CDLL(self.lib_name) Modified: pypy/branch/asmgcc-64/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/cpyext/methodobject.py (original) +++ pypy/branch/asmgcc-64/pypy/module/cpyext/methodobject.py Mon Aug 2 20:58:49 2010 @@ -100,7 +100,11 @@ return generic_cpy_call(space, self.ml.c_ml_meth, w_self, w_arg) def get_doc(space, self): - return space.wrap(rffi.charp2str(self.ml.c_ml_doc)) + doc = self.ml.c_ml_doc + if doc: + return space.wrap(rffi.charp2str(doc)) + else: + return space.w_None class W_PyCMethodObject(W_PyCFunctionObject): Modified: pypy/branch/asmgcc-64/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/asmgcc-64/pypy/module/posix/interp_posix.py Mon Aug 2 20:58:49 2010 @@ -1,8 +1,9 @@ from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped from pypy.rlib import rposix +from pypy.rlib.objectmodel import specialize from pypy.rlib.rarithmetic import r_longlong from pypy.rlib.unroll import unrolling_iterable -from pypy.interpreter.error import OperationError, wrap_oserror +from pypy.interpreter.error import OperationError, wrap_oserror, wrap_oserror2 from pypy.rpython.module.ll_os import RegisterOs from pypy.rpython.module import ll_os_stat from pypy.rpython.lltypesystem import rffi, lltype @@ -12,15 +13,78 @@ import os, sys _WIN = sys.platform == 'win32' -def open(space, fname, flag, mode=0777): +class FileEncoder: + def __init__(self, space, w_obj): + self.space = space + self.w_obj = w_obj + + def as_bytes(self): + from pypy.module.sys.interp_encoding import getfilesystemencoding + space = self.space + w_bytes = space.call_method(self.w_obj, 'encode', + getfilesystemencoding(space)) + return space.str_w(w_bytes) + + def as_unicode(self): + return self.space.unicode_w(self.w_obj) + +class FileDecoder: + def __init__(self, space, w_obj): + self.space = space + self.w_obj = w_obj + + def as_bytes(self): + return self.space.str_w(self.w_obj) + + def as_unicode(self): + from pypy.module.sys.interp_encoding import getfilesystemencoding + space = self.space + w_unicode = space.call_method(self.w_obj, 'decode', + getfilesystemencoding(space)) + return space.unicode_w(w_unicode) + + at specialize.memo() +def dispatch_filename(func, tag=0): + def dispatch(space, w_fname, *args): + if space.isinstance_w(w_fname, space.w_unicode): + fname = FileEncoder(space, w_fname) + return func(fname, *args) + else: + fname = space.str_w(w_fname) + return func(fname, *args) + return dispatch + + at specialize.memo() +def dispatch_filename_2(func): + def dispatch(space, w_fname1, w_fname2, *args): + if space.isinstance_w(w_fname1, space.w_unicode): + fname1 = FileEncoder(space, w_fname1) + if space.isinstance_w(w_fname2, space.w_unicode): + fname2 = FileEncoder(space, w_fname2) + return func(fname1, fname2, *args) + else: + fname2 = FileDecoder(space, w_fname2) + return func(fname1, fname2, *args) + else: + fname1 = FileDecoder(space, w_fname1) + if space.isinstance_w(w_fname2, space.w_unicode): + fname2 = FileEncoder(space, w_fname2) + return func(fname1, fname2, *args) + else: + fname2 = FileDecoder(space, w_fname2) + return func(fname1, fname2, *args) + return dispatch + +def open(space, w_fname, flag, mode=0777): """Open a file (for low level IO). Return a file descriptor (a small integer).""" - try: - fd = os.open(fname, flag, mode) + try: + fd = dispatch_filename(rposix.open)( + space, w_fname, flag, mode) except OSError, e: - raise wrap_oserror(space, e, fname) + raise wrap_oserror2(space, e, w_fname) return space.wrap(fd) -open.unwrap_spec = [ObjSpace, 'path', "c_int", "c_int"] +open.unwrap_spec = [ObjSpace, W_Root, "c_int", "c_int"] def lseek(space, fd, pos, how): """Set the current position of a file descriptor. Return the new position. @@ -159,7 +223,7 @@ return build_stat_result(space, st) fstat.unwrap_spec = [ObjSpace, "c_int"] -def stat(space, path): +def stat(space, w_path): """Perform a stat system call on the given path. Return an object with (at least) the following attributes: st_mode @@ -175,22 +239,22 @@ """ try: - st = os.stat(path) + st = dispatch_filename(rposix.stat)(space, w_path) except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) else: return build_stat_result(space, st) -stat.unwrap_spec = [ObjSpace, 'path'] +stat.unwrap_spec = [ObjSpace, W_Root] -def lstat(space, path): +def lstat(space, w_path): "Like stat(path), but do no follow symbolic links." try: - st = os.lstat(path) + st = dispatch_filename(rposix.lstat)(space, w_path) except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) else: return build_stat_result(space, st) -lstat.unwrap_spec = [ObjSpace, 'path'] +lstat.unwrap_spec = [ObjSpace, W_Root] class StatState(object): def __init__(self, space): @@ -231,7 +295,7 @@ raise wrap_oserror(space, e) dup2.unwrap_spec = [ObjSpace, "c_int", "c_int"] -def access(space, path, mode): +def access(space, w_path, mode): """ access(path, mode) -> 1 if granted, 0 otherwise @@ -242,12 +306,12 @@ existence, or the inclusive-OR of R_OK, W_OK, and X_OK. """ try: - ok = os.access(path, mode) - except OSError, e: - raise wrap_oserror(space, e, path) + ok = dispatch_filename(rposix.access)(space, w_path, mode) + except OSError, e: + raise wrap_oserror2(space, e, w_path) else: return space.wrap(ok) -access.unwrap_spec = [ObjSpace, str, "c_int"] +access.unwrap_spec = [ObjSpace, W_Root, "c_int"] def times(space): @@ -278,32 +342,38 @@ return space.wrap(rc) system.unwrap_spec = [ObjSpace, str] -def unlink(space, path): +def unlink(space, w_path): """Remove a file (same as remove(path)).""" try: - os.unlink(path) - except OSError, e: - raise wrap_oserror(space, e, path) -unlink.unwrap_spec = [ObjSpace, 'path'] + dispatch_filename(rposix.unlink)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +unlink.unwrap_spec = [ObjSpace, W_Root] -def remove(space, path): +def remove(space, w_path): """Remove a file (same as unlink(path)).""" try: - os.unlink(path) - except OSError, e: - raise wrap_oserror(space, e, path) -remove.unwrap_spec = [ObjSpace, 'path'] + dispatch_filename(rposix.unlink)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +remove.unwrap_spec = [ObjSpace, W_Root] -def _getfullpathname(space, path): +def _getfullpathname(space, w_path): """helper for ntpath.abspath """ - posix = __import__(os.name) # nt specific try: - fullpath = posix._getfullpathname(path) + if space.isinstance_w(w_path, space.w_unicode): + path = FileEncoder(space, w_path) + fullpath = rposix._getfullpathname(path) + w_fullpath = space.wrap(fullpath) + else: + path = space.str_w(w_path) + fullpath = rposix._getfullpathname(path) + w_fullpath = space.wrap(fullpath) except OSError, e: - raise wrap_oserror(space, e, path) - else: - return space.wrap(fullpath) -_getfullpathname.unwrap_spec = [ObjSpace, str] + raise wrap_oserror2(space, e, w_path) + else: + return w_fullpath +_getfullpathname.unwrap_spec = [ObjSpace, W_Root] def getcwd(space): """Return the current working directory.""" @@ -315,35 +385,46 @@ return space.wrap(cur) getcwd.unwrap_spec = [ObjSpace] -def getcwdu(space): - """Return the current working directory as a unicode string.""" - # XXX ascii encoding for now - return space.call_method(getcwd(space), 'decode') +if sys.platform == 'win32': + def getcwdu(space): + """Return the current working directory as a unicode string.""" + try: + cur = os.getcwdu() + except OSError, e: + raise wrap_oserror(space, e) + else: + return space.wrap(cur) +else: + def getcwdu(space): + """Return the current working directory as a unicode string.""" + filesystemencoding = space.sys.filesystemencoding + return space.call_method(getcwd(space), 'decode', + space.wrap(filesystemencoding)) getcwdu.unwrap_spec = [ObjSpace] -def chdir(space, path): +def chdir(space, w_path): """Change the current working directory to the specified path.""" try: - os.chdir(path) - except OSError, e: - raise wrap_oserror(space, e, path) -chdir.unwrap_spec = [ObjSpace, str] + dispatch_filename(rposix.chdir)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +chdir.unwrap_spec = [ObjSpace, W_Root] -def mkdir(space, path, mode=0777): +def mkdir(space, w_path, mode=0777): """Create a directory.""" try: - os.mkdir(path, mode) - except OSError, e: - raise wrap_oserror(space, e, path) -mkdir.unwrap_spec = [ObjSpace, str, "c_int"] + dispatch_filename(rposix.mkdir)(space, w_path, mode) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +mkdir.unwrap_spec = [ObjSpace, W_Root, "c_int"] -def rmdir(space, path): +def rmdir(space, w_path): """Remove a directory.""" try: - os.rmdir(path) - except OSError, e: - raise wrap_oserror(space, e, path) -rmdir.unwrap_spec = [ObjSpace, str] + dispatch_filename(rposix.rmdir)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +rmdir.unwrap_spec = [ObjSpace, W_Root] def strerror(space, errno): """Translate an error code to a message string.""" @@ -410,7 +491,7 @@ unsetenv.unwrap_spec = [ObjSpace, str] -def listdir(space, dirname): +def listdir(space, w_dirname): """Return a list containing the names of the entries in the directory. \tpath: path of directory to list @@ -418,12 +499,18 @@ The list is in arbitrary order. It does not include the special entries '.' and '..' even if they are present in the directory.""" try: - result = os.listdir(dirname) + if space.isinstance_w(w_dirname, space.w_unicode): + dirname = FileEncoder(space, w_dirname) + result = rposix.listdir(dirname) + result_w = [space.wrap(s) for s in result] + else: + dirname = space.str_w(w_dirname) + result = rposix.listdir(dirname) + result_w = [space.wrap(s) for s in result] except OSError, e: - raise wrap_oserror(space, e, dirname) - result_w = [space.wrap(s) for s in result] + raise wrap_oserror2(space, e, w_dirname) return space.newlist(result_w) -listdir.unwrap_spec = [ObjSpace, str] +listdir.unwrap_spec = [ObjSpace, W_Root] def pipe(space): "Create a pipe. Returns (read_end, write_end)." @@ -434,21 +521,21 @@ return space.newtuple([space.wrap(fd1), space.wrap(fd2)]) pipe.unwrap_spec = [ObjSpace] -def chmod(space, path, mode): +def chmod(space, w_path, mode): "Change the access permissions of a file." - try: - os.chmod(path, mode) - except OSError, e: - raise wrap_oserror(space, e, path) -chmod.unwrap_spec = [ObjSpace, str, "c_int"] + try: + dispatch_filename(rposix.chmod)(space, w_path, mode) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +chmod.unwrap_spec = [ObjSpace, W_Root, "c_int"] -def rename(space, old, new): +def rename(space, w_old, w_new): "Rename a file or directory." - try: - os.rename(old, new) - except OSError, e: + try: + dispatch_filename_2(rposix.rename)(space, w_old, w_new) + except OSError, e: raise wrap_oserror(space, e) -rename.unwrap_spec = [ObjSpace, str, str] +rename.unwrap_spec = [ObjSpace, W_Root, W_Root] def umask(space, mask): "Set the current numeric umask and return the previous umask." @@ -576,7 +663,7 @@ raise wrap_oserror(space, e) execve.unwrap_spec = [ObjSpace, str, W_Root, W_Root] -def utime(space, path, w_tuple): +def utime(space, w_path, w_tuple): """ utime(path, (atime, mtime)) utime(path, None) @@ -585,10 +672,10 @@ """ if space.is_w(w_tuple, space.w_None): try: - os.utime(path, None) + dispatch_filename(rposix.utime, 1)(space, w_path, None) return except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) try: msg = "utime() arg 2 must be a tuple (atime, mtime) or None" args_w = space.fixedview(w_tuple) @@ -596,14 +683,14 @@ raise OperationError(space.w_TypeError, space.wrap(msg)) actime = space.float_w(args_w[0]) modtime = space.float_w(args_w[1]) - os.utime(path, (actime, modtime)) + dispatch_filename(rposix.utime, 2)(space, w_path, (actime, modtime)) except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) except OperationError, e: if not e.match(space, space.w_TypeError): raise raise OperationError(space.w_TypeError, space.wrap(msg)) -utime.unwrap_spec = [ObjSpace, str, W_Root] +utime.unwrap_spec = [ObjSpace, W_Root, W_Root] def setsid(space): """setsid() -> pid Modified: pypy/branch/asmgcc-64/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/posix/test/test_posix2.py (original) +++ pypy/branch/asmgcc-64/pypy/module/posix/test/test_posix2.py Mon Aug 2 20:58:49 2010 @@ -32,6 +32,9 @@ # even when running on top of CPython 2.4. os.stat_float_times(True) + # Initialize sys.filesystemencoding + space.call_method(space.getbuiltinmodule('sys'), 'getfilesystemencoding') + def need_sparse_files(): if sys.platform == 'darwin': py.test.skip("no sparse files on default Mac OS X file system") @@ -706,6 +709,28 @@ except OSError: pass +class AppTestUnicodeFilename: + def setup_class(cls): + ufilename = (unicode(udir.join('test_unicode_filename_')) + + u'\u65e5\u672c.txt') # "Japan" + try: + f = file(ufilename, 'w') + except UnicodeEncodeError: + py.test.skip("encoding not good enough") + f.write("test") + f.close() + cls.space = space + cls.w_filename = space.wrap(ufilename) + cls.w_posix = space.appexec([], GET_POSIX) + + def test_open(self): + fd = self.posix.open(self.filename, self.posix.O_RDONLY) + try: + content = self.posix.read(fd, 50) + finally: + self.posix.close(fd) + assert content == "test" + class TestPexpect(object): # XXX replace with AppExpectTest class as soon as possible Modified: pypy/branch/asmgcc-64/pypy/module/signal/interp_signal.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/signal/interp_signal.py (original) +++ pypy/branch/asmgcc-64/pypy/module/signal/interp_signal.py Mon Aug 2 20:58:49 2010 @@ -7,6 +7,7 @@ from pypy.translator.tool.cbuild import ExternalCompilationInfo import py from pypy.tool import autopath +from pypy.rlib import jit def setup(): for key, value in cpy_signal.__dict__.items(): @@ -159,10 +160,12 @@ return space.wrap(SIG_DFL) getsignal.unwrap_spec = [ObjSpace, int] + at jit.dont_look_inside def alarm(space, timeout): return space.wrap(c_alarm(timeout)) alarm.unwrap_spec = [ObjSpace, int] + at jit.dont_look_inside def pause(space): c_pause() return space.w_None @@ -173,6 +176,7 @@ raise OperationError(space.w_ValueError, space.wrap("signal number out of range")) + at jit.dont_look_inside def signal(space, signum, w_handler): """ signal(sig, action) -> action Modified: pypy/branch/asmgcc-64/pypy/objspace/std/callmethod.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/objspace/std/callmethod.py (original) +++ pypy/branch/asmgcc-64/pypy/objspace/std/callmethod.py Mon Aug 2 20:58:49 2010 @@ -12,7 +12,7 @@ from pypy.interpreter import function from pypy.objspace.descroperation import object_getattribute -from pypy.rlib import rstack # for resume points +from pypy.rlib import jit, rstack # for resume points # This module exports two extra methods for StdObjSpaceFrame implementing # the LOOKUP_METHOD and CALL_METHOD opcodes in an efficient way, as well @@ -56,16 +56,42 @@ f.pushvalue(w_value) f.pushvalue(None) -def CALL_METHOD(f, nargs, *ignored): - # 'nargs' is the argument count excluding the implicit 'self' - w_self = f.peekvalue(nargs) - w_callable = f.peekvalue(nargs + 1) - n = nargs + (w_self is not None) - try: - w_result = f.space.call_valuestack(w_callable, n, f) - rstack.resume_point("CALL_METHOD", f, nargs, returns=w_result) - finally: - f.dropvalues(nargs + 2) + at jit.unroll_safe +def CALL_METHOD(f, oparg, *ignored): + # opargs contains the arg, and kwarg count, excluding the implicit 'self' + n_args = oparg & 0xff + n_kwargs = (oparg >> 8) & 0xff + w_self = f.peekvalue(n_args + (2 * n_kwargs)) + w_callable = f.peekvalue(n_args + (2 * n_kwargs) + 1) + n = n_args + (w_self is not None) + + if not n_kwargs: + try: + w_result = f.space.call_valuestack(w_callable, n, f) + rstack.resume_point("CALL_METHOD_no_kwargs", f, n_args, returns=w_result) + finally: + f.dropvalues(n_args + 2) + else: + keywords = [None] * n_kwargs + keywords_w = [None] * n_kwargs + while True: + n_kwargs -= 1 + if n_kwargs < 0: + break + w_value = f.popvalue() + w_key = f.popvalue() + key = f.space.str_w(w_key) + keywords[n_kwargs] = key + keywords_w[n_kwargs] = w_value + + arguments = f.popvalues(n) + args = f.argument_factory(arguments, keywords, keywords_w, None, None) + + try: + w_result = f.space.call_args(w_callable, args) + rstack.resume_point("CALL_METHOD", f, w_self, returns=w_result) + finally: + f.dropvalues(1 + (w_self is None)) f.pushvalue(w_result) Modified: pypy/branch/asmgcc-64/pypy/objspace/std/test/test_callmethod.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/objspace/std/test/test_callmethod.py (original) +++ pypy/branch/asmgcc-64/pypy/objspace/std/test/test_callmethod.py Mon Aug 2 20:58:49 2010 @@ -106,6 +106,15 @@ else: raise Exception("did not raise?") """ + + def test_kwargs(self): + exec """if 1: + class C(object): + def f(self, a): + return a + 2 + + assert C().f(a=3) == 5 + """ class AppTestCallMethodWithGetattributeShortcut(AppTestCallMethod): Modified: pypy/branch/asmgcc-64/pypy/rlib/objectmodel.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rlib/objectmodel.py (original) +++ pypy/branch/asmgcc-64/pypy/rlib/objectmodel.py Mon Aug 2 20:58:49 2010 @@ -19,32 +19,80 @@ # def f(... # -class _AttachSpecialization(object): +class _Specialize(object): + def memo(self): + """ Specialize functions based on argument values. All arguments has + to be constant at the compile time. The whole function call is replaced + by a call result then. + """ + def decorated_func(func): + func._annspecialcase_ = 'specialize:memo' + return func + return decorated_func - def __init__(self, tag): - self.tag = tag + def arg(self, *args): + """ Specialize function based on values of given positions of arguments. + They must be compile-time constants in order to work. + + There will be a copy of provided function for each combination + of given arguments on positions in args (that can lead to + exponential behavior!). + """ + def decorated_func(func): + func._annspecialcase_ = 'specialize:arg' + self._wrap(args) + return func - def __call__(self, *args): - if not args: - args = "" - else: - args = "("+','.join([repr(arg) for arg in args]) +")" - specialcase = "specialize:%s%s" % (self.tag, args) - - def specialize_decorator(func): - "NOT_RPYTHON" - func._annspecialcase_ = specialcase + return decorated_func + + def argtype(self, *args): + """ Specialize function based on types of arguments on given positions. + + There will be a copy of provided function for each combination + of given arguments on positions in args (that can lead to + exponential behavior!). + """ + def decorated_func(func): + func._annspecialcase_ = 'specialize:argtype' + self._wrap(args) return func - return specialize_decorator - -class _Specialize(object): + return decorated_func - def __getattr__(self, name): - return _AttachSpecialization(name) + def ll(self): + """ This is version of argtypes that cares about low-level types + (so it'll get additional copies for two different types of pointers + for example). Same warnings about exponential behavior apply. + """ + def decorated_func(func): + func._annspecialcase_ = 'specialize:ll' + return func + + return decorated_func + + def ll_and_arg(self, arg): + """ XXX what does that do? + """ + def decorated_func(func): + func._annspecialcase_ = 'specialize:ll_and_arg(%d)' % arg + return func + + return decorated_func + + def _wrap(self, args): + return "("+','.join([repr(arg) for arg in args]) +")" specialize = _Specialize() +def enforceargs(*args): + """ Decorate a function with forcing of RPython-level types on arguments. + None means no enforcing. + + XXX shouldn't we also add asserts in function body? + """ + def decorator(f): + f._annenforceargs_ = args + return f + return decorator + # ____________________________________________________________ class Symbolic(object): Modified: pypy/branch/asmgcc-64/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rlib/rarithmetic.py (original) +++ pypy/branch/asmgcc-64/pypy/rlib/rarithmetic.py Mon Aug 2 20:58:49 2010 @@ -33,7 +33,7 @@ """ -import math +import sys, math from pypy.rpython import extregistry from pypy.rlib import objectmodel @@ -113,14 +113,23 @@ "NOT_RPYTHON" return _local_ovfcheck(int(long(a) << b)) -FL_MAXINT = float(LONG_TEST-1) -FL_MININT = float(-LONG_TEST) - -def ovfcheck_float_to_int(x): - _, intp = math.modf(x) - if FL_MININT < intp < FL_MAXINT: - return int(intp) - raise OverflowError +# Strange things happening for float to int on 64 bit: +# int(float(i)) != i because of rounding issues. +# These are the minimum and maximum float value that can +# successfully be casted to an int. +if sys.maxint == 2147483647: + def ovfcheck_float_to_int(x): + if -2147483649.0 < x < 2147483648.0: + return int(x) + raise OverflowError +else: + # The following values are not quite +/-sys.maxint. + # Note the "<= x <" here, as opposed to "< x <" above. + # This is justified by test_typed in translator/c/test. + def ovfcheck_float_to_int(x): + if -9223372036854776832.0 <= x < 9223372036854775296.0: + return int(x) + raise OverflowError def compute_restype(self_type, other_type): if self_type is other_type: Modified: pypy/branch/asmgcc-64/pypy/rlib/rdynload.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rlib/rdynload.py (original) +++ pypy/branch/asmgcc-64/pypy/rlib/rdynload.py Mon Aug 2 20:58:49 2010 @@ -18,8 +18,6 @@ if _WIN32: from pypy.rlib import rwin32 - -if _WIN32: includes = ['windows.h'] else: includes = ['dlfcn.h'] Modified: pypy/branch/asmgcc-64/pypy/rlib/rposix.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rlib/rposix.py (original) +++ pypy/branch/asmgcc-64/pypy/rlib/rposix.py Mon Aug 2 20:58:49 2010 @@ -3,6 +3,7 @@ from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rlib.rarithmetic import intmask +from pypy.rlib.objectmodel import specialize class CConstantErrno(CConstant): # these accessors are used when calling get_errno() or set_errno() @@ -42,3 +43,102 @@ os.close(fd) except OSError: pass + +#___________________________________________________________________ +# Wrappers around posix functions, that accept either strings, or +# instances with a "as_bytes()" method. +# - pypy.modules.posix.interp_posix passes an object containing a unicode path +# which can encode itself with sys.filesystemencoding. +# - but pypy.rpython.module.ll_os.py on Windows will replace these functions +# with other wrappers that directly handle unicode strings. + at specialize.argtype(0) +def open(path, flags, mode): + if isinstance(path, str): + return os.open(path, flags, mode) + else: + return os.open(path.as_bytes(), flags, mode) + + at specialize.argtype(0) +def stat(path): + if isinstance(path, str): + return os.stat(path) + else: + return os.stat(path.as_bytes()) + + at specialize.argtype(0) +def lstat(path): + if isinstance(path, str): + return os.lstat(path) + else: + return os.lstat(path.as_bytes()) + + at specialize.argtype(0) +def unlink(path): + if isinstance(path, str): + return os.unlink(path) + else: + return os.unlink(path.as_bytes()) + + at specialize.argtype(0, 1) +def rename(path1, path2): + if isinstance(path1, str): + return os.rename(path1, path2) + else: + return os.rename(path1.as_bytes(), path2.as_bytes()) + + at specialize.argtype(0) +def listdir(dirname): + if isinstance(dirname, str): + return os.listdir(dirname) + else: + return os.listdir(dirname.as_bytes()) + + at specialize.argtype(0) +def access(path, mode): + if isinstance(path, str): + return os.access(path, mode) + else: + return os.access(path.as_bytes(), mode) + + at specialize.argtype(0) +def chmod(path, mode): + if isinstance(path, str): + return os.chmod(path, mode) + else: + return os.chmod(path.as_bytes(), mode) + + at specialize.argtype(0, 1) +def utime(path, times): + if isinstance(path, str): + return os.utime(path, times) + else: + return os.utime(path.as_bytes(), times) + + at specialize.argtype(0) +def chdir(path): + if isinstance(path, str): + return os.chdir(path) + else: + return os.chdir(path.as_bytes()) + + at specialize.argtype(0) +def mkdir(path, mode=0777): + if isinstance(path, str): + return os.mkdir(path, mode) + else: + return os.mkdir(path.as_bytes(), mode) + + at specialize.argtype(0) +def rmdir(path): + if isinstance(path, str): + return os.rmdir(path) + else: + return os.rmdir(path.as_bytes()) + +if os.name == 'nt': + import nt + def _getfullpathname(path): + if isinstance(path, str): + return nt._getfullpathname(path) + else: + return nt._getfullpathname(path.as_bytes()) Modified: pypy/branch/asmgcc-64/pypy/rlib/rwin32.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rlib/rwin32.py (original) +++ pypy/branch/asmgcc-64/pypy/rlib/rwin32.py Mon Aug 2 20:58:49 2010 @@ -31,6 +31,7 @@ DWORD = rffi_platform.SimpleType("DWORD", rffi.UINT) BOOL = rffi_platform.SimpleType("BOOL", rffi.LONG) BYTE = rffi_platform.SimpleType("BYTE", rffi.UCHAR) + WCHAR = rffi_platform.SimpleType("WCHAR", rffi.UCHAR) INT = rffi_platform.SimpleType("INT", rffi.INT) LONG = rffi_platform.SimpleType("LONG", rffi.LONG) PLONG = rffi_platform.SimpleType("PLONG", rffi.LONGP) @@ -38,6 +39,8 @@ LPCVOID = rffi_platform.SimpleType("LPCVOID", rffi.VOIDP) LPSTR = rffi_platform.SimpleType("LPSTR", rffi.CCHARP) LPCSTR = rffi_platform.SimpleType("LPCSTR", rffi.CCHARP) + LPWSTR = rffi_platform.SimpleType("LPWSTR", rffi.CWCHARP) + LPCWSTR = rffi_platform.SimpleType("LPCWSTR", rffi.CWCHARP) LPDWORD = rffi_platform.SimpleType("LPDWORD", rffi.INTP) SIZE_T = rffi_platform.SimpleType("SIZE_T", rffi.SIZE_T) ULONG_PTR = rffi_platform.SimpleType("ULONG_PTR", rffi.ULONG) @@ -87,6 +90,10 @@ GetLastError = winexternal('GetLastError', [], DWORD) SetLastError = winexternal('SetLastError', [DWORD], lltype.Void) + # In tests, the first call to GetLastError is always wrong, because error + # is hidden by operations in ll2ctypes. Call it now. + GetLastError() + LoadLibrary = winexternal('LoadLibraryA', [rffi.CCHARP], rffi.VOIDP) GetProcAddress = winexternal('GetProcAddress', [rffi.VOIDP, rffi.CCHARP], @@ -129,13 +136,29 @@ } return 0; }''') - exename = static_platform.compile( - [cfile], ExternalCompilationInfo(), - outputfilename = "dosmaperr", - standalone=True) - output = os.popen(str(exename)) - errors = dict(map(int, line.split()) - for line in output) + try: + exename = static_platform.compile( + [cfile], ExternalCompilationInfo(), + outputfilename = "dosmaperr", + standalone=True) + except WindowsError: + # Fallback for the mingw32 compiler + errors = { + 2: 2, 3: 2, 4: 24, 5: 13, 6: 9, 7: 12, 8: 12, 9: 12, 10: 7, + 11: 8, 15: 2, 16: 13, 17: 18, 18: 2, 19: 13, 20: 13, 21: 13, + 22: 13, 23: 13, 24: 13, 25: 13, 26: 13, 27: 13, 28: 13, + 29: 13, 30: 13, 31: 13, 32: 13, 33: 13, 34: 13, 35: 13, + 36: 13, 53: 2, 65: 13, 67: 2, 80: 17, 82: 13, 83: 13, 89: 11, + 108: 13, 109: 32, 112: 28, 114: 9, 128: 10, 129: 10, 130: 9, + 132: 13, 145: 41, 158: 13, 161: 2, 164: 11, 167: 13, 183: 17, + 188: 8, 189: 8, 190: 8, 191: 8, 192: 8, 193: 8, 194: 8, + 195: 8, 196: 8, 197: 8, 198: 8, 199: 8, 200: 8, 201: 8, + 202: 8, 206: 2, 215: 11, 1816: 12, + } + else: + output = os.popen(str(exename)) + errors = dict(map(int, line.split()) + for line in output) return errors, errno.EINVAL # A bit like strerror... Modified: pypy/branch/asmgcc-64/pypy/rlib/streamio.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rlib/streamio.py (original) +++ pypy/branch/asmgcc-64/pypy/rlib/streamio.py Mon Aug 2 20:58:49 2010 @@ -38,7 +38,9 @@ # import os, sys +from pypy.rlib.objectmodel import specialize from pypy.rlib.rarithmetic import r_longlong, intmask +from pypy.rlib import rposix from os import O_RDONLY, O_WRONLY, O_RDWR, O_CREAT, O_TRUNC O_BINARY = getattr(os, "O_BINARY", 0) @@ -71,6 +73,7 @@ return s.join(string.split(c)) + at specialize.argtype(0) def open_file_as_stream(path, mode="r", buffering=-1): os_flags, universal, reading, writing, basemode, binary = decode_mode(mode) stream = open_path_helper(path, os_flags, basemode == "a") @@ -89,9 +92,10 @@ return construct_stream_tower(stream, buffering, universal, reading, writing, binary) + at specialize.argtype(0) def open_path_helper(path, os_flags, append): # XXX for now always return DiskFile - fd = os.open(path, os_flags, 0666) + fd = rposix.open(path, os_flags, 0666) if append: try: os.lseek(fd, 0, 2) Modified: pypy/branch/asmgcc-64/pypy/rlib/test/test_objectmodel.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rlib/test/test_objectmodel.py (original) +++ pypy/branch/asmgcc-64/pypy/rlib/test/test_objectmodel.py Mon Aug 2 20:58:49 2010 @@ -404,6 +404,13 @@ assert f._annspecialcase_ == 'specialize:arg(1)' +def test_enforceargs_decorator(): + @enforceargs(int, str, None) + def f(a, b, c): + pass + + assert f._annenforceargs_ == (int, str, None) + def getgraph(f, argtypes): from pypy.translator.translator import TranslationContext, graphof from pypy.translator.backendopt.all import backend_optimizations Modified: pypy/branch/asmgcc-64/pypy/rlib/test/test_rarithmetic.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rlib/test/test_rarithmetic.py (original) +++ pypy/branch/asmgcc-64/pypy/rlib/test/test_rarithmetic.py Mon Aug 2 20:58:49 2010 @@ -271,26 +271,29 @@ assert ovfcheck_float_to_int(13.0) == 13 assert ovfcheck_float_to_int(-1.0) == -1 assert ovfcheck_float_to_int(-13.0) == -13 - # strange things happening for float to int on 64 bit - maxint32 = 2 ** 31 - 1 - assert ovfcheck_float_to_int(float(maxint32-1)) == maxint32-1 - #assert ovfcheck_float_to_int(float(maxint32)) == maxint32 - assert ovfcheck_float_to_int(float(-maxint32)) == -maxint32 - #assert ovfcheck_float_to_int(float(-maxint32-1)) == -maxint32-1 - - try: - ovfcheck_float_to_int(float(-sys.maxint-1)-1) - except OverflowError: - pass - else: - assert False - - try: - ovfcheck_float_to_int(float(sys.maxint)+1) - except OverflowError: - pass - else: - assert False + + # strange things happening for float to int on 64 bit: + # int(float(i)) != i because of rounding issues + x = sys.maxint + while int(float(x)) > sys.maxint: + x -= 1 + assert ovfcheck_float_to_int(float(x)) == int(float(x)) + + x = sys.maxint + 1 + while int(float(x)) <= sys.maxint: + x += 1 + py.test.raises(OverflowError, ovfcheck_float_to_int, x) + + x = -sys.maxint-1 + while int(float(x)) < -sys.maxint-1: + x += 1 + assert ovfcheck_float_to_int(float(x)) == int(float(x)) + + x = -sys.maxint-1 + while int(float(x)) >= -sys.maxint-1: + x -= 1 + py.test.raises(OverflowError, ovfcheck_float_to_int, x) + def test_abs(): assert type(abs(r_longlong(1))) is r_longlong Modified: pypy/branch/asmgcc-64/pypy/rpython/extfunc.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rpython/extfunc.py (original) +++ pypy/branch/asmgcc-64/pypy/rpython/extfunc.py Mon Aug 2 20:58:49 2010 @@ -52,7 +52,10 @@ return super(ExtRegistryEntry, self).__getattr__(attr) raise exc, exc_inst, tb -def registering(func): +def registering(func, condition=True): + if not condition: + return lambda method: None + def decorator(method): method._registering_func = func return method @@ -63,11 +66,9 @@ func = getattr(ns, name) except AttributeError: condition = False + func = None - if condition: - return registering(func) - else: - return lambda method: None + return registering(func, condition=condition) class LazyRegisteringMeta(type): def __new__(self, _name, _type, _vars): @@ -167,8 +168,6 @@ return signature_args def compute_result_annotation(self, *args_s): - if hasattr(self, 'ann_hook'): - self.ann_hook() self.normalize_args(*args_s) # check arguments return self.signature_result @@ -235,7 +234,6 @@ def register_external(function, args, result=None, export_name=None, llimpl=None, ooimpl=None, llfakeimpl=None, oofakeimpl=None, - annotation_hook=None, sandboxsafe=False): """ function: the RPython function that will be rendered as an external function (e.g.: math.floor) @@ -244,7 +242,6 @@ export_name: the name of the function as it will be seen by the backends llimpl, ooimpl: optional; if provided, these RPython functions are called instead of the target function llfakeimpl, oofakeimpl: optional; if provided, they are called by the llinterpreter - annotationhook: optional; a callable that is called during annotation, useful for genc hacks sandboxsafe: use True if the function performs no I/O (safe for --sandbox) """ @@ -271,8 +268,6 @@ lltypefakeimpl = staticmethod(llfakeimpl) if oofakeimpl: ootypefakeimpl = staticmethod(oofakeimpl) - if annotation_hook: - ann_hook = staticmethod(annotation_hook) if export_name: FunEntry.__name__ = export_name Modified: pypy/branch/asmgcc-64/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/asmgcc-64/pypy/rpython/lltypesystem/rffi.py Mon Aug 2 20:58:49 2010 @@ -176,6 +176,15 @@ # XXX leaks if a str2charp() fails with MemoryError # and was not the first in this function freeme = arg + elif TARGET == CWCHARP: + if arg is None: + arg = lltype.nullptr(CWCHARP.TO) # None => (wchar_t*)NULL + freeme = arg + elif isinstance(arg, unicode): + arg = unicode2wcharp(arg) + # XXX leaks if a unicode2wcharp() fails with MemoryError + # and was not the first in this function + freeme = arg elif _isfunctype(TARGET) and not _isllptr(arg): # XXX pass additional arguments if invoke_around_handlers: Modified: pypy/branch/asmgcc-64/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/asmgcc-64/pypy/rpython/lltypesystem/rstr.py Mon Aug 2 20:58:49 2010 @@ -2,7 +2,7 @@ from pypy.tool.pairtype import pairtype from pypy.rpython.error import TyperError from pypy.rlib.objectmodel import malloc_zero_filled, we_are_translated -from pypy.rlib.objectmodel import _hash_string +from pypy.rlib.objectmodel import _hash_string, enforceargs from pypy.rlib.debug import ll_assert from pypy.rlib.jit import purefunction from pypy.rpython.robject import PyObjRepr, pyobj_repr @@ -56,6 +56,7 @@ llmemory.itemoffsetof(TP.chars, 0) + llmemory.sizeof(CHAR_TP) * item) + @enforceargs(None, None, int, int, int) def copy_string_contents(src, dst, srcstart, dststart, length): assert srcstart >= 0 assert dststart >= 0 @@ -674,6 +675,7 @@ res_index += item_len i += 1 return result + ll_join_strs._annenforceargs_ = [int, None] def ll_join_chars(length, chars): # no need to optimize this, will be replaced by string builder Modified: pypy/branch/asmgcc-64/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/asmgcc-64/pypy/rpython/module/ll_os.py Mon Aug 2 20:58:49 2010 @@ -6,12 +6,15 @@ # might be found in doc/rffi.txt import os, sys, errno +import py from pypy.rpython.module.support import ll_strcpy, OOSupport -from pypy.tool.sourcetools import func_with_new_name +from pypy.tool.sourcetools import func_with_new_name, func_renamer from pypy.rlib.rarithmetic import r_longlong -from pypy.rpython.extfunc import BaseLazyRegistering +from pypy.rpython.extfunc import ( + BaseLazyRegistering, lazy_register, register_external) from pypy.rpython.extfunc import registering, registering_if, extdef -from pypy.annotation.model import SomeInteger, SomeString, SomeTuple, SomeFloat +from pypy.annotation.model import ( + SomeInteger, SomeString, SomeTuple, SomeFloat, SomeUnicodeString) from pypy.annotation.model import s_ImpossibleValue, s_None, s_Bool from pypy.rpython.lltypesystem import rffi from pypy.rpython.lltypesystem import lltype @@ -26,7 +29,99 @@ from pypy.rpython.lltypesystem.rstr import STR from pypy.rpython.annlowlevel import llstr from pypy.rlib import rgc -from pypy.rlib.objectmodel import keepalive_until_here +from pypy.rlib.objectmodel import keepalive_until_here, specialize + +def monkeypatch_rposix(posixfunc, unicodefunc, signature): + func_name = posixfunc.__name__ + + if hasattr(signature, '_default_signature_'): + signature = signature._default_signature_ + arglist = ['arg%d' % (i,) for i in range(len(signature))] + transformed_arglist = arglist[:] + for i, arg in enumerate(signature): + if arg is unicode: + transformed_arglist[i] = transformed_arglist[i] + '.as_unicode()' + + args = ', '.join(arglist) + transformed_args = ', '.join(transformed_arglist) + main_arg = 'arg%d' % (signature.index(unicode),) + + source = py.code.Source(""" + def %(func_name)s(%(args)s): + if isinstance(%(main_arg)s, str): + return posixfunc(%(args)s) + else: + return unicodefunc(%(transformed_args)s) + """ % locals()) + miniglobals = {'posixfunc' : posixfunc, + 'unicodefunc': unicodefunc, + '__name__': __name__, # for module name propagation + } + exec source.compile() in miniglobals + new_func = miniglobals[func_name] + specialized_args = [i for i in range(len(signature)) + if signature[i] in (unicode, None)] + new_func = specialize.argtype(*specialized_args)(new_func) + + # Monkeypatch the function in pypy.rlib.rposix + setattr(rposix, func_name, new_func) + +class StringTraits: + str = str + CHAR = rffi.CHAR + CCHARP = rffi.CCHARP + charp2str = staticmethod(rffi.charp2str) + str2charp = staticmethod(rffi.str2charp) + free_charp = staticmethod(rffi.free_charp) + + @staticmethod + def posix_function_name(name): + return underscore_on_windows + name + + @staticmethod + def ll_os_name(name): + return 'll_os.ll_os_' + name + +class UnicodeTraits: + str = unicode + CHAR = rffi.WCHAR_T + CCHARP = rffi.CWCHARP + charp2str = staticmethod(rffi.wcharp2unicode) + str2charp = staticmethod(rffi.unicode2wcharp) + free_charp = staticmethod(rffi.free_wcharp) + + @staticmethod + def posix_function_name(name): + return underscore_on_windows + 'w' + name + + @staticmethod + def ll_os_name(name): + return 'll_os.ll_os_w' + name + +def registering_str_unicode(posixfunc, condition=True): + if not condition: + return registering(None, condition=False) + + func_name = posixfunc.__name__ + + def register_posixfunc(self, method): + val = method(self, StringTraits()) + register_external(posixfunc, *val.def_args, **val.def_kwds) + + if sys.platform == 'win32': + val = method(self, UnicodeTraits()) + @func_renamer(func_name + "_unicode") + def unicodefunc(*args): + return posixfunc(*args) + register_external(unicodefunc, *val.def_args, **val.def_kwds) + signature = val.def_args[0] + monkeypatch_rposix(posixfunc, unicodefunc, signature) + + def decorator(method): + decorated = lambda self: register_posixfunc(self, method) + decorated._registering_func = posixfunc + return decorated + return decorator posix = __import__(os.name) @@ -282,8 +377,8 @@ return extdef([int, int], s_None, llimpl=dup2_llimpl, export_name="ll_os.ll_os_dup2") - @registering(os.utime) - def register_os_utime(self): + @registering_str_unicode(os.utime) + def register_os_utime(self, traits): UTIMBUFP = lltype.Ptr(self.UTIMBUF) os_utime = self.llexternal('utime', [rffi.CCHARP, UTIMBUFP], rffi.INT) @@ -336,6 +431,9 @@ # tp is known to be None, and one version where it is known # to be a tuple of 2 floats. if not _WIN32: + assert traits.str is str + + @specialize.argtype(1) def os_utime_llimpl(path, tp): if tp is None: error = os_utime(path, lltype.nullptr(UTIMBUFP.TO)) @@ -346,85 +444,13 @@ if error == -1: raise OSError(rposix.get_errno(), "os_utime failed") else: - from pypy.rlib import rwin32 - from pypy.rpython.module.ll_os_stat import time_t_to_FILE_TIME - - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h'], - ) - - FILE_WRITE_ATTRIBUTES = platform.ConstantInteger( - 'FILE_WRITE_ATTRIBUTES') - OPEN_EXISTING = platform.ConstantInteger( - 'OPEN_EXISTING') - FILE_FLAG_BACKUP_SEMANTICS = platform.ConstantInteger( - 'FILE_FLAG_BACKUP_SEMANTICS') - globals().update(platform.configure(CConfig)) - - CreateFile = rffi.llexternal( - 'CreateFileA', - [rwin32.LPCSTR, rwin32.DWORD, rwin32.DWORD, - rwin32.LPSECURITY_ATTRIBUTES, rwin32.DWORD, rwin32.DWORD, - rwin32.HANDLE], - rwin32.HANDLE, - calling_conv='win') - - GetSystemTime = rffi.llexternal( - 'GetSystemTime', - [lltype.Ptr(rwin32.SYSTEMTIME)], - lltype.Void, - calling_conv='win') - - SystemTimeToFileTime = rffi.llexternal( - 'SystemTimeToFileTime', - [lltype.Ptr(rwin32.SYSTEMTIME), - lltype.Ptr(rwin32.FILETIME)], - rwin32.BOOL, - calling_conv='win') - - SetFileTime = rffi.llexternal( - 'SetFileTime', - [rwin32.HANDLE, - lltype.Ptr(rwin32.FILETIME), - lltype.Ptr(rwin32.FILETIME), - lltype.Ptr(rwin32.FILETIME)], - rwin32.BOOL, - calling_conv = 'win') + from pypy.rpython.module.ll_win32file import make_utime_impl + os_utime_llimpl = make_utime_impl(traits) - def os_utime_llimpl(path, tp): - hFile = CreateFile(path, - FILE_WRITE_ATTRIBUTES, 0, - None, OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS, 0) - if hFile == rwin32.INVALID_HANDLE_VALUE: - raise rwin32.lastWindowsError() - ctime = lltype.nullptr(rwin32.FILETIME) - atime = lltype.malloc(rwin32.FILETIME, flavor='raw') - mtime = lltype.malloc(rwin32.FILETIME, flavor='raw') - try: - if tp is None: - now = lltype.malloc(rwin32.SYSTEMTIME, flavor='raw') - try: - GetSystemTime(now) - if (not SystemTimeToFileTime(now, atime) or - not SystemTimeToFileTime(now, mtime)): - raise rwin32.lastWindowsError() - finally: - lltype.free(now, flavor='raw') - else: - actime, modtime = tp - time_t_to_FILE_TIME(actime, atime) - time_t_to_FILE_TIME(modtime, mtime) - if not SetFileTime(hFile, ctime, atime, mtime): - raise rwin32.lastWindowsError() - finally: - rwin32.CloseHandle(hFile) - lltype.free(atime, flavor='raw') - lltype.free(mtime, flavor='raw') - os_utime_llimpl._annspecialcase_ = 'specialize:argtype(1)' - - s_string = SomeString() + if traits.str is str: + s_string = SomeString() + else: + s_string = SomeUnicodeString() s_tuple_of_2_floats = SomeTuple([SomeFloat(), SomeFloat()]) def os_utime_normalize_args(s_path, s_times): @@ -445,12 +471,12 @@ else: raise Exception("os.utime() arg 2 must be None or a tuple of " "2 floats, got %s" % (s_times,)) + os_utime_normalize_args._default_signature_ = [traits.str, None] return extdef(os_utime_normalize_args, s_None, "ll_os.ll_os_utime", llimpl=os_utime_llimpl) - @registering(os.times) def register_os_times(self): if sys.platform.startswith('win'): @@ -687,22 +713,21 @@ def register_os_setsid(self): return self.extdef_for_os_function_returning_int('setsid') - @registering(os.open) - def register_os_open(self): - os_open = self.llexternal(underscore_on_windows+'open', - [rffi.CCHARP, rffi.INT, rffi.MODE_T], + @registering_str_unicode(os.open) + def register_os_open(self, traits): + os_open = self.llexternal(traits.posix_function_name('open'), + [traits.CCHARP, rffi.INT, rffi.MODE_T], rffi.INT) - def os_open_llimpl(path, flags, mode): result = rffi.cast(rffi.LONG, os_open(path, flags, mode)) if result == -1: raise OSError(rposix.get_errno(), "os_open failed") return result - def os_open_oofakeimpl(o_path, flags, mode): - return os.open(o_path._str, flags, mode) + def os_open_oofakeimpl(path, flags, mode): + return os.open(OOSupport.from_rstr(path), flags, mode) - return extdef([str, int, int], int, "ll_os.ll_os_open", + return extdef([traits.str, int, int], int, traits.ll_os_name('open'), llimpl=os_open_llimpl, oofakeimpl=os_open_oofakeimpl) # ------------------------------- os.read ------------------------------- @@ -862,10 +887,10 @@ llimpl=fdatasync_llimpl, export_name="ll_os.ll_os_fdatasync") - @registering(os.access) - def register_os_access(self): - os_access = self.llexternal(underscore_on_windows + 'access', - [rffi.CCHARP, rffi.INT], + @registering_str_unicode(os.access) + def register_os_access(self, traits): + os_access = self.llexternal(traits.posix_function_name('access'), + [traits.CCHARP, rffi.INT], rffi.INT) if sys.platform.startswith('win'): @@ -882,44 +907,22 @@ def os_access_oofakeimpl(path, mode): return os.access(OOSupport.from_rstr(path), mode) - return extdef([str, int], s_Bool, llimpl=access_llimpl, - export_name="ll_os.ll_os_access", + return extdef([traits.str, int], s_Bool, llimpl=access_llimpl, + export_name=traits.ll_os_name("access"), oofakeimpl=os_access_oofakeimpl) - @registering_if(posix, '_getfullpathname') - def register_posix__getfullpathname(self): - from pypy.rlib import rwin32 + @registering_str_unicode(getattr(posix, '_getfullpathname', None), + condition=sys.platform=='win32') + def register_posix__getfullpathname(self, traits): # this nt function is not exposed via os, but needed # to get a correct implementation of os.abspath - # XXX why do we ignore WINAPI conventions everywhere? - LPSTRP = rffi.CArrayPtr(rwin32.LPSTR) - # XXX unicode? - GetFullPathName = self.llexternal( - 'GetFullPathNameA', - [rwin32.LPCSTR, - rwin32.DWORD, - rwin32.LPSTR, - rffi.CArrayPtr(rwin32.LPSTR)], - rwin32.DWORD) - - def _getfullpathname_llimpl(lpFileName): - nBufferLength = rwin32.MAX_PATH + 1 - lpBuffer = lltype.malloc(rwin32.LPSTR.TO, nBufferLength, flavor='raw') - try: - res = GetFullPathName( - lpFileName, rffi.cast(rwin32.DWORD, nBufferLength), - lpBuffer, lltype.nullptr(LPSTRP.TO)) - if res == 0: - raise rwin32.lastWindowsError("_getfullpathname failed") - result = rffi.charp2str(lpBuffer) - return result - finally: - lltype.free(lpBuffer, flavor='raw') + from pypy.rpython.module.ll_win32file import make_getfullpathname_impl + getfullpathname_llimpl = make_getfullpathname_impl(traits) - return extdef([str], # a single argument which is a str - str, # returns a string - "ll_os.posix__getfullpathname", - llimpl=_getfullpathname_llimpl) + return extdef([traits.str], # a single argument which is a str + traits.str, # returns a string + traits.ll_os_name('_getfullpathname'), + llimpl=getfullpathname_llimpl) @registering(os.getcwd) def register_os_getcwd(self): @@ -953,71 +956,42 @@ "ll_os.ll_os_getcwd", llimpl=os_getcwd_llimpl, oofakeimpl=os_getcwd_oofakeimpl) - @registering(os.listdir) - def register_os_listdir(self): - # we need a different approach on Windows and on Posix - if sys.platform.startswith('win'): - from pypy.rlib import rwin32 - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h'] - ) - WIN32_FIND_DATA = platform.Struct('struct _WIN32_FIND_DATAA', - [('cFileName', lltype.FixedSizeArray(rffi.CHAR, 1))]) - ERROR_FILE_NOT_FOUND = platform.ConstantInteger( - 'ERROR_FILE_NOT_FOUND') - ERROR_NO_MORE_FILES = platform.ConstantInteger( - 'ERROR_NO_MORE_FILES') + @registering(os.getcwdu, condition=sys.platform=='win32') + def register_os_getcwdu(self): + os_wgetcwd = self.llexternal(underscore_on_windows + 'wgetcwd', + [rffi.CWCHARP, rffi.SIZE_T], + rffi.CWCHARP) - config = platform.configure(CConfig) - WIN32_FIND_DATA = config['WIN32_FIND_DATA'] - ERROR_FILE_NOT_FOUND = config['ERROR_FILE_NOT_FOUND'] - ERROR_NO_MORE_FILES = config['ERROR_NO_MORE_FILES'] - LPWIN32_FIND_DATA = lltype.Ptr(WIN32_FIND_DATA) - - FindFirstFile = self.llexternal('FindFirstFile', - [rwin32.LPCSTR, LPWIN32_FIND_DATA], - rwin32.HANDLE) - FindNextFile = self.llexternal('FindNextFile', - [rwin32.HANDLE, LPWIN32_FIND_DATA], - rwin32.BOOL) - FindClose = self.llexternal('FindClose', - [rwin32.HANDLE], - rwin32.BOOL) + def os_getcwd_llimpl(): + bufsize = 256 + while True: + buf = lltype.malloc(rffi.CWCHARP.TO, bufsize, flavor='raw') + res = os_wgetcwd(buf, rffi.cast(rffi.SIZE_T, bufsize)) + if res: + break # ok + error = rposix.get_errno() + lltype.free(buf, flavor='raw') + if error != errno.ERANGE: + raise OSError(error, "getcwd failed") + # else try again with a larger buffer, up to some sane limit + bufsize *= 4 + if bufsize > 1024*1024: # xxx hard-coded upper limit + raise OSError(error, "getcwd result too large") + result = rffi.wcharp2unicode(res) + lltype.free(buf, flavor='raw') + return result - def os_listdir_llimpl(path): - if path and path[-1] not in ('/', '\\', ':'): - path += '/' - path += '*.*' - filedata = lltype.malloc(WIN32_FIND_DATA, flavor='raw') - try: - result = [] - hFindFile = FindFirstFile(path, filedata) - if hFindFile == rwin32.INVALID_HANDLE_VALUE: - error = rwin32.GetLastError() - if error == ERROR_FILE_NOT_FOUND: - return result - else: - raise WindowsError(error, "FindFirstFile failed") - while True: - name = rffi.charp2str(rffi.cast(rffi.CCHARP, - filedata.c_cFileName)) - if name != "." and name != "..": # skip these - result.append(name) - if not FindNextFile(hFindFile, filedata): - break - # FindNextFile sets error to ERROR_NO_MORE_FILES if - # it got to the end of the directory - error = rwin32.GetLastError() - FindClose(hFindFile) - if error == ERROR_NO_MORE_FILES: - return result - else: - raise WindowsError(error, "FindNextFile failed") - finally: - lltype.free(filedata, flavor='raw') + return extdef([], unicode, + "ll_os.ll_os_wgetcwd", llimpl=os_getcwd_llimpl) + @registering_str_unicode(os.listdir) + def register_os_listdir(self, traits): + # we need a different approach on Windows and on Posix + if sys.platform.startswith('win'): + from pypy.rpython.module.ll_win32file import make_listdir_impl + os_listdir_llimpl = make_listdir_impl(traits) else: + assert traits.str is str compilation_info = ExternalCompilationInfo( includes = ['sys/types.h', 'dirent.h'] ) @@ -1057,9 +1031,9 @@ raise OSError(error, "os_readdir failed") return result - return extdef([str], # a single argument which is a str - [str], # returns a list of strings - "ll_os.ll_os_listdir", + return extdef([traits.str], # a single argument which is a str + [traits.str], # returns a list of strings + traits.ll_os_name('listdir'), llimpl=os_listdir_llimpl) @registering(os.pipe) @@ -1234,38 +1208,40 @@ return extdef([str], int, llimpl=system_llimpl, export_name="ll_os.ll_os_system") - @registering(os.unlink) - def register_os_unlink(self): - os_unlink = self.llexternal(underscore_on_windows+'unlink', [rffi.CCHARP], rffi.INT) + @registering_str_unicode(os.unlink) + def register_os_unlink(self, traits): + os_unlink = self.llexternal(traits.posix_function_name('unlink'), + [traits.CCHARP], rffi.INT) def unlink_llimpl(pathname): res = rffi.cast(lltype.Signed, os_unlink(pathname)) if res < 0: raise OSError(rposix.get_errno(), "os_unlink failed") - return extdef([str], s_None, llimpl=unlink_llimpl, - export_name="ll_os.ll_os_unlink") + return extdef([traits.str], s_None, llimpl=unlink_llimpl, + export_name=traits.ll_os_name('unlink')) - @registering(os.chdir) - def register_os_chdir(self): - os_chdir = self.llexternal(underscore_on_windows+'chdir', [rffi.CCHARP], rffi.INT) + @registering_str_unicode(os.chdir) + def register_os_chdir(self, traits): + os_chdir = self.llexternal(traits.posix_function_name('chdir'), + [traits.CCHARP], rffi.INT) def chdir_llimpl(path): res = rffi.cast(lltype.Signed, os_chdir(path)) if res < 0: raise OSError(rposix.get_errno(), "os_chdir failed") - return extdef([str], s_None, llimpl=chdir_llimpl, - export_name="ll_os.ll_os_chdir") + return extdef([traits.str], s_None, llimpl=chdir_llimpl, + export_name=traits.ll_os_name('chdir')) - @registering(os.mkdir) - def register_os_mkdir(self): + @registering_str_unicode(os.mkdir) + def register_os_mkdir(self, traits): if os.name == 'nt': ARG2 = [] # no 'mode' argument on Windows - just ignored else: ARG2 = [rffi.MODE_T] - os_mkdir = self.llexternal(underscore_on_windows+'mkdir', - [rffi.CCHARP]+ARG2, rffi.INT) + os_mkdir = self.llexternal(traits.posix_function_name('mkdir'), + [traits.CCHARP] + ARG2, rffi.INT) IGNORE_MODE = len(ARG2) == 0 def mkdir_llimpl(pathname, mode): @@ -1277,46 +1253,47 @@ if res < 0: raise OSError(rposix.get_errno(), "os_mkdir failed") - return extdef([str, int], s_None, llimpl=mkdir_llimpl, - export_name="ll_os.ll_os_mkdir") + return extdef([traits.str, int], s_None, llimpl=mkdir_llimpl, + export_name=traits.ll_os_name('mkdir')) - @registering(os.rmdir) - def register_os_rmdir(self): - os_rmdir = self.llexternal(underscore_on_windows+'rmdir', [rffi.CCHARP], rffi.INT) + @registering_str_unicode(os.rmdir) + def register_os_rmdir(self, traits): + os_rmdir = self.llexternal(traits.posix_function_name('rmdir'), + [traits.CCHARP], rffi.INT) def rmdir_llimpl(pathname): res = rffi.cast(lltype.Signed, os_rmdir(pathname)) if res < 0: raise OSError(rposix.get_errno(), "os_rmdir failed") - return extdef([str], s_None, llimpl=rmdir_llimpl, - export_name="ll_os.ll_os_rmdir") + return extdef([traits.str], s_None, llimpl=rmdir_llimpl, + export_name=traits.ll_os_name('rmdir')) - @registering(os.chmod) - def register_os_chmod(self): - os_chmod = self.llexternal(underscore_on_windows+'chmod', [rffi.CCHARP, rffi.MODE_T], - rffi.INT) + @registering_str_unicode(os.chmod) + def register_os_chmod(self, traits): + os_chmod = self.llexternal(traits.posix_function_name('chmod'), + [traits.CCHARP, rffi.MODE_T], rffi.INT) def chmod_llimpl(path, mode): res = rffi.cast(lltype.Signed, os_chmod(path, rffi.cast(rffi.MODE_T, mode))) if res < 0: raise OSError(rposix.get_errno(), "os_chmod failed") - return extdef([str, int], s_None, llimpl=chmod_llimpl, - export_name="ll_os.ll_os_chmod") + return extdef([traits.str, int], s_None, llimpl=chmod_llimpl, + export_name=traits.ll_os_name('chmod')) - @registering(os.rename) - def register_os_rename(self): - os_rename = self.llexternal('rename', [rffi.CCHARP, rffi.CCHARP], - rffi.INT) + @registering_str_unicode(os.rename) + def register_os_rename(self, traits): + os_rename = self.llexternal(traits.posix_function_name('rename'), + [traits.CCHARP, traits.CCHARP], rffi.INT) def rename_llimpl(oldpath, newpath): res = rffi.cast(lltype.Signed, os_rename(oldpath, newpath)) if res < 0: raise OSError(rposix.get_errno(), "os_rename failed") - return extdef([str, str], s_None, llimpl=rename_llimpl, - export_name="ll_os.ll_os_rename") + return extdef([traits.str, traits.str], s_None, llimpl=rename_llimpl, + export_name=traits.ll_os_name('rename')) @registering(os.umask) def register_os_umask(self): @@ -1425,17 +1402,17 @@ @registering(os.fstat) def register_os_fstat(self): from pypy.rpython.module import ll_os_stat - ll_os_stat.register_stat_variant('fstat') + return ll_os_stat.register_stat_variant('fstat', StringTraits()) - @registering(os.stat) - def register_os_stat(self): + @registering_str_unicode(os.stat) + def register_os_stat(self, traits): from pypy.rpython.module import ll_os_stat - ll_os_stat.register_stat_variant('stat') + return ll_os_stat.register_stat_variant('stat', traits) - @registering(os.lstat) - def register_os_lstat(self): + @registering_str_unicode(os.lstat) + def register_os_lstat(self, traits): from pypy.rpython.module import ll_os_stat - ll_os_stat.register_stat_variant('lstat') + return ll_os_stat.register_stat_variant('lstat', traits) # ------------------------------- os.W* --------------------------------- Modified: pypy/branch/asmgcc-64/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/branch/asmgcc-64/pypy/rpython/module/ll_os_stat.py Mon Aug 2 20:58:49 2010 @@ -5,13 +5,14 @@ import os, sys from pypy.annotation import model as annmodel from pypy.tool.pairtype import pairtype -from pypy.tool.sourcetools import func_with_new_name +from pypy.tool.sourcetools import func_with_new_name, func_renamer from pypy.rpython import extregistry -from pypy.rpython.extfunc import register_external +from pypy.rpython.extfunc import register_external, extdef from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.tool import rffi_platform as platform from pypy.rpython.lltypesystem.rtupletype import TUPLE_TYPE from pypy.rlib import rposix +from pypy.rlib.objectmodel import specialize from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rpython.annlowlevel import hlstr @@ -211,13 +212,27 @@ return make_stat_result(result) -def register_stat_variant(name): - if sys.platform.startswith('win'): - _functions = {'stat': '_stati64', - 'fstat': '_fstati64', - 'lstat': '_stati64'} # no lstat on Windows - c_func_name = _functions[name] - elif sys.platform.startswith('linux'): +def register_stat_variant(name, traits): + if name != 'fstat': + arg_is_path = True + s_arg = traits.str + ARG1 = traits.CCHARP + else: + arg_is_path = False + s_arg = int + ARG1 = rffi.INT + + if sys.platform == 'win32': + # See Win32 implementation below + posix_stat_llimpl = make_win32_stat_impl(name, traits) + + return extdef( + [s_arg], s_StatResult, traits.ll_os_name(name), + llimpl=posix_stat_llimpl) + + assert traits.str is str + + if sys.platform.startswith('linux'): # because we always use _FILE_OFFSET_BITS 64 - this helps things work that are not a c compiler _functions = {'stat': 'stat64', 'fstat': 'fstat64', @@ -226,22 +241,26 @@ else: c_func_name = name - arg_is_path = (name != 'fstat') + posix_mystat = rffi.llexternal(c_func_name, + [ARG1, STAT_STRUCT], rffi.INT, + compilation_info=compilation_info) + @func_renamer('os_%s_llimpl' % (name,)) def posix_stat_llimpl(arg): stresult = lltype.malloc(STAT_STRUCT.TO, flavor='raw') try: if arg_is_path: - arg = rffi.str2charp(arg) + arg = traits.str2charp(arg) error = rffi.cast(rffi.LONG, posix_mystat(arg, stresult)) if arg_is_path: - rffi.free_charp(arg) + traits.free_charp(arg) if error != 0: raise OSError(rposix.get_errno(), "os_?stat failed") return build_stat_result(stresult) finally: lltype.free(stresult, flavor='raw') + @func_renamer('os_%s_fake' % (name,)) def posix_fakeimpl(arg): if s_arg == str: arg = hlstr(arg) @@ -259,40 +278,17 @@ setattr(ll_tup, 'item%d' % i, val) return ll_tup - if arg_is_path: - s_arg = str - ARG1 = rffi.CCHARP - else: - s_arg = int - ARG1 = rffi.INT + return extdef( + [s_arg], s_StatResult, "ll_os.ll_os_%s" % (name,), + llimpl=posix_stat_llimpl, llfakeimpl=posix_fakeimpl) - if sys.platform != 'win32': - posix_mystat = rffi.llexternal(c_func_name, - [ARG1, STAT_STRUCT], rffi.INT, - compilation_info=compilation_info) - - register_external( - getattr(os, name), [s_arg], s_StatResult, - "ll_os.ll_os_%s" % (name,), - llimpl=func_with_new_name(posix_stat_llimpl, - 'os_%s_llimpl' % (name,)), - llfakeimpl=func_with_new_name(posix_fakeimpl, - 'os_%s_fake' % (name,)), - ) - else: - # See Win32 implementation below - register_external( - getattr(os, name), [s_arg], s_StatResult, - "ll_os.ll_os_%s" % (name,), - llimpl=func_with_new_name(globals()['win32_%s_llimpl' % (name,)], - 'os_%s_llimpl' % (name,)), - ) +def make_win32_stat_impl(name, traits): + from pypy.rlib import rwin32 + from pypy.rpython.module.ll_win32file import make_win32_traits + win32traits = make_win32_traits(traits) -# ____________________________________________________________ -if sys.platform == 'win32': # The CRT of Windows has a number of flaws wrt. its stat() implementation: - # - for when we implement subsecond resolution in RPython, time stamps - # would be restricted to second resolution + # - time stamps are restricted to second resolution # - file modification times suffer from forth-and-back conversions between # UTC and local time # Therefore, we implement our own stat, based on the Win32 API directly. @@ -302,122 +298,18 @@ assert len(STAT_FIELDS) == 10 # no extra fields on Windows - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h', 'winbase.h', 'sys/stat.h'], - ) - - GetFileExInfoStandard = platform.ConstantInteger( - 'GetFileExInfoStandard') - FILE_ATTRIBUTE_DIRECTORY = platform.ConstantInteger( - 'FILE_ATTRIBUTE_DIRECTORY') - FILE_ATTRIBUTE_READONLY = platform.ConstantInteger( - 'FILE_ATTRIBUTE_READONLY') - ERROR_SHARING_VIOLATION = platform.ConstantInteger( - 'ERROR_SHARING_VIOLATION') - _S_IFDIR = platform.ConstantInteger('_S_IFDIR') - _S_IFREG = platform.ConstantInteger('_S_IFREG') - _S_IFCHR = platform.ConstantInteger('_S_IFCHR') - _S_IFIFO = platform.ConstantInteger('_S_IFIFO') - FILE_TYPE_UNKNOWN = platform.ConstantInteger('FILE_TYPE_UNKNOWN') - FILE_TYPE_CHAR = platform.ConstantInteger('FILE_TYPE_CHAR') - FILE_TYPE_PIPE = platform.ConstantInteger('FILE_TYPE_PIPE') - - WIN32_FILE_ATTRIBUTE_DATA = platform.Struct( - 'WIN32_FILE_ATTRIBUTE_DATA', - [('dwFileAttributes', rwin32.DWORD), - ('nFileSizeHigh', rwin32.DWORD), - ('nFileSizeLow', rwin32.DWORD), - ('ftCreationTime', rwin32.FILETIME), - ('ftLastAccessTime', rwin32.FILETIME), - ('ftLastWriteTime', rwin32.FILETIME)]) - - BY_HANDLE_FILE_INFORMATION = platform.Struct( - 'BY_HANDLE_FILE_INFORMATION', - [('dwFileAttributes', rwin32.DWORD), - ('nFileSizeHigh', rwin32.DWORD), - ('nFileSizeLow', rwin32.DWORD), - ('nNumberOfLinks', rwin32.DWORD), - ('nFileIndexHigh', rwin32.DWORD), - ('nFileIndexLow', rwin32.DWORD), - ('ftCreationTime', rwin32.FILETIME), - ('ftLastAccessTime', rwin32.FILETIME), - ('ftLastWriteTime', rwin32.FILETIME)]) - - WIN32_FIND_DATA = platform.Struct( - 'WIN32_FIND_DATAA', - # Only interesting fields - [('dwFileAttributes', rwin32.DWORD), - ('nFileSizeHigh', rwin32.DWORD), - ('nFileSizeLow', rwin32.DWORD), - ('ftCreationTime', rwin32.FILETIME), - ('ftLastAccessTime', rwin32.FILETIME), - ('ftLastWriteTime', rwin32.FILETIME)]) - - globals().update(platform.configure(CConfig)) - GET_FILEEX_INFO_LEVELS = rffi.ULONG # an enumeration - - GetFileAttributesEx = rffi.llexternal( - 'GetFileAttributesExA', - [rffi.CCHARP, GET_FILEEX_INFO_LEVELS, - lltype.Ptr(WIN32_FILE_ATTRIBUTE_DATA)], - rwin32.BOOL, - calling_conv='win') - - GetFileInformationByHandle = rffi.llexternal( - 'GetFileInformationByHandle', - [rwin32.HANDLE, lltype.Ptr(BY_HANDLE_FILE_INFORMATION)], - rwin32.BOOL, - calling_conv='win') - - GetFileType = rffi.llexternal( - 'GetFileType', - [rwin32.HANDLE], - rwin32.DWORD, - calling_conv='win') - - FindFirstFile = rffi.llexternal( - 'FindFirstFileA', - [rffi.CCHARP, lltype.Ptr(WIN32_FIND_DATA)], - rwin32.HANDLE, - calling_conv='win') - - FindClose = rffi.llexternal( - 'FindClose', - [rwin32.HANDLE], - rwin32.BOOL, - calling_conv='win') - def attributes_to_mode(attributes): m = 0 - if attributes & FILE_ATTRIBUTE_DIRECTORY: - m |= _S_IFDIR | 0111 # IFEXEC for user,group,other + if attributes & win32traits.FILE_ATTRIBUTE_DIRECTORY: + m |= win32traits._S_IFDIR | 0111 # IFEXEC for user,group,other else: - m |= _S_IFREG - if attributes & FILE_ATTRIBUTE_READONLY: + m |= win32traits._S_IFREG + if attributes & win32traits.FILE_ATTRIBUTE_READONLY: m |= 0444 else: m |= 0666 return m - def make_longlong(high, low): - return (lltype.r_longlong(high) << 32) + lltype.r_longlong(low) - - # Seconds between 1.1.1601 and 1.1.1970 - secs_between_epochs = lltype.r_longlong(11644473600) - - def FILE_TIME_to_time_t_nsec(filetime): - ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime) - # FILETIME is in units of 100 nsec - nsec = (ft % 10000000) * 100 - time = (ft / 10000000) - secs_between_epochs - return time, nsec - - def time_t_to_FILE_TIME(time, filetime): - ft = lltype.r_longlong((time + secs_between_epochs) * 10000000) - filetime.c_dwHighDateTime = lltype.r_uint(ft >> 32) - filetime.c_dwLowDateTime = lltype.r_uint(ft & ((1 << 32) - 1)) - def attribute_data_to_stat(info): st_mode = attributes_to_mode(info.c_dwFileAttributes) st_size = make_longlong(info.c_nFileSizeHigh, info.c_nFileSizeLow) @@ -456,65 +348,94 @@ return make_stat_result(result) def attributes_from_dir(l_path, data): - filedata = lltype.malloc(WIN32_FIND_DATA, flavor='raw') - hFindFile = FindFirstFile(l_path, filedata) - if hFindFile == rwin32.INVALID_HANDLE_VALUE: - return 0 - FindClose(hFindFile) - data.c_dwFileAttributes = filedata.c_dwFileAttributes - rffi.structcopy(data.c_ftCreationTime, filedata.c_ftCreationTime) - rffi.structcopy(data.c_ftLastAccessTime, filedata.c_ftLastAccessTime) - rffi.structcopy(data.c_ftLastWriteTime, filedata.c_ftLastWriteTime) - data.c_nFileSizeHigh = filedata.c_nFileSizeHigh - data.c_nFileSizeLow = filedata.c_nFileSizeLow - return 1 + filedata = lltype.malloc(win32traits.WIN32_FIND_DATA, flavor='raw') + try: + hFindFile = win32traits.FindFirstFile(l_path, filedata) + if hFindFile == rwin32.INVALID_HANDLE_VALUE: + return 0 + win32traits.FindClose(hFindFile) + data.c_dwFileAttributes = filedata.c_dwFileAttributes + rffi.structcopy(data.c_ftCreationTime, filedata.c_ftCreationTime) + rffi.structcopy(data.c_ftLastAccessTime, filedata.c_ftLastAccessTime) + rffi.structcopy(data.c_ftLastWriteTime, filedata.c_ftLastWriteTime) + data.c_nFileSizeHigh = filedata.c_nFileSizeHigh + data.c_nFileSizeLow = filedata.c_nFileSizeLow + return 1 + finally: + lltype.free(filedata, flavor='raw') def win32_stat_llimpl(path): - data = lltype.malloc(WIN32_FILE_ATTRIBUTE_DATA, flavor='raw') + data = lltype.malloc(win32traits.WIN32_FILE_ATTRIBUTE_DATA, flavor='raw') try: - l_path = rffi.str2charp(path) - res = GetFileAttributesEx(l_path, GetFileExInfoStandard, data) + l_path = traits.str2charp(path) + res = win32traits.GetFileAttributesEx(l_path, win32traits.GetFileExInfoStandard, data) errcode = rwin32.GetLastError() if res == 0: - if errcode == ERROR_SHARING_VIOLATION: + if errcode == win32traits.ERROR_SHARING_VIOLATION: res = attributes_from_dir(l_path, data) errcode = rwin32.GetLastError() - rffi.free_charp(l_path) + traits.free_charp(l_path) if res == 0: raise WindowsError(errcode, "os_stat failed") return attribute_data_to_stat(data) finally: lltype.free(data, flavor='raw') - win32_lstat_llimpl = win32_stat_llimpl def win32_fstat_llimpl(fd): handle = rwin32._get_osfhandle(fd) - filetype = GetFileType(handle) - if filetype == FILE_TYPE_CHAR: + filetype = win32traits.GetFileType(handle) + if filetype == win32traits.FILE_TYPE_CHAR: # console or LPT device - return make_stat_result((_S_IFCHR, + return make_stat_result((win32traits._S_IFCHR, 0, 0, 0, 0, 0, 0, 0, 0, 0)) - elif filetype == FILE_TYPE_PIPE: + elif filetype == win32traits.FILE_TYPE_PIPE: # socket or named pipe - return make_stat_result((_S_IFIFO, + return make_stat_result((win32traits._S_IFIFO, 0, 0, 0, 0, 0, 0, 0, 0, 0)) - elif filetype == FILE_TYPE_UNKNOWN: + elif filetype == win32traits.FILE_TYPE_UNKNOWN: error = rwin32.GetLastError() if error != 0: raise WindowsError(error, "os_fstat failed") # else: unknown but valid file # normal disk file (FILE_TYPE_DISK) - info = lltype.malloc(BY_HANDLE_FILE_INFORMATION, flavor='raw', - zero=True) + info = lltype.malloc(win32traits.BY_HANDLE_FILE_INFORMATION, + flavor='raw', zero=True) try: - res = GetFileInformationByHandle(handle, info) + res = win32traits.GetFileInformationByHandle(handle, info) if res == 0: raise WindowsError(rwin32.GetLastError(), "os_fstat failed") return by_handle_info_to_stat(info) finally: lltype.free(info, flavor='raw') + if name == 'fstat': + return win32_fstat_llimpl + else: + return win32_stat_llimpl + + +#__________________________________________________ +# Helper functions for win32 + +def make_longlong(high, low): + return (lltype.r_longlong(high) << 32) + lltype.r_longlong(low) + +# Seconds between 1.1.1601 and 1.1.1970 +secs_between_epochs = lltype.r_longlong(11644473600) + +def FILE_TIME_to_time_t_nsec(filetime): + ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime) + # FILETIME is in units of 100 nsec + nsec = (ft % 10000000) * 100 + time = (ft / 10000000) - secs_between_epochs + return time, nsec + +def time_t_to_FILE_TIME(time, filetime): + ft = lltype.r_longlong((time + secs_between_epochs) * 10000000) + filetime.c_dwHighDateTime = lltype.r_uint(ft >> 32) + filetime.c_dwLowDateTime = lltype.r_uint(ft & ((1 << 32) - 1)) + Modified: pypy/branch/asmgcc-64/pypy/rpython/module/test/test_ll_os_stat.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rpython/module/test/test_ll_os_stat.py (original) +++ pypy/branch/asmgcc-64/pypy/rpython/module/test/test_ll_os_stat.py Mon Aug 2 20:58:49 2010 @@ -1,4 +1,4 @@ -from pypy.rpython.module import ll_os_stat +from pypy.rpython.module import ll_os_stat, ll_os import sys, os import py @@ -8,14 +8,18 @@ py.test.skip("win32 specific tests") def test_stat(self): - stat = ll_os_stat.win32_stat_llimpl + stat = ll_os_stat.make_win32_stat_impl('stat', ll_os.StringTraits()) + wstat = ll_os_stat.make_win32_stat_impl('stat', ll_os.UnicodeTraits()) def check(f): - assert stat(f).st_mtime == os.stat(f).st_mtime + expected = os.stat(f).st_mtime + assert stat(f).st_mtime == expected + assert wstat(unicode(f)).st_mtime == expected check('c:/') check('c:/temp') check('c:/pagefile.sys') def test_fstat(self): - stat = ll_os_stat.win32_fstat_llimpl(0) # stdout + fstat = ll_os_stat.make_win32_stat_impl('fstat', ll_os.StringTraits()) + stat = fstat(0) # stdout assert stat.st_mode != 0 Modified: pypy/branch/asmgcc-64/pypy/rpython/rstr.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rpython/rstr.py (original) +++ pypy/branch/asmgcc-64/pypy/rpython/rstr.py Mon Aug 2 20:58:49 2010 @@ -288,7 +288,11 @@ if not hop.args_s[1].is_constant(): raise TyperError("encoding must be constant") encoding = hop.args_s[1].const - v_self = hop.inputarg(self.repr, 0) + if encoding == "ascii": + expect = self.lowleveltype # can be a UniChar + else: + expect = self.repr # must be a regular unicode string + v_self = hop.inputarg(expect, 0) hop.exception_is_here() if encoding == "ascii": return hop.gendirectcall(self.ll_str, v_self) @@ -415,7 +419,17 @@ sourcevars.append((v_item, r_arg)) return r_str.ll.do_stringformat(hop, sourcevars) - + + +class __extend__(AbstractCharRepr): + def ll_str(self, ch): + return self.ll.ll_chr2str(ch) + +class __extend__(AbstractUniCharRepr): + def ll_str(self, ch): + # xxx suboptimal, maybe + return str(unicode(ch)) + class __extend__(AbstractCharRepr, AbstractUniCharRepr): @@ -433,9 +447,6 @@ get_ll_fasthash_function = get_ll_hash_function - def ll_str(self, ch): - return self.ll.ll_chr2str(ch) - def rtype_len(_, hop): return hop.inputconst(Signed, 1) Modified: pypy/branch/asmgcc-64/pypy/rpython/test/test_extfunc.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rpython/test/test_extfunc.py (original) +++ pypy/branch/asmgcc-64/pypy/rpython/test/test_extfunc.py Mon Aug 2 20:58:49 2010 @@ -6,150 +6,164 @@ from pypy.annotation.policy import AnnotatorPolicy from pypy.rpython.test.test_llinterp import interpret -def b(x): - return eval("x+40") +class TestExtFuncEntry: -class BTestFuncEntry(ExtFuncEntry): - _about_ = b - name = 'b' - signature_args = [annmodel.SomeInteger()] - signature_result = annmodel.SomeInteger() - -def test_annotation_b(): - def f(): - return b(1) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeInteger) - -def test_rtyping_b(): - def f(): - return b(2) - - res = interpret(f, []) - assert res == 42 - -def c(y, x): - yyy - -class CTestFuncEntry(ExtFuncEntry): - _about_ = c - name = 'ccc' - signature_args = [annmodel.SomeInteger()] * 2 - signature_result = annmodel.SomeInteger() - - def lltypeimpl(y, x): - return y + x - lltypeimpl = staticmethod(lltypeimpl) - -def test_interp_c(): - def f(): - return c(3, 4) - - res = interpret(f, []) - assert res == 7 - -def d(y): - return eval("y()") - -class DTestFuncEntry(ExtFuncEntry): - _about_ = d - name = 'd' - signature_args = [annmodel.SomeGenericCallable(args=[], result= - annmodel.SomeFloat())] - signature_result = annmodel.SomeFloat() - -def test_callback(): - def callback(): - return 2.5 - - def f(): - return d(callback) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeFloat) - assert a.translator._graphof(callback) - -def dd(): - pass - -register_external(dd, [int], int) - -def test_register_external_signature(): - def f(): - return dd(3) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeInteger) - - -def function_with_tuple_arg(): - """ - Dummy function which is declared via register_external to take a tuple as - an argument so that register_external's behavior for tuple-taking functions - can be verified. - """ -register_external(function_with_tuple_arg, [(int,)], int) - -def test_register_external_tuple_args(): - """ - Verify the annotation of a registered external function which takes a tuple - argument. - """ - def f(): - return function_with_tuple_arg((1,)) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - - # Not a very good assertion, but at least it means _something_ happened. - assert isinstance(s, annmodel.SomeInteger) - -def function_with_list(): - pass -register_external(function_with_list, [[int]], int) - -def function_returning_list(): - pass -register_external(function_returning_list, [], [int]) - -def test_register_external_return_goes_back(): - """ - Check whether it works to pass the same list from one external - fun to another - [bookkeeper and list joining issues] - """ - def f(): - return function_with_list(function_returning_list()) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeInteger) - -def function_withspecialcase(arg): - return repr(arg) -register_external(function_withspecialcase, args=None, result=str) - -def test_register_external_specialcase(): - def f(): - x = function_withspecialcase - return x(33) + x("aaa") + x([]) + "\n" - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeString) + def test_basic(self): + """ + A ExtFuncEntry provides an annotation for a function, no need to flow + its graph. + """ + def b(x): + "NOT_RPYTHON" + return eval("x+40") + + class BTestFuncEntry(ExtFuncEntry): + _about_ = b + name = 'b' + signature_args = [annmodel.SomeInteger()] + signature_result = annmodel.SomeInteger() + + def f(): + return b(2) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeInteger) + + res = interpret(f, []) + assert res == 42 + + def test_lltypeimpl(self): + """ + interpret() calls lltypeimpl instead of of the function/ + """ + def c(y, x): + yyy + + class CTestFuncEntry(ExtFuncEntry): + _about_ = c + name = 'ccc' + signature_args = [annmodel.SomeInteger()] * 2 + signature_result = annmodel.SomeInteger() + + def lltypeimpl(y, x): + return y + x + lltypeimpl = staticmethod(lltypeimpl) + + def f(): + return c(3, 4) + + res = interpret(f, []) + assert res == 7 + + def test_callback(self): + """ + Verify annotation when a callback function is in the arguments list. + """ + def d(y): + return eval("y()") + + class DTestFuncEntry(ExtFuncEntry): + _about_ = d + name = 'd' + signature_args = [annmodel.SomeGenericCallable(args=[], result= + annmodel.SomeFloat())] + signature_result = annmodel.SomeFloat() + + def callback(): + return 2.5 + + def f(): + return d(callback) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeFloat) + assert a.translator._graphof(callback) + + def test_register_external_signature(self): + """ + Test the standard interface for external functions. + """ + def dd(): + pass + register_external(dd, [int], int) + + def f(): + return dd(3) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeInteger) + + def test_register_external_tuple_args(self): + """ + Verify the annotation of a registered external function which takes a + tuple argument. + """ + + def function_with_tuple_arg(): + """ + Dummy function which is declared via register_external to take a + tuple as an argument so that register_external's behavior for + tuple-taking functions can be verified. + """ + register_external(function_with_tuple_arg, [(int,)], int) + + def f(): + return function_with_tuple_arg((1,)) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + + # Not a very good assertion, but at least it means _something_ happened. + assert isinstance(s, annmodel.SomeInteger) + + def test_register_external_return_goes_back(self): + """ + Check whether it works to pass the same list from one external + fun to another + [bookkeeper and list joining issues] + """ + def function_with_list(): + pass + register_external(function_with_list, [[int]], int) + + def function_returning_list(): + pass + register_external(function_returning_list, [], [int]) + + def f(): + return function_with_list(function_returning_list()) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeInteger) + + def test_register_external_specialcase(self): + """ + When args=None, the external function accepts any arguments unmodified. + """ + def function_withspecialcase(arg): + return repr(arg) + register_external(function_withspecialcase, args=None, result=str) + + def f(): + x = function_withspecialcase + return x(33) + x("aaa") + x([]) + "\n" + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeString) Modified: pypy/branch/asmgcc-64/pypy/rpython/test/test_rstr.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rpython/test/test_rstr.py (original) +++ pypy/branch/asmgcc-64/pypy/rpython/test/test_rstr.py Mon Aug 2 20:58:49 2010 @@ -863,6 +863,24 @@ res = self.interpret(f, [1]) assert self.ll_to_string(res) == "hello" + def test_str_unichar(self): + def f(i): + c = u"abc" + return str(c[i])[0] + assert self.interpret(f, [1]) == "b" + + def test_encode_char(self): + def f(i): + c = u"abc" + return c[i].encode("ascii") + assert self.ll_to_string(self.interpret(f, [0])) == "a" + + def test_encode_char_latin1(self): + def f(i): + c = u"abc" + return c[i].encode("latin-1") + assert self.ll_to_string(self.interpret(f, [0])) == "a" + def FIXME_test_str_to_pystringobj(): def f(n): if n >= 0: Modified: pypy/branch/asmgcc-64/pypy/translator/c/test/test_typed.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/c/test/test_typed.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/c/test/test_typed.py Mon Aug 2 20:58:49 2010 @@ -789,3 +789,37 @@ return u'hello' + unichr(i) f = self.getcompiled(func, [int]) assert f(0x1234) == u'hello\u1234' + + def test_ovfcheck_float_to_int(self): + from pypy.rlib.rarithmetic import ovfcheck_float_to_int + + def func(fl): + try: + return ovfcheck_float_to_int(fl) + except OverflowError: + return -666 + f = self.getcompiled(func, [float]) + assert f(-123.0) == -123 + + for frac in [0.0, 0.01, 0.99]: + # strange things happening for float to int on 64 bit: + # int(float(i)) != i because of rounding issues + x = sys.maxint + while int(x + frac) > sys.maxint: + x -= 1 + assert f(x + frac) == int(x + frac) + + x = sys.maxint + while int(x - frac) <= sys.maxint: + x += 1 + assert f(x - frac) == -666 + + x = -sys.maxint-1 + while int(x - frac) < -sys.maxint-1: + x += 1 + assert f(x - frac) == int(x - frac) + + x = -sys.maxint-1 + while int(x + frac) >= -sys.maxint-1: + x -= 1 + assert f(x + frac) == -666 Modified: pypy/branch/asmgcc-64/pypy/translator/goal/app_main.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/goal/app_main.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/goal/app_main.py Mon Aug 2 20:58:49 2010 @@ -223,7 +223,6 @@ path = os.getenv('PYTHONPATH') if path: newpath = path.split(os.pathsep) + newpath - newpath.insert(0, '') # remove duplicates _seen = {} del sys.path[:] @@ -327,6 +326,10 @@ except: print >> sys.stderr, "'import site' failed" + # update sys.path *after* loading site.py, in case there is a + # "site.py" file in the script's directory. + sys.path.insert(0, '') + if warnoptions: sys.warnoptions.append(warnoptions) from warnings import _processoptions Modified: pypy/branch/asmgcc-64/pypy/translator/goal/test2/test_app_main.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/goal/test2/test_app_main.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/goal/test2/test_app_main.py Mon Aug 2 20:58:49 2010 @@ -5,6 +5,7 @@ import sys, os, re import autopath from pypy.tool.udir import udir +from contextlib import contextmanager banner = sys.version.splitlines()[0] @@ -326,8 +327,9 @@ class TestNonInteractive: def run(self, cmdline, senddata='', expect_prompt=False, - expect_banner=False): - cmdline = '%s "%s" %s' % (sys.executable, app_main, cmdline) + expect_banner=False, python_flags=''): + cmdline = '%s %s "%s" %s' % (sys.executable, python_flags, + app_main, cmdline) print 'POPEN:', cmdline child_in, child_out_err = os.popen4(cmdline) child_in.write(senddata) @@ -449,6 +451,43 @@ assert data == '\x00(STDOUT)\n\x00' # from stdout child_out_err.close() + def test_proper_sys_path(self, tmpdir): + + @contextmanager + def chdir_and_unset_pythonpath(new_cwd): + old_cwd = new_cwd.chdir() + old_pythonpath = os.getenv('PYTHONPATH') + os.unsetenv('PYTHONPATH') + try: + yield + finally: + old_cwd.chdir() + os.putenv('PYTHONPATH', old_pythonpath) + + tmpdir.join('site.py').write('print "SHOULD NOT RUN"') + runme_py = tmpdir.join('runme.py') + runme_py.write('print "some text"') + + cmdline = str(runme_py) + + with chdir_and_unset_pythonpath(tmpdir): + data = self.run(cmdline, python_flags='-S') + + assert data == "some text\n" + + runme2_py = tmpdir.mkdir('otherpath').join('runme2.py') + runme2_py.write('print "some new text"\n' + 'import sys\n' + 'print sys.path\n') + + cmdline2 = str(runme2_py) + + with chdir_and_unset_pythonpath(tmpdir): + data = self.run(cmdline2, python_flags='-S') + + assert data.startswith("some new text\n") + assert repr(str(tmpdir.join('otherpath'))) in data + class AppTestAppMain: Modified: pypy/branch/asmgcc-64/pypy/translator/platform/__init__.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/platform/__init__.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/platform/__init__.py Mon Aug 2 20:58:49 2010 @@ -161,7 +161,7 @@ return (library_dirs + self.link_flags + export_flags + link_files + list(eci.link_extra) + libraries) - def _exportsymbols_link_flags(self, eci): + def _exportsymbols_link_flags(self, eci, relto=None): if eci.export_symbols: raise ValueError("This platform does not support export symbols") return [] Modified: pypy/branch/asmgcc-64/pypy/translator/platform/darwin.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/platform/darwin.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/platform/darwin.py Mon Aug 2 20:58:49 2010 @@ -56,7 +56,7 @@ include_dirs = self._includedirs(eci.include_dirs) return (args + frameworks + include_dirs) - def _exportsymbols_link_flags(self, eci): + def _exportsymbols_link_flags(self, eci, relto=None): if not eci.export_symbols: return [] @@ -65,6 +65,9 @@ for sym in eci.export_symbols: f.write("_%s\n" % (sym,)) f.close() + + if relto: + response_file = relto.bestrelpath(response_file) return ["-Wl,-exported_symbols_list,%s" % (response_file,)] class Darwin_i386(Darwin): Modified: pypy/branch/asmgcc-64/pypy/translator/platform/posix.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/platform/posix.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/platform/posix.py Mon Aug 2 20:58:49 2010 @@ -39,7 +39,7 @@ def _link_args_from_eci(self, eci, standalone): return Platform._link_args_from_eci(self, eci, standalone) - def _exportsymbols_link_flags(self, eci): + def _exportsymbols_link_flags(self, eci, relto=None): if not eci.export_symbols: return [] @@ -50,6 +50,9 @@ f.write("%s;\n" % (sym,)) f.write("};") f.close() + + if relto: + response_file = relto.bestrelpath(response_file) return ["-Wl,--export-dynamic,--version-script=%s" % (response_file,)] def _link(self, cc, ofiles, link_args, standalone, exe_name): @@ -90,7 +93,7 @@ if shared: linkflags = self._args_for_shared(linkflags) - linkflags += self._exportsymbols_link_flags(eci) + linkflags += self._exportsymbols_link_flags(eci, relto=path) if shared: libname = exe_name.new(ext='').basename Modified: pypy/branch/asmgcc-64/pypy/translator/platform/windows.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/platform/windows.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/platform/windows.py Mon Aug 2 20:58:49 2010 @@ -141,7 +141,7 @@ # Windows needs to resolve all symbols even for DLLs return super(MsvcPlatform, self)._link_args_from_eci(eci, standalone=True) - def _exportsymbols_link_flags(self, eci): + def _exportsymbols_link_flags(self, eci, relto=None): if not eci.export_symbols: return [] @@ -150,6 +150,9 @@ for sym in eci.export_symbols: f.write("/EXPORT:%s\n" % (sym,)) f.close() + + if relto: + response_file = relto.bestrelpath(response_file) return ["@%s" % (response_file,)] def _compile_c_file(self, cc, cfile, compile_args): @@ -219,7 +222,7 @@ if shared: linkflags = self._args_for_shared(linkflags) + [ '/EXPORT:$(PYPY_MAIN_FUNCTION)'] - linkflags += self._exportsymbols_link_flags(eci) + linkflags += self._exportsymbols_link_flags(eci, relto=path) if shared: so_name = exe_name.new(purebasename='lib' + exe_name.purebasename, From agaynor at codespeak.net Mon Aug 2 21:56:56 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Mon, 2 Aug 2010 21:56:56 +0200 (CEST) Subject: [pypy-svn] r76441 - pypy/extradoc/planning Message-ID: <20100802195656.ABC84282B9C@codespeak.net> Author: agaynor Date: Mon Aug 2 21:56:54 2010 New Revision: 76441 Added: pypy/extradoc/planning/gc.txt Log: Added some notes on carding-marking GC. Added: pypy/extradoc/planning/gc.txt ============================================================================== --- (empty file) +++ pypy/extradoc/planning/gc.txt Mon Aug 2 21:56:54 2010 @@ -0,0 +1,46 @@ +Card marking GC for PyPy +======================== + +With a generational GC one needs to keep track of references from the old +generation to the young one using a write barrier. Currently this is +implemented with a list of old objects that contain pointers to young objects. +Then, when the nursery is collected each remembered old object is scanned for +any pointers it contains to young objects, and those are promoted out of the +nursery. The problem with this is exceptionally large objects with pointers +from the old generation to the young one: + + def main(): + objs = [] + for i in xrange(10000000): + objs.append(i) + # A bunch of stuff here. + +If that loop does enough things on each iteration to force a garbage collection +then every time through the loop ``objs`` will contain a pointer to a young +object (``i``) and will be scanned, in it's entirety for young pointers. This +results in the loop taking quadratic time, where it ought to be linear. The +solution to this is a strategy named card marking. + +In a card marking generational collector, instead of storing a list of old +objects, the heap is partitioned into sections, called cards (generally these +are smaller than a page of memory), and the write barrier marks the card for +the region of memory that contains the pointer. Then, during collection +instead of scanning objects for pointers, the regions of memory identified by +marked cards are scanned. This bounds the amount of space that must be scanned +by the size of a card, rather than by the size of an object (which is variable, +and could grow infinitely large). + +In principle this sounds nice, but there are a few issues: + + * We have pre-built constants (PBCs), which don't live in our contiguous heap. + Since the write barrier is supposed to be very cheap, we don't want it to + need to special case PBCs. There are two proposed solutions: + * Copy all PBCs to the heap on startup. This requires double the memory for + PBCs. (According to fijal, this is what the JVM does). + * Don't pre-build PBCs, instead allocate them fresh on startup. The OO type + backends do this, and it results in very slow startup time (5-6 seconds), + though there are likely other factors involved. + * Currently the hybrid GC allocates large objects externally to the heap, + this causes the same problem as PBCs. + * The JVM apparently also allocates large objects externally, and pays a + similar price for it. From hakanardo at codespeak.net Tue Aug 3 11:05:31 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Tue, 3 Aug 2010 11:05:31 +0200 (CEST) Subject: [pypy-svn] r76445 - pypy/branch/interplevel-array/pypy/module/array Message-ID: <20100803090531.5D279282B9C@codespeak.net> Author: hakanardo Date: Tue Aug 3 11:05:27 2010 New Revision: 76445 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py Log: now compiling Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Tue Aug 3 11:05:27 2010 @@ -296,9 +296,7 @@ idx, stop, step = space.decode_index(w_idx, self.len) assert step == 0 item = self.buffer[idx] - tc=mytype.typecode - if (tc == 'b' or tc == 'B' or tc == 'h' or tc == 'H' or - tc == 'i' or tc == 'l'): + if mytype.typecode in 'bBhHil': item = rffi.cast(lltype.Signed, item) elif mytype.typecode == 'f': item = float(item) @@ -555,7 +553,7 @@ def cmp__Array_ANY(space, self, other): if isinstance(other, W_ArrayBase): w_lst1 = array_tolist__Array(space, self) - w_lst2 = array_tolist__Array(space, other) + w_lst2 = space.call_method(other, 'tolist') return space.cmp(w_lst1, w_lst2) else: raise OperationError(space.w_NotImplementedError, space.wrap('')) From jcreigh at codespeak.net Tue Aug 3 14:43:45 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Tue, 3 Aug 2010 14:43:45 +0200 (CEST) Subject: [pypy-svn] r76446 - pypy/branch/asmgcc-64/pypy/translator/c/gcc Message-ID: <20100803124345.6602A282BDE@codespeak.net> Author: jcreigh Date: Tue Aug 3 14:43:42 2010 New Revision: 76446 Modified: pypy/branch/asmgcc-64/pypy/translator/c/gcc/trackgcroot.py Log: try to detect elf vs elf64 Modified: pypy/branch/asmgcc-64/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/c/gcc/trackgcroot.py Tue Aug 3 14:43:42 2010 @@ -15,6 +15,8 @@ from pypy.translator.c.gcc.instruction import LOC_EBP_PLUS, LOC_EBP_MINUS from pypy.translator.c.gcc.instruction import LOC_ESP_PLUS +IS_64_BITS = sys.maxint > 2147483647 + class FunctionGcRootTracker(object): skip = 0 COMMENT = "([#;].*)?" @@ -1844,9 +1846,10 @@ elif sys.platform == 'win32': format = 'mingw32' else: - # FIXME - # format = 'elf' - format = 'elf64' + if IS_64_BITS: + format = 'elf64' + else: + format = 'elf' entrypoint = 'main' while len(sys.argv) > 1: if sys.argv[1] == '-v': From jcreigh at codespeak.net Tue Aug 3 18:25:33 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Tue, 3 Aug 2010 18:25:33 +0200 (CEST) Subject: [pypy-svn] r76447 - in pypy/trunk/pypy/jit/backend/x86: . test Message-ID: <20100803162533.064DD282BDE@codespeak.net> Author: jcreigh Date: Tue Aug 3 18:25:31 2010 New Revision: 76447 Modified: pypy/trunk/pypy/jit/backend/x86/regloc.py pypy/trunk/pypy/jit/backend/x86/rx86.py pypy/trunk/pypy/jit/backend/x86/test/test_regloc.py pypy/trunk/pypy/jit/backend/x86/test/test_rx86.py Log: Fix JIT test failure related to jumping from a large "negative" address to a large "positive" address, or vice versa Modified: pypy/trunk/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regloc.py Tue Aug 3 18:25:31 2010 @@ -4,6 +4,7 @@ from pypy.jit.backend.x86.arch import WORD from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.objectmodel import specialize +from pypy.rlib.rarithmetic import intmask # # This module adds support for "locations", which can be either in a Const, @@ -252,7 +253,7 @@ if code == possible_code: val = getattr(loc, "value_" + possible_code)() if possible_code == 'i': - offset = val - (self.tell() + 5) + offset = intmask(val - (self.tell() + 5)) if rx86.fits_in_32bits(offset): _rx86_getattr(self, name + "_l")(val) else: Modified: pypy/trunk/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/rx86.py (original) +++ pypy/trunk/pypy/jit/backend/x86/rx86.py Tue Aug 3 18:25:31 2010 @@ -2,6 +2,7 @@ from pypy.rlib.objectmodel import ComputedIntSymbolic, we_are_translated from pypy.rlib.objectmodel import specialize from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib.rarithmetic import intmask from pypy.rpython.lltypesystem import rffi BYTE_REG_FLAG = 0x20 @@ -138,7 +139,7 @@ def encode_relative(mc, target, _, orbyte): assert orbyte == 0 - offset = target - (mc.tell() + 4) + offset = intmask(target - (mc.tell() + 4)) mc.writeimm32(offset) return 0 Modified: pypy/trunk/pypy/jit/backend/x86/test/test_regloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_regloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_regloc.py Tue Aug 3 18:25:31 2010 @@ -1,7 +1,9 @@ +import struct from pypy.jit.backend.x86.regloc import * from pypy.jit.backend.x86.test.test_rx86 import CodeBuilder32, CodeBuilder64, assert_encodes_as from pypy.jit.backend.x86.assembler import heap -from pypy.jit.backend.x86.arch import IS_X86_64 +from pypy.jit.backend.x86.arch import IS_X86_64, IS_X86_32 +from pypy.rlib.rarithmetic import intmask import py.test class LocationCodeBuilder32(CodeBuilder32, LocationCodeBuilder): @@ -21,6 +23,27 @@ assert_encodes_as(cb32, "CMP16", (ecx, ebx), '\x66\x39\xD9') assert_encodes_as(cb32, "CMP16", (ecx, ImmedLoc(12345)), '\x66\x81\xF9\x39\x30') +def test_jmp_wraparound(): + if not IS_X86_32: + py.test.skip() + + pos_addr = intmask(0x7FFFFF00) + neg_addr = intmask(0x800000BB) + + # JMP to "negative" address from "positive" address + s = cb32() + s.base_address = pos_addr + s.JMP(ImmedLoc(neg_addr)) + expected_ofs = neg_addr - (pos_addr+5) + assert s.getvalue() == '\xE9' + struct.pack(" Author: fijal Date: Tue Aug 3 18:53:11 2010 New Revision: 76450 Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/test/test_runner.py Log: Merge improved-asm-logging (worked, wow, great) Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Tue Aug 3 18:53:11 2010 @@ -25,12 +25,13 @@ from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.jit.backend.x86 import rx86, regloc, codebuf -from pypy.jit.metainterp.resoperation import rop +from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.backend.x86.support import values_array from pypy.rlib.debug import debug_print from pypy.rlib import rgc from pypy.jit.backend.x86.jump import remap_frame_layout from pypy.rlib.streamio import open_file_as_stream +from pypy.jit.metainterp.history import ConstInt, BoxInt # darwin requires the stack to be 16 bytes aligned on calls. Same for gcc 4.5.0, # better safe than sorry @@ -150,6 +151,8 @@ # XXX: 32 is pulled out of the air return 32 + len(self.desc_bytes) +DEBUG_COUNTER = rffi.CArray(lltype.Signed) + class Assembler386(object): mc = None mc_size = MachineCodeBlockWrapper.MC_DEFAULT_SIZE @@ -170,8 +173,7 @@ self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) self.fail_boxes_float = values_array(lltype.Float, failargs_limit) self.fail_ebp = 0 - self.loop_run_counter = values_array(lltype.Signed, 10000) - self.loop_names = [] + self.loop_run_counters = [] # if we have 10000 loops, we have some other problems I guess self.float_const_neg_addr = 0 self.float_const_abs_addr = 0 @@ -179,8 +181,8 @@ self.malloc_fixedsize_slowpath2 = 0 self.pending_guard_tokens = None self.setup_failure_recovery() - self._loop_counter = 0 self._debug = False + self.debug_counter_descr = cpu.arraydescrof(DEBUG_COUNTER) def leave_jitted_hook(self): ptrs = self.fail_boxes_ptr.ar @@ -237,9 +239,9 @@ output_log = self._output_loop_log assert output_log is not None f = open_file_as_stream(output_log, "w") - for i in range(self._loop_counter): - f.write(self.loop_names[i] + ":" + - str(self.loop_run_counter.getitem(i)) + "\n") + for i in range(len(self.loop_run_counters)): + name, arr = self.loop_run_counters[i] + f.write(name + ":" + str(arr[0]) + "\n") f.close() def _build_float_constants(self): @@ -304,6 +306,7 @@ regalloc = RegAlloc(self, self.cpu.translate_support_code) + operations = self._inject_debugging_code(operations) arglocs = regalloc.prepare_loop(inputargs, operations, looptoken) looptoken._x86_arglocs = arglocs @@ -343,6 +346,7 @@ assert ([loc.assembler() for loc in arglocs] == [loc.assembler() for loc in faildescr._x86_debug_faillocs]) regalloc = RegAlloc(self, self.cpu.translate_support_code) + operations = self._inject_debugging_code(operations) fail_depths = faildescr._x86_current_depths regalloc.prepare_bridge(fail_depths, inputargs, arglocs, operations) @@ -387,11 +391,12 @@ funcname = op.args[0]._get_str() break else: - funcname = "" % self._loop_counter + funcname = "" % len(self.loop_run_counters) # invent the counter, so we don't get too confused if self._debug: - self.loop_names.append(funcname) - self._loop_counter += 1 + arr = lltype.malloc(DEBUG_COUNTER, 1, flavor='raw') + arr[0] = 0 + self.loop_run_counters.append((funcname, arr)) return funcname def patch_jump_for_descr(self, faildescr, adr_new_target): @@ -414,17 +419,31 @@ mc.valgrind_invalidated() mc.done() - def _assemble(self, regalloc, operations): - self._regalloc = regalloc + def _inject_debugging_code(self, operations): if self._debug: # before doing anything, let's increase a counter - # we need one register free (a bit of a hack, but whatever) - self.mc.PUSH(eax) - adr = self.loop_run_counter.get_addr_for_num(self._loop_counter - 1) - self.mc.MOV(eax, heap(adr)) - self.mc.ADD(eax, imm(1)) - self.mc.MOV(heap(adr), eax) - self.mc.POP(eax) + c_adr = ConstInt(rffi.cast(lltype.Signed, + self.loop_run_counters[-1][1])) + box = BoxInt() + box2 = BoxInt() + ops = [ResOperation(rop.GETARRAYITEM_RAW, [c_adr, ConstInt(0)], + box, descr=self.debug_counter_descr), + ResOperation(rop.INT_ADD, [box, ConstInt(1)], box2), + ResOperation(rop.SETARRAYITEM_RAW, [c_adr, ConstInt(0), + box2], + None, descr=self.debug_counter_descr)] + operations = ops + operations + # # we need one register free (a bit of a hack, but whatever) + # self.mc.PUSH(eax) + # adr = rffi.cast(lltype.Signed, self.loop_run_counters[-1][1]) + # self.mc.MOV(eax, heap(adr)) + # self.mc.ADD(eax, imm(1)) + # self.mc.MOV(heap(adr), eax) + # self.mc.POP(eax) + return operations + + def _assemble(self, regalloc, operations): + self._regalloc = regalloc regalloc.walk_operations(operations) self.mc.done() if we_are_translated() or self.cpu.dont_keepalive_stuff: Modified: pypy/trunk/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_runner.py Tue Aug 3 18:53:11 2010 @@ -484,8 +484,9 @@ self.cpu.set_future_value_int(0, 0) self.cpu.execute_token(ops.token) # check debugging info - assert self.cpu.assembler.loop_names == ["xyz"] - assert self.cpu.assembler.loop_run_counter.getitem(0) == 10 + name, arr = self.cpu.assembler.loop_run_counters[0] + assert name == 'xyz' + assert arr[0] == 10 self.cpu.finish_once() lines = py.path.local(self.logfile + ".count").readlines() assert lines[0] == 'xyz:10\n' From fijal at codespeak.net Tue Aug 3 18:53:40 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 3 Aug 2010 18:53:40 +0200 (CEST) Subject: [pypy-svn] r76451 - pypy/branch/improved-asm-logging Message-ID: <20100803165340.2BCDC282BDE@codespeak.net> Author: fijal Date: Tue Aug 3 18:53:38 2010 New Revision: 76451 Removed: pypy/branch/improved-asm-logging/ Log: Remove merged branch From jcreigh at codespeak.net Tue Aug 3 18:56:50 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Tue, 3 Aug 2010 18:56:50 +0200 (CEST) Subject: [pypy-svn] r76452 - pypy/trunk/pypy/jit/backend/llsupport/test Message-ID: <20100803165650.4E648282BDE@codespeak.net> Author: jcreigh Date: Tue Aug 3 18:56:48 2010 New Revision: 76452 Modified: pypy/trunk/pypy/jit/backend/llsupport/test/test_regalloc.py Log: fix regalloc tests Modified: pypy/trunk/pypy/jit/backend/llsupport/test/test_regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/test/test_regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/test/test_regalloc.py Tue Aug 3 18:56:48 2010 @@ -1,5 +1,5 @@ -from pypy.jit.metainterp.history import BoxInt, ConstInt, BoxFloat +from pypy.jit.metainterp.history import BoxInt, ConstInt, BoxFloat, INT, FLOAT from pypy.jit.backend.llsupport.regalloc import FrameManager from pypy.jit.backend.llsupport.regalloc import RegisterManager as BaseRegMan @@ -26,9 +26,20 @@ def convert_to_imm(self, v): return v +class FakeFramePos(object): + def __init__(self, pos, box_type): + self.pos = pos + self.box_type = box_type + + def frame_size(self): + if self.box_type == FLOAT: + return 2 + else: + return 1 + class TFrameManager(FrameManager): - def frame_pos(self, i, size): - return i + def frame_pos(self, i, box_type): + return FakeFramePos(i, box_type) class MockAsm(object): def __init__(self): @@ -146,8 +157,8 @@ rm.next_instruction() # allocate a stack position b0, b1, b2, b3, b4 = boxes - sp = fm.loc(b0, 1) - assert sp == 0 + sp = fm.loc(b0) + assert sp.pos == 0 loc = rm.make_sure_var_in_reg(b0) assert isinstance(loc, FakeReg) rm._check_invariants() @@ -207,13 +218,13 @@ asm = MockAsm() rm = RegisterManager(longevity, frame_manager=fm, assembler=asm) rm.next_instruction() - fm.loc(b0, 1) + fm.loc(b0) rm.force_result_in_reg(b1, b0) rm._check_invariants() loc = rm.loc(b1) assert isinstance(loc, FakeReg) loc = rm.loc(b0) - assert isinstance(loc, int) + assert isinstance(loc, FakeFramePos) assert len(asm.moves) == 1 def test_return_constant(self): @@ -304,7 +315,7 @@ def test_different_frame_width(self): class XRegisterManager(RegisterManager): - reg_width = 2 + pass fm = TFrameManager() b0 = BoxInt() From hakanardo at codespeak.net Tue Aug 3 19:29:10 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Tue, 3 Aug 2010 19:29:10 +0200 (CEST) Subject: [pypy-svn] r76457 - pypy/branch/interplevel-array/pypy/module/array Message-ID: <20100803172910.95F3A282BDE@codespeak.net> Author: hakanardo Date: Tue Aug 3 19:29:09 2010 New Revision: 76457 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py Log: pep8 Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Tue Aug 3 19:29:09 2010 @@ -1,7 +1,8 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.rpython.lltypesystem import lltype, rffi -from pypy.interpreter.gateway import interp2app, ObjSpace, W_Root, ApplevelClass +from pypy.interpreter.gateway import interp2app, ObjSpace, W_Root, \ + ApplevelClass from pypy.rlib.jit import dont_look_inside from pypy.rlib import rgc from pypy.rlib.unroll import unrolling_iterable @@ -13,6 +14,7 @@ from pypy.objspace.std.model import W_Object from pypy.interpreter.argument import Arguments, Signature + def w_array(space, w_cls, typecode, w_initializer=None, w_args=None): if len(w_args.arguments_w) > 0: msg = 'array() takes at most 2 arguments' @@ -20,7 +22,7 @@ if len(typecode) != 1: msg = 'array() argument 1 must be char, not str' raise OperationError(space.w_TypeError, space.wrap(msg)) - typecode=typecode[0] + typecode = typecode[0] for tc in unroll_typecodes: if typecode == tc: @@ -72,6 +74,8 @@ def descr_itemsize(space, self): return space.wrap(self.itemsize) + + def descr_typecode(space, self): return space.wrap(self.typecode) @@ -88,10 +92,12 @@ class W_ArrayBase(W_Object): typedef = type_typedef + @staticmethod def register(typeorder): typeorder[W_ArrayBase] = [] + class TypeCode(object): def __init__(self, itemtype, unwrap, canoverflow=False, signed=False): self.itemtype = itemtype @@ -105,17 +111,16 @@ if self.canoverflow: assert self.bytes <= rffi.sizeof(rffi.ULONG) - if self.bytes == rffi.sizeof(rffi.ULONG) and not signed and self.unwrap == 'int_w': + if self.bytes == rffi.sizeof(rffi.ULONG) and not signed and \ + self.unwrap == 'int_w': # Treat this type as a ULONG self.unwrap = 'bigint_w' self.canoverflow = False - def _freeze_(self): # hint for the annotator: track individual constant instances return True - types = { 'c': TypeCode(lltype.Char, 'str_w'), 'u': TypeCode(lltype.UniChar, 'unicode_w'), @@ -126,14 +131,18 @@ 'i': TypeCode(rffi.INT, 'int_w', True, True), 'I': TypeCode(rffi.UINT, 'int_w', True), 'l': TypeCode(rffi.LONG, 'int_w', True, True), - 'L': TypeCode(rffi.ULONG, 'bigint_w'), # Overflow handled by rbigint.touint() which - # corresponds to the C-type unsigned long + 'L': TypeCode(rffi.ULONG, 'bigint_w'), # Overflow handled by + # rbigint.touint() which + # corresponds to the + # C-type unsigned long 'f': TypeCode(lltype.SingleFloat, 'float_w'), 'd': TypeCode(lltype.Float, 'float_w'), } -for k, v in types.items(): v.typecode=k +for k, v in types.items(): + v.typecode = k unroll_typecodes = unrolling_iterable(types.keys()) + def make_array(mytype): class W_Array(W_ArrayBase): itemsize = mytype.bytes @@ -157,13 +166,15 @@ try: item = item.touint() except (ValueError, OverflowError): - msg = 'unsigned %d-byte integer out of range' % mytype.bytes - raise OperationError(space.w_OverflowError, space.wrap(msg)) + msg = 'unsigned %d-byte integer out of range' % \ + mytype.bytes + raise OperationError(space.w_OverflowError, + space.wrap(msg)) elif mytype.unwrap == 'str_w' or mytype.unwrap == 'unicode_w': if len(item) != 1: msg = 'array item must be char' raise OperationError(space.w_TypeError, space.wrap(msg)) - item=item[0] + item = item[0] if mytype.canoverflow: msg = None @@ -182,25 +193,25 @@ msg = ('unsigned %d-byte integer is greater' ' than maximum' % mytype.bytes) if msg is not None: - raise OperationError(space.w_OverflowError, space.wrap(msg)) + raise OperationError(space.w_OverflowError, + space.wrap(msg)) return rffi.cast(mytype.itemtype, item) - def __del__(self): self.setlen(0) - def setlen(self, size): if size > 0: - if size > self.allocated or size < self.allocated/2: + if size > self.allocated or size < self.allocated / 2: if size < 9: some = 3 else: some = 6 some += size >> 3 self.allocated = size + some - new_buffer = lltype.malloc(mytype.arraytype, self.allocated, flavor='raw') - for i in range(min(size,self.len)): + new_buffer = lltype.malloc(mytype.arraytype, + self.allocated, flavor='raw') + for i in range(min(size, self.len)): new_buffer[i] = self.buffer[i] else: self.len = size @@ -215,7 +226,6 @@ self.buffer = new_buffer self.len = size - def fromsequence(self, w_seq): space = self.space oldlen = self.len @@ -244,13 +254,13 @@ def fromstring(self, w_s): space = self.space s = space.str_w(w_s) - if len(s)%self.itemsize !=0: + if len(s) % self.itemsize != 0: msg = 'string length not a multiple of item size' raise OperationError(space.w_ValueError, space.wrap(msg)) oldlen = self.len new = len(s) / mytype.bytes self.setlen(oldlen + new) - cbuf =self.charbuf() + cbuf = self.charbuf() for i in range(len(s)): cbuf[oldlen * mytype.bytes + i] = s[i] @@ -281,12 +291,9 @@ else: self.fromsequence(w_iterable) - - def charbuf(self): return rffi.cast(rffi.CCHARP, self.buffer) - # Basic get/set/append/extend methods def len__Array(space, self): @@ -307,14 +314,15 @@ if step < 0: w_lst = array_tolist__Array(space, self) w_lst = space.getitem(w_lst, w_slice) - w_a=mytype.w_class(self.space) + w_a = mytype.w_class(self.space) w_a.fromsequence(w_lst) elif step == 0: - raise ValueError, 'getitem__Array_Slice with step zero' + raise ValueError('getitem__Array_Slice with step zero') else: size = (stop - start) / step - if (stop - start) % step > 0: size += 1 - w_a=mytype.w_class(self.space) + if (stop - start) % step > 0: + size += 1 + w_a = mytype.w_class(self.space) w_a.setlen(size) j = 0 for i in range(start, stop, step): @@ -328,7 +336,7 @@ def setitem__Array_ANY_ANY(space, self, w_idx, w_item): idx, stop, step = space.decode_index(w_idx, self.len) if step != 0: - msg='can only assign array to array slice' + msg = 'can only assign array to array slice' raise OperationError(self.space.w_TypeError, self.space.wrap(msg)) item = self.item_w(w_item) self.buffer[idx] = item @@ -336,7 +344,8 @@ def setitem__Array_Slice_Array(space, self, w_idx, w_item): start, stop, step = self.space.decode_index(w_idx, self.len) size = (stop - start) / step - if (stop - start) % step > 0: size += 1 + if (stop - start) % step > 0: + size += 1 if w_item.len != size or step < 0: w_lst = array_tolist__Array(space, self) w_item = space.call_method(w_item, 'tolist') @@ -344,7 +353,7 @@ self.setlen(0) self.fromsequence(w_lst) elif step == 0: - raise ValueError, 'setitem__Array_Slice with step zero' + raise ValueError('setitem__Array_Slice with step zero') else: j = 0 for i in range(start, stop, step): @@ -354,17 +363,14 @@ def setslice__Array_ANY_ANY_ANY(space, self, w_i, w_j, w_x): space.setitem(self, space.newslice(w_i, w_j, space.w_None), w_x) - def array_append__Array_ANY(space, self, w_x): x = self.item_w(w_x) self.setlen(self.len + 1) self.buffer[self.len - 1] = x - def array_extend__Array_ANY(space, self, w_iterable): self.extend(w_iterable) - # List interface def array_count__Array_ANY(space, self, w_val): cnt = 0 @@ -373,7 +379,6 @@ if space.is_true(space.eq(w_item, w_val)): cnt += 1 return space.wrap(cnt) - def array_index__Array_ANY(space, self, w_val): cnt = 0 @@ -384,16 +389,15 @@ msg = 'array.index(x): x not in list' raise OperationError(space.w_ValueError, space.wrap(msg)) - def array_reverse__Array(space, self): b = self.buffer - for i in range(self.len/2): - b[i], b[self.len - i -1] = b[self.len - i -1], b[i] - + for i in range(self.len / 2): + b[i], b[self.len - i - 1] = b[self.len - i - 1], b[i] def array_pop__Array_ANY(space, self, w_idx): i = space.int_w(w_idx) - if i < 0: i += self.len + if i < 0: + i += self.len if i < 0 or i >= self.len: msg = 'pop index out of range' raise OperationError(space.w_IndexError, space.wrap(msg)) @@ -401,30 +405,30 @@ while i < self.len - 1: self.buffer[i] = self.buffer[i + 1] i += 1 - self.setlen(self.len-1) + self.setlen(self.len - 1) return w_val - def array_remove__Array_ANY(space, self, w_val): w_idx = array_index__Array_ANY(space, self, w_val) array_pop__Array_ANY(space, self, w_idx) - def array_insert__Array_ANY_ANY(space, self, w_idx, w_val): idx = space.int_w(w_idx) - if idx < 0: idx += self.len - if idx < 0: idx = 0 - if idx > self.len: idx = self.len + if idx < 0: + idx += self.len + if idx < 0: + idx = 0 + if idx > self.len: + idx = self.len val = self.item_w(w_val) self.setlen(self.len + 1) - i = self.len-1 + i = self.len - 1 while i > idx: - self.buffer[i] = self.buffer[i-1] + self.buffer[i] = self.buffer[i - 1] i -= 1 self.buffer[i] = val - def delitem__Array_ANY(space, self, w_idx): w_lst = array_tolist__Array(space, self) space.delitem(w_lst, w_idx) @@ -467,9 +471,9 @@ def inplace_mul__Array_ANY(space, self, w_repeat): repeat = space.int_w(w_repeat) - oldlen=self.len + oldlen = self.len self.setlen(self.len * repeat) - for r in range(1,repeat): + for r in range(1, repeat): for i in range(oldlen): self.buffer[r * oldlen + i] = self.buffer[i] return self @@ -477,7 +481,7 @@ # Convertions def array_tolist__Array(space, self): - w_l=space.newlist([]) + w_l = space.newlist([]) for i in range(self.len): w_l.append(getitem__Array_ANY(space, self, space.wrap(i))) return w_l @@ -491,16 +495,14 @@ def array_tostring__Array(space, self): cbuf = self.charbuf() s = '' - i=0 + i = 0 while i < self.len * mytype.bytes: s += cbuf[i] - i+=1 + i += 1 return self.space.wrap(s) - - def array_fromfile__Array_ANY_ANY(space, self, w_f, w_n): - if space.type(w_f).name != 'file': # FIXME: this cant be the right way? + if space.type(w_f).name != 'file': msg = "arg1 must be open file" raise OperationError(space.w_TypeError, space.wrap(msg)) n = space.int_w(w_n) @@ -510,8 +512,9 @@ item = space.str_w(w_item) if len(item) < size: n = len(item) % self.itemsize - elems = max(0,len(item)-(len(item) % self.itemsize)) - if n != 0: item = item[0:elems] + elems = max(0, len(item) - (len(item) % self.itemsize)) + if n != 0: + item = item[0:elems] w_item = space.wrap(item) array_fromstring__Array_ANY(space, self, w_item) msg = "not enough items in file" @@ -519,13 +522,14 @@ array_fromstring__Array_ANY(space, self, w_item) def array_tofile__Array_ANY(space, self, w_f): - if space.type(w_f).name != 'file': # FIXME: this cant be the right way? + if space.type(w_f).name != 'file': msg = "arg1 must be open file" raise OperationError(space.w_TypeError, space.wrap(msg)) w_s = array_tostring__Array(space, self) space.call_method(w_f, 'write', w_s) if mytype.typecode == 'u': + def array_fromunicode__Array_Unicode(space, self, w_ustr): # XXX the following probable bug is not emulated: # CPython accepts a non-unicode string or a buffer, and then @@ -540,6 +544,7 @@ u += self.buffer[i] return space.wrap(u) else: + def array_fromunicode__Array_Unicode(space, self, w_ustr): msg = "fromunicode() may only be called on type 'u' arrays" raise OperationError(space.w_ValueError, space.wrap(msg)) @@ -548,7 +553,6 @@ msg = "tounicode() may only be called on type 'u' arrays" raise OperationError(space.w_ValueError, space.wrap(msg)) - # Compare methods def cmp__Array_ANY(space, self, other): if isinstance(other, W_ArrayBase): @@ -557,7 +561,6 @@ return space.cmp(w_lst1, w_lst2) else: raise OperationError(space.w_NotImplementedError, space.wrap('')) - # Misc methods @@ -579,12 +582,12 @@ args = [space.wrap(mytype.typecode)] return space.newtuple([space.type(self), space.newtuple(args)]) - def array_byteswap__Array(space, self): if mytype.bytes not in [1, 2, 4, 8]: - msg="byteswap not supported for this array" + msg = "byteswap not supported for this array" raise OperationError(space.w_RuntimeError, space.wrap(msg)) - if self.len == 0: return + if self.len == 0: + return bytes = self.charbuf() tmp = [bytes[0]] * mytype.bytes for start in range(0, self.len * mytype.bytes, mytype.bytes): @@ -593,7 +596,6 @@ tmp[i] = bytes[start + i] for i in range(mytype.bytes): bytes[stop - i] = tmp[i] - def repr__Array(space, self): if self.len == 0: @@ -620,12 +622,12 @@ mytype.w_class = W_Array # Annotator seems to mess up if the names are not unique - name = 'ArrayType'+mytype.typecode + name = 'ArrayType' + mytype.typecode W_Array.__name__ = 'W_' + name import re - for n,f in locals().items(): - new,n = re.subn('_Array_', '_%s_' % name, n) - if n>0: + for n, f in locals().items(): + new, n = re.subn('_Array_', '_%s_' % name, n) + if n > 0: f.__name__ = new from pypy.objspace.std.sliceobject import W_SliceObject From fijal at codespeak.net Tue Aug 3 19:30:49 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 3 Aug 2010 19:30:49 +0200 (CEST) Subject: [pypy-svn] r76458 - pypy/trunk/pypy/objspace/std Message-ID: <20100803173049.B3374282BDE@codespeak.net> Author: fijal Date: Tue Aug 3 19:30:48 2010 New Revision: 76458 Modified: pypy/trunk/pypy/objspace/std/callmethod.py Log: hopefully fix stackless Modified: pypy/trunk/pypy/objspace/std/callmethod.py ============================================================================== --- pypy/trunk/pypy/objspace/std/callmethod.py (original) +++ pypy/trunk/pypy/objspace/std/callmethod.py Tue Aug 3 19:30:48 2010 @@ -68,7 +68,8 @@ if not n_kwargs: try: w_result = f.space.call_valuestack(w_callable, n, f) - rstack.resume_point("CALL_METHOD_no_kwargs", f, n_args, returns=w_result) + rstack.resume_point("CALL_METHOD", f, w_self, n_args, + returns=w_result) finally: f.dropvalues(n_args + 2) else: @@ -89,7 +90,8 @@ try: w_result = f.space.call_args(w_callable, args) - rstack.resume_point("CALL_METHOD", f, w_self, returns=w_result) + rstack.resume_point("CALL_METHOD", f, w_self, n_args, + returns=w_result) finally: f.dropvalues(1 + (w_self is None)) f.pushvalue(w_result) From wlav at codespeak.net Tue Aug 3 23:22:06 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Tue, 3 Aug 2010 23:22:06 +0200 (CEST) Subject: [pypy-svn] r76460 - in pypy/branch/reflex-support/pypy/module/cppyy: . include src test Message-ID: <20100803212206.23A44282BE0@codespeak.net> Author: wlav Date: Tue Aug 3 23:22:05 2010 New Revision: 76460 Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py pypy/branch/reflex-support/pypy/module/cppyy/converter.py pypy/branch/reflex-support/pypy/module/cppyy/helper.py pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/pythonify.py pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py Log: Support for public data members of type (unsigned) char and renaming various names for consistency. Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/capi.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/capi.py Tue Aug 3 23:22:05 2010 @@ -85,19 +85,36 @@ "cppyy_method_name", [C_TYPEHANDLE, rffi.INT], rffi.CCHARP, compilation_info=eci) -c_result_type_method = rffi.llexternal( - "cppyy_result_type_method", +c_method_result_type = rffi.llexternal( + "cppyy_method_result_type", [C_TYPEHANDLE, rffi.INT], rffi.CCHARP, compilation_info=eci) -c_num_args_method = rffi.llexternal( - "cppyy_num_args_method", +c_method_num_args = rffi.llexternal( + "cppyy_method_num_args", [C_TYPEHANDLE, rffi.INT], rffi.INT, compilation_info=eci) -c_arg_type_method = rffi.llexternal( - "cppyy_arg_type_method", +c_method_arg_type = rffi.llexternal( + "cppyy_method_arg_type", [C_TYPEHANDLE, rffi.INT, rffi.INT], rffi.CCHARP, compilation_info=eci) +c_num_data_members = rffi.llexternal( + "cppyy_num_data_members", + [C_TYPEHANDLE], rffi.INT, + compilation_info=eci) +c_data_member_name = rffi.llexternal( + "cppyy_data_member_name", + [C_TYPEHANDLE, rffi.INT], rffi.CCHARP, + compilation_info=eci) +c_data_member_type = rffi.llexternal( + "cppyy_data_member_type", + [C_TYPEHANDLE, rffi.INT], rffi.CCHARP, + compilation_info=eci) +c_data_member_offset = rffi.llexternal( + "cppyy_data_member_offset", + [C_TYPEHANDLE, rffi.INT], rffi.INT, + compilation_info=eci) + c_is_constructor = rffi.llexternal( "cppyy_is_constructor", [C_TYPEHANDLE, rffi.INT], rffi.INT, Modified: pypy/branch/reflex-support/pypy/module/cppyy/converter.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/converter.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/converter.py Tue Aug 3 23:22:05 2010 @@ -10,6 +10,9 @@ def convert_argument(self, space, w_obj): raise NotImplementedError("abstract base class") + def from_memory(self, space, w_obj, offset): + raise NotImplementedError("abstract base class") + def free_argument(self, arg): lltype.free(arg, flavor='raw') @@ -41,6 +44,17 @@ x = rffi.str2charp(arg) return rffi.cast(rffi.VOIDP, x) + def from_memory(self, space, w_obj, offset): + obj = space.interpclass_w(space.findattr(w_obj, space.wrap("_cppinstance"))) + fieldptr = lltype.direct_ptradd(obj.rawobject, offset) + return space.wrap(fieldptr[0]) + + def to_memory(self, space, w_obj, w_value, offset): + obj = space.interpclass_w(space.findattr(w_obj, space.wrap("_cppinstance"))) + fieldptr = lltype.direct_ptradd(obj.rawobject, offset) + print w_value + fieldptr[0] = space.str_w(w_value) + class IntConverter(TypeConverter): def convert_argument(self, space, w_obj): arg = space.c_int_w(w_obj) @@ -87,6 +101,7 @@ space.wrap("cannot pass %s as %s" % ( space.type(w_obj).getname(space, "?"), self.cpptype.name))) + def free_argument(self, arg): pass Modified: pypy/branch/reflex-support/pypy/module/cppyy/helper.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/helper.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/helper.py Tue Aug 3 23:22:05 2010 @@ -3,17 +3,25 @@ def compound(name): name = "".join(rstring.split(name, "const")) # poor man's replace i = _find_qualifier_index(name) + if name[-1] == "]": # array type + return "*" return "".join(name[i:].split(" ")) def _find_qualifier_index(name): i = len(name) for i in range(len(name) - 1, -1, -1): c = name[i] - if c.isalnum() or c == ">": + if c.isalnum() or c == ">" or c == "]": break return i + 1 def clean_type(name): assert name.find("const") == -1 i = _find_qualifier_index(name) - return name[:i].strip(' ') + name = name[:i].strip(' ') + if name[-1] == "]": + for i in range(len(name) - 1, -1, -1): + c = name[i] + if c == "[": + return name[:i] + return name Modified: pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h Tue Aug 3 23:22:05 2010 @@ -14,6 +14,7 @@ void* cppyy_allocate(cppyy_typehandle_t handle); void cppyy_deallocate(cppyy_typehandle_t handle, cppyy_object_t instance); + /* method/function dispatching */ void cppyy_call_v(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); int cppyy_call_b(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); char cppyy_call_c(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); @@ -24,11 +25,18 @@ void cppyy_destruct(cppyy_typehandle_t handle, cppyy_object_t self); cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_typehandle_t handle, int method_index); + /* method/function reflection information */ int cppyy_num_methods(cppyy_typehandle_t handle); char* cppyy_method_name(cppyy_typehandle_t handle, int method_index); - char* cppyy_result_type_method(cppyy_typehandle_t handle, int method_index); - int cppyy_num_args_method(cppyy_typehandle_t handle, int method_index); - char* cppyy_arg_type_method(cppyy_typehandle_t handle, int method_index, int index); + char* cppyy_method_result_type(cppyy_typehandle_t handle, int method_index); + int cppyy_method_num_args(cppyy_typehandle_t handle, int method_index); + char* cppyy_method_arg_type(cppyy_typehandle_t handle, int method_index, int index); + + /* data member reflection information */ + int cppyy_num_data_members(cppyy_typehandle_t handle); + char* cppyy_data_member_name(cppyy_typehandle_t handle, int data_member_index); + char* cppyy_data_member_type(cppyy_typehandle_t handle, int data_member_index); + size_t cppyy_data_member_offset(cppyy_typehandle_t handle, int data_member_index); int cppyy_is_constructor(cppyy_typehandle_t handle, int method_index); int cppyy_is_static(cppyy_typehandle_t handle, int method_index); Modified: pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/interp_cppyy.py Tue Aug 3 23:22:05 2010 @@ -35,7 +35,8 @@ if handle: cpptype = W_CPPType(space, name, handle) state.cpptype_cache[name] = cpptype - cpptype._find_func_members() + cpptype._find_methods() + cpptype._find_data_members() return cpptype raise OperationError(space.w_TypeError, space.wrap("no such C++ class %s" % name)) @@ -213,6 +214,28 @@ ) +class W_CPPDataMember(Wrappable): + _immutable_fields_ = ["converter", "offset"] + + def __init__(self, space, cpptype, offset): + self.space = space + self.converter = converter.get_converter(self.space, cpptype) + self.offset = offset + + def __get__(self, args_w): + return self.converter.from_memory(self.space, args_w[0], self.offset) + + def __set__(self, args_w): + self.converter.to_memory(self.space, args_w[0], args_w[1], self.offset) + return None + +W_CPPDataMember.typedef = TypeDef( + 'CPPDataMember', + __get__ = interp2app(W_CPPDataMember.__get__, unwrap_spec=['self', 'args_w']), + __set__ = interp2app(W_CPPDataMember.__set__, unwrap_spec=['self', 'args_w']), +) + + class W_CPPType(Wrappable): _immutable_fields_ = ["name","handle"] @@ -220,30 +243,33 @@ self.space = space self.name = name self.handle = handle - self.function_members = {} - # Do not call "self._find_func_members()" here, so that a distinction can be - # made between testing for existence (i.e. existence in the cache of classes) - # and actual use. Point being that a class can use itself, e.g. as a return - # type or an argument to one of its methods. + self.methods = {} + # Do not call "self._find_methods()" here, so that a distinction can + # be made between testing for existence (i.e. existence in the cache + # of classes) and actual use. Point being that a class can use itself, + # e.g. as a return type or an argument to one of its methods. + + self.data_members = {} + # Idem self.methods: a type could hold itself by pointer. - def _find_func_members(self): - num_func_members = capi.c_num_methods(self.handle) + def _find_methods(self): + num_methods = capi.c_num_methods(self.handle) args_temp = {} - for i in range(num_func_members): - func_member_name = capi.charp2str_free(capi.c_method_name(self.handle, i)) + for i in range(num_methods): + method_name = capi.charp2str_free(capi.c_method_name(self.handle, i)) cppfunction = self._make_cppfunction(i) - overload = args_temp.setdefault(func_member_name, []) + overload = args_temp.setdefault(method_name, []) overload.append(cppfunction) for name, functions in args_temp.iteritems(): overload = W_CPPOverload(self.space, name, functions[:]) - self.function_members[name] = overload + self.methods[name] = overload def _make_cppfunction(self, method_index): - result_type = capi.charp2str_free(capi.c_result_type_method(self.handle, method_index)) - num_args = capi.c_num_args_method(self.handle, method_index) + result_type = capi.charp2str_free(capi.c_method_result_type(self.handle, method_index)) + num_args = capi.c_method_num_args(self.handle, method_index) argtypes = [] for i in range(num_args): - argtype = capi.charp2str_free(capi.c_arg_type_method(self.handle, method_index, i)) + argtype = capi.charp2str_free(capi.c_method_arg_type(self.handle, method_index, i)) argtypes.append(argtype) if capi.c_is_constructor(self.handle, method_index): cls = CPPConstructor @@ -253,12 +279,28 @@ cls = CPPMethod return cls(self, method_index, result_type, argtypes) - def get_function_members(self): - return self.space.newlist([self.space.wrap(name) for name in self.function_members]) + def _find_data_members(self): + num_data_members = capi.c_num_data_members(self.handle) + for i in range(num_data_members): + data_member_name = capi.charp2str_free(capi.c_data_member_name(self.handle, i)) + cpptype = capi.charp2str_free(capi.c_data_member_type(self.handle, i)) + offset = capi.c_data_member_offset(self.handle, i) + data_member = W_CPPDataMember(self.space, cpptype, offset) + self.data_members[data_member_name] = data_member + + def get_method_names(self): + return self.space.newlist([self.space.wrap(name) for name in self.methods]) @jit.purefunction def get_overload(self, name): - return self.function_members[name] + return self.methods[name] + + def get_data_member_names(self): + return self.space.newlist([self.space.wrap(name) for name in self.data_members]) + + @jit.purefunction + def get_data_member(self, name): + return self.data_members[name] def invoke(self, name, args_w): overload = self.get_overload(name) @@ -270,8 +312,10 @@ W_CPPType.typedef = TypeDef( 'CPPType', - get_function_members = interp2app(W_CPPType.get_function_members, unwrap_spec=['self']), + get_method_names = interp2app(W_CPPType.get_method_names, unwrap_spec=['self']), get_overload = interp2app(W_CPPType.get_overload, unwrap_spec=['self', str]), + get_data_member_names = interp2app(W_CPPType.get_data_member_names, unwrap_spec=['self']), + get_data_member = interp2app(W_CPPType.get_data_member, unwrap_spec=['self', str]), invoke = interp2app(W_CPPType.invoke, unwrap_spec=['self', str, 'args_w']), construct = interp2app(W_CPPType.construct, unwrap_spec=['self', 'args_w']), ) Modified: pypy/branch/reflex-support/pypy/module/cppyy/pythonify.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/pythonify.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/pythonify.py Tue Aug 3 23:22:05 2010 @@ -54,13 +54,16 @@ # TODO: handle base classes cpptype = cppyy._type_byname(name) d = {"_cppyyclass" : cpptype} - for f in cpptype.get_function_members(): + for f in cpptype.get_method_names(): cppol = cpptype.get_overload(f) if cppol.is_static(): d[f] = make_static_function(cpptype, f, cppol.get_returntype()) else: d[f] = make_method(f, cppol.get_returntype()) + for dm in cpptype.get_data_member_names(): + d[dm] = cpptype.get_data_member(dm) + pycpptype = CppyyClass(name, (CppyyObject,), d) return pycpptype @@ -77,6 +80,7 @@ self.__dict__[attr] = cppclass return cppclass except TypeError, e: + import traceback raise AttributeError("'gbl' object has no attribute '%s'" % attr) Modified: pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Tue Aug 3 23:22:05 2010 @@ -18,6 +18,7 @@ } +/* method/function dispatching -------------------------------------------- */ void cppyy_call_v(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]) { std::vector arguments(args, args+numargs); @@ -96,6 +97,7 @@ } +/* method/function reflection information --------------------------------- */ int cppyy_num_methods(cppyy_typehandle_t handle) { Reflex::Type t((Reflex::TypeName*)handle); // for (int i = 0; i < (int)t.FunctionMemberSize(); i++) { @@ -120,7 +122,7 @@ return name_char; } -char* cppyy_result_type_method(cppyy_typehandle_t handle, int method_index) { +char* cppyy_method_result_type(cppyy_typehandle_t handle, int method_index) { Reflex::Type t((Reflex::TypeName*)handle); Reflex::Member m = t.FunctionMemberAt(method_index); Reflex::Type rt = m.TypeOf().ReturnType(); @@ -130,13 +132,13 @@ return name_char; } -int cppyy_num_args_method(cppyy_typehandle_t handle, int method_index) { +int cppyy_method_num_args(cppyy_typehandle_t handle, int method_index) { Reflex::Type t((Reflex::TypeName*)handle); Reflex::Member m = t.FunctionMemberAt(method_index); return m.FunctionParameterSize(); } -char* cppyy_arg_type_method(cppyy_typehandle_t handle, int method_index, int arg_index) { +char* cppyy_method_arg_type(cppyy_typehandle_t handle, int method_index, int arg_index) { Reflex::Type t((Reflex::TypeName*)handle); Reflex::Member m = t.FunctionMemberAt(method_index); Reflex::Type at = m.TypeOf().FunctionParameterAt(arg_index); @@ -147,6 +149,37 @@ } +/* data member reflection information ------------------------------------- */ +int cppyy_num_data_members(cppyy_typehandle_t handle) { + Reflex::Type t((Reflex::TypeName*)handle); + return t.DataMemberSize(); +} + +char* cppyy_data_member_name(cppyy_typehandle_t handle, int data_member_index) { + Reflex::Type t((Reflex::TypeName*)handle); + Reflex::Member m = t.DataMemberAt(data_member_index); + std::string name = m.Name(); + char* name_char = (char*)malloc(name.size() + 1); + strcpy(name_char, name.c_str()); + return name_char; +} + +char* cppyy_data_member_type(cppyy_typehandle_t handle, int data_member_index) { + Reflex::Type t((Reflex::TypeName*)handle); + Reflex::Member m = t.DataMemberAt(data_member_index); + std::string name = m.TypeOf().Name(Reflex::FINAL|Reflex::SCOPED|Reflex::QUALIFIED); + char* name_char = (char*)malloc(name.size() + 1); + strcpy(name_char, name.c_str()); + return name_char; +} + +size_t cppyy_data_member_offset(cppyy_typehandle_t handle, int data_member_index) { + Reflex::Type t((Reflex::TypeName*)handle); + Reflex::Member m = t.DataMemberAt(data_member_index); + return m.Offset(); +} + + int cppyy_is_constructor(cppyy_typehandle_t handle, int method_index) { Reflex::Type t((Reflex::TypeName*)handle); Reflex::Member m = t.FunctionMemberAt(method_index); Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_cppyy.py Tue Aug 3 23:22:05 2010 @@ -21,7 +21,7 @@ w_cppyyclass = interp_cppyy.type_byname(space, "example01") w_cppyyclass2 = interp_cppyy.type_byname(space, "example01") assert space.is_w(w_cppyyclass, w_cppyyclass2) - adddouble = w_cppyyclass.function_members["staticAddToDouble"] + adddouble = w_cppyyclass.methods["staticAddToDouble"] func, = adddouble.functions assert isinstance(func.executor, executor.DoubleExecutor) assert func.arg_types == ["double"] Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py Tue Aug 3 23:22:05 2010 @@ -53,19 +53,19 @@ """ raises(TypeError, 'c.set_bool(10)') + # char types through functions c.set_char('c'); assert c.get_char() == 'c' c.set_uchar('e'); assert c.get_uchar() == 'e' - """ - # char types + + # char types through data members c.m_char = 'b'; assert c.get_char() == 'b' - c.m_char = 40; assert c.get_char() == chr(40) + #c.m_char = 40; assert c.get_char() == chr(40) c.set_char('c'); assert c.m_char == 'c' - c.set_char(41); assert c.m_char == chr(41) + #c.set_char(41); assert c.m_char == chr(41) c.m_uchar = 'd'; assert c.get_uchar() == 'd' - c.m_uchar = 42; assert c.get_uchar() == chr(42) + #c.m_uchar = 42; assert c.get_uchar() == chr(42) c.set_uchar('e'); assert c.m_uchar == 'e' - c.set_uchar(43); assert c.m_uchar == chr(43) - """ + #c.set_uchar(43); assert c.m_uchar == chr(43) raises(TypeError, 'c.set_char("string")') # raises(TypeError, 'c.set_uchar(-1)') From wlav at codespeak.net Tue Aug 3 23:49:18 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Tue, 3 Aug 2010 23:49:18 +0200 (CEST) Subject: [pypy-svn] r76461 - in pypy/branch/reflex-support/pypy/module/cppyy: . test Message-ID: <20100803214918.9CC6C282BE0@codespeak.net> Author: wlav Date: Tue Aug 3 23:49:16 2010 New Revision: 76461 Modified: pypy/branch/reflex-support/pypy/module/cppyy/converter.py pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py Log: Enable passing int value to C char Modified: pypy/branch/reflex-support/pypy/module/cppyy/converter.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/converter.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/converter.py Tue Aug 3 23:49:16 2010 @@ -1,6 +1,7 @@ from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype from pypy.rlib.rarithmetic import r_singlefloat +from pypy.objspace.std.intobject import W_IntObject from pypy.module.cppyy import helper, capi @@ -36,11 +37,23 @@ return rffi.cast(rffi.VOIDP, x) class CharConverter(TypeConverter): - def convert_argument(self, space, w_obj): - arg = space.str_w(w_obj) - if len(arg) != 1: + def _from_space(self, space, w_value): + # allow int to pass to char and make sure that str is of length 1 + if type(w_value) == W_IntObject: + try: + value = chr(space.c_int_w(w_value)) + except ValueError, e: + raise OperationError(space.w_TypeError, space.wrap(str(e))) + else: + value = space.str_w(w_value) + + if len(value) != 1: raise OperationError(space.w_TypeError, - space.wrap("char expecter, got string of size %d" % len(arg))) + space.wrap("char expecter, got string of size %d" % len(value))) + return value + + def convert_argument(self, space, w_obj): + arg = self._from_space(space, w_obj) x = rffi.str2charp(arg) return rffi.cast(rffi.VOIDP, x) @@ -52,8 +65,7 @@ def to_memory(self, space, w_obj, w_value, offset): obj = space.interpclass_w(space.findattr(w_obj, space.wrap("_cppinstance"))) fieldptr = lltype.direct_ptradd(obj.rawobject, offset) - print w_value - fieldptr[0] = space.str_w(w_value) + fieldptr[0] = self._from_space(space, w_value) class IntConverter(TypeConverter): def convert_argument(self, space, w_obj): Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py Tue Aug 3 23:49:16 2010 @@ -59,17 +59,18 @@ # char types through data members c.m_char = 'b'; assert c.get_char() == 'b' - #c.m_char = 40; assert c.get_char() == chr(40) + c.m_char = 40; assert c.get_char() == chr(40) c.set_char('c'); assert c.m_char == 'c' - #c.set_char(41); assert c.m_char == chr(41) + c.set_char(41); assert c.m_char == chr(41) c.m_uchar = 'd'; assert c.get_uchar() == 'd' - #c.m_uchar = 42; assert c.get_uchar() == chr(42) + c.m_uchar = 42; assert c.get_uchar() == chr(42) c.set_uchar('e'); assert c.m_uchar == 'e' - #c.set_uchar(43); assert c.m_uchar == chr(43) + c.set_uchar(43); assert c.m_uchar == chr(43) raises(TypeError, 'c.set_char("string")') -# raises(TypeError, 'c.set_uchar(-1)') + raises(TypeError, 'c.set_char(500)') raises(TypeError, 'c.set_uchar("string")') +# raises(TypeError, 'c.set_uchar(-1)') """ # integer types From wlav at codespeak.net Wed Aug 4 00:33:33 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Wed, 4 Aug 2010 00:33:33 +0200 (CEST) Subject: [pypy-svn] r76462 - in pypy/branch/reflex-support/pypy/module/cppyy: . test Message-ID: <20100803223333.CD817282BE0@codespeak.net> Author: wlav Date: Wed Aug 4 00:33:26 2010 New Revision: 76462 Modified: pypy/branch/reflex-support/pypy/module/cppyy/converter.py pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py Log: Enable float and double data members Modified: pypy/branch/reflex-support/pypy/module/cppyy/converter.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/converter.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/converter.py Wed Aug 4 00:33:26 2010 @@ -8,12 +8,19 @@ _converters = {} class TypeConverter(object): + def _get_fieldptr(self, space, w_obj, offset): + obj = space.interpclass_w(space.findattr(w_obj, space.wrap("_cppinstance"))) + return lltype.direct_ptradd(obj.rawobject, offset) + def convert_argument(self, space, w_obj): raise NotImplementedError("abstract base class") def from_memory(self, space, w_obj, offset): raise NotImplementedError("abstract base class") + def to_memory(self, space, w_obj, w_value, offset): + raise NotImplementedError("abstract base class") + def free_argument(self, arg): lltype.free(arg, flavor='raw') @@ -26,6 +33,7 @@ raise OperationError(space.w_TypeError, space.wrap('no converter available for type "%s"' % self.name)) + class BoolConverter(TypeConverter): def convert_argument(self, space, w_obj): arg = space.c_int_w(w_obj) @@ -58,13 +66,11 @@ return rffi.cast(rffi.VOIDP, x) def from_memory(self, space, w_obj, offset): - obj = space.interpclass_w(space.findattr(w_obj, space.wrap("_cppinstance"))) - fieldptr = lltype.direct_ptradd(obj.rawobject, offset) + fieldptr = self._get_fieldptr(space, w_obj, offset) return space.wrap(fieldptr[0]) def to_memory(self, space, w_obj, w_value, offset): - obj = space.interpclass_w(space.findattr(w_obj, space.wrap("_cppinstance"))) - fieldptr = lltype.direct_ptradd(obj.rawobject, offset) + fieldptr = self._get_fieldptr(space, w_obj, offset) fieldptr[0] = self._from_space(space, w_value) class IntConverter(TypeConverter): @@ -81,6 +87,16 @@ x[0] = r_singlefloat(arg) return rffi.cast(rffi.VOIDP, x) + def from_memory(self, space, w_obj, offset): + fieldptr = self._get_fieldptr(space, w_obj, offset) + floatptr = rffi.cast(rffi.FLOATP, fieldptr) + return space.wrap(float(floatptr[0])) + + def to_memory(self, space, w_obj, w_value, offset): + fieldptr = self._get_fieldptr(space, w_obj, offset) + floatptr = rffi.cast(rffi.FLOATP, fieldptr) + floatptr[0] = r_singlefloat(space.float_w(w_value)) + class DoubleConverter(TypeConverter): def convert_argument(self, space, w_obj): arg = space.float_w(w_obj) @@ -88,6 +104,17 @@ x[0] = arg return rffi.cast(rffi.VOIDP, x) + def from_memory(self, space, w_obj, offset): + fieldptr = self._get_fieldptr(space, w_obj, offset) + doubleptr = rffi.cast(rffi.DOUBLEP, fieldptr) + return space.wrap(doubleptr[0]) + + def to_memory(self, space, w_obj, w_value, offset): + fieldptr = self._get_fieldptr(space, w_obj, offset) + doubleptr = rffi.cast(rffi.DOUBLEP, fieldptr) + doubleptr[0] = space.float_w(w_value) + + class CStringConverter(TypeConverter): def convert_argument(self, space, w_obj): arg = space.str_w(w_obj) Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py Wed Aug 4 00:33:26 2010 @@ -70,7 +70,7 @@ raises(TypeError, 'c.set_char("string")') raises(TypeError, 'c.set_char(500)') raises(TypeError, 'c.set_uchar("string")') -# raises(TypeError, 'c.set_uchar(-1)') +# TODO: raises(TypeError, 'c.set_uchar(-1)') """ # integer types @@ -84,15 +84,17 @@ assert eval('c.m_%s' % names[i]) == i """ - # float types + # float types through functions c.set_float( 0.123 ); assert round(c.get_float() - 0.123, 5) == 0 c.set_double( 0.456 ); assert round(c.get_double() - 0.456, 8) == 0 - """ + + # float types through data members c.m_float = 0.123; assert round(c.get_float() - 0.123, 5) == 0 c.set_float( 0.234 ); assert round(c.m_float - 0.234, 5) == 0 c.m_double = 0.456; assert round(c.get_double() - 0.456, 8) == 0 c.set_double( 0.567 ); assert round(c.m_double - 0.567, 8) == 0 + """ # arrays; there will be pointer copies, so destroy the current ones c.destroy_arrays() From getxsick at codespeak.net Wed Aug 4 01:49:56 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Wed, 4 Aug 2010 01:49:56 +0200 (CEST) Subject: [pypy-svn] r76464 - in pypy/branch/fast-ctypes/pypy: module/jitffi rlib Message-ID: <20100803234956.D6FF9282BAD@codespeak.net> Author: getxsick Date: Wed Aug 4 01:49:55 2010 New Revision: 76464 Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: use space.fromcache for compiled loops Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Wed Aug 4 01:49:55 2010 @@ -61,6 +61,9 @@ W_Root, str]) ) +class Cache(object): + def __init__(self, space): + self.cache = {} class W_Get(Wrappable): def __init__(self, space, cpu, lib, func, args_type, res_type='v'): @@ -68,19 +71,29 @@ if res_type == 'i': self.rget = rjitffi._Get(cpu, lib, func, args_type, - res_type, self.wrap_int) + res_type, self.wrap_int, cache=True) elif res_type == 'f': self.rget = rjitffi._Get(cpu, lib, func, args_type, - res_type, self.wrap_float) + res_type, self.wrap_float, cache=True) elif res_type == 'v': self.rget = rjitffi._Get(cpu, lib, func, args_type, - res_type, self.wrap_void) + res_type, self.wrap_void, cache=True) else: raise OperationError( space.w_ValueError, space.wrap('Unsupported type of result: %s' % res_type)) + # grab from the cache if possible + arg_classes = ''.join(args_type) + key = (res_type, arg_classes) + cache = space.fromcache(Cache).cache + try: + self.rget.looptoken = cache[key] + except KeyError: + self.rget.gen_looptaken() + cache[key] = self.rget.looptoken + def call_w(self, space, w_args=None): if not space.is_w(w_args, space.w_None): i = 0 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Wed Aug 4 01:49:55 2010 @@ -33,56 +33,51 @@ rffi.free_charp(name_ptr) class _Get(object): - def __init__(self, cpu, lib, func, args_type, res_type='v', push_result=None): + def __init__(self, cpu, lib, func, args_type, res_type='v', + push_result=None, cache=False): assert isinstance(args_type, list) self.args_type = args_type self.res_type = res_type self.push_result = push_result self.cpu = cpu + self.bargs = [] lib = lib.handler - bargs = [] - self._cache = {} try: self.funcaddr = rffi.cast(lltype.Signed, rdynload.dlsym(lib, func)) except KeyError: raise ValueError("Cannot find symbol %s", func) - bargs.append(BoxInt()) + self.bargs.append(BoxInt()) + if not cache: + self.gen_looptaken() + self.setup_stack() - # grab from the cache if possible - arg_classes = ''.join(self.args_type) - key = (self.res_type, arg_classes) - try: - self.looptoken = self._cache[key] - except KeyError: - for arg in self.args_type: - if arg == 'i': - bargs.append(BoxInt()) - elif arg == 'f': - bargs.append(BoxFloat()) - else: - raise ValueError(arg) - - if self.res_type == 'i': - bres = BoxInt() - elif self.res_type == 'f': - bres = BoxFloat() - elif self.res_type == 'v': - bres = NULLBOX + def gen_looptaken(self): + for arg in self.args_type: + if arg == 'i': + self.bargs.append(BoxInt()) + elif arg == 'f': + self.bargs.append(BoxFloat()) else: - raise ValueError(self.res_type) + raise ValueError(arg) - calldescr = self.get_calldescr() - self.looptoken = LoopToken() - bargs = list(bargs) # make sure it's not resized before ResOperation - oplist = [ResOperation(rop.CALL, bargs, bres, descr=calldescr), - ResOperation(rop.FINISH, [bres], None, - descr=BasicFailDescr(0))] - self.cpu.compile_loop(bargs, oplist, self.looptoken) + if self.res_type == 'i': + bres = BoxInt() + elif self.res_type == 'f': + bres = BoxFloat() + elif self.res_type == 'v': + bres = NULLBOX + else: + raise ValueError(self.res_type) - # add to the cache - self._cache[key] = self.looptoken - self.setup_stack() + calldescr = self.get_calldescr() + self.looptoken = LoopToken() + # make sure it's not resized before ResOperation + self.bargs = list(self.bargs) + oplist = [ResOperation(rop.CALL, self.bargs, bres, descr=calldescr), + ResOperation(rop.FINISH, [bres], None, + descr=BasicFailDescr(0))] + self.cpu.compile_loop(self.bargs, oplist, self.looptoken) def get_calldescr(self): if self.res_type == 'i': From hakanardo at codespeak.net Wed Aug 4 11:22:35 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Wed, 4 Aug 2010 11:22:35 +0200 (CEST) Subject: [pypy-svn] r76466 - in pypy/branch/interplevel-array/pypy/module/array: . test Message-ID: <20100804092235.A9A42282BAD@codespeak.net> Author: hakanardo Date: Wed Aug 4 11:22:32 2010 New Revision: 76466 Modified: pypy/branch/interplevel-array/pypy/module/array/__init__.py pypy/branch/interplevel-array/pypy/module/array/interp_array.py pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: pep8 Modified: pypy/branch/interplevel-array/pypy/module/array/__init__.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/__init__.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/__init__.py Wed Aug 4 11:22:32 2010 @@ -6,10 +6,11 @@ print mytype.w_class registerimplementation(mytype.w_class) + class Module(MixedModule): interpleveldefs = { - 'array' : 'interp_array.W_ArrayBase', + 'array': 'interp_array.W_ArrayBase', } appleveldefs = { Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Wed Aug 4 11:22:32 2010 @@ -23,7 +23,7 @@ msg = 'array() argument 1 must be char, not str' raise OperationError(space.w_TypeError, space.wrap(msg)) typecode = typecode[0] - + for tc in unroll_typecodes: if typecode == tc: a = space.allocate_instance(types[tc].w_class, w_cls) @@ -83,7 +83,7 @@ type_typedef = StdTypeDef( 'array', __new__ = interp2app(w_array), - __module__ = 'array', + __module__ = 'array', itemsize = GetSetProperty(descr_itemsize), typecode = GetSetProperty(descr_typecode), ) @@ -118,7 +118,7 @@ self.canoverflow = False def _freeze_(self): - # hint for the annotator: track individual constant instances + # hint for the annotator: track individual constant instances return True types = { @@ -129,7 +129,7 @@ 'h': TypeCode(rffi.SHORT, 'int_w', True, True), 'H': TypeCode(rffi.USHORT, 'int_w', True), 'i': TypeCode(rffi.INT, 'int_w', True, True), - 'I': TypeCode(rffi.UINT, 'int_w', True), + 'I': TypeCode(rffi.UINT, 'int_w', True), 'l': TypeCode(rffi.LONG, 'int_w', True, True), 'L': TypeCode(rffi.ULONG, 'bigint_w'), # Overflow handled by # rbigint.touint() which @@ -222,7 +222,7 @@ new_buffer = lltype.nullptr(mytype.arraytype) if self.buffer: - lltype.free(self.buffer, flavor='raw') + lltype.free(self.buffer, flavor='raw') self.buffer = new_buffer self.len = size @@ -290,7 +290,7 @@ raise OperationError(space.w_TypeError, space.wrap(msg)) else: self.fromsequence(w_iterable) - + def charbuf(self): return rffi.cast(rffi.CCHARP, self.buffer) @@ -362,7 +362,7 @@ def setslice__Array_ANY_ANY_ANY(space, self, w_i, w_j, w_x): space.setitem(self, space.newslice(w_i, w_j, space.w_None), w_x) - + def array_append__Array_ANY(space, self, w_x): x = self.item_w(w_x) self.setlen(self.len + 1) @@ -437,7 +437,7 @@ def delslice__Array_ANY_ANY(space, self, w_i, w_j): return space.delitem(self, space.newslice(w_i, w_j, space.w_None)) - + # Add and mul methods def add__Array_Array(space, self, other): @@ -479,7 +479,7 @@ return self # Convertions - + def array_tolist__Array(space, self): w_l = space.newlist([]) for i in range(self.len): @@ -506,7 +506,7 @@ msg = "arg1 must be open file" raise OperationError(space.w_TypeError, space.wrap(msg)) n = space.int_w(w_n) - + size = self.itemsize * n w_item = space.call_method(w_f, 'read', space.wrap(size)) item = space.str_w(w_item) @@ -537,7 +537,7 @@ # string arguments at multiples of the unicode byte size. # Let's only accept unicode arguments for now. self.fromsequence(w_ustr) - + def array_tounicode__Array(space, self): u = u"" for i in range(self.len): @@ -552,7 +552,7 @@ def array_tounicode__Array(space, self): msg = "tounicode() may only be called on type 'u' arrays" raise OperationError(space.w_ValueError, space.wrap(msg)) - + # Compare methods def cmp__Array_ANY(space, self, other): if isinstance(other, W_ArrayBase): @@ -563,7 +563,7 @@ raise OperationError(space.w_NotImplementedError, space.wrap('')) # Misc methods - + def buffer__Array(space, self): from pypy.interpreter.buffer import StringLikeBuffer w_s = array_tostring__Array(space, self) @@ -611,12 +611,12 @@ else: r = space.repr(array_tolist__Array(space, self)) s = "array('%s', %s)" % (self.typecode, space.str_w(r)) - return space.wrap(s) + return space.wrap(s) init_signature = Signature(['typecode', 'initializer']) init_defaults = [None, None] - - def init__Array(space, self, args): + + def init__Array(space, self, args): args.parse_obj(None, 'array', init_signature, init_defaults) mytype.w_class = W_Array @@ -630,7 +630,7 @@ if n > 0: f.__name__ = new - from pypy.objspace.std.sliceobject import W_SliceObject + from pypy.objspace.std.sliceobject import W_SliceObject from pypy.objspace.std.listobject import W_ListObject from pypy.objspace.std.unicodeobject import W_UnicodeObject register_all(locals(), globals()) Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Wed Aug 4 11:22:32 2010 @@ -52,13 +52,13 @@ assert a == b raises(TypeError, self.array, 'i', a) - a = self.array('i', (1,2,3)) - b = self.array('h', (1,2,3)) + a = self.array('i', (1, 2, 3)) + b = self.array('h', (1, 2, 3)) assert a == b for tc in 'bhilBHILfd': assert self.array(tc).typecode == tc - + def test_value_range(self): values = (-129, 128, -128, 127, 0, 255, -1, 256, -32768, 32767, -32769, 32768, 65535, 65536, @@ -102,12 +102,12 @@ for tc in 'BHIL': a = self.array(tc) - vals=[0, 2 ** a.itemsize - 1] + vals = [0, 2 ** a.itemsize - 1] a.fromlist(vals) assert a.tolist() == vals a = self.array(tc.lower()) - vals=[ -1 * (2 ** a.itemsize) / 2, (2 ** a.itemsize) / 2 - 1] + vals = [-1 * (2 ** a.itemsize) / 2, (2 ** a.itemsize) / 2 - 1] a.fromlist(vals) assert a.tolist() == vals @@ -126,10 +126,14 @@ assert len(a) == len(values) def test_itemsize(self): - for t in 'cbB': assert(self.array(t).itemsize >= 1) - for t in 'uhHiI': assert(self.array(t).itemsize >= 2) - for t in 'lLf': assert(self.array(t).itemsize >= 4) - for t in 'd': assert(self.array(t).itemsize >= 8) + for t in 'cbB': + assert(self.array(t).itemsize >= 1) + for t in 'uhHiI': + assert(self.array(t).itemsize >= 2) + for t in 'lLf': + assert(self.array(t).itemsize >= 4) + for t in 'd': + assert(self.array(t).itemsize >= 8) inttypes = 'bhil' for t in inttypes: @@ -148,7 +152,7 @@ assert a[0] == 1 and a[1] == v and a[2] == 3 raises(OverflowError, a.append, -1) raises(OverflowError, a.append, 2 ** (8 * b)) - + def test_fromstring(self): a = self.array('c') a.fromstring('Hi!') @@ -156,18 +160,18 @@ for t in 'bBhHiIlLfd': a = self.array(t) - a.fromstring('\x00' * a.itemsize*2) + a.fromstring('\x00' * a.itemsize * 2) assert len(a) == 2 and a[0] == 0 and a[1] == 0 if a.itemsize > 1: - raises(ValueError, a.fromstring, '\x00' * (a.itemsize-1)) - raises(ValueError, a.fromstring, '\x00' * (a.itemsize+1)) - raises(ValueError, a.fromstring, '\x00' * (2*a.itemsize-1)) - raises(ValueError, a.fromstring, '\x00' * (2*a.itemsize+1)) - b = self.array(t, '\x00' * a.itemsize*2) - assert len(b) == 2 and b[0] == 0 and b[1] == 0 + raises(ValueError, a.fromstring, '\x00' * (a.itemsize - 1)) + raises(ValueError, a.fromstring, '\x00' * (a.itemsize + 1)) + raises(ValueError, a.fromstring, '\x00' * (2 * a.itemsize - 1)) + raises(ValueError, a.fromstring, '\x00' * (2 * a.itemsize + 1)) + b = self.array(t, '\x00' * a.itemsize * 2) + assert len(b) == 2 and b[0] == 0 and b[1] == 0 def test_fromfile(self): - + ## class myfile(object): ## def __init__(self, c, s): ## self.c = c @@ -175,30 +179,29 @@ ## def read(self,n): ## return self.c*min(n,self.s) def myfile(c, s): - f=open('/tmp/deleteme', 'w') - f.write(c*s) + f = open('/tmp/deleteme', 'w') + f.write(c * s) f.close() return open('/tmp/deleteme', 'r') - f=open('/dev/zero','r') + f = open('/dev/zero', 'r') for t in 'bBhHiIlLfd': a = self.array(t) - a.fromfile(f,2) - assert len(a)==2 and a[0]==0 and a[1]==0 + a.fromfile(f, 2) + assert len(a) == 2 and a[0] == 0 and a[1] == 0 a = self.array('b') - a.fromfile(myfile('\x01', 20),2) - assert len(a)==2 and a[0]==1 and a[1]==1 + a.fromfile(myfile('\x01', 20), 2) + assert len(a) == 2 and a[0] == 1 and a[1] == 1 a = self.array('h') - a.fromfile(myfile('\x01', 20),2) - assert len(a)==2 and a[0]==257 and a[1]==257 + a.fromfile(myfile('\x01', 20), 2) + assert len(a) == 2 and a[0] == 257 and a[1] == 257 - for i in (0,1): + for i in (0, 1): a = self.array('h') - raises(EOFError, a.fromfile, myfile('\x01', 2+i),2) - assert len(a)==1 and a[0]==257 - + raises(EOFError, a.fromfile, myfile('\x01', 2 + i), 2) + assert len(a) == 1 and a[0] == 257 def test_fromlist(self): a = self.array('b') @@ -219,7 +222,7 @@ raises(OverflowError, a.extend, (1, 2, 400)) assert len(a) == 2 and a[0] == 1 and a[1] == 2 - raises(TypeError, a.extend, self.array('i',(7,8))) + raises(TypeError, a.extend, self.array('i', (7, 8))) assert len(a) == 2 and a[0] == 1 and a[1] == 2 def gen(): @@ -240,88 +243,90 @@ raises(ValueError, self.array('i').fromunicode, unicode('hi')) a = self.array('u') a.fromunicode(unicode('hi')) - assert len(a) == 2 and a[0] == 'h' and a[1]=='i' + assert len(a) == 2 and a[0] == 'h' and a[1] == 'i' b = self.array('u', unicode('hi')) - assert len(b) == 2 and b[0] == 'h' and b[1]=='i' - + assert len(b) == 2 and b[0] == 'h' and b[1] == 'i' + def test_sequence(self): - a=self.array('i', [1,2,3,4]) - assert len(a)==4 + a = self.array('i', [1, 2, 3, 4]) + assert len(a) == 4 assert a[0] == 1 and a[1] == 2 and a[2] == 3 and a[3] == 4 assert a[-4] == 1 and a[-3] == 2 and a[-2] == 3 and a[-1] == 4 - a[-2]=5 + a[-2] = 5 assert a[0] == 1 and a[1] == 2 and a[2] == 5 and a[3] == 4 - for i in (4, -5): raises(IndexError, a.__getitem__, i) + for i in (4, -5): + raises(IndexError, a.__getitem__, i) b = a[0:2] assert len(b) == 2 and b[0] == 1 and b[1] == 2 - b[0]=6 + b[0] = 6 assert len(b) == 2 and b[0] == 6 and b[1] == 2 assert a[0] == 1 and a[1] == 2 and a[2] == 5 and a[3] == 4 assert a.itemsize == b.itemsize b = a[0:100] - assert len(b)==4 + assert len(b) == 4 assert b[0] == 1 and b[1] == 2 and b[2] == 5 and b[3] == 4 l1 = [2 * i + 1 for i in range(10)] a1 = self.array('i', l1) for start in range(10): for stop in range(start, 10): - for step in range(1,10): + for step in range(1, 10): l2 = l1[start:stop:step] a2 = a1[start:stop:step] assert len(l2) == len(a2) - for i in range(len(l2)): assert l2[i] == a2[i] + for i in range(len(l2)): + assert l2[i] == a2[i] - a=self.array('i', [1,2,3,4]) - a[1:3]=self.array('i', [5,6]) - assert len(a)==4 + a = self.array('i', [1, 2, 3, 4]) + a[1:3] = self.array('i', [5, 6]) + assert len(a) == 4 assert a[0] == 1 and a[1] == 5 and a[2] == 6 and a[3] == 4 - a[0:-1:2]=self.array('i', [7,8]) + a[0:-1:2] = self.array('i', [7, 8]) assert a[0] == 7 and a[1] == 5 and a[2] == 8 and a[3] == 4 try: - a[1:2:4]=self.array('i', [5,6,7]) + a[1:2:4] = self.array('i', [5, 6, 7]) assert False except ValueError: pass try: - a[1:3]=self.array('I', [5,6]) + a[1:3] = self.array('I', [5, 6]) assert False except TypeError: pass try: - a[1:3]=[5,6] + a[1:3] = [5, 6] assert False except TypeError: pass - a=self.array('i', [1,2,3]) - assert a.__getslice__(1,2) == a[1:2] - a.__setslice__(1,2, self.array('i',(7,))) + a = self.array('i', [1, 2, 3]) + assert a.__getslice__(1, 2) == a[1:2] + a.__setslice__(1, 2, self.array('i', (7,))) assert a[0] == 1 and a[1] == 7 and a[2] == 3 def test_resizingslice(self): - a=self.array('i', [1, 2, 3]) + a = self.array('i', [1, 2, 3]) a[1:2] = self.array('i', [7, 8, 9]) assert repr(a) == "array('i', [1, 7, 8, 9, 3])" a[1:2] = self.array('i', [10]) assert repr(a) == "array('i', [1, 10, 8, 9, 3])" a[1:2] = self.array('i') assert repr(a) == "array('i', [1, 8, 9, 3])" - + a[1:3] = self.array('i', [11, 12, 13]) assert repr(a) == "array('i', [1, 11, 12, 13, 3])" a[1:3] = self.array('i', [14]) assert repr(a) == "array('i', [1, 14, 13, 3])" a[1:3] = self.array('i') assert repr(a) == "array('i', [1, 3])" - + a[1:1] = self.array('i', [15, 16, 17]) assert repr(a) == "array('i', [1, 15, 16, 17, 3])" a[1:1] = self.array('i', [18]) @@ -339,18 +344,19 @@ assert repr(a[2:-1:-1]) == "array('i')" assert repr(a[-1:0:-1]) == "array('i', [20, 21])" - for a in range(-4,5): - for b in range(-4,5): + for a in range(-4, 5): + for b in range(-4, 5): for c in [-4, -3, -2, -1, 1, 2, 3, 4]: lst = [1, 2, 3] - arr=self.array('i', lst) - assert repr(arr[a:b:c]) == repr(self.array('i', lst[a:b:c])) - for vals in ([4,5], [6], []): + arr = self.array('i', lst) + assert repr(arr[a:b:c]) == \ + repr(self.array('i', lst[a:b:c])) + for vals in ([4, 5], [6], []): try: ok = False - lst[a:b:c]=vals + lst[a:b:c] = vals ok = True - arr[a:b:c]=self.array('i', vals) + arr[a:b:c] = self.array('i', vals) assert repr(arr) == repr(self.array('i', lst)) except ValueError: assert not ok @@ -360,47 +366,45 @@ if sys.version_info >= (2, 6): py.test.skip('arrays can handle more slice opps than lists in 2.6') - for a in range(-4,5): - for b in range(-4,5): + for a in range(-4, 5): + for b in range(-4, 5): for c in [-4, -3, -2, -1, 1, 2, 3, 4]: lst = [1, 2, 3] - arr=self.array('i', lst) - for vals in ([4,5], [6], []): + arr = self.array('i', lst) + for vals in ([4, 5], [6], []): try: - lst[a:b:c]=vals + lst[a:b:c] = vals except ValueError: raises(ValueError, "arr[a:b:c]=self.array('i', vals)") - - def test_toxxx(self): - a = self.array('i', [1,2,3]) - l = a.tolist() - assert type(l) is list and len(l)==3 + a = self.array('i', [1, 2, 3]) + l = a.tolist() + assert type(l) is list and len(l) == 3 assert a[0] == 1 and a[1] == 2 and a[2] == 3 b = self.array('i', a.tostring()) assert len(b) == 3 and b[0] == 1 and b[1] == 2 and b[2] == 3 assert self.array('c', ('h', 'i')).tostring() == 'hi' - a = self.array('i',[0,0,0]) - assert a.tostring() == '\x00'*3*a.itemsize + a = self.array('i', [0, 0, 0]) + assert a.tostring() == '\x00' * 3 * a.itemsize s = self.array('i', [1, 2, 3]).tostring() assert '\x00' in s assert '\x01' in s assert '\x02' in s assert '\x03' in s - a=self.array('i', s) - assert a[0]==1 and a[1]==2 and a[2]==3 + a = self.array('i', s) + assert a[0] == 1 and a[1] == 2 and a[2] == 3 - unpack=self.struct.unpack + unpack = self.struct.unpack values = (-129, 128, -128, 127, 0, 255, -1, 256, -32760, 32760) s = self.array('i', values).tostring() - fmt = 'i'*len(values) - a=unpack(fmt, s) - assert a==values + fmt = 'i' * len(values) + a = unpack(fmt, s) + assert a == values for tcodes, values in (('bhilfd', (-128, 127, 0, 1, 7, -10)), ('BHILfd', (127, 0, 1, 7, 255, 169)), @@ -408,35 +412,34 @@ for tc in tcodes: values += ((2 ** self.array(tc).itemsize) / 2 - 1, ) s = self.array(tc, values).tostring() - a=unpack(tc*len(values), s) - assert a==values + a = unpack(tc * len(values), s) + assert a == values - f=open('/tmp/deleteme', 'w') + f = open('/tmp/deleteme', 'w') self.array('c', ('h', 'i')).tofile(f) f.close() assert open('/tmp/deleteme', 'r').readline() == 'hi' - a=self.array('c') - a.fromfile(open('/tmp/deleteme', 'r'),2) + a = self.array('c') + a.fromfile(open('/tmp/deleteme', 'r'), 2) assert repr(a) == "array('c', 'hi')" - - + raises(ValueError, self.array('i').tounicode) - assert self.array('u', unicode('hello')).tounicode() == unicode('hello') + assert self.array('u', unicode('hello')).tounicode() == \ + unicode('hello') def test_buffer(self): a = self.array('h', 'Hi') buf = buffer(a) assert buf[1] == 'i' raises(TypeError, buf.__setitem__, 1, 'o') - - + def test_list_methods(self): assert repr(self.array('i')) == "array('i')" assert repr(self.array('i', [1, 2, 3])) == "array('i', [1, 2, 3])" assert repr(self.array('h')) == "array('h')" - - a=self.array('i', [1, 2, 3, 1, 2, 1]) + + a = self.array('i', [1, 2, 3, 1, 2, 1]) assert a.count(1) == 3 assert a.count(2) == 2 assert a.index(3) == 2 @@ -446,10 +449,10 @@ a.reverse() assert repr(a) == "array('i', [1, 2, 1, 3, 2, 1])" - b=self.array('i', [1, 2, 3, 1, 2]) + b = self.array('i', [1, 2, 3, 1, 2]) b.reverse() assert repr(b) == "array('i', [2, 1, 3, 2, 1])" - + a.remove(3) assert repr(a) == "array('i', [1, 2, 1, 2, 1])" a.remove(1) @@ -464,19 +467,19 @@ a.pop(-2) assert repr(a) == "array('i', [2])" - a.insert(1,7) + a.insert(1, 7) assert repr(a) == "array('i', [2, 7])" - a.insert(0,8) - a.insert(-1,9) + a.insert(0, 8) + a.insert(-1, 9) assert repr(a) == "array('i', [8, 2, 9, 7])" - a.insert(100,10) + a.insert(100, 10) assert repr(a) == "array('i', [8, 2, 9, 7, 10])" - a.insert(-100,20) + a.insert(-100, 20) assert repr(a) == "array('i', [20, 8, 2, 9, 7, 10])" def test_compare(self): - for v1,v2,tt in (([1, 2, 3], [1, 3, 2], 'bhilBHIL'), + for v1, v2, tt in (([1, 2, 3], [1, 3, 2], 'bhilBHIL'), ('abc', 'acb', 'c'), (unicode('abc'), unicode('acb'), 'u')): for t in tt: @@ -525,11 +528,11 @@ assert cmp(a, c) < 0 assert cmp(b, a) == 0 assert cmp(c, a) > 0 - + def test_reduce(self): import pickle a = self.array('i', [1, 2, 3]) - s = pickle.dumps(a,1) + s = pickle.dumps(a, 1) b = pickle.loads(s) assert a == b @@ -552,14 +555,14 @@ assert repr(b) == "array('i', [1, 2, 3])" for tc in 'bhilBHIL': - a=self.array(tc, [1, 2, 3]) + a = self.array(tc, [1, 2, 3]) a.byteswap() - assert len(a)==3 - assert a[0] == 1 * (256 ** (a.itemsize-1)) - assert a[1] == 2 * (256 ** (a.itemsize-1)) - assert a[2] == 3 * (256 ** (a.itemsize-1)) + assert len(a) == 3 + assert a[0] == 1 * (256 ** (a.itemsize - 1)) + assert a[1] == 2 * (256 ** (a.itemsize - 1)) + assert a[2] == 3 * (256 ** (a.itemsize - 1)) a.byteswap() - assert len(a)==3 + assert len(a) == 3 assert a[0] == 1 assert a[1] == 2 assert a[2] == 3 @@ -611,6 +614,7 @@ class addable(object): def __add__(self, other): return "add" + def __radd__(self, other): return "radd" @@ -626,7 +630,7 @@ del a[1:3] assert repr(a) == "array('i', [1, 4, 5])" - a.__delslice__(0,2) + a.__delslice__(0, 2) assert repr(a) == "array('i', [5])" def test_iter(self): @@ -641,12 +645,16 @@ class lier(object): def __init__(self, n): self.n = n + def __len__(self): return 3 + def next(self): self.n -= 1 - if self.n < 0: raise StopIteration + if self.n < 0: + raise StopIteration return self.n + def __iter__(self): return self @@ -669,17 +677,18 @@ print type(self.array('b')) assert len(self.array('b')) == 0 - a=self.array('i') + a = self.array('i') a.append(7) assert len(a) == 1 - array=self.array + array = self.array + class adder(array): def __getitem__(self, i): print 25 return array.__getitem__(self, i) + 1 - a=adder('i', (1,2,3)) + a = adder('i', (1, 2, 3)) print type(a) assert len(a) == 3 assert a[0] == 2 @@ -688,10 +697,13 @@ class mya(self.array): def fromlist(self, lst): self.append(7) + def fromstring(self, lst): self.append('8') + def fromunicode(self, lst): self.append(u'9') + def extend(self, lst): self.append(10) @@ -720,15 +732,17 @@ class mya(self.array): def tolist(self): return 'list' + def tostring(self): return 'str' + def tounicode(self): return 'unicode' assert mya('i', [1, 2, 3]).tolist() == 'list' assert mya('c', 'hi').tostring() == 'str' assert mya('u', u'hi').tounicode() == 'unicode' - + assert repr(mya('c', 'hi')) == "array('c', 'hi')" assert repr(mya('u', u'hi')) == "array('u', u'hi')" assert repr(mya('i', [1, 2, 3])) == "array('i', [1, 2, 3])" @@ -760,12 +774,10 @@ return _rawffi """) - def test_buffer_info(self): a = self.array('c', 'Hi!') - bi=a.buffer_info() - assert bi[0] != 0 + bi = a.buffer_info() + assert bi[0] != 0 assert bi[1] == 3 data = self.rffi.charp2string(bi[0]) assert data[0:3] == 'Hi!' - From fijal at codespeak.net Wed Aug 4 14:12:48 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 4 Aug 2010 14:12:48 +0200 (CEST) Subject: [pypy-svn] r76467 - pypy/trunk/pypy/objspace/std Message-ID: <20100804121248.B896D282BF6@codespeak.net> Author: fijal Date: Wed Aug 4 14:12:45 2010 New Revision: 76467 Modified: pypy/trunk/pypy/objspace/std/callmethod.py Log: Another attempt to fix stackless (?) Modified: pypy/trunk/pypy/objspace/std/callmethod.py ============================================================================== --- pypy/trunk/pypy/objspace/std/callmethod.py (original) +++ pypy/trunk/pypy/objspace/std/callmethod.py Wed Aug 4 14:12:45 2010 @@ -68,8 +68,6 @@ if not n_kwargs: try: w_result = f.space.call_valuestack(w_callable, n, f) - rstack.resume_point("CALL_METHOD", f, w_self, n_args, - returns=w_result) finally: f.dropvalues(n_args + 2) else: @@ -90,10 +88,9 @@ try: w_result = f.space.call_args(w_callable, args) - rstack.resume_point("CALL_METHOD", f, w_self, n_args, - returns=w_result) finally: f.dropvalues(1 + (w_self is None)) + rstack.resume_point("CALL_METHOD", f, returns=w_result) f.pushvalue(w_result) From hakanardo at codespeak.net Wed Aug 4 16:47:37 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Wed, 4 Aug 2010 16:47:37 +0200 (CEST) Subject: [pypy-svn] r76468 - in pypy/branch/interplevel-array/pypy: jit/backend/test jit/tl module/pypyjit/test translator/platform Message-ID: <20100804144737.D7A40282BAD@codespeak.net> Author: hakanardo Date: Wed Aug 4 16:47:35 2010 New Revision: 76468 Modified: pypy/branch/interplevel-array/pypy/jit/backend/test/runner_test.py pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py pypy/branch/interplevel-array/pypy/module/pypyjit/test/test_pypy_c.py pypy/branch/interplevel-array/pypy/translator/platform/posix.py Log: 16-bit arrayitem test Modified: pypy/branch/interplevel-array/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/test/runner_test.py Wed Aug 4 16:47:35 2010 @@ -638,6 +638,21 @@ assert r.value == 1 def test_array_basic(self): + a_box, A = self.alloc_array_of(rffi.SHORT, 342) + arraydescr = self.cpu.arraydescrof(A) + assert not arraydescr.is_array_of_pointers() + # + r = self.execute_operation(rop.ARRAYLEN_GC, [a_box], + 'int', descr=arraydescr) + assert r.value == 342 + r = self.execute_operation(rop.SETARRAYITEM_GC, [a_box, BoxInt(310), + BoxInt(744)], + 'void', descr=arraydescr) + assert r is None + r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(310)], + 'int', descr=arraydescr) + assert r.value == 744 + a_box, A = self.alloc_array_of(lltype.Signed, 342) arraydescr = self.cpu.arraydescrof(A) assert not arraydescr.is_array_of_pointers() Modified: pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py (original) +++ pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py Wed Aug 4 16:47:35 2010 @@ -47,7 +47,7 @@ i+=1 return sa - img=array('i',(1,2,3,4)) + img=array('h',(1,2,3,4)) print f(img) except Exception, e: print "Exception: ", type(e) Modified: pypy/branch/interplevel-array/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/interplevel-array/pypy/module/pypyjit/test/test_pypy_c.py Wed Aug 4 16:47:35 2010 @@ -778,6 +778,34 @@ return sa '''%(e1, e2), n, ([], res)) + def test_array_sum(self): + #for tc in 'bhilBHILfd': + for tc in 'd': + self.run_source(''' + from array import array + + def main(): + img = array("%s", range(127) * 5) * 484 + l, i = 0, 0 + while i < 640 * 480: + l += img[i] + i += 1 + return l + ''' % tc, 38, ([], 19352859)) + + def test_array_sum_char(self): + self.run_source(''' + from array import array + + def main(): + img = array("c", ' ') * 640 * 480 + l, i = 0, 0 + while i < 640 * 480: + l += ord(img[i]) + i += 1 + return l + ''', 60, ([], 9830400)) + class AppTestJIT(PyPyCJITTests): def setup_class(cls): if not option.runappdirect: Modified: pypy/branch/interplevel-array/pypy/translator/platform/posix.py ============================================================================== --- pypy/branch/interplevel-array/pypy/translator/platform/posix.py (original) +++ pypy/branch/interplevel-array/pypy/translator/platform/posix.py Wed Aug 4 16:47:35 2010 @@ -14,7 +14,10 @@ def __init__(self, cc=None): if cc is None: - cc = 'gcc' + try: + cc = os.environ['CC'] + except KeyError: + cc = 'gcc' self.cc = cc def _libs(self, libraries): From hakanardo at codespeak.net Wed Aug 4 16:53:51 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Wed, 4 Aug 2010 16:53:51 +0200 (CEST) Subject: [pypy-svn] r76469 - in pypy/branch/interplevel-array: . lib-python lib-python/modified-2.5.2 lib-python/modified-2.5.2/ctypes lib_pypy lib_pypy/_ctypes lib_pypy/pypy_test pypy/config pypy/interpreter pypy/interpreter/astcompiler pypy/interpreter/astcompiler/tools pypy/interpreter/test pypy/jit/backend pypy/jit/backend/llgraph pypy/jit/backend/llsupport pypy/jit/backend/llsupport/test pypy/jit/backend/test pypy/jit/backend/x86 pypy/jit/backend/x86/test pypy/jit/backend/x86/tool pypy/jit/metainterp pypy/jit/metainterp/test pypy/jit/tool pypy/module/__builtin__ pypy/module/_ast pypy/module/_ast/test pypy/module/_file pypy/module/_file/test pypy/module/_rawffi/test pypy/module/array pypy/module/cpyext pypy/module/posix pypy/module/posix/test pypy/module/signal pypy/module/test_lib_pypy/ctypes_tests pypy/objspace/std pypy/objspace/std/test pypy/rlib pypy/rlib/test pypy/rpython pypy/rpython/lltypesystem pypy/rpython/module pypy/rpython/module/test pypy/rpython/test pypy/translator/c/test pypy/translator/goal pypy/translator/goal/test2 pypy/translator/platform Message-ID: <20100804145351.E9577282BAD@codespeak.net> Author: hakanardo Date: Wed Aug 4 16:53:45 2010 New Revision: 76469 Added: pypy/branch/interplevel-array/pypy/jit/backend/x86/arch.py - copied unchanged from r76467, pypy/trunk/pypy/jit/backend/x86/arch.py pypy/branch/interplevel-array/pypy/jit/backend/x86/regloc.py - copied unchanged from r76467, pypy/trunk/pypy/jit/backend/x86/regloc.py pypy/branch/interplevel-array/pypy/jit/backend/x86/rx86.py - copied unchanged from r76467, pypy/trunk/pypy/jit/backend/x86/rx86.py pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_regloc.py - copied unchanged from r76467, pypy/trunk/pypy/jit/backend/x86/test/test_regloc.py pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_rx86.py - copied unchanged from r76467, pypy/trunk/pypy/jit/backend/x86/test/test_rx86.py pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py - copied unchanged from r76467, pypy/trunk/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py - copied unchanged from r76467, pypy/trunk/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py pypy/branch/interplevel-array/pypy/jit/backend/x86/tool/instruction_encoding.sh - copied unchanged from r76467, pypy/trunk/pypy/jit/backend/x86/tool/instruction_encoding.sh pypy/branch/interplevel-array/pypy/jit/tool/gen-trace-mode-keywords.py - copied unchanged from r76467, pypy/trunk/pypy/jit/tool/gen-trace-mode-keywords.py pypy/branch/interplevel-array/pypy/jit/tool/pypytrace-mode.el - copied unchanged from r76467, pypy/trunk/pypy/jit/tool/pypytrace-mode.el pypy/branch/interplevel-array/pypy/rlib/test/test_rposix.py - copied unchanged from r76467, pypy/trunk/pypy/rlib/test/test_rposix.py pypy/branch/interplevel-array/pypy/rpython/module/ll_win32file.py - copied unchanged from r76467, pypy/trunk/pypy/rpython/module/ll_win32file.py Removed: pypy/branch/interplevel-array/pypy/jit/backend/x86/ri386.py pypy/branch/interplevel-array/pypy/jit/backend/x86/ri386setup.py pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_ri386.py pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py Modified: pypy/branch/interplevel-array/ (props changed) pypy/branch/interplevel-array/lib-python/ (props changed) pypy/branch/interplevel-array/lib-python/conftest.py pypy/branch/interplevel-array/lib-python/modified-2.5.2/ctypes/__init__.py pypy/branch/interplevel-array/lib-python/modified-2.5.2/site.py pypy/branch/interplevel-array/lib_pypy/ (props changed) pypy/branch/interplevel-array/lib_pypy/_ctypes/__init__.py pypy/branch/interplevel-array/lib_pypy/_ctypes/array.py pypy/branch/interplevel-array/lib_pypy/_ctypes/builtin.py pypy/branch/interplevel-array/lib_pypy/_ctypes/primitive.py pypy/branch/interplevel-array/lib_pypy/datetime.py pypy/branch/interplevel-array/lib_pypy/dbm.py (props changed) pypy/branch/interplevel-array/lib_pypy/pypy_test/test_ctypes_support.py pypy/branch/interplevel-array/lib_pypy/pypy_test/test_datetime.py pypy/branch/interplevel-array/lib_pypy/pypy_test/test_functools.py (props changed) pypy/branch/interplevel-array/pypy/config/translationoption.py pypy/branch/interplevel-array/pypy/interpreter/astcompiler/assemble.py pypy/branch/interplevel-array/pypy/interpreter/astcompiler/ast.py pypy/branch/interplevel-array/pypy/interpreter/astcompiler/codegen.py pypy/branch/interplevel-array/pypy/interpreter/astcompiler/consts.py pypy/branch/interplevel-array/pypy/interpreter/astcompiler/symtable.py pypy/branch/interplevel-array/pypy/interpreter/astcompiler/tools/asdl_py.py pypy/branch/interplevel-array/pypy/interpreter/baseobjspace.py pypy/branch/interplevel-array/pypy/interpreter/error.py pypy/branch/interplevel-array/pypy/interpreter/gateway.py pypy/branch/interplevel-array/pypy/interpreter/pyopcode.py pypy/branch/interplevel-array/pypy/interpreter/test/test_compiler.py pypy/branch/interplevel-array/pypy/interpreter/test/test_gateway.py pypy/branch/interplevel-array/pypy/jit/backend/detect_cpu.py pypy/branch/interplevel-array/pypy/jit/backend/llgraph/llimpl.py pypy/branch/interplevel-array/pypy/jit/backend/llsupport/descr.py pypy/branch/interplevel-array/pypy/jit/backend/llsupport/gc.py pypy/branch/interplevel-array/pypy/jit/backend/llsupport/regalloc.py pypy/branch/interplevel-array/pypy/jit/backend/llsupport/test/test_descr.py pypy/branch/interplevel-array/pypy/jit/backend/llsupport/test/test_regalloc.py pypy/branch/interplevel-array/pypy/jit/backend/test/runner_test.py pypy/branch/interplevel-array/pypy/jit/backend/x86/assembler.py pypy/branch/interplevel-array/pypy/jit/backend/x86/codebuf.py pypy/branch/interplevel-array/pypy/jit/backend/x86/jump.py pypy/branch/interplevel-array/pypy/jit/backend/x86/regalloc.py pypy/branch/interplevel-array/pypy/jit/backend/x86/runner.py pypy/branch/interplevel-array/pypy/jit/backend/x86/test/conftest.py pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_assembler.py pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_basic.py pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_gc_integration.py pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_jump.py pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_recompilation.py pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_regalloc.py pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_regalloc2.py pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_runner.py pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_symbolic_x86.py pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_zll_random.py pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_zrpy_gc.py pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_ztranslation.py pypy/branch/interplevel-array/pypy/jit/backend/x86/tool/viewcode.py pypy/branch/interplevel-array/pypy/jit/metainterp/optimizeopt.py pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_loop.py pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_loop_spec.py pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_recursive.py pypy/branch/interplevel-array/pypy/module/__builtin__/compiling.py pypy/branch/interplevel-array/pypy/module/_ast/__init__.py pypy/branch/interplevel-array/pypy/module/_ast/test/test_ast.py pypy/branch/interplevel-array/pypy/module/_file/interp_file.py pypy/branch/interplevel-array/pypy/module/_file/test/test_file.py pypy/branch/interplevel-array/pypy/module/_rawffi/test/test__rawffi.py pypy/branch/interplevel-array/pypy/module/array/interp_array_try1.py (props changed) pypy/branch/interplevel-array/pypy/module/cpyext/methodobject.py pypy/branch/interplevel-array/pypy/module/posix/interp_posix.py pypy/branch/interplevel-array/pypy/module/posix/test/test_posix2.py pypy/branch/interplevel-array/pypy/module/signal/interp_signal.py pypy/branch/interplevel-array/pypy/module/test_lib_pypy/ctypes_tests/ (props changed) pypy/branch/interplevel-array/pypy/objspace/std/callmethod.py pypy/branch/interplevel-array/pypy/objspace/std/test/test_callmethod.py pypy/branch/interplevel-array/pypy/rlib/objectmodel.py pypy/branch/interplevel-array/pypy/rlib/rarithmetic.py pypy/branch/interplevel-array/pypy/rlib/rdynload.py pypy/branch/interplevel-array/pypy/rlib/rposix.py pypy/branch/interplevel-array/pypy/rlib/rwin32.py pypy/branch/interplevel-array/pypy/rlib/streamio.py pypy/branch/interplevel-array/pypy/rlib/test/test_objectmodel.py pypy/branch/interplevel-array/pypy/rpython/extfunc.py pypy/branch/interplevel-array/pypy/rpython/lltypesystem/rffi.py pypy/branch/interplevel-array/pypy/rpython/lltypesystem/rstr.py pypy/branch/interplevel-array/pypy/rpython/module/ll_os.py pypy/branch/interplevel-array/pypy/rpython/module/ll_os_stat.py pypy/branch/interplevel-array/pypy/rpython/module/test/test_ll_os_stat.py pypy/branch/interplevel-array/pypy/rpython/rstr.py pypy/branch/interplevel-array/pypy/rpython/test/test_extfunc.py pypy/branch/interplevel-array/pypy/rpython/test/test_rstr.py pypy/branch/interplevel-array/pypy/translator/c/test/test_typed.py pypy/branch/interplevel-array/pypy/translator/goal/app_main.py pypy/branch/interplevel-array/pypy/translator/goal/test2/test_app_main.py pypy/branch/interplevel-array/pypy/translator/platform/__init__.py pypy/branch/interplevel-array/pypy/translator/platform/darwin.py pypy/branch/interplevel-array/pypy/translator/platform/posix.py pypy/branch/interplevel-array/pypy/translator/platform/windows.py Log: svn merge -r76023:76467 svn+ssh://hakanardo at codespeak.net/svn/pypy/trunk Modified: pypy/branch/interplevel-array/lib-python/conftest.py ============================================================================== --- pypy/branch/interplevel-array/lib-python/conftest.py (original) +++ pypy/branch/interplevel-array/lib-python/conftest.py Wed Aug 4 16:53:45 2010 @@ -464,11 +464,7 @@ RegrTest('test_coding.py'), RegrTest('test_complex_args.py'), RegrTest('test_contextlib.py', usemodules="thread"), - # we skip test ctypes, since we adapted it massively in order - # to test what we want to support. There are real failures, - # but it's about missing features that we don't want to support - # now - RegrTest('test_ctypes.py', skip="we have a replacement"), + RegrTest('test_ctypes.py', usemodules="_rawffi"), RegrTest('test_defaultdict.py'), RegrTest('test_email_renamed.py'), RegrTest('test_exception_variations.py'), Modified: pypy/branch/interplevel-array/lib-python/modified-2.5.2/ctypes/__init__.py ============================================================================== --- pypy/branch/interplevel-array/lib-python/modified-2.5.2/ctypes/__init__.py (original) +++ pypy/branch/interplevel-array/lib-python/modified-2.5.2/ctypes/__init__.py Wed Aug 4 16:53:45 2010 @@ -471,7 +471,7 @@ # functions -from _ctypes import _memmove_addr, _memset_addr, _string_at_addr, _cast_addr +from _ctypes import _memmove_addr, _memset_addr, _cast_addr ## void *memmove(void *, const void *, size_t); memmove = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)(_memmove_addr) @@ -490,24 +490,34 @@ def cast(obj, typ): return _cast(obj, obj, typ) -_string_at = CFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr) +try: + from _ctypes import _string_at_addr +except ImportError: + from _ctypes import _string_at +else: + _string_at = CFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr) + def string_at(ptr, size=-1): """string_at(addr[, size]) -> string Return the string at addr.""" return _string_at(ptr, size) +def wstring_at(ptr, size=-1): + """wstring_at(addr[, size]) -> string + + Return the string at addr.""" + return _wstring_at(ptr, size) + try: from _ctypes import _wstring_at_addr except ImportError: - pass + try: + from _ctypes import _wstring_at + except ImportError: + del wstring_at else: _wstring_at = CFUNCTYPE(py_object, c_void_p, c_int)(_wstring_at_addr) - def wstring_at(ptr, size=-1): - """wstring_at(addr[, size]) -> string - - Return the string at addr.""" - return _wstring_at(ptr, size) if _os.name in ("nt", "ce"): # COM stuff Modified: pypy/branch/interplevel-array/lib-python/modified-2.5.2/site.py ============================================================================== --- pypy/branch/interplevel-array/lib-python/modified-2.5.2/site.py (original) +++ pypy/branch/interplevel-array/lib-python/modified-2.5.2/site.py Wed Aug 4 16:53:45 2010 @@ -175,7 +175,7 @@ def addsitepackages(known_paths): """Add site-packages to sys.path, in a PyPy-specific way.""" - if hasattr(sys, 'pypy_version_info'): + if hasattr(sys, 'pypy_version_info') and hasattr(sys, 'prefix'): from distutils.sysconfig import get_python_lib sitedir = get_python_lib(standard_lib=False) if os.path.isdir(sitedir): Modified: pypy/branch/interplevel-array/lib_pypy/_ctypes/__init__.py ============================================================================== --- pypy/branch/interplevel-array/lib_pypy/_ctypes/__init__.py (original) +++ pypy/branch/interplevel-array/lib_pypy/_ctypes/__init__.py Wed Aug 4 16:53:45 2010 @@ -7,8 +7,8 @@ from _ctypes.dll import dlopen from _ctypes.structure import Structure from _ctypes.array import Array -from _ctypes.builtin import _memmove_addr, _string_at_addr, _memset_addr,\ - set_conversion_mode, _wstring_at_addr +from _ctypes.builtin import _memmove_addr, _string_at, _memset_addr,\ + set_conversion_mode, _wstring_at from _ctypes.union import Union import os as _os Modified: pypy/branch/interplevel-array/lib_pypy/_ctypes/array.py ============================================================================== --- pypy/branch/interplevel-array/lib_pypy/_ctypes/array.py (original) +++ pypy/branch/interplevel-array/lib_pypy/_ctypes/array.py Wed Aug 4 16:53:45 2010 @@ -4,7 +4,6 @@ from _ctypes.basics import _CData, cdata_from_address, _CDataMeta, sizeof from _ctypes.basics import keepalive_key, store_reference, ensure_objects from _ctypes.basics import CArgObject -from _ctypes.builtin import _string_at_addr, _wstring_at_addr def _create_unicode(buffer, maxlength): res = [] Modified: pypy/branch/interplevel-array/lib_pypy/_ctypes/builtin.py ============================================================================== --- pypy/branch/interplevel-array/lib_pypy/_ctypes/builtin.py (original) +++ pypy/branch/interplevel-array/lib_pypy/_ctypes/builtin.py Wed Aug 4 16:53:45 2010 @@ -8,10 +8,10 @@ _memmove_addr = _rawffi.get_libc().getaddressindll('memmove') _memset_addr = _rawffi.get_libc().getaddressindll('memset') -def _string_at_addr(addr, lgt): +def _string_at(addr, lgt): # address here can be almost anything import ctypes - arg = ctypes.c_char_p._CData_value(addr) + arg = ctypes.c_void_p._CData_value(addr) return _rawffi.charp2rawstring(arg, lgt) def set_conversion_mode(encoding, errors): @@ -20,9 +20,9 @@ ConvMode.encoding = encoding return old_cm -def _wstring_at_addr(addr, lgt): +def _wstring_at(addr, lgt): import ctypes - arg = ctypes.c_wchar_p._CData_value(addr) + arg = ctypes.c_void_p._CData_value(addr) # XXX purely applevel if lgt == -1: lgt = sys.maxint Modified: pypy/branch/interplevel-array/lib_pypy/_ctypes/primitive.py ============================================================================== --- pypy/branch/interplevel-array/lib_pypy/_ctypes/primitive.py (original) +++ pypy/branch/interplevel-array/lib_pypy/_ctypes/primitive.py Wed Aug 4 16:53:45 2010 @@ -86,6 +86,8 @@ return value if isinstance(value, _Pointer): return cls.from_address(value._buffer.buffer) + if isinstance(value, (int, long)): + return cls(value) FROM_PARAM_BY_TYPE = { 'z': from_param_char_p, @@ -141,13 +143,13 @@ result.value = property(_getvalue, _setvalue) elif tp == 'Z': # c_wchar_p - from _ctypes import Array, _Pointer, _wstring_at_addr + from _ctypes import Array, _Pointer, _wstring_at def _getvalue(self): addr = self._buffer[0] if addr == 0: return None else: - return _wstring_at_addr(addr, -1) + return _wstring_at(addr, -1) def _setvalue(self, value): if isinstance(value, basestring): @@ -216,14 +218,14 @@ SysAllocStringLen = windll.oleaut32.SysAllocStringLen SysStringLen = windll.oleaut32.SysStringLen SysFreeString = windll.oleaut32.SysFreeString - from _ctypes import _wstring_at_addr + from _ctypes import _wstring_at def _getvalue(self): addr = self._buffer[0] if addr == 0: return None else: size = SysStringLen(addr) - return _wstring_at_addr(addr, size) + return _wstring_at(addr, size) def _setvalue(self, value): if isinstance(value, basestring): @@ -254,18 +256,21 @@ from_address = cdata_from_address def from_param(self, value): + if isinstance(value, self): + return value + from_param_f = FROM_PARAM_BY_TYPE.get(self._type_) if from_param_f: res = from_param_f(self, value) if res is not None: return res - - if isinstance(value, self): - return value - try: - return self(value) - except (TypeError, ValueError): - return super(SimpleType, self).from_param(value) + else: + try: + return self(value) + except (TypeError, ValueError): + pass + + return super(SimpleType, self).from_param(value) def _CData_output(self, resbuffer, base=None, index=-1): output = super(SimpleType, self)._CData_output(resbuffer, base, index) Modified: pypy/branch/interplevel-array/lib_pypy/datetime.py ============================================================================== --- pypy/branch/interplevel-array/lib_pypy/datetime.py (original) +++ pypy/branch/interplevel-array/lib_pypy/datetime.py Wed Aug 4 16:53:45 2010 @@ -1412,7 +1412,7 @@ def utcfromtimestamp(cls, t): "Construct a UTC datetime from a POSIX timestamp (like time.time())." - if 1 - (t % 1.0) < 0.000001: + if 1 - (t % 1.0) < 0.0000005: t = float(int(t)) + 1 if t < 0: t -= 1 Modified: pypy/branch/interplevel-array/lib_pypy/pypy_test/test_ctypes_support.py ============================================================================== --- pypy/branch/interplevel-array/lib_pypy/pypy_test/test_ctypes_support.py (original) +++ pypy/branch/interplevel-array/lib_pypy/pypy_test/test_ctypes_support.py Wed Aug 4 16:53:45 2010 @@ -20,3 +20,14 @@ assert get_errno() != 0 set_errno(0) assert get_errno() == 0 + +def test_argument_conversion_and_checks(): + import ctypes + libc = ctypes.cdll.LoadLibrary("libc.so.6") + libc.strlen.argtypes = ctypes.c_char_p, + libc.strlen.restype = ctypes.c_size_t + assert libc.strlen("eggs") == 4 + + # Should raise ArgumentError, not segfault + py.test.raises(ctypes.ArgumentError, libc.strlen, False) + Modified: pypy/branch/interplevel-array/lib_pypy/pypy_test/test_datetime.py ============================================================================== --- pypy/branch/interplevel-array/lib_pypy/pypy_test/test_datetime.py (original) +++ pypy/branch/interplevel-array/lib_pypy/pypy_test/test_datetime.py Wed Aug 4 16:53:45 2010 @@ -15,4 +15,18 @@ expected = datetime.datetime(*(time.strptime(string, format)[0:6])) got = datetime.datetime.strptime(string, format) assert expected == got + +def test_datetime_rounding(): + b = 0.0000001 + a = 0.9999994 + + assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 + assert datetime.datetime.utcfromtimestamp(a).second == 0 + a += b + assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 + assert datetime.datetime.utcfromtimestamp(a).second == 0 + a += b + assert datetime.datetime.utcfromtimestamp(a).microsecond == 0 + assert datetime.datetime.utcfromtimestamp(a).second == 1 + Modified: pypy/branch/interplevel-array/pypy/config/translationoption.py ============================================================================== --- pypy/branch/interplevel-array/pypy/config/translationoption.py (original) +++ pypy/branch/interplevel-array/pypy/config/translationoption.py Wed Aug 4 16:53:45 2010 @@ -342,6 +342,10 @@ 'jit': 'hybrid extraopts jit', } +# For now, 64-bit JIT requires boehm +if IS_64_BITS: + OPT_TABLE['jit'] = OPT_TABLE['jit'].replace('hybrid', 'boehm') + def set_opt_level(config, level): """Apply optimization suggestions on the 'config'. The optimizations depend on the selected level and possibly on the backend. Modified: pypy/branch/interplevel-array/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/interplevel-array/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/interplevel-array/pypy/interpreter/astcompiler/assemble.py Wed Aug 4 16:53:45 2010 @@ -566,22 +566,22 @@ return (oparg % 256) + 2 * (oparg / 256) def _compute_CALL_FUNCTION(arg): - return _num_args(arg) + return -_num_args(arg) def _compute_CALL_FUNCTION_VAR(arg): - return _num_args(arg) - 1 + return -_num_args(arg) - 1 def _compute_CALL_FUNCTION_KW(arg): - return _num_args(arg) - 1 + return -_num_args(arg) - 1 def _compute_CALL_FUNCTION_VAR_KW(arg): - return _num_args(arg) - 2 + return -_num_args(arg) - 2 def _compute_CALL_LIKELY_BUILTIN(arg): return -(arg & 0xFF) + 1 def _compute_CALL_METHOD(arg): - return -arg - 1 + return -_num_args(arg) - 1 _stack_effect_computers = {} Modified: pypy/branch/interplevel-array/pypy/interpreter/astcompiler/ast.py ============================================================================== --- pypy/branch/interplevel-array/pypy/interpreter/astcompiler/ast.py (original) +++ pypy/branch/interplevel-array/pypy/interpreter/astcompiler/ast.py Wed Aug 4 16:53:45 2010 @@ -230,6 +230,7 @@ visitor.visit_FunctionDef(self) def mutate_over(self, visitor): + self.args = self.args.mutate_over(visitor) if self.body: visitor._mutate_sequence(self.body) if self.decorators: @@ -784,6 +785,8 @@ def mutate_over(self, visitor): if self.body: visitor._mutate_sequence(self.body) + if self.handlers: + visitor._mutate_sequence(self.handlers) if self.orelse: visitor._mutate_sequence(self.orelse) return visitor.visit_TryExcept(self) @@ -927,6 +930,8 @@ visitor.visit_Import(self) def mutate_over(self, visitor): + if self.names: + visitor._mutate_sequence(self.names) return visitor.visit_Import(self) def sync_app_attrs(self, space): @@ -965,6 +970,8 @@ visitor.visit_ImportFrom(self) def mutate_over(self, visitor): + if self.names: + visitor._mutate_sequence(self.names) return visitor.visit_ImportFrom(self) def sync_app_attrs(self, space): @@ -1282,6 +1289,7 @@ visitor.visit_Lambda(self) def mutate_over(self, visitor): + self.args = self.args.mutate_over(visitor) self.body = self.body.mutate_over(visitor) return visitor.visit_Lambda(self) @@ -1398,6 +1406,8 @@ def mutate_over(self, visitor): self.elt = self.elt.mutate_over(visitor) + if self.generators: + visitor._mutate_sequence(self.generators) return visitor.visit_ListComp(self) def sync_app_attrs(self, space): @@ -1437,6 +1447,8 @@ def mutate_over(self, visitor): self.elt = self.elt.mutate_over(visitor) + if self.generators: + visitor._mutate_sequence(self.generators) return visitor.visit_GeneratorExp(self) def sync_app_attrs(self, space): @@ -1562,6 +1574,8 @@ self.func = self.func.mutate_over(visitor) if self.args: visitor._mutate_sequence(self.args) + if self.keywords: + visitor._mutate_sequence(self.keywords) if self.starargs: self.starargs = self.starargs.mutate_over(visitor) if self.kwargs: @@ -2293,6 +2307,13 @@ self.w_ifs = None self.initialization_state = 7 + def mutate_over(self, visitor): + self.target = self.target.mutate_over(visitor) + self.iter = self.iter.mutate_over(visitor) + if self.ifs: + visitor._mutate_sequence(self.ifs) + return visitor.visit_comprehension(self) + def walkabout(self, visitor): visitor.visit_comprehension(self) @@ -2327,6 +2348,15 @@ self.col_offset = col_offset self.initialization_state = 31 + def mutate_over(self, visitor): + if self.type: + self.type = self.type.mutate_over(visitor) + if self.name: + self.name = self.name.mutate_over(visitor) + if self.body: + visitor._mutate_sequence(self.body) + return visitor.visit_excepthandler(self) + def walkabout(self, visitor): visitor.visit_excepthandler(self) @@ -2366,6 +2396,13 @@ self.w_defaults = None self.initialization_state = 15 + def mutate_over(self, visitor): + if self.args: + visitor._mutate_sequence(self.args) + if self.defaults: + visitor._mutate_sequence(self.defaults) + return visitor.visit_arguments(self) + def walkabout(self, visitor): visitor.visit_arguments(self) @@ -2407,6 +2444,10 @@ self.value = value self.initialization_state = 3 + def mutate_over(self, visitor): + self.value = self.value.mutate_over(visitor) + return visitor.visit_keyword(self) + def walkabout(self, visitor): visitor.visit_keyword(self) @@ -2426,6 +2467,9 @@ self.asname = asname self.initialization_state = 3 + def mutate_over(self, visitor): + return visitor.visit_alias(self) + def walkabout(self, visitor): visitor.visit_alias(self) Modified: pypy/branch/interplevel-array/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/interplevel-array/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/interplevel-array/pypy/interpreter/astcompiler/codegen.py Wed Aug 4 16:53:45 2010 @@ -258,9 +258,11 @@ # Load decorators first, but apply them after the function is created. if func.decorators: self.visit_sequence(func.decorators) - if func.args.defaults: - self.visit_sequence(func.args.defaults) - num_defaults = len(func.args.defaults) + args = func.args + assert isinstance(args, ast.arguments) + if args.defaults: + self.visit_sequence(args.defaults) + num_defaults = len(args.defaults) else: num_defaults = 0 code = self.sub_scope(FunctionCodeGenerator, func.name, func, @@ -274,9 +276,11 @@ def visit_Lambda(self, lam): self.update_position(lam.lineno) - if lam.args.defaults: - self.visit_sequence(lam.args.defaults) - default_count = len(lam.args.defaults) + args = lam.args + assert isinstance(args, ast.arguments) + if args.defaults: + self.visit_sequence(args.defaults) + default_count = len(args.defaults) else: default_count = 0 code = self.sub_scope(LambdaCodeGenerator, "", lam, lam.lineno) @@ -956,9 +960,12 @@ elif call_type == 3: op = ops.CALL_FUNCTION_VAR_KW self.emit_op_arg(op, arg) + + def _call_has_no_star_args(self, call): + return not call.starargs and not call.kwargs def _call_has_simple_args(self, call): - return not call.starargs and not call.kwargs and not call.keywords + return self._call_has_no_star_args(call) and not call.keywords def _optimize_builtin_call(self, call): if not self.space.config.objspace.opcodes.CALL_LIKELY_BUILTIN or \ @@ -984,7 +991,7 @@ def _optimize_method_call(self, call): if not self.space.config.objspace.opcodes.CALL_METHOD or \ - not self._call_has_simple_args(call) or \ + not self._call_has_no_star_args(call) or \ not isinstance(call.func, ast.Attribute): return False attr_lookup = call.func @@ -996,7 +1003,12 @@ arg_count = len(call.args) else: arg_count = 0 - self.emit_op_arg(ops.CALL_METHOD, arg_count) + if call.keywords: + self.visit_sequence(call.keywords) + kwarg_count = len(call.keywords) + else: + kwarg_count = 0 + self.emit_op_arg(ops.CALL_METHOD, (kwarg_count << 8) | arg_count) return True def _listcomp_generator(self, list_name, gens, gen_index, elt): @@ -1275,9 +1287,11 @@ else: self.add_const(self.space.w_None) start = 0 - if func.args.args: - self._handle_nested_args(func.args.args) - self.argcount = len(func.args.args) + args = func.args + assert isinstance(args, ast.arguments) + if args.args: + self._handle_nested_args(args.args) + self.argcount = len(args.args) for i in range(start, len(func.body)): func.body[i].walkabout(self) @@ -1286,9 +1300,11 @@ def _compile(self, lam): assert isinstance(lam, ast.Lambda) - if lam.args.args: - self._handle_nested_args(lam.args.args) - self.argcount = len(lam.args.args) + args = lam.args + assert isinstance(args, ast.arguments) + if args.args: + self._handle_nested_args(args.args) + self.argcount = len(args.args) # Prevent a string from being the first constant and thus a docstring. self.add_const(self.space.w_None) lam.body.walkabout(self) Modified: pypy/branch/interplevel-array/pypy/interpreter/astcompiler/consts.py ============================================================================== --- pypy/branch/interplevel-array/pypy/interpreter/astcompiler/consts.py (original) +++ pypy/branch/interplevel-array/pypy/interpreter/astcompiler/consts.py Wed Aug 4 16:53:45 2010 @@ -18,4 +18,4 @@ PyCF_SOURCE_IS_UTF8 = 0x0100 PyCF_DONT_IMPLY_DEDENT = 0x0200 -PyCF_AST_ONLY = 0x0400 +PyCF_ONLY_AST = 0x0400 Modified: pypy/branch/interplevel-array/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/interplevel-array/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/interplevel-array/pypy/interpreter/astcompiler/symtable.py Wed Aug 4 16:53:45 2010 @@ -353,8 +353,10 @@ def visit_FunctionDef(self, func): self.note_symbol(func.name, SYM_ASSIGNED) # Function defaults and decorators happen in the outer scope. - if func.args.defaults: - self.visit_sequence(func.args.defaults) + args = func.args + assert isinstance(args, ast.arguments) + if args.defaults: + self.visit_sequence(args.defaults) if func.decorators: self.visit_sequence(func.decorators) new_scope = FunctionScope(func.name, func.lineno, func.col_offset) @@ -420,8 +422,10 @@ self.note_symbol(name, SYM_GLOBAL) def visit_Lambda(self, lamb): - if lamb.args.defaults: - self.visit_sequence(lamb.args.defaults) + args = lamb.args + assert isinstance(args, ast.arguments) + if args.defaults: + self.visit_sequence(args.defaults) new_scope = FunctionScope("lambda", lamb.lineno, lamb.col_offset) self.push_scope(new_scope, lamb) lamb.args.walkabout(self) Modified: pypy/branch/interplevel-array/pypy/interpreter/astcompiler/tools/asdl_py.py ============================================================================== --- pypy/branch/interplevel-array/pypy/interpreter/astcompiler/tools/asdl_py.py (original) +++ pypy/branch/interplevel-array/pypy/interpreter/astcompiler/tools/asdl_py.py Wed Aug 4 16:53:45 2010 @@ -100,6 +100,7 @@ self.emit("") self.make_constructor(product.fields, product) self.emit("") + self.make_mutate_over(product, name) self.emit("def walkabout(self, visitor):", 1) self.emit("visitor.visit_%s(self)" % (name,), 2) self.emit("") @@ -183,6 +184,26 @@ have_everything = self.data.required_masks[node] | \ self.data.optional_masks[node] self.emit("self.initialization_state = %i" % (have_everything,), 2) + + def make_mutate_over(self, cons, name): + self.emit("def mutate_over(self, visitor):", 1) + for field in cons.fields: + if (field.type.value not in asdl.builtin_types and + field.type.value not in self.data.simple_types): + if field.opt or field.seq: + level = 3 + self.emit("if self.%s:" % (field.name,), 2) + else: + level = 2 + if field.seq: + sub = (field.name,) + self.emit("visitor._mutate_sequence(self.%s)" % sub, level) + else: + sub = (field.name, field.name) + self.emit("self.%s = self.%s.mutate_over(visitor)" % sub, + level) + self.emit("return visitor.visit_%s(self)" % (name,), 2) + self.emit("") def visitConstructor(self, cons, base, extra_attributes): self.emit("class %s(%s):" % (cons.name, base)) @@ -199,24 +220,7 @@ self.emit("def walkabout(self, visitor):", 1) self.emit("visitor.visit_%s(self)" % (cons.name,), 2) self.emit("") - self.emit("def mutate_over(self, visitor):", 1) - for field in cons.fields: - if field.type.value not in asdl.builtin_types and \ - field.type.value not in self.data.prod_simple: - if field.opt or field.seq: - level = 3 - self.emit("if self.%s:" % (field.name,), 2) - else: - level = 2 - if field.seq: - sub = (field.name,) - self.emit("visitor._mutate_sequence(self.%s)" % sub, level) - else: - sub = (field.name, field.name) - self.emit("self.%s = self.%s.mutate_over(visitor)" % sub, - level) - self.emit("return visitor.visit_%s(self)" % (cons.name,), 2) - self.emit("") + self.make_mutate_over(cons, cons.name) self.make_var_syncer(cons.fields + self.data.cons_attributes[cons], cons, cons.name) Modified: pypy/branch/interplevel-array/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/interplevel-array/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/interplevel-array/pypy/interpreter/baseobjspace.py Wed Aug 4 16:53:45 2010 @@ -1102,17 +1102,6 @@ self.wrap('argument must be a unicode')) return self.unicode_w(w_obj) - def path_w(self, w_obj): - """ Like str_w, but if the object is unicode, encode it using - filesystemencoding - """ - filesystemencoding = self.sys.filesystemencoding - if (filesystemencoding and - self.is_true(self.isinstance(w_obj, self.w_unicode))): - w_obj = self.call_method(w_obj, "encode", - self.wrap(filesystemencoding)) - return self.str_w(w_obj) - def bool_w(self, w_obj): # Unwraps a bool, also accepting an int for compatibility. # This is here mostly just for gateway.int_unwrapping_space_method(). Modified: pypy/branch/interplevel-array/pypy/interpreter/error.py ============================================================================== --- pypy/branch/interplevel-array/pypy/interpreter/error.py (original) +++ pypy/branch/interplevel-array/pypy/interpreter/error.py Wed Aug 4 16:53:45 2010 @@ -344,7 +344,7 @@ else: _WINDOWS = True - def wrap_windowserror(space, e, filename=None): + def wrap_windowserror(space, e, w_filename=None): from pypy.rlib import rwin32 winerror = e.winerror @@ -353,19 +353,19 @@ except ValueError: msg = 'Windows Error %d' % winerror exc = space.w_WindowsError - if filename is not None: + if w_filename is not None: w_error = space.call_function(exc, space.wrap(winerror), - space.wrap(msg), space.wrap(filename)) + space.wrap(msg), w_filename) else: w_error = space.call_function(exc, space.wrap(winerror), space.wrap(msg)) return OperationError(exc, w_error) -def wrap_oserror(space, e, filename=None, exception_name='w_OSError'): +def wrap_oserror2(space, e, w_filename=None, exception_name='w_OSError'): assert isinstance(e, OSError) if _WINDOWS and isinstance(e, WindowsError): - return wrap_windowserror(space, e, filename) + return wrap_windowserror(space, e, w_filename) errno = e.errno try: @@ -373,10 +373,21 @@ except ValueError: msg = 'error %d' % errno exc = getattr(space, exception_name) - if filename is not None: + if w_filename is not None: w_error = space.call_function(exc, space.wrap(errno), - space.wrap(msg), space.wrap(filename)) + space.wrap(msg), w_filename) else: - w_error = space.call_function(exc, space.wrap(errno), space.wrap(msg)) + w_error = space.call_function(exc, space.wrap(errno), + space.wrap(msg)) return OperationError(exc, w_error) +wrap_oserror2._annspecialcase_ = 'specialize:arg(3)' + +def wrap_oserror(space, e, filename=None, exception_name='w_OSError'): + if filename is not None: + return wrap_oserror2(space, e, space.wrap(filename), + exception_name=exception_name) + else: + return wrap_oserror2(space, e, None, + exception_name=exception_name) wrap_oserror._annspecialcase_ = 'specialize:arg(3)' + Modified: pypy/branch/interplevel-array/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/interplevel-array/pypy/interpreter/gateway.py (original) +++ pypy/branch/interplevel-array/pypy/interpreter/gateway.py Wed Aug 4 16:53:45 2010 @@ -137,9 +137,6 @@ def visit_c_nonnegint(self, el, app_sig): self.checked_space_method(el, app_sig) - def visit_path(self, el, app_sig): - self.checked_space_method(el, app_sig) - def visit__Wrappable(self, el, app_sig): name = el.__name__ argname = self.orig_arg() @@ -241,9 +238,6 @@ def visit_bufferstr(self, typ): self.run_args.append("space.bufferstr_w(%s)" % (self.scopenext(),)) - def visit_path(self, typ): - self.run_args.append("space.path_w(%s)" % (self.scopenext(),)) - def visit_nonnegint(self, typ): self.run_args.append("space.nonnegint_w(%s)" % (self.scopenext(),)) @@ -371,9 +365,6 @@ def visit_bufferstr(self, typ): self.unwrap.append("space.bufferstr_w(%s)" % (self.nextarg(),)) - def visit_path(self, typ): - self.unwrap.append("space.path_w(%s)" % (self.nextarg(),)) - def visit_nonnegint(self, typ): self.unwrap.append("space.nonnegint_w(%s)" % (self.nextarg(),)) Modified: pypy/branch/interplevel-array/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/interplevel-array/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/interplevel-array/pypy/interpreter/pyopcode.py Wed Aug 4 16:53:45 2010 @@ -211,10 +211,6 @@ next_instr = block.handle(self, unroller) return next_instr # now inside a 'finally' block - if opcode == self.opcodedesc.YIELD_VALUE.index: - #self.last_instr = intmask(next_instr - 1) XXX clean up! - raise Yield - if opcode == self.opcodedesc.END_FINALLY.index: unroller = self.end_finally() if isinstance(unroller, SuspendedUnroller): @@ -239,7 +235,7 @@ if not opdesc.is_enabled(space): continue if opdesc.methodname in ( - 'EXTENDED_ARG', 'RETURN_VALUE', 'YIELD_VALUE', + 'EXTENDED_ARG', 'RETURN_VALUE', 'END_FINALLY', 'JUMP_ABSOLUTE'): continue # opcodes implemented above @@ -814,6 +810,9 @@ self.space.str_w(w_name)) self.pushvalue(w_obj) + def YIELD_VALUE(self, oparg, next_instr): + raise Yield + def jump_absolute(self, jumpto, next_instr, ec): return jumpto Modified: pypy/branch/interplevel-array/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/branch/interplevel-array/pypy/interpreter/test/test_compiler.py (original) +++ pypy/branch/interplevel-array/pypy/interpreter/test/test_compiler.py Wed Aug 4 16:53:45 2010 @@ -4,6 +4,7 @@ from pypy.interpreter.pycode import PyCode from pypy.interpreter.error import OperationError from pypy.interpreter.argument import Arguments +from pypy.conftest import gettestobjspace class BaseTestCompiler: def setup_method(self, method): @@ -838,6 +839,47 @@ sys.stdout = save_stdout output = s.getvalue() assert "STOP_CODE" not in output + + def test_optimize_list_comp(self): + source = """def _f(a): + return [x for x in a if None] + """ + exec source + code = _f.func_code + + import StringIO, sys, dis + s = StringIO.StringIO() + out = sys.stdout + sys.stdout = s + try: + dis.dis(code) + finally: + sys.stdout = out + output = s.getvalue() + assert "LOAD_GLOBAL" not in output + +class AppTestCallMethod(object): + def setup_class(cls): + cls.space = gettestobjspace(**{'objspace.opcodes.CALL_METHOD': True}) + + def test_call_method_kwargs(self): + source = """def _f(a): + return a.f(a=a) + """ + exec source + code = _f.func_code + + import StringIO, sys, dis + s = StringIO.StringIO() + out = sys.stdout + sys.stdout = s + try: + dis.dis(code) + finally: + sys.stdout = out + output = s.getvalue() + assert "CALL_METHOD" in output + class AppTestExceptions: def test_indentation_error(self): Modified: pypy/branch/interplevel-array/pypy/interpreter/test/test_gateway.py ============================================================================== --- pypy/branch/interplevel-array/pypy/interpreter/test/test_gateway.py (original) +++ pypy/branch/interplevel-array/pypy/interpreter/test/test_gateway.py Wed Aug 4 16:53:45 2010 @@ -454,16 +454,6 @@ assert len(l) == 1 assert space.eq_w(l[0], w("foo")) - def test_interp2app_unwrap_spec_path(self, monkeypatch): - space = self.space - def g(space, p): - return p - - app_g = gateway.interp2app(g, unwrap_spec=[gateway.ObjSpace, 'path']) - w_app_g = space.wrap(app_g) - monkeypatch.setattr(space.sys, "filesystemencoding", "utf-8") - w_res = space.call_function(w_app_g, space.wrap(u"?")) - def test_interp2app_classmethod(self): space = self.space w = space.wrap Modified: pypy/branch/interplevel-array/pypy/jit/backend/detect_cpu.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/detect_cpu.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/detect_cpu.py Wed Aug 4 16:53:45 2010 @@ -56,6 +56,8 @@ return "pypy.jit.backend.x86.runner", "CPU" elif backend_name == 'x86-without-sse2': return "pypy.jit.backend.x86.runner", "CPU386_NO_SSE2" + elif backend_name == 'x86_64': + return "pypy.jit.backend.x86.runner", "CPU_X86_64" elif backend_name == 'cli': return "pypy.jit.backend.cli.runner", "CliCPU" elif backend_name == 'llvm': Modified: pypy/branch/interplevel-array/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/llgraph/llimpl.py Wed Aug 4 16:53:45 2010 @@ -421,11 +421,9 @@ global _last_exception assert _last_exception is None, "exception left behind" verbose = True - operations = self.loop.operations - opindex = 0 + self.opindex = 0 while True: - self.opindex = opindex - op = operations[opindex] + op = self.loop.operations[self.opindex] args = [self.getenv(v) for v in op.args] if not op.is_final(): try: @@ -439,8 +437,8 @@ args = [self.getenv(v) for v in op.fail_args if v] assert len(op.jump_target.inputargs) == len(args) self.env = dict(zip(op.jump_target.inputargs, args)) - operations = op.jump_target.operations - opindex = 0 + self.loop = op.jump_target + self.opindex = 0 continue else: self._populate_fail_args(op) @@ -465,14 +463,13 @@ raise Exception("op.result.concretetype is %r" % (RESTYPE,)) self.env[op.result] = x - opindex += 1 + self.opindex += 1 continue if op.opnum == rop.JUMP: assert len(op.jump_target.inputargs) == len(args) self.env = dict(zip(op.jump_target.inputargs, args)) self.loop = op.jump_target - operations = self.loop.operations - opindex = 0 + self.opindex = 0 _stats.exec_jumps += 1 elif op.opnum == rop.FINISH: if self.verbose: Modified: pypy/branch/interplevel-array/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/llsupport/descr.py Wed Aug 4 16:53:45 2010 @@ -67,9 +67,11 @@ class BaseFieldDescr(AbstractDescr): offset = 0 # help translation + name = '' _clsname = '' - def __init__(self, offset): + def __init__(self, name, offset): + self.name = name self.offset = offset def sort_key(self): @@ -88,7 +90,7 @@ return self._is_float_field def repr_of_descr(self): - return '<%s %s>' % (self._clsname, self.offset) + return '<%s %s %s>' % (self._clsname, self.name, self.offset) class NonGcPtrFieldDescr(BaseFieldDescr): @@ -113,7 +115,8 @@ offset, _ = symbolic.get_field_token(STRUCT, fieldname, gccache.translate_support_code) FIELDTYPE = getattr(STRUCT, fieldname) - fielddescr = getFieldDescrClass(FIELDTYPE)(offset) + name = '%s.%s' % (STRUCT._name, fieldname) + fielddescr = getFieldDescrClass(FIELDTYPE)(name, offset) cachedict = cache.setdefault(STRUCT, {}) cachedict[fieldname] = fielddescr return fielddescr Modified: pypy/branch/interplevel-array/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/llsupport/gc.py Wed Aug 4 16:53:45 2010 @@ -351,7 +351,7 @@ gcrootmap = cls() self.gcrootmap = gcrootmap self.gcrefs = GcRefList() - self.single_gcref_descr = GcPtrFieldDescr(0) + self.single_gcref_descr = GcPtrFieldDescr('', 0) # make a TransformerLayoutBuilder and save it on the translator # where it can be fished and reused by the FrameworkGCTransformer Modified: pypy/branch/interplevel-array/pypy/jit/backend/llsupport/regalloc.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/llsupport/regalloc.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/llsupport/regalloc.py Wed Aug 4 16:53:45 2010 @@ -22,18 +22,19 @@ def get(self, box): return self.frame_bindings.get(box, None) - def loc(self, box, size): + def loc(self, box): res = self.get(box) if res is not None: return res - newloc = self.frame_pos(self.frame_depth, size) + newloc = self.frame_pos(self.frame_depth, box.type) self.frame_bindings[box] = newloc - self.frame_depth += size + # Objects returned by frame_pos must support frame_size() + self.frame_depth += newloc.frame_size() return newloc # abstract methods that need to be overwritten for specific assemblers @staticmethod - def frame_pos(loc, size): + def frame_pos(loc, type): raise NotImplementedError("Purely abstract") class RegisterManager(object): @@ -43,7 +44,6 @@ all_regs = [] no_lower_byte_regs = [] save_around_call_regs = [] - reg_width = 1 # in terms of stack space eaten def __init__(self, longevity, frame_manager=None, assembler=None): self.free_regs = self.all_regs[:] @@ -148,7 +148,7 @@ loc = self.reg_bindings[v_to_spill] del self.reg_bindings[v_to_spill] if self.frame_manager.get(v_to_spill) is None: - newloc = self.frame_manager.loc(v_to_spill, self.reg_width) + newloc = self.frame_manager.loc(v_to_spill) self.assembler.regalloc_mov(loc, newloc) return loc @@ -204,7 +204,7 @@ try: return self.reg_bindings[box] except KeyError: - return self.frame_manager.loc(box, self.reg_width) + return self.frame_manager.loc(box) def return_constant(self, v, forbidden_vars=[], selected_reg=None, imm_fine=True): @@ -260,7 +260,7 @@ self.reg_bindings[v] = loc self.assembler.regalloc_mov(prev_loc, loc) else: - loc = self.frame_manager.loc(v, self.reg_width) + loc = self.frame_manager.loc(v) self.assembler.regalloc_mov(prev_loc, loc) def force_result_in_reg(self, result_v, v, forbidden_vars=[]): @@ -280,7 +280,7 @@ self.free_regs = [reg for reg in self.free_regs if reg is not loc] return loc if v not in self.reg_bindings: - prev_loc = self.frame_manager.loc(v, self.reg_width) + prev_loc = self.frame_manager.loc(v) loc = self.force_allocate_reg(v, forbidden_vars) self.assembler.regalloc_mov(prev_loc, loc) assert v in self.reg_bindings @@ -300,7 +300,7 @@ def _sync_var(self, v): if not self.frame_manager.get(v): reg = self.reg_bindings[v] - to = self.frame_manager.loc(v, self.reg_width) + to = self.frame_manager.loc(v) self.assembler.regalloc_mov(reg, to) # otherwise it's clean Modified: pypy/branch/interplevel-array/pypy/jit/backend/llsupport/test/test_descr.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/llsupport/test/test_descr.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/llsupport/test/test_descr.py Wed Aug 4 16:53:45 2010 @@ -53,6 +53,10 @@ assert descr_y.__class__ is GcPtrFieldDescr assert descr_z.__class__ is NonGcPtrFieldDescr assert descr_f.__class__ is clsf + assert descr_x.name == 'S.x' + assert descr_y.name == 'S.y' + assert descr_z.name == 'S.z' + assert descr_f.name == 'S.f' if not tsc: assert descr_x.offset < descr_y.offset < descr_z.offset assert descr_x.sort_key() < descr_y.sort_key() < descr_z.sort_key() @@ -228,11 +232,11 @@ # descr2 = get_field_descr(c0, S, 'y') o, _ = symbolic.get_field_token(S, 'y', False) - assert descr2.repr_of_descr() == '' % o + assert descr2.repr_of_descr() == '' % o # descr2i = get_field_descr(c0, S, 'x') o, _ = symbolic.get_field_token(S, 'x', False) - assert descr2i.repr_of_descr() == '' % o + assert descr2i.repr_of_descr() == '' % o # descr3 = get_array_descr(c0, lltype.GcArray(lltype.Ptr(S))) assert descr3.repr_of_descr() == '' Modified: pypy/branch/interplevel-array/pypy/jit/backend/llsupport/test/test_regalloc.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/llsupport/test/test_regalloc.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/llsupport/test/test_regalloc.py Wed Aug 4 16:53:45 2010 @@ -1,5 +1,5 @@ -from pypy.jit.metainterp.history import BoxInt, ConstInt, BoxFloat +from pypy.jit.metainterp.history import BoxInt, ConstInt, BoxFloat, INT, FLOAT from pypy.jit.backend.llsupport.regalloc import FrameManager from pypy.jit.backend.llsupport.regalloc import RegisterManager as BaseRegMan @@ -26,9 +26,20 @@ def convert_to_imm(self, v): return v +class FakeFramePos(object): + def __init__(self, pos, box_type): + self.pos = pos + self.box_type = box_type + + def frame_size(self): + if self.box_type == FLOAT: + return 2 + else: + return 1 + class TFrameManager(FrameManager): - def frame_pos(self, i, size): - return i + def frame_pos(self, i, box_type): + return FakeFramePos(i, box_type) class MockAsm(object): def __init__(self): @@ -146,8 +157,8 @@ rm.next_instruction() # allocate a stack position b0, b1, b2, b3, b4 = boxes - sp = fm.loc(b0, 1) - assert sp == 0 + sp = fm.loc(b0) + assert sp.pos == 0 loc = rm.make_sure_var_in_reg(b0) assert isinstance(loc, FakeReg) rm._check_invariants() @@ -207,13 +218,13 @@ asm = MockAsm() rm = RegisterManager(longevity, frame_manager=fm, assembler=asm) rm.next_instruction() - fm.loc(b0, 1) + fm.loc(b0) rm.force_result_in_reg(b1, b0) rm._check_invariants() loc = rm.loc(b1) assert isinstance(loc, FakeReg) loc = rm.loc(b0) - assert isinstance(loc, int) + assert isinstance(loc, FakeFramePos) assert len(asm.moves) == 1 def test_return_constant(self): @@ -304,7 +315,7 @@ def test_different_frame_width(self): class XRegisterManager(RegisterManager): - reg_width = 2 + pass fm = TFrameManager() b0 = BoxInt() Modified: pypy/branch/interplevel-array/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/test/runner_test.py Wed Aug 4 16:53:45 2010 @@ -461,6 +461,25 @@ [funcbox] + args, 'float', descr=calldescr) assert abs(res.value - 4.6) < 0.0001 + + def test_call_many_arguments(self): + # Test calling a function with a large number of arguments (more than + # 6, which will force passing some arguments on the stack on 64-bit) + + def func(*args): + assert len(args) == 16 + # Try to sum up args in a way that would probably detect a + # transposed argument + return sum(arg * (2**i) for i, arg in enumerate(args)) + + FUNC = self.FuncType([lltype.Signed]*16, lltype.Signed) + FPTR = self.Ptr(FUNC) + calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + func_ptr = llhelper(FPTR, func) + args = range(16) + funcbox = self.get_funcbox(self.cpu, func_ptr) + res = self.execute_operation(rop.CALL, [funcbox] + map(BoxInt, args), 'int', descr=calldescr) + assert res.value == func(*args) def test_call_stack_alignment(self): # test stack alignment issues, notably for Mac OS/X. @@ -993,6 +1012,8 @@ else: assert 0 operations.append(ResOperation(opnum, boxargs, boxres)) + # Unique-ify inputargs + inputargs = list(set(inputargs)) faildescr = BasicFailDescr(1) operations.append(ResOperation(rop.FINISH, [], None, descr=faildescr)) @@ -1065,9 +1086,11 @@ descr=BasicFailDescr(5))] operations[1].fail_args = [] looptoken = LoopToken() - self.cpu.compile_loop(list(testcase), operations, + # Use "set" to unique-ify inputargs + unique_testcase_list = list(set(testcase)) + self.cpu.compile_loop(unique_testcase_list, operations, looptoken) - for i, box in enumerate(testcase): + for i, box in enumerate(unique_testcase_list): self.cpu.set_future_value_float(i, box.value) fail = self.cpu.execute_token(looptoken) if fail.identifier != 5 - (expected_id^expected): @@ -1710,7 +1733,7 @@ def test_assembler_call(self): called = [] def assembler_helper(failindex, virtualizable): - assert self.cpu.get_latest_value_int(0) == 10 + assert self.cpu.get_latest_value_int(0) == 97 called.append(failindex) return 4 + 9 @@ -1723,33 +1746,41 @@ _assembler_helper_ptr) ops = ''' - [i0, i1] - i2 = int_add(i0, i1) - finish(i2)''' + [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] + i10 = int_add(i0, i1) + i11 = int_add(i10, i2) + i12 = int_add(i11, i3) + i13 = int_add(i12, i4) + i14 = int_add(i13, i5) + i15 = int_add(i14, i6) + i16 = int_add(i15, i7) + i17 = int_add(i16, i8) + i18 = int_add(i17, i9) + finish(i18)''' loop = parse(ops) looptoken = LoopToken() looptoken.outermost_jitdriver_sd = FakeJitDriverSD() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) - ARGS = [lltype.Signed, lltype.Signed] + ARGS = [lltype.Signed] * 10 RES = lltype.Signed self.cpu.portal_calldescr = self.cpu.calldescrof( lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES) - self.cpu.set_future_value_int(0, 1) - self.cpu.set_future_value_int(1, 2) + for i in range(10): + self.cpu.set_future_value_int(i, i+1) res = self.cpu.execute_token(looptoken) - assert self.cpu.get_latest_value_int(0) == 3 + assert self.cpu.get_latest_value_int(0) == 55 ops = ''' - [i4, i5] - i6 = int_add(i4, 1) - i3 = call_assembler(i6, i5, descr=looptoken) + [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] + i10 = int_add(i0, 42) + i11 = call_assembler(i10, i1, i2, i3, i4, i5, i6, i7, i8, i9, descr=looptoken) guard_not_forced()[] - finish(i3) + finish(i11) ''' loop = parse(ops, namespace=locals()) othertoken = LoopToken() self.cpu.compile_loop(loop.inputargs, loop.operations, othertoken) - self.cpu.set_future_value_int(0, 4) - self.cpu.set_future_value_int(1, 5) + for i in range(10): + self.cpu.set_future_value_int(i, i+1) res = self.cpu.execute_token(othertoken) assert self.cpu.get_latest_value_int(0) == 13 assert called Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/assembler.py Wed Aug 4 16:53:45 2010 @@ -1,4 +1,4 @@ -import sys +import sys, os from pypy.jit.backend.llsupport import symbolic from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxFloat from pypy.jit.metainterp.history import AbstractFailDescr, INT, REF, FLOAT,\ @@ -7,23 +7,35 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.annlowlevel import llhelper from pypy.tool.uid import fixid -from pypy.jit.backend.x86.regalloc import RegAlloc, WORD,\ - X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs, FRAME_FIXED_SIZE,\ - FORCE_INDEX_OFS +from pypy.jit.backend.x86.regalloc import RegAlloc, \ + X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs + +from pypy.jit.backend.x86.arch import FRAME_FIXED_SIZE, FORCE_INDEX_OFS, WORD, IS_X86_32, IS_X86_64 + +from pypy.jit.backend.x86.regloc import (eax, ecx, edx, ebx, + esp, ebp, esi, edi, + xmm0, xmm1, xmm2, xmm3, + xmm4, xmm5, xmm6, xmm7, + r8, r9, r10, r11, + r12, r13, r14, r15, + X86_64_SCRATCH_REG, + X86_64_XMM_SCRATCH_REG, + RegLoc, StackLoc, + ImmedLoc, AddressLoc, imm) + from pypy.rlib.objectmodel import we_are_translated, specialize -from pypy.jit.backend.x86 import codebuf -from pypy.jit.backend.x86.ri386 import * -from pypy.jit.metainterp.resoperation import rop +from pypy.jit.backend.x86 import rx86, regloc, codebuf +from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.backend.x86.support import values_array from pypy.rlib.debug import debug_print from pypy.rlib import rgc - -# our calling convention - we pass first 6 args in registers -# and the rest stays on the stack +from pypy.jit.backend.x86.jump import remap_frame_layout +from pypy.rlib.streamio import open_file_as_stream +from pypy.jit.metainterp.history import ConstInt, BoxInt # darwin requires the stack to be 16 bytes aligned on calls. Same for gcc 4.5.0, # better safe than sorry -CALL_ALIGN = 4 +CALL_ALIGN = 16 // WORD def align_stack_words(words): return (words + CALL_ALIGN - 1) & ~(CALL_ALIGN-1) @@ -31,16 +43,36 @@ class MachineCodeBlockWrapper(object): MC_DEFAULT_SIZE = 1024*1024 - def __init__(self, bigsize, profile_agent=None): + def __init__(self, assembler, bigsize, profile_agent=None): + self.assembler = assembler self.old_mcs = [] # keepalive self.bigsize = bigsize self._mc = self._instantiate_mc() self.function_name = None self.profile_agent = profile_agent + self.reset_reserved_bytes() def _instantiate_mc(self): # hook for testing return codebuf.MachineCodeBlock(self.bigsize) + def ensure_bytes_available(self, num_bytes): + if self.bytes_free() <= (self._reserved_bytes + num_bytes): + self.make_new_mc() + + def reserve_bytes(self, num_bytes): + self.ensure_bytes_available(num_bytes) + self._reserved_bytes += num_bytes + + def reset_reserved_bytes(self): + # XXX er.... pretty random number, just to be sure + # not to write half-instruction + self._reserved_bytes = 64 + + def get_relative_pos(self): + return self._mc.get_relative_pos() + + def overwrite(self, pos, listofchars): + return self._mc.overwrite(pos, listofchars) def bytes_free(self): return self._mc._size - self._mc.get_relative_pos() @@ -62,12 +94,25 @@ def make_new_mc(self): new_mc = self._instantiate_mc() debug_print('[new machine code block at', new_mc.tell(), ']') - self._mc.JMP(rel32(new_mc.tell())) + + if IS_X86_64: + # The scratch register is sometimes used as a temporary + # register, but the JMP below might clobber it. Rather than risk + # subtle bugs, we preserve the scratch register across the jump. + self._mc.PUSH_r(X86_64_SCRATCH_REG.value) + + self._mc.JMP(imm(new_mc.tell())) + + if IS_X86_64: + # Restore scratch reg + new_mc.POP_r(X86_64_SCRATCH_REG.value) if self.function_name is not None: self.end_function(done=False) self.start_pos = new_mc.get_relative_pos() + self.assembler.write_pending_failure_recoveries() + self._mc.done() self.old_mcs.append(self._mc) self._mc = new_mc @@ -81,24 +126,39 @@ def _new_method(name): def method(self, *args): - # XXX er.... pretty random number, just to be sure - # not to write half-instruction - if self.bytes_free() < 64: + if self.bytes_free() < self._reserved_bytes: self.make_new_mc() getattr(self._mc, name)(*args) method.func_name = name return method +for _name in rx86.all_instructions + regloc.all_extra_instructions: + setattr(MachineCodeBlockWrapper, _name, _new_method(_name)) + for name in dir(codebuf.MachineCodeBlock): if name.upper() == name or name == "writechr": setattr(MachineCodeBlockWrapper, name, _new_method(name)) +class GuardToken(object): + def __init__(self, faildescr, failargs, fail_locs, exc, desc_bytes): + self.faildescr = faildescr + self.failargs = failargs + self.fail_locs = fail_locs + self.exc = exc + self.desc_bytes = desc_bytes + + def recovery_stub_size(self): + # XXX: 32 is pulled out of the air + return 32 + len(self.desc_bytes) + +DEBUG_COUNTER = rffi.CArray(lltype.Signed) + class Assembler386(object): mc = None - mc2 = None mc_size = MachineCodeBlockWrapper.MC_DEFAULT_SIZE _float_constants = None _regalloc = None + _output_loop_log = None def __init__(self, cpu, translate_support_code=False, failargs_limit=1000): @@ -113,18 +173,26 @@ self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) self.fail_boxes_float = values_array(lltype.Float, failargs_limit) self.fail_ebp = 0 - self.loc_float_const_neg = None - self.loc_float_const_abs = None + self.loop_run_counters = [] + # if we have 10000 loops, we have some other problems I guess + self.float_const_neg_addr = 0 + self.float_const_abs_addr = 0 self.malloc_fixedsize_slowpath1 = 0 self.malloc_fixedsize_slowpath2 = 0 + self.pending_guard_tokens = None self.setup_failure_recovery() + self._debug = False + self.debug_counter_descr = cpu.arraydescrof(DEBUG_COUNTER) def leave_jitted_hook(self): ptrs = self.fail_boxes_ptr.ar llop.gc_assume_young_pointers(lltype.Void, llmemory.cast_ptr_to_adr(ptrs)) - def make_sure_mc_exists(self): + def set_debug(self, v): + self._debug = v + + def setup(self): if self.mc is None: # the address of the function called by 'new' gc_ll_descr = self.cpu.gc_ll_descr @@ -143,11 +211,7 @@ ll_new_unicode = gc_ll_descr.get_funcptr_for_newunicode() self.malloc_unicode_func_addr = rffi.cast(lltype.Signed, ll_new_unicode) - # done - # we generate the loop body in 'mc' - # 'mc2' is for guard recovery code - self.mc = MachineCodeBlockWrapper(self.mc_size, self.cpu.profile_agent) - self.mc2 = MachineCodeBlockWrapper(self.mc_size) + self.mc = MachineCodeBlockWrapper(self, self.mc_size, self.cpu.profile_agent) self._build_failure_recovery(False) self._build_failure_recovery(True) if self.cpu.supports_floats: @@ -157,46 +221,72 @@ self._build_float_constants() if hasattr(gc_ll_descr, 'get_malloc_fixedsize_slowpath_addr'): self._build_malloc_fixedsize_slowpath() + s = os.environ.get('PYPYLOG') + if s: + if s.find(':') != -1: + s = s.split(':')[-1] + self.set_debug(True) + self._output_loop_log = s + ".count" + # Intialize here instead of __init__ to prevent + # pending_guard_tokens from being considered a prebuilt object, + # which sometimes causes memory leaks since the prebuilt list is + # still considered a GC root after we re-assign + # pending_guard_tokens in write_pending_failure_recoveries + self.pending_guard_tokens = [] + + def finish_once(self): + if self._debug: + output_log = self._output_loop_log + assert output_log is not None + f = open_file_as_stream(output_log, "w") + for i in range(len(self.loop_run_counters)): + name, arr = self.loop_run_counters[i] + f.write(name + ":" + str(arr[0]) + "\n") + f.close() def _build_float_constants(self): - # 11 words: 8 words for the data, and up to 3 words for alignment - addr = lltype.malloc(rffi.CArray(lltype.Signed), 11, flavor='raw') + # 44 bytes: 32 bytes for the data, and up to 12 bytes for alignment + addr = lltype.malloc(rffi.CArray(lltype.Char), 44, flavor='raw') if not we_are_translated(): self._keepalive_malloced_float_consts = addr float_constants = rffi.cast(lltype.Signed, addr) float_constants = (float_constants + 15) & ~15 # align to 16 bytes - addr = rffi.cast(rffi.CArrayPtr(lltype.Signed), float_constants) - addr[0] = 0 # \ - addr[1] = -2147483648 # / for neg - addr[2] = 0 # - addr[3] = 0 # - addr[4] = -1 # \ - addr[5] = 2147483647 # / for abs - addr[6] = 0 # - addr[7] = 0 # - self.loc_float_const_neg = heap64(float_constants) - self.loc_float_const_abs = heap64(float_constants + 16) + addr = rffi.cast(rffi.CArrayPtr(lltype.Char), float_constants) + qword_padding = '\x00\x00\x00\x00\x00\x00\x00\x00' + # 0x8000000000000000 + neg_const = '\x00\x00\x00\x00\x00\x00\x00\x80' + # 0x7FFFFFFFFFFFFFFF + abs_const = '\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x7F' + data = neg_const + qword_padding + abs_const + qword_padding + for i in range(len(data)): + addr[i] = data[i] + self.float_const_neg_addr = float_constants + self.float_const_abs_addr = float_constants + 16 def _build_malloc_fixedsize_slowpath(self): - mc = self.mc2._mc # ---------- first helper for the slow path of malloc ---------- - self.malloc_fixedsize_slowpath1 = mc.tell() + self.malloc_fixedsize_slowpath1 = self.mc.tell() if self.cpu.supports_floats: # save the XMM registers in - for i in range(8): # the *caller* frame, from esp+8 - mc.MOVSD(mem64(esp, 8+8*i), xmm_registers[i]) - mc.SUB(edx, eax) # compute the size we want - mc.MOV(mem(esp, 4), edx) # save it as the new argument + for i in range(self.cpu.NUM_REGS):# the *caller* frame, from esp+8 + self.mc.MOVSD_sx((WORD*2)+8*i, i) + self.mc.SUB_rr(edx.value, eax.value) # compute the size we want + if IS_X86_32: + self.mc.MOV_sr(WORD, edx.value) # save it as the new argument + elif IS_X86_64: + # FIXME: We can't just clobber rdi like this, can we? + self.mc.MOV_rr(edi.value, edx.value) + addr = self.cpu.gc_ll_descr.get_malloc_fixedsize_slowpath_addr() - mc.JMP(rel32(addr)) # tail call to the real malloc + self.mc.JMP(imm(addr)) # tail call to the real malloc # ---------- second helper for the slow path of malloc ---------- - self.malloc_fixedsize_slowpath2 = mc.tell() + self.malloc_fixedsize_slowpath2 = self.mc.tell() if self.cpu.supports_floats: # restore the XMM registers - for i in range(8): # from where they were saved - mc.MOVSD(xmm_registers[i], mem64(esp, 8+8*i)) + for i in range(self.cpu.NUM_REGS):# from where they were saved + self.mc.MOVSD_xs(i, (WORD*2)+8*i) nursery_free_adr = self.cpu.gc_ll_descr.get_nursery_free_addr() - mc.MOV(edx, heap(nursery_free_adr)) # load this in EDX - mc.RET() - self.mc2.done() + self.mc.MOV(edx, heap(nursery_free_adr)) # load this in EDX + self.mc.RET() + self.mc.done() def assemble_loop(self, inputargs, operations, looptoken): """adds the following attributes to looptoken: @@ -207,15 +297,18 @@ _x86_param_depth _x86_arglocs """ + if not we_are_translated(): + # Arguments should be unique + assert len(set(inputargs)) == len(inputargs) + + self.setup() funcname = self._find_debug_merge_point(operations) - self.make_sure_mc_exists() + regalloc = RegAlloc(self, self.cpu.translate_support_code) + operations = self._inject_debugging_code(operations) arglocs = regalloc.prepare_loop(inputargs, operations, looptoken) looptoken._x86_arglocs = arglocs - needed_mem = len(arglocs[0]) * 16 + 16 - if needed_mem >= self.mc.bytes_free(): - self.mc.make_new_mc() # profile support name = "Loop # %s: %s" % (looptoken.number, funcname) @@ -230,29 +323,30 @@ self._patch_stackadjust(adr_stackadjust, frame_depth+param_depth) looptoken._x86_frame_depth = frame_depth looptoken._x86_param_depth = param_depth - # we need to make sure here that we don't overload an mc badly. - # a safe estimate is that we need at most 16 bytes per arg - needed_mem = len(arglocs[0]) * 16 + 16 - if needed_mem >= self.mc.bytes_free(): - self.mc.make_new_mc() + looptoken._x86_direct_bootstrap_code = self.mc.tell() self._assemble_bootstrap_direct_call(arglocs, curadr, frame_depth+param_depth) debug_print("Loop #", looptoken.number, "has address", looptoken._x86_loop_code, "to", self.mc.tell()) self.mc.end_function() + self.write_pending_failure_recoveries() - def assemble_bridge(self, faildescr, inputargs, operations): + if not we_are_translated(): + # Arguments should be unique + assert len(set(inputargs)) == len(inputargs) + + self.setup() funcname = self._find_debug_merge_point(operations) - self.make_sure_mc_exists() arglocs = self.rebuild_faillocs_from_descr( faildescr._x86_failure_recovery_bytecode) if not we_are_translated(): assert ([loc.assembler() for loc in arglocs] == [loc.assembler() for loc in faildescr._x86_debug_faillocs]) regalloc = RegAlloc(self, self.cpu.translate_support_code) + operations = self._inject_debugging_code(operations) fail_depths = faildescr._x86_current_depths regalloc.prepare_bridge(fail_depths, inputargs, arglocs, operations) @@ -276,25 +370,82 @@ descr_number, "has address", adr_bridge, "to", self.mc.tell()) self.mc.end_function() + self.write_pending_failure_recoveries() + + def write_pending_failure_recoveries(self): + for tok in self.pending_guard_tokens: + # Okay to write to _mc because we've already made sure that + # there's enough space by "reserving" bytes. + addr = self.generate_quick_failure(self.mc._mc, tok.faildescr, tok.failargs, tok.fail_locs, tok.exc, tok.desc_bytes) + tok.faildescr._x86_adr_recovery_stub = addr + self.patch_jump_for_descr(tok.faildescr, addr) + + self.pending_guard_tokens = [] + self.mc.reset_reserved_bytes() + self.mc.done() def _find_debug_merge_point(self, operations): + for op in operations: if op.opnum == rop.DEBUG_MERGE_POINT: - return op.args[0]._get_str() - return "" + funcname = op.args[0]._get_str() + break + else: + funcname = "" % len(self.loop_run_counters) + # invent the counter, so we don't get too confused + if self._debug: + arr = lltype.malloc(DEBUG_COUNTER, 1, flavor='raw') + arr[0] = 0 + self.loop_run_counters.append((funcname, arr)) + return funcname def patch_jump_for_descr(self, faildescr, adr_new_target): adr_jump_offset = faildescr._x86_adr_jump_offset - mc = codebuf.InMemoryCodeBuilder(adr_jump_offset, adr_jump_offset + 4) - mc.write(packimm32(adr_new_target - adr_jump_offset - 4)) + adr_recovery_stub = faildescr._x86_adr_recovery_stub + offset = adr_new_target - (adr_jump_offset + 4) + # If the new target fits within a rel32 of the jump, just patch + # that. Otherwise, leave the original rel32 to the recovery stub in + # place, but clobber the recovery stub with a jump to the real + # target. + if rx86.fits_in_32bits(offset): + mc = codebuf.InMemoryCodeBuilder(adr_jump_offset, adr_jump_offset + 4) + mc.writeimm32(offset) + else: + # "mov r11, addr; jmp r11" is 13 bytes + mc = codebuf.InMemoryCodeBuilder(adr_recovery_stub, adr_recovery_stub + 13) + mc.MOV_ri(X86_64_SCRATCH_REG.value, adr_new_target) + mc.JMP_r(X86_64_SCRATCH_REG.value) + mc.valgrind_invalidated() mc.done() + def _inject_debugging_code(self, operations): + if self._debug: + # before doing anything, let's increase a counter + c_adr = ConstInt(rffi.cast(lltype.Signed, + self.loop_run_counters[-1][1])) + box = BoxInt() + box2 = BoxInt() + ops = [ResOperation(rop.GETARRAYITEM_RAW, [c_adr, ConstInt(0)], + box, descr=self.debug_counter_descr), + ResOperation(rop.INT_ADD, [box, ConstInt(1)], box2), + ResOperation(rop.SETARRAYITEM_RAW, [c_adr, ConstInt(0), + box2], + None, descr=self.debug_counter_descr)] + operations = ops + operations + # # we need one register free (a bit of a hack, but whatever) + # self.mc.PUSH(eax) + # adr = rffi.cast(lltype.Signed, self.loop_run_counters[-1][1]) + # self.mc.MOV(eax, heap(adr)) + # self.mc.ADD(eax, imm(1)) + # self.mc.MOV(heap(adr), eax) + # self.mc.POP(eax) + return operations + def _assemble(self, regalloc, operations): self._regalloc = regalloc regalloc.walk_operations(operations) self.mc.done() - self.mc2.done() if we_are_translated() or self.cpu.dont_keepalive_stuff: self._regalloc = None # else keep it around for debugging frame_depth = regalloc.fm.frame_depth @@ -309,7 +460,7 @@ def _patchable_stackadjust(self): # stack adjustment LEA - self.mc.LEA(esp, fixedsize_ebp_ofs(0)) + self.mc.LEA32_rb(esp.value, 0) return self.mc.tell() - 4 def _patch_stackadjust(self, adr_lea, reserved_depth): @@ -318,23 +469,34 @@ # Compute the correct offset for the instruction LEA ESP, [EBP-4*words]. # Given that [EBP] is where we saved EBP, i.e. in the last word # of our fixed frame, then the 'words' value is: - words = (FRAME_FIXED_SIZE - 1) + reserved_depth + words = (self.cpu.FRAME_FIXED_SIZE - 1) + reserved_depth # align, e.g. for Mac OS X aligned_words = align_stack_words(words+2)-2 # 2 = EIP+EBP - mc.write(packimm32(-WORD * aligned_words)) + mc.writeimm32(-WORD * aligned_words) mc.done() def _call_header(self): - self.mc.PUSH(ebp) - self.mc.MOV(ebp, esp) - self.mc.PUSH(ebx) - self.mc.PUSH(esi) - self.mc.PUSH(edi) + self.mc.PUSH_r(ebp.value) + self.mc.MOV_rr(ebp.value, esp.value) + for regloc in self.cpu.CALLEE_SAVE_REGISTERS: + self.mc.PUSH_r(regloc.value) + # NB. the shape of the frame is hard-coded in get_basic_shape() too. # Also, make sure this is consistent with FRAME_FIXED_SIZE. return self._patchable_stackadjust() + def _call_footer(self): + self.mc.LEA_rb(esp.value, -len(self.cpu.CALLEE_SAVE_REGISTERS) * WORD) + + for i in range(len(self.cpu.CALLEE_SAVE_REGISTERS)-1, -1, -1): + self.mc.POP_r(self.cpu.CALLEE_SAVE_REGISTERS[i].value) + + self.mc.POP_r(ebp.value) + self.mc.RET() + def _assemble_bootstrap_direct_call(self, arglocs, jmpadr, stackdepth): + if IS_X86_64: + return self._assemble_bootstrap_direct_call_64(arglocs, jmpadr, stackdepth) # XXX pushing ebx esi and edi is a bit pointless, since we store # all regsiters anyway, for the case of guard_not_forced # XXX this can be improved greatly. Right now it'll behave like @@ -345,23 +507,81 @@ self._patch_stackadjust(adr_stackadjust, stackdepth) for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] - if isinstance(loc, REG): - self.mc.MOV(loc, mem(ebp, (2 + i) * WORD)) + if isinstance(loc, RegLoc): + assert not loc.is_xmm + self.mc.MOV_rb(loc.value, (2 + i) * WORD) loc = floatlocs[i] - if isinstance(loc, XMMREG): - self.mc.MOVSD(loc, mem64(ebp, (1 + i) * 2 * WORD)) + if isinstance(loc, RegLoc): + assert loc.is_xmm + self.mc.MOVSD_xb(loc.value, (1 + i) * 2 * WORD) tmp = eax xmmtmp = xmm0 for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] - if loc is not None and not isinstance(loc, REG): - self.mc.MOV(tmp, mem(ebp, (2 + i) * WORD)) + if loc is not None and not isinstance(loc, RegLoc): + self.mc.MOV_rb(tmp.value, (2 + i) * WORD) self.mc.MOV(loc, tmp) loc = floatlocs[i] - if loc is not None and not isinstance(loc, XMMREG): - self.mc.MOVSD(xmmtmp, mem64(ebp, (1 + i) * 2 * WORD)) - self.mc.MOVSD(loc, xmmtmp) - self.mc.JMP(rel32(jmpadr)) + if loc is not None and not isinstance(loc, RegLoc): + self.mc.MOVSD_xb(xmmtmp.value, (1 + i) * 2 * WORD) + assert isinstance(loc, StackLoc) + self.mc.MOVSD_bx(loc.value, xmmtmp.value) + self.mc.JMP_l(jmpadr) + return adr_stackadjust + + def _assemble_bootstrap_direct_call_64(self, arglocs, jmpadr, stackdepth): + # XXX: Very similar to _emit_call_64 + + src_locs = [] + dst_locs = [] + xmm_src_locs = [] + xmm_dst_locs = [] + get_from_stack = [] + + # In reverse order for use with pop() + unused_gpr = [r9, r8, ecx, edx, esi, edi] + unused_xmm = [xmm7, xmm6, xmm5, xmm4, xmm3, xmm2, xmm1, xmm0] + + nonfloatlocs, floatlocs = arglocs + adr_stackadjust = self._call_header() + self._patch_stackadjust(adr_stackadjust, stackdepth) + + # The lists are padded with Nones + assert len(nonfloatlocs) == len(floatlocs) + + for i in range(len(nonfloatlocs)): + loc = nonfloatlocs[i] + if loc is not None: + if len(unused_gpr) > 0: + src_locs.append(unused_gpr.pop()) + dst_locs.append(loc) + else: + get_from_stack.append((loc, False)) + + floc = floatlocs[i] + if floc is not None: + if len(unused_xmm) > 0: + xmm_src_locs.append(unused_xmm.pop()) + xmm_dst_locs.append(floc) + else: + get_from_stack.append((floc, True)) + + remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG) + remap_frame_layout(self, xmm_src_locs, xmm_dst_locs, X86_64_XMM_SCRATCH_REG) + + for i in range(len(get_from_stack)): + loc, is_xmm = get_from_stack[i] + if is_xmm: + self.mc.MOVSD_xb(X86_64_XMM_SCRATCH_REG.value, (2 + i) * WORD) + self.mc.MOVSD(loc, X86_64_XMM_SCRATCH_REG) + else: + self.mc.MOV_rb(X86_64_SCRATCH_REG.value, (2 + i) * WORD) + # XXX: We're assuming that "loc" won't require regloc to + # clobber the scratch register + self.mc.MOV(loc, X86_64_SCRATCH_REG) + + self.mc.JMP(imm(jmpadr)) + return adr_stackadjust def _assemble_bootstrap_code(self, inputargs, arglocs): @@ -369,11 +589,12 @@ adr_stackadjust = self._call_header() tmp = X86RegisterManager.all_regs[0] xmmtmp = X86XMMRegisterManager.all_regs[0] + self.mc._mc.begin_reuse_scratch_register() for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] if loc is None: continue - if isinstance(loc, REG): + if isinstance(loc, RegLoc): target = loc else: target = tmp @@ -387,17 +608,20 @@ adr = self.fail_boxes_int.get_addr_for_num(i) self.mc.MOV(target, heap(adr)) if target is not loc: - self.mc.MOV(loc, target) + assert isinstance(loc, StackLoc) + self.mc.MOV_br(loc.value, target.value) for i in range(len(floatlocs)): loc = floatlocs[i] if loc is None: continue adr = self.fail_boxes_float.get_addr_for_num(i) - if isinstance(loc, REG): - self.mc.MOVSD(loc, heap64(adr)) + if isinstance(loc, RegLoc): + self.mc.MOVSD(loc, heap(adr)) else: - self.mc.MOVSD(xmmtmp, heap64(adr)) - self.mc.MOVSD(loc, xmmtmp) + self.mc.MOVSD(xmmtmp, heap(adr)) + assert isinstance(loc, StackLoc) + self.mc.MOVSD_bx(loc.value, xmmtmp.value) + self.mc._mc.end_reuse_scratch_register() return adr_stackadjust def dump(self, text): @@ -410,27 +634,10 @@ finally: Box._extended_display = _prev - def _start_block(self): - # Return a 'mc' that can be used to write an "atomic" block, - # i.e. one that will not contain any JMP. - mc = self.mc._mc - if not we_are_translated(): - self._block_started_mc = (self.mc, mc.tell()) - self.mc = "block started" - return mc - - def _stop_block(self): - if not we_are_translated(): - assert self.mc == "block started" - self.mc, orgpos = self._block_started_mc - assert 0 <= self.mc._mc.tell() - orgpos <= 58, ( - "too many bytes in _start_block/_stop_block pair") - del self._block_started_mc - # ------------------------------------------------------------ def mov(self, from_loc, to_loc): - if isinstance(from_loc, XMMREG) or isinstance(to_loc, XMMREG): + if (isinstance(from_loc, RegLoc) and from_loc.is_xmm) or (isinstance(to_loc, RegLoc) and to_loc.is_xmm): self.mc.MOVSD(to_loc, from_loc) else: self.mc.MOV(to_loc, from_loc) @@ -438,24 +645,24 @@ regalloc_mov = mov # legacy interface def regalloc_push(self, loc): - if isinstance(loc, XMMREG): - self.mc.SUB(esp, imm(2*WORD)) - self.mc.MOVSD(mem64(esp, 0), loc) - elif isinstance(loc, MODRM64): + if isinstance(loc, RegLoc) and loc.is_xmm: + self.mc.SUB_ri(esp.value, 2*WORD) + self.mc.MOVSD_sx(0, loc.value) + elif WORD == 4 and isinstance(loc, StackLoc) and loc.width == 8: # XXX evil trick - self.mc.PUSH(mem(ebp, get_ebp_ofs(loc.position))) - self.mc.PUSH(mem(ebp, get_ebp_ofs(loc.position + 1))) + self.mc.PUSH_b(get_ebp_ofs(loc.position)) + self.mc.PUSH_b(get_ebp_ofs(loc.position + 1)) else: self.mc.PUSH(loc) def regalloc_pop(self, loc): - if isinstance(loc, XMMREG): - self.mc.MOVSD(loc, mem64(esp, 0)) - self.mc.ADD(esp, imm(2*WORD)) - elif isinstance(loc, MODRM64): + if isinstance(loc, RegLoc) and loc.is_xmm: + self.mc.MOVSD_xs(loc.value, 0) + self.mc.ADD_ri(esp.value, 2*WORD) + elif WORD == 4 and isinstance(loc, StackLoc) and loc.width == 8: # XXX evil trick - self.mc.POP(mem(ebp, get_ebp_ofs(loc.position + 1))) - self.mc.POP(mem(ebp, get_ebp_ofs(loc.position))) + self.mc.POP_b(get_ebp_ofs(loc.position + 1)) + self.mc.POP_b(get_ebp_ofs(loc.position)) else: self.mc.POP(loc) @@ -472,14 +679,14 @@ faildescr._x86_current_depths = current_depths failargs = guard_op.fail_args guard_opnum = guard_op.opnum - failaddr = self.implement_guard_recovery(guard_opnum, - faildescr, failargs, - faillocs) + guard_token = self.implement_guard_recovery(guard_opnum, + faildescr, failargs, + faillocs) if op is None: dispatch_opnum = guard_opnum else: dispatch_opnum = op.opnum - res = genop_guard_list[dispatch_opnum](self, op, guard_op, failaddr, + res = genop_guard_list[dispatch_opnum](self, op, guard_op, guard_token, arglocs, resloc) faildescr._x86_adr_jump_offset = res @@ -506,103 +713,158 @@ rl = result_loc.lowest8bits() if isinstance(op.args[0], Const): self.mc.CMP(arglocs[1], arglocs[0]) - getattr(self.mc, 'SET' + rev_cond)(rl) + self.mc.SET_ir(rx86.Conditions[rev_cond], rl.value) else: self.mc.CMP(arglocs[0], arglocs[1]) - getattr(self.mc, 'SET' + cond)(rl) - self.mc.MOVZX(result_loc, rl) + self.mc.SET_ir(rx86.Conditions[cond], rl.value) + self.mc.MOVZX8_rr(result_loc.value, rl.value) return genop_cmp def _cmpop_float(cond, is_ne=False): def genop_cmp(self, op, arglocs, result_loc): self.mc.UCOMISD(arglocs[0], arglocs[1]) - rl = result_loc.lowest8bits() - rh = result_loc.higher8bits() - getattr(self.mc, 'SET' + cond)(rl) + tmp1 = result_loc.lowest8bits() + if IS_X86_32: + tmp2 = result_loc.higher8bits() + elif IS_X86_64: + tmp2 = X86_64_SCRATCH_REG.lowest8bits() + + self.mc.SET_ir(rx86.Conditions[cond], tmp1.value) if is_ne: - self.mc.SETP(rh) - self.mc.OR(rl, rh) + self.mc.SET_ir(rx86.Conditions['P'], tmp2.value) + self.mc.OR8_rr(tmp1.value, tmp2.value) else: - self.mc.SETNP(rh) - self.mc.AND(rl, rh) - self.mc.MOVZX(result_loc, rl) + self.mc.SET_ir(rx86.Conditions['NP'], tmp2.value) + self.mc.AND8_rr(tmp1.value, tmp2.value) + self.mc.MOVZX8_rr(result_loc.value, tmp1.value) return genop_cmp def _cmpop_guard(cond, rev_cond, false_cond, false_rev_cond): - def genop_cmp_guard(self, op, guard_op, addr, arglocs, result_loc): + def genop_cmp_guard(self, op, guard_op, guard_token, arglocs, result_loc): guard_opnum = guard_op.opnum if isinstance(op.args[0], Const): self.mc.CMP(arglocs[1], arglocs[0]) if guard_opnum == rop.GUARD_FALSE: - name = 'J' + rev_cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(guard_token, rev_cond) else: - name = 'J' + false_rev_cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(guard_token, false_rev_cond) else: self.mc.CMP(arglocs[0], arglocs[1]) if guard_opnum == rop.GUARD_FALSE: - name = 'J' + cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(guard_token, cond) else: - name = 'J' + false_cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(guard_token, false_cond) return genop_cmp_guard def _cmpop_guard_float(cond, false_cond, need_jp): - def genop_cmp_guard_float(self, op, guard_op, addr, arglocs, + def genop_cmp_guard_float(self, op, guard_op, guard_token, arglocs, result_loc): guard_opnum = guard_op.opnum self.mc.UCOMISD(arglocs[0], arglocs[1]) + # 16 is enough space for the rel8 jumps below and the rel32 + # jump in implement_guard + self.mc.ensure_bytes_available(16 + guard_token.recovery_stub_size()) if guard_opnum == rop.GUARD_FALSE: - mc = self.mc._mc - name = 'J' + cond if need_jp: - mc.JP(rel8(6)) - getattr(mc, name)(rel32(addr)) - return mc.tell() - 4 + self.mc.J_il8(rx86.Conditions['P'], 6) + return self.implement_guard(guard_token, cond) else: if need_jp: - mc = self.mc._mc - mc.JP(rel8(2)) - getattr(mc, 'J' + cond)(rel8(5)) - return self.implement_guard(addr, mc.JMP) - name = 'J' + false_cond - return self.implement_guard(addr, getattr(self.mc, name)) + self.mc.J_il8(rx86.Conditions['P'], 2) + self.mc.J_il8(rx86.Conditions[cond], 5) + return self.implement_guard(guard_token) + return self.implement_guard(guard_token, false_cond) return genop_cmp_guard_float - @specialize.arg(5) - def _emit_call(self, x, arglocs, start=0, tmp=eax, force_mc=False, - mc=None): - if not force_mc: - mc = self.mc + def _emit_call(self, x, arglocs, start=0, tmp=eax): + if IS_X86_64: + return self._emit_call_64(x, arglocs, start) + p = 0 n = len(arglocs) for i in range(start, n): loc = arglocs[i] - if isinstance(loc, REG): - if isinstance(loc, XMMREG): - mc.MOVSD(mem64(esp, p), loc) + if isinstance(loc, RegLoc): + if loc.is_xmm: + self.mc.MOVSD_sx(p, loc.value) else: - mc.MOV(mem(esp, p), loc) + self.mc.MOV_sr(p, loc.value) p += round_up_to_4(loc.width) p = 0 for i in range(start, n): loc = arglocs[i] - if not isinstance(loc, REG): - if isinstance(loc, MODRM64): - mc.MOVSD(xmm0, loc) - mc.MOVSD(mem64(esp, p), xmm0) + if not isinstance(loc, RegLoc): + if loc.width == 8: + self.mc.MOVSD(xmm0, loc) + self.mc.MOVSD_sx(p, xmm0.value) else: - mc.MOV(tmp, loc) - mc.MOV(mem(esp, p), tmp) + self.mc.MOV(tmp, loc) + self.mc.MOV_sr(p, tmp.value) p += round_up_to_4(loc.width) self._regalloc.reserve_param(p//WORD) - mc.CALL(x) + # x is a location + self.mc.CALL(x) self.mark_gc_roots() + + def _emit_call_64(self, x, arglocs, start=0): + src_locs = [] + dst_locs = [] + xmm_src_locs = [] + xmm_dst_locs = [] + pass_on_stack = [] + + # In reverse order for use with pop() + unused_gpr = [r9, r8, ecx, edx, esi, edi] + unused_xmm = [xmm7, xmm6, xmm5, xmm4, xmm3, xmm2, xmm1, xmm0] + + for i in range(start, len(arglocs)): + loc = arglocs[i] + assert isinstance(loc, RegLoc) or isinstance(loc, ImmedLoc) or isinstance(loc, StackLoc) + + # XXX: Should be much simplier to tell whether a location is a float! + if (isinstance(loc, RegLoc) and loc.is_xmm) or (isinstance(loc, StackLoc) and loc.type == FLOAT): + if len(unused_xmm) > 0: + xmm_src_locs.append(loc) + xmm_dst_locs.append(unused_xmm.pop()) + else: + pass_on_stack.append(loc) + else: + if len(unused_gpr) > 0: + src_locs.append(loc) + dst_locs.append(unused_gpr.pop()) + else: + pass_on_stack.append(loc) + # Emit instructions to pass the stack arguments + # XXX: Would be nice to let remap_frame_layout take care of this, but + # we'd need to create something like StackLoc, but relative to esp, + # and I don't know if it's worth it. + for i in range(len(pass_on_stack)): + loc = pass_on_stack[i] + if not isinstance(loc, RegLoc): + if isinstance(loc, StackLoc) and loc.type == FLOAT: + self.mc.MOVSD(X86_64_XMM_SCRATCH_REG, loc) + self.mc.MOVSD_sx(i*WORD, X86_64_XMM_SCRATCH_REG.value) + else: + self.mc.MOV(X86_64_SCRATCH_REG, loc) + self.mc.MOV_sr(i*WORD, X86_64_SCRATCH_REG.value) + else: + # It's a register + if loc.is_xmm: + self.mc.MOVSD_sx(i*WORD, loc.value) + else: + self.mc.MOV_sr(i*WORD, loc.value) + + # Handle register arguments + remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG) + remap_frame_layout(self, xmm_src_locs, xmm_dst_locs, X86_64_XMM_SCRATCH_REG) + + self._regalloc.reserve_param(len(pass_on_stack)) + self.mc.CALL(x) + self.mark_gc_roots() + def call(self, addr, args, res): - self._emit_call(rel32(addr), args) + self._emit_call(imm(addr), args) assert res is eax genop_int_neg = _unaryop("NEG") @@ -613,6 +875,9 @@ genop_int_and = _binaryop("AND", True) genop_int_or = _binaryop("OR", True) genop_int_xor = _binaryop("XOR", True) + genop_int_lshift = _binaryop("SHL") + genop_int_rshift = _binaryop("SAR") + genop_uint_rshift = _binaryop("SHR") genop_float_add = _binaryop("ADDSD", True) genop_float_sub = _binaryop('SUBSD') genop_float_mul = _binaryop('MULSD', True) @@ -659,26 +924,27 @@ genop_guard_float_gt = _cmpop_guard_float("A", "BE", False) genop_guard_float_ge = _cmpop_guard_float("AE", "B", False) - def genop_guard_float_ne(self, op, guard_op, addr, arglocs, result_loc): + def genop_guard_float_ne(self, op, guard_op, guard_token, arglocs, result_loc): guard_opnum = guard_op.opnum self.mc.UCOMISD(arglocs[0], arglocs[1]) - mc = self.mc._mc + # 16 is enough space for the rel8 jumps below and the rel32 + # jump in implement_guard + self.mc.ensure_bytes_available(16 + guard_token.recovery_stub_size()) if guard_opnum == rop.GUARD_TRUE: - mc.JP(rel8(6)) - mc.JE(rel32(addr)) - return mc.tell() - 4 - else: - mc.JP(rel8(2)) - mc.JE(rel8(5)) - return self.implement_guard(addr, mc.JMP) + self.mc.J_il8(rx86.Conditions['P'], 6) + return self.implement_guard(guard_token, 'E') + else: + self.mc.J_il8(rx86.Conditions['P'], 2) + self.mc.J_il8(rx86.Conditions['E'], 5) + return self.implement_guard(guard_token) def genop_float_neg(self, op, arglocs, resloc): # Following what gcc does: res = x ^ 0x8000000000000000 - self.mc.XORPD(arglocs[0], self.loc_float_const_neg) + self.mc.XORPD(arglocs[0], heap(self.float_const_neg_addr)) def genop_float_abs(self, op, arglocs, resloc): # Following what gcc does: res = x & 0x7FFFFFFFFFFFFFFF - self.mc.ANDPD(arglocs[0], self.loc_float_const_abs) + self.mc.ANDPD(arglocs[0], heap(self.float_const_abs_addr)) def genop_cast_float_to_int(self, op, arglocs, resloc): self.mc.CVTTSD2SI(resloc, arglocs[0]) @@ -686,70 +952,56 @@ def genop_cast_int_to_float(self, op, arglocs, resloc): self.mc.CVTSI2SD(resloc, arglocs[0]) - def genop_int_lshift(self, op, arglocs, resloc): - loc, loc2 = arglocs - if loc2 is ecx: - loc2 = cl - self.mc.SHL(loc, loc2) - - def genop_int_rshift(self, op, arglocs, resloc): - loc, loc2 = arglocs - if loc2 is ecx: - loc2 = cl - self.mc.SAR(loc, loc2) - - def genop_uint_rshift(self, op, arglocs, resloc): - loc, loc2 = arglocs - if loc2 is ecx: - loc2 = cl - self.mc.SHR(loc, loc2) - - def genop_guard_int_is_true(self, op, guard_op, addr, arglocs, resloc): + def genop_guard_int_is_true(self, op, guard_op, guard_token, arglocs, resloc): guard_opnum = guard_op.opnum - self.mc.CMP(arglocs[0], imm8(0)) + self.mc.CMP(arglocs[0], imm(0)) if guard_opnum == rop.GUARD_TRUE: - return self.implement_guard(addr, self.mc.JZ) + return self.implement_guard(guard_token, 'Z') else: - return self.implement_guard(addr, self.mc.JNZ) + return self.implement_guard(guard_token, 'NZ') def genop_int_is_true(self, op, arglocs, resloc): - self.mc.CMP(arglocs[0], imm8(0)) + self.mc.CMP(arglocs[0], imm(0)) rl = resloc.lowest8bits() - self.mc.SETNE(rl) - self.mc.MOVZX(resloc, rl) + self.mc.SET_ir(rx86.Conditions['NE'], rl.value) + self.mc.MOVZX8(resloc, rl) - def genop_guard_int_is_zero(self, op, guard_op, addr, arglocs, resloc): + def genop_guard_int_is_zero(self, op, guard_op, guard_token, arglocs, resloc): guard_opnum = guard_op.opnum - self.mc.CMP(arglocs[0], imm8(0)) + self.mc.CMP(arglocs[0], imm(0)) if guard_opnum == rop.GUARD_TRUE: - return self.implement_guard(addr, self.mc.JNZ) + return self.implement_guard(guard_token, 'NZ') else: - return self.implement_guard(addr, self.mc.JZ) + return self.implement_guard(guard_token, 'Z') def genop_int_is_zero(self, op, arglocs, resloc): - self.mc.CMP(arglocs[0], imm8(0)) + self.mc.CMP(arglocs[0], imm(0)) rl = resloc.lowest8bits() - self.mc.SETE(rl) - self.mc.MOVZX(resloc, rl) + self.mc.SET_ir(rx86.Conditions['E'], rl.value) + self.mc.MOVZX8(resloc, rl) def genop_same_as(self, op, arglocs, resloc): self.mov(arglocs[0], resloc) #genop_cast_ptr_to_int = genop_same_as def genop_int_mod(self, op, arglocs, resloc): - self.mc.CDQ() - self.mc.IDIV(ecx) + if IS_X86_32: + self.mc.CDQ() + elif IS_X86_64: + self.mc.CQO() + + self.mc.IDIV_r(ecx.value) genop_int_floordiv = genop_int_mod def genop_uint_floordiv(self, op, arglocs, resloc): - self.mc.XOR(edx, edx) - self.mc.DIV(ecx) + self.mc.XOR_rr(edx.value, edx.value) + self.mc.DIV_r(ecx.value) def genop_new_with_vtable(self, op, arglocs, result_loc): assert result_loc is eax loc_vtable = arglocs[-1] - assert isinstance(loc_vtable, IMM32) + assert isinstance(loc_vtable, ImmedLoc) arglocs = arglocs[:-1] self.call(self.malloc_func_addr, arglocs, eax) # xxx ignore NULL returns for now @@ -757,7 +1009,9 @@ def set_vtable(self, loc, loc_vtable): if self.cpu.vtable_offset is not None: - self.mc.MOV(mem(loc, self.cpu.vtable_offset), loc_vtable) + assert isinstance(loc, RegLoc) + assert isinstance(loc_vtable, ImmedLoc) + self.mc.MOV_mi((loc.value, self.cpu.vtable_offset), loc_vtable.value) # XXX genop_new is abused for all varsized mallocs with Boehm, for now # (instead of genop_new_array, genop_newstr, genop_newunicode) @@ -779,16 +1033,19 @@ def genop_getfield_gc(self, op, arglocs, resloc): base_loc, ofs_loc, size_loc = arglocs - assert isinstance(size_loc, IMM32) + assert isinstance(size_loc, ImmedLoc) + assert isinstance(resloc, RegLoc) size = size_loc.value - if size == 1: - self.mc.MOVZX(resloc, addr8_add(base_loc, ofs_loc)) + + source_addr = AddressLoc(base_loc, ofs_loc) + if resloc.is_xmm: + self.mc.MOVSD(resloc, source_addr) + elif size == 1: + self.mc.MOVZX8(resloc, source_addr) elif size == 2: - self.mc.MOVZX(resloc, addr_add(base_loc, ofs_loc)) + self.mc.MOVZX16(resloc, source_addr) elif size == WORD: - self.mc.MOV(resloc, addr_add(base_loc, ofs_loc)) - elif size == 8: - self.mc.MOVSD(resloc, addr64_add(base_loc, ofs_loc)) + self.mc.MOV(resloc, source_addr) else: raise NotImplementedError("getfield size = %d" % size) @@ -798,16 +1055,16 @@ def genop_getarrayitem_gc(self, op, arglocs, resloc): base_loc, ofs_loc, scale, ofs = arglocs - assert isinstance(ofs, IMM32) - assert isinstance(scale, IMM32) + assert isinstance(ofs, ImmedLoc) + assert isinstance(scale, ImmedLoc) if op.result.type == FLOAT: - self.mc.MOVSD(resloc, addr64_add(base_loc, ofs_loc, ofs.value, + self.mc.MOVSD(resloc, addr_add(base_loc, ofs_loc, ofs.value, scale.value)) else: if scale.value == 0: - self.mc.MOVZX(resloc, addr8_add(base_loc, ofs_loc, ofs.value, + self.mc.MOVZX8(resloc, addr_add(base_loc, ofs_loc, ofs.value, scale.value)) - elif scale.value == 2: + elif (1 << scale.value) == WORD: self.mc.MOV(resloc, addr_add(base_loc, ofs_loc, ofs.value, scale.value)) else: @@ -819,34 +1076,33 @@ def genop_discard_setfield_gc(self, op, arglocs): base_loc, ofs_loc, size_loc, value_loc = arglocs - assert isinstance(size_loc, IMM32) + assert isinstance(size_loc, ImmedLoc) size = size_loc.value - if size == WORD * 2: - self.mc.MOVSD(addr64_add(base_loc, ofs_loc), value_loc) + dest_addr = AddressLoc(base_loc, ofs_loc) + if isinstance(value_loc, RegLoc) and value_loc.is_xmm: + self.mc.MOVSD(dest_addr, value_loc) elif size == WORD: - self.mc.MOV(addr_add(base_loc, ofs_loc), value_loc) + self.mc.MOV(dest_addr, value_loc) elif size == 2: - self.mc.MOV16(addr_add(base_loc, ofs_loc), value_loc) + self.mc.MOV16(dest_addr, value_loc) elif size == 1: - self.mc.MOV(addr8_add(base_loc, ofs_loc), value_loc.lowest8bits()) + self.mc.MOV8(dest_addr, value_loc.lowest8bits()) else: print "[asmgen]setfield addr size %d" % size raise NotImplementedError("Addr size %d" % size) def genop_discard_setarrayitem_gc(self, op, arglocs): base_loc, ofs_loc, value_loc, scale_loc, baseofs = arglocs - assert isinstance(baseofs, IMM32) - assert isinstance(scale_loc, IMM32) + assert isinstance(baseofs, ImmedLoc) + assert isinstance(scale_loc, ImmedLoc) + dest_addr = AddressLoc(base_loc, ofs_loc, scale_loc.value, baseofs.value) if op.args[2].type == FLOAT: - self.mc.MOVSD(addr64_add(base_loc, ofs_loc, baseofs.value, - scale_loc.value), value_loc) + self.mc.MOVSD(dest_addr, value_loc) else: - if scale_loc.value == 2: - self.mc.MOV(addr_add(base_loc, ofs_loc, baseofs.value, - scale_loc.value), value_loc) + if (1 << scale_loc.value) == WORD: + self.mc.MOV(dest_addr, value_loc) elif scale_loc.value == 0: - self.mc.MOV(addr8_add(base_loc, ofs_loc, baseofs.value, - scale_loc.value), value_loc.lowest8bits()) + self.mc.MOV8(dest_addr, value_loc.lowest8bits()) else: raise NotImplementedError("scale = %d" % scale_loc.value) @@ -855,17 +1111,17 @@ basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) assert itemsize == 1 - self.mc.MOV(addr8_add(base_loc, ofs_loc, basesize), - val_loc.lowest8bits()) + dest_addr = AddressLoc(base_loc, ofs_loc, 0, basesize) + self.mc.MOV8(dest_addr, val_loc.lowest8bits()) def genop_discard_unicodesetitem(self, op, arglocs): base_loc, ofs_loc, val_loc = arglocs basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) if itemsize == 4: - self.mc.MOV(addr_add(base_loc, ofs_loc, basesize, 2), val_loc) + self.mc.MOV32(AddressLoc(base_loc, ofs_loc, 2, basesize), val_loc) elif itemsize == 2: - self.mc.MOV16(addr_add(base_loc, ofs_loc, basesize, 1), val_loc) + self.mc.MOV16(AddressLoc(base_loc, ofs_loc, 1, basesize), val_loc) else: assert 0, itemsize @@ -886,7 +1142,7 @@ def genop_arraylen_gc(self, op, arglocs, resloc): base_loc, ofs_loc = arglocs - assert isinstance(ofs_loc, IMM32) + assert isinstance(ofs_loc, ImmedLoc) self.mc.MOV(resloc, addr_add_const(base_loc, ofs_loc.value)) def genop_strgetitem(self, op, arglocs, resloc): @@ -894,83 +1150,83 @@ basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) assert itemsize == 1 - self.mc.MOVZX(resloc, addr8_add(base_loc, ofs_loc, basesize)) + self.mc.MOVZX8(resloc, AddressLoc(base_loc, ofs_loc, 0, basesize)) def genop_unicodegetitem(self, op, arglocs, resloc): base_loc, ofs_loc = arglocs basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) if itemsize == 4: - self.mc.MOV(resloc, addr_add(base_loc, ofs_loc, basesize, 2)) + self.mc.MOV32(resloc, AddressLoc(base_loc, ofs_loc, 2, basesize)) elif itemsize == 2: - self.mc.MOVZX(resloc, addr_add(base_loc, ofs_loc, basesize, 1)) + self.mc.MOVZX16(resloc, AddressLoc(base_loc, ofs_loc, 1, basesize)) else: assert 0, itemsize - def genop_guard_guard_true(self, ign_1, guard_op, addr, locs, ign_2): + def genop_guard_guard_true(self, ign_1, guard_op, guard_token, locs, ign_2): loc = locs[0] self.mc.TEST(loc, loc) - return self.implement_guard(addr, self.mc.JZ) + return self.implement_guard(guard_token, 'Z') genop_guard_guard_nonnull = genop_guard_guard_true - def genop_guard_guard_no_exception(self, ign_1, guard_op, addr, + def genop_guard_guard_no_exception(self, ign_1, guard_op, guard_token, locs, ign_2): self.mc.CMP(heap(self.cpu.pos_exception()), imm(0)) - return self.implement_guard(addr, self.mc.JNZ) + return self.implement_guard(guard_token, 'NZ') - def genop_guard_guard_exception(self, ign_1, guard_op, addr, + def genop_guard_guard_exception(self, ign_1, guard_op, guard_token, locs, resloc): loc = locs[0] loc1 = locs[1] self.mc.MOV(loc1, heap(self.cpu.pos_exception())) self.mc.CMP(loc1, loc) - addr = self.implement_guard(addr, self.mc.JNE) + addr = self.implement_guard(guard_token, 'NE') if resloc is not None: self.mc.MOV(resloc, heap(self.cpu.pos_exc_value())) self.mc.MOV(heap(self.cpu.pos_exception()), imm(0)) self.mc.MOV(heap(self.cpu.pos_exc_value()), imm(0)) return addr - def _gen_guard_overflow(self, guard_op, addr): + def _gen_guard_overflow(self, guard_op, guard_token): guard_opnum = guard_op.opnum if guard_opnum == rop.GUARD_NO_OVERFLOW: - return self.implement_guard(addr, self.mc.JO) + return self.implement_guard(guard_token, 'O') elif guard_opnum == rop.GUARD_OVERFLOW: - return self.implement_guard(addr, self.mc.JNO) + return self.implement_guard(guard_token, 'NO') else: print "int_xxx_ovf followed by", guard_op.getopname() raise AssertionError - def genop_guard_int_add_ovf(self, op, guard_op, addr, arglocs, result_loc): + def genop_guard_int_add_ovf(self, op, guard_op, guard_token, arglocs, result_loc): self.genop_int_add(op, arglocs, result_loc) - return self._gen_guard_overflow(guard_op, addr) + return self._gen_guard_overflow(guard_op, guard_token) - def genop_guard_int_sub_ovf(self, op, guard_op, addr, arglocs, result_loc): + def genop_guard_int_sub_ovf(self, op, guard_op, guard_token, arglocs, result_loc): self.genop_int_sub(op, arglocs, result_loc) - return self._gen_guard_overflow(guard_op, addr) + return self._gen_guard_overflow(guard_op, guard_token) - def genop_guard_int_mul_ovf(self, op, guard_op, addr, arglocs, result_loc): + def genop_guard_int_mul_ovf(self, op, guard_op, guard_token, arglocs, result_loc): self.genop_int_mul(op, arglocs, result_loc) - return self._gen_guard_overflow(guard_op, addr) + return self._gen_guard_overflow(guard_op, guard_token) - def genop_guard_guard_false(self, ign_1, guard_op, addr, locs, ign_2): + def genop_guard_guard_false(self, ign_1, guard_op, guard_token, locs, ign_2): loc = locs[0] self.mc.TEST(loc, loc) - return self.implement_guard(addr, self.mc.JNZ) + return self.implement_guard(guard_token, 'NZ') genop_guard_guard_isnull = genop_guard_guard_false - def genop_guard_guard_value(self, ign_1, guard_op, addr, locs, ign_2): + def genop_guard_guard_value(self, ign_1, guard_op, guard_token, locs, ign_2): if guard_op.args[0].type == FLOAT: assert guard_op.args[1].type == FLOAT self.mc.UCOMISD(locs[0], locs[1]) else: self.mc.CMP(locs[0], locs[1]) - return self.implement_guard(addr, self.mc.JNE) + return self.implement_guard(guard_token, 'NE') - def _cmp_guard_class(self, mc, locs): + def _cmp_guard_class(self, locs): offset = self.cpu.vtable_offset if offset is not None: - mc.CMP(mem(locs[0], offset), locs[1]) + self.mc.CMP(mem(locs[0], offset), locs[1]) else: # XXX hard-coded assumption: to go from an object to its class # we use the following algorithm: @@ -979,7 +1235,7 @@ # - multiply by 4 and use it as an offset in type_info_group # - add 16 bytes, to go past the TYPE_INFO structure loc = locs[1] - assert isinstance(loc, IMM32) + assert isinstance(loc, ImmedLoc) classptr = loc.value # here, we have to go back from 'classptr' to the value expected # from reading the 16 bits in the object header @@ -988,37 +1244,37 @@ type_info_group = llop.gc_get_type_info_group(llmemory.Address) type_info_group = rffi.cast(lltype.Signed, type_info_group) expected_typeid = (classptr - sizeof_ti - type_info_group) >> 2 - mc.CMP16(mem(locs[0], 0), imm32(expected_typeid)) + self.mc.CMP16(mem(locs[0], 0), ImmedLoc(expected_typeid)) - def genop_guard_guard_class(self, ign_1, guard_op, addr, locs, ign_2): - mc = self._start_block() - self._cmp_guard_class(mc, locs) - self._stop_block() - return self.implement_guard(addr, self.mc.JNE) + def genop_guard_guard_class(self, ign_1, guard_op, guard_token, locs, ign_2): + self.mc.ensure_bytes_available(256) + self._cmp_guard_class(locs) + return self.implement_guard(guard_token, 'NE') def genop_guard_guard_nonnull_class(self, ign_1, guard_op, - addr, locs, ign_2): - mc = self._start_block() - mc.CMP(locs[0], imm8(1)) - mc.JB(rel8_patched_later) - jb_location = mc.get_relative_pos() - self._cmp_guard_class(mc, locs) + guard_token, locs, ign_2): + self.mc.ensure_bytes_available(256) + self.mc.CMP(locs[0], imm(1)) + # Patched below + self.mc.J_il8(rx86.Conditions['B'], 0) + jb_location = self.mc.get_relative_pos() + self._cmp_guard_class(locs) # patch the JB above - offset = mc.get_relative_pos() - jb_location + offset = self.mc.get_relative_pos() - jb_location assert 0 < offset <= 127 - mc.overwrite(jb_location-1, [chr(offset)]) - self._stop_block() + self.mc.overwrite(jb_location-1, [chr(offset)]) # - return self.implement_guard(addr, self.mc.JNE) + return self.implement_guard(guard_token, 'NE') def implement_guard_recovery(self, guard_opnum, faildescr, failargs, fail_locs): exc = (guard_opnum == rop.GUARD_EXCEPTION or guard_opnum == rop.GUARD_NO_EXCEPTION or guard_opnum == rop.GUARD_NOT_FORCED) - return self.generate_quick_failure(faildescr, failargs, fail_locs, exc) + desc_bytes = self.failure_recovery_description(failargs, fail_locs) + return GuardToken(faildescr, failargs, fail_locs, exc, desc_bytes) - def generate_quick_failure(self, faildescr, failargs, fail_locs, exc): + def generate_quick_failure(self, mc, faildescr, failargs, fail_locs, exc, desc_bytes): """Generate the initial code for handling a failure. We try to keep it as compact as possible. The idea is that this code is executed at most once (and very often, zero times); when @@ -1026,38 +1282,43 @@ really handle recovery from this particular failure. """ fail_index = self.cpu.get_fail_descr_number(faildescr) - bytes_needed = 20 + 5 * len(failargs) # conservative estimate - if self.mc2.bytes_free() < bytes_needed: - self.mc2.make_new_mc() - mc = self.mc2._mc addr = mc.tell() withfloats = False for box in failargs: if box is not None and box.type == FLOAT: withfloats = True break - mc.CALL(rel32(self.failure_recovery_code[exc + 2 * withfloats])) + mc.CALL(imm(self.failure_recovery_code[exc + 2 * withfloats])) # write tight data that describes the failure recovery faildescr._x86_failure_recovery_bytecode = mc.tell() - self.write_failure_recovery_description(mc, failargs, fail_locs) + for byte in desc_bytes: + mc.writechr(ord(byte)) # write the fail_index too - mc.write(packimm32(fail_index)) + mc.writeimm32(fail_index) # for testing the decoding, write a final byte 0xCC if not we_are_translated(): mc.writechr(0xCC) faildescr._x86_debug_faillocs = [loc for loc in fail_locs if loc is not None] + + # Make sure the recovery stub is at least 16 bytes long (for the + # case where we overwrite the recovery stub with a 64-bit absolute + # jump) + while mc.tell() - addr < 16: + mc.writechr(0x00) return addr DESCR_REF = 0x00 DESCR_INT = 0x01 DESCR_FLOAT = 0x02 DESCR_SPECIAL = 0x03 - CODE_FROMSTACK = 4*8 + # XXX: 4*8 works on i386, should we optimize for that case? + CODE_FROMSTACK = 4*16 CODE_STOP = 0 | DESCR_SPECIAL CODE_HOLE = 4 | DESCR_SPECIAL - def write_failure_recovery_description(self, mc, failargs, locs): + def failure_recovery_description(self, failargs, locs): + desc_bytes = [] for i in range(len(failargs)): arg = failargs[i] if arg is not None: @@ -1070,24 +1331,30 @@ else: raise AssertionError("bogus kind") loc = locs[i] - if isinstance(loc, MODRM): + if isinstance(loc, StackLoc): n = self.CODE_FROMSTACK//4 + loc.position else: - assert isinstance(loc, REG) - n = loc.op + assert isinstance(loc, RegLoc) + n = loc.value n = kind + 4*n while n > 0x7F: - mc.writechr((n & 0x7F) | 0x80) + desc_bytes.append(chr((n & 0x7F) | 0x80)) n >>= 7 else: n = self.CODE_HOLE - mc.writechr(n) - mc.writechr(self.CODE_STOP) + desc_bytes.append(chr(n)) + desc_bytes.append(chr(self.CODE_STOP)) # assert that the fail_boxes lists are big enough assert len(failargs) <= self.fail_boxes_int.SIZE + return desc_bytes + + def write_failure_recovery_description(self, mc, failargs, locs): + for byte in self.failure_recovery_description(failargs, locs): + mc.writechr(ord(byte)) def rebuild_faillocs_from_descr(self, bytecode): from pypy.jit.backend.x86.regalloc import X86FrameManager + descr_to_box_type = [REF, INT, FLOAT] bytecode = rffi.cast(rffi.UCHARP, bytecode) arglocs = [] while 1: @@ -1112,7 +1379,7 @@ size = 2 else: size = 1 - loc = X86FrameManager.frame_pos(code, size) + loc = X86FrameManager.frame_pos(code, descr_to_box_type[kind]) elif code == self.CODE_STOP: break elif code == self.CODE_HOLE: @@ -1122,16 +1389,16 @@ kind = code & 3 code >>= 2 if kind == self.DESCR_FLOAT: - loc = xmm_registers[code] + loc = regloc.XMMREGLOCS[code] else: - loc = registers[code] + loc = regloc.REGLOCS[code] arglocs.append(loc) return arglocs[:] @rgc.no_collect def grab_frame_values(self, bytecode, frame_addr, allregisters): # no malloc allowed here!! - self.fail_ebp = allregisters[16 + ebp.op] + self.fail_ebp = allregisters[16 + ebp.value] num = 0 value_hi = 0 while 1: @@ -1154,7 +1421,7 @@ code = (code - self.CODE_FROMSTACK) >> 2 stackloc = frame_addr + get_ebp_ofs(code) value = rffi.cast(rffi.LONGP, stackloc)[0] - if kind == self.DESCR_FLOAT: + if kind == self.DESCR_FLOAT and WORD == 4: value_hi = value value = rffi.cast(rffi.LONGP, stackloc - 4)[0] else: @@ -1168,8 +1435,11 @@ break code >>= 2 if kind == self.DESCR_FLOAT: - value = allregisters[2*code] - value_hi = allregisters[2*code + 1] + if WORD == 4: + value = allregisters[2*code] + value_hi = allregisters[2*code + 1] + else: + value = allregisters[code] else: value = allregisters[16 + code] @@ -1180,7 +1450,8 @@ tgt = self.fail_boxes_ptr.get_addr_for_num(num) elif kind == self.DESCR_FLOAT: tgt = self.fail_boxes_float.get_addr_for_num(num) - rffi.cast(rffi.LONGP, tgt)[1] = value_hi + if WORD == 4: + rffi.cast(rffi.LONGP, tgt)[1] = value_hi else: assert 0, "bogus kind" rffi.cast(rffi.LONGP, tgt)[0] = value @@ -1189,7 +1460,8 @@ if not we_are_translated(): assert bytecode[4] == 0xCC self.fail_boxes_count = num - fail_index = rffi.cast(rffi.LONGP, bytecode)[0] + fail_index = rffi.cast(rffi.INTP, bytecode)[0] + fail_index = rffi.cast(lltype.Signed, fail_index) return fail_index def setup_failure_recovery(self): @@ -1200,8 +1472,8 @@ # original value of the registers, optionally the original # value of XMM registers, and finally a reference to the # recovery bytecode. See _build_failure_recovery() for details. - stack_at_ebp = registers[ebp.op] - bytecode = rffi.cast(rffi.UCHARP, registers[8]) + stack_at_ebp = registers[ebp.value] + bytecode = rffi.cast(rffi.UCHARP, registers[self.cpu.NUM_REGS]) allregisters = rffi.ptradd(registers, -16) return self.grab_frame_values(bytecode, stack_at_ebp, allregisters) @@ -1216,23 +1488,23 @@ self.failure_recovery_func) failure_recovery_func = rffi.cast(lltype.Signed, failure_recovery_func) - mc = self.mc2._mc + mc = self.mc._mc # Assume that we are called at the beginning, when there is no risk # that 'mc' runs out of space. Checked by asserts in mc.write(). recovery_addr = mc.tell() - mc.PUSH(edi) - mc.PUSH(esi) - mc.PUSH(ebp) - mc.PUSH(esp) # <-- not really used, but needed to take up the space - mc.PUSH(ebx) - mc.PUSH(edx) - mc.PUSH(ecx) - mc.PUSH(eax) - mc.MOV(esi, esp) + + # Push all general purpose registers + for gpr in range(self.cpu.NUM_REGS-1, -1, -1): + mc.PUSH_r(gpr) + + # ebx/rbx is callee-save in both i386 and x86-64 + mc.MOV_rr(ebx.value, esp.value) + if withfloats: - mc.SUB(esp, imm(8*8)) - for i in range(8): - mc.MOVSD(mem64(esp, 8*i), xmm_registers[i]) + # Push all float registers + mc.SUB_ri(esp.value, self.cpu.NUM_REGS*8) + for i in range(self.cpu.NUM_REGS): + mc.MOVSD_sx(8*i, i) # we call a provided function that will # - call our on_leave_jitted_hook which will mark @@ -1240,7 +1512,7 @@ # avoid unwarranted freeing # - optionally save exception depending on the flag addr = self.cpu.get_on_leave_jitted_int(save_exception=exc) - mc.CALL(rel32(addr)) + mc.CALL(imm(addr)) # the following call saves all values from the stack and from # registers to the right 'fail_boxes_' location. @@ -1250,50 +1522,58 @@ # bytecode, pushed just before by the CALL instruction written by # generate_quick_failure(). XXX misaligned stack in the call, but # it's ok because failure_recovery_func is not calling anything more - mc.PUSH(esi) - mc.CALL(rel32(failure_recovery_func)) + + # XXX + if IS_X86_32: + mc.PUSH_r(ebx.value) + elif IS_X86_64: + mc.MOV_rr(edi.value, ebx.value) + # XXX: Correct to only align the stack on 64-bit? + mc.AND_ri(esp.value, -16) + else: + raise AssertionError("Shouldn't happen") + + mc.CALL(imm(failure_recovery_func)) # returns in eax the fail_index # now we return from the complete frame, which starts from - # _assemble_bootstrap_code(). The LEA below throws away most - # of the frame, including all the PUSHes that we did just above. - mc.LEA(esp, addr_add(ebp, imm(-3 * WORD))) - mc.POP(edi) # [ebp-12] - mc.POP(esi) # [ebp-8] - mc.POP(ebx) # [ebp-4] - mc.POP(ebp) # [ebp] - mc.RET() - self.mc2.done() + # _assemble_bootstrap_code(). The LEA in _call_footer below throws + # away most of the frame, including all the PUSHes that we did just + # above. + + self._call_footer() + self.mc.done() self.failure_recovery_code[exc + 2 * withfloats] = recovery_addr def generate_failure(self, fail_index, locs, exc, locs_are_ref): - mc = self.mc + self.mc._mc.begin_reuse_scratch_register() for i in range(len(locs)): loc = locs[i] - if isinstance(loc, REG): - if loc.width == 8: + if isinstance(loc, RegLoc): + if loc.is_xmm: adr = self.fail_boxes_float.get_addr_for_num(i) - mc.MOVSD(heap64(adr), loc) + self.mc.MOVSD(heap(adr), loc) else: if locs_are_ref[i]: adr = self.fail_boxes_ptr.get_addr_for_num(i) else: adr = self.fail_boxes_int.get_addr_for_num(i) - mc.MOV(heap(adr), loc) + self.mc.MOV(heap(adr), loc) for i in range(len(locs)): loc = locs[i] - if not isinstance(loc, REG): - if loc.width == 8: - mc.MOVSD(xmm0, loc) + if not isinstance(loc, RegLoc): + if isinstance(loc, StackLoc) and loc.type == FLOAT: + self.mc.MOVSD_xb(xmm0.value, loc.value) adr = self.fail_boxes_float.get_addr_for_num(i) - mc.MOVSD(heap64(adr), xmm0) + self.mc.MOVSD(heap(adr), xmm0) else: if locs_are_ref[i]: adr = self.fail_boxes_ptr.get_addr_for_num(i) else: adr = self.fail_boxes_int.get_addr_for_num(i) - mc.MOV(eax, loc) - mc.MOV(heap(adr), eax) + self.mc.MOV(eax, loc) + self.mc.MOV(heap(adr), eax) + self.mc._mc.end_reuse_scratch_register() # we call a provided function that will # - call our on_leave_jitted_hook which will mark @@ -1301,28 +1581,31 @@ # avoid unwarranted freeing # - optionally save exception depending on the flag addr = self.cpu.get_on_leave_jitted_int(save_exception=exc) - mc.CALL(rel32(addr)) + self.mc.CALL(imm(addr)) + + self.mc.MOV_ri(eax.value, fail_index) - mc.LEA(esp, addr_add(ebp, imm(-3 * WORD))) - mc.MOV(eax, imm(fail_index)) - mc.POP(edi) # [ebp-12] - mc.POP(esi) # [ebp-8] - mc.POP(ebx) # [ebp-4] - mc.POP(ebp) # [ebp] - mc.RET() - - @specialize.arg(2) - def implement_guard(self, addr, emit_jump): - emit_jump(rel32(addr)) + # exit function + self._call_footer() + + def implement_guard(self, guard_token, condition=None): + self.mc.reserve_bytes(guard_token.recovery_stub_size()) + self.pending_guard_tokens.append(guard_token) + # XXX: These jumps are patched later, the self.mc.tell() are just + # dummy values + if condition: + self.mc.J_il(rx86.Conditions[condition], self.mc.tell()) + else: + self.mc.JMP_l(self.mc.tell()) return self.mc.tell() - 4 def genop_call(self, op, arglocs, resloc): sizeloc = arglocs[0] - assert isinstance(sizeloc, IMM32) + assert isinstance(sizeloc, ImmedLoc) size = sizeloc.value if isinstance(op.args[0], Const): - x = rel32(op.args[0].getint()) + x = imm(op.args[0].getint()) else: x = arglocs[1] if x is eax: @@ -1332,35 +1615,35 @@ self._emit_call(x, arglocs, 2, tmp=tmp) - if isinstance(resloc, MODRM64): - self.mc.FSTP(resloc) + if isinstance(resloc, StackLoc) and resloc.width == 8 and IS_X86_32: + self.mc.FSTP_b(resloc.value) elif size == 1: - self.mc.AND(eax, imm(0xff)) + self.mc.AND_ri(eax.value, 0xff) elif size == 2: - self.mc.AND(eax, imm(0xffff)) + self.mc.AND_ri(eax.value, 0xffff) - def genop_guard_call_may_force(self, op, guard_op, addr, + def genop_guard_call_may_force(self, op, guard_op, guard_token, arglocs, result_loc): faildescr = guard_op.descr fail_index = self.cpu.get_fail_descr_number(faildescr) - self.mc.MOV(mem(ebp, FORCE_INDEX_OFS), imm(fail_index)) + self.mc.MOV_bi(FORCE_INDEX_OFS, fail_index) self.genop_call(op, arglocs, result_loc) - self.mc.CMP(mem(ebp, FORCE_INDEX_OFS), imm(0)) - return self.implement_guard(addr, self.mc.JL) + self.mc.CMP_bi(FORCE_INDEX_OFS, 0) + return self.implement_guard(guard_token, 'L') - def genop_guard_call_assembler(self, op, guard_op, addr, + def genop_guard_call_assembler(self, op, guard_op, guard_token, arglocs, result_loc): faildescr = guard_op.descr fail_index = self.cpu.get_fail_descr_number(faildescr) - self.mc.MOV(mem(ebp, FORCE_INDEX_OFS), imm(fail_index)) + self.mc.MOV_bi(FORCE_INDEX_OFS, fail_index) descr = op.descr assert isinstance(descr, LoopToken) assert len(arglocs) - 2 == len(descr._x86_arglocs[0]) # # Write a call to the direct_bootstrap_code of the target assembler - self._emit_call(rel32(descr._x86_direct_bootstrap_code), arglocs, 2, + self._emit_call(imm(descr._x86_direct_bootstrap_code), arglocs, 2, tmp=eax) - mc = self._start_block() + self.mc.ensure_bytes_available(256) if op.result is None: assert result_loc is None value = self.cpu.done_with_this_frame_void_v @@ -1376,26 +1659,27 @@ value = self.cpu.done_with_this_frame_float_v else: raise AssertionError(kind) - mc.CMP(eax, imm(value)) - mc.JE(rel8_patched_later) # goto B if we get 'done_with_this_frame' - je_location = mc.get_relative_pos() + self.mc.CMP_ri(eax.value, value) + # patched later + self.mc.J_il8(rx86.Conditions['E'], 0) # goto B if we get 'done_with_this_frame' + je_location = self.mc.get_relative_pos() # # Path A: use assembler_helper_adr jd = descr.outermost_jitdriver_sd assert jd is not None asm_helper_adr = self.cpu.cast_adr_to_int(jd.assembler_helper_adr) - self._emit_call(rel32(asm_helper_adr), [eax, arglocs[1]], 0, - tmp=ecx, force_mc=True, mc=mc) - if isinstance(result_loc, MODRM64): - mc.FSTP(result_loc) + self._emit_call(imm(asm_helper_adr), [eax, arglocs[1]], 0, + tmp=ecx) + if IS_X86_32 and isinstance(result_loc, StackLoc) and result_loc.type == FLOAT: + self.mc.FSTP_b(result_loc.value) #else: result_loc is already either eax or None, checked below - mc.JMP(rel8_patched_later) # done - jmp_location = mc.get_relative_pos() + self.mc.JMP_l8(0) # jump to done, patched later + jmp_location = self.mc.get_relative_pos() # # Path B: fast path. Must load the return value, and reset the token offset = jmp_location - je_location assert 0 < offset <= 127 - mc.overwrite(je_location - 1, [chr(offset)]) + self.mc.overwrite(je_location - 1, [chr(offset)]) # # Reset the vable token --- XXX really too much special logic here:-( if jd.index_of_virtualizable >= 0: @@ -1403,8 +1687,8 @@ fielddescr = jd.vable_token_descr assert isinstance(fielddescr, BaseFieldDescr) ofs = fielddescr.offset - mc.MOV(eax, arglocs[1]) - mc.MOV(addr_add(eax, imm(ofs)), imm(0)) + self.mc.MOV(eax, arglocs[1]) + self.mc.MOV_mi((eax.value, ofs), 0) # in the line above, TOKEN_NONE = 0 # if op.result is not None: @@ -1413,27 +1697,26 @@ if kind == FLOAT: xmmtmp = X86XMMRegisterManager.all_regs[0] adr = self.fail_boxes_float.get_addr_for_num(0) - mc.MOVSD(xmmtmp, heap64(adr)) - mc.MOVSD(result_loc, xmmtmp) + self.mc.MOVSD(xmmtmp, heap(adr)) + self.mc.MOVSD(result_loc, xmmtmp) else: assert result_loc is eax if kind == INT: adr = self.fail_boxes_int.get_addr_for_num(0) - mc.MOV(eax, heap(adr)) + self.mc.MOV(eax, heap(adr)) elif kind == REF: adr = self.fail_boxes_ptr.get_addr_for_num(0) - mc.XOR(eax, eax) - mc.XCHG(eax, heap(adr)) + self.mc.XOR_rr(eax.value, eax.value) + self.mc.XCHG(eax, heap(adr)) else: raise AssertionError(kind) # # Here we join Path A and Path B again - offset = mc.get_relative_pos() - jmp_location + offset = self.mc.get_relative_pos() - jmp_location assert 0 <= offset <= 127 - mc.overwrite(jmp_location - 1, [chr(offset)]) - self._stop_block() - self.mc.CMP(mem(ebp, FORCE_INDEX_OFS), imm(0)) - return self.implement_guard(addr, self.mc.JL) + self.mc.overwrite(jmp_location - 1, [chr(offset)]) + self.mc.CMP_bi(FORCE_INDEX_OFS, 0) + return self.implement_guard(guard_token, 'L') def genop_discard_cond_call_gc_wb(self, op, arglocs): # use 'mc._mc' directly instead of 'mc', to avoid @@ -1443,31 +1726,41 @@ cls = self.cpu.gc_ll_descr.has_write_barrier_class() assert cls is not None and isinstance(descr, cls) loc_base = arglocs[0] - mc = self._start_block() - mc.TEST(mem8(loc_base, descr.jit_wb_if_flag_byteofs), - imm8(descr.jit_wb_if_flag_singlebyte)) - mc.JZ(rel8_patched_later) - jz_location = mc.get_relative_pos() + self.mc.ensure_bytes_available(256) + self.mc.TEST8_mi((loc_base.value, descr.jit_wb_if_flag_byteofs), + descr.jit_wb_if_flag_singlebyte) + self.mc.J_il8(rx86.Conditions['Z'], 0) # patched later + jz_location = self.mc.get_relative_pos() # the following is supposed to be the slow path, so whenever possible # we choose the most compact encoding over the most efficient one. for i in range(len(arglocs)-1, -1, -1): - mc.PUSH(arglocs[i]) + self.mc.PUSH(arglocs[i]) + + if IS_X86_64: + # We clobber these registers to pass the arguments, but that's + # okay, because consider_cond_call_gc_wb makes sure that any + # caller-save registers with values in them are present in arglocs, + # so they are saved on the stack above and restored below + self.mc.MOV_rs(edi.value, 0) + self.mc.MOV_rs(esi.value, 8) + # misaligned stack in the call, but it's ok because the write barrier # is not going to call anything more. Also, this assumes that the # write barrier does not touch the xmm registers. - mc.CALL(rel32(descr.get_write_barrier_fn(self.cpu))) + self.mc.CALL(imm(descr.get_write_barrier_fn(self.cpu))) for i in range(len(arglocs)): loc = arglocs[i] - assert isinstance(loc, REG) - mc.POP(loc) + assert isinstance(loc, RegLoc) + self.mc.POP(loc) # patch the JZ above - offset = mc.get_relative_pos() - jz_location + offset = self.mc.get_relative_pos() - jz_location assert 0 < offset <= 127 - mc.overwrite(jz_location-1, [chr(offset)]) - self._stop_block() + self.mc.overwrite(jz_location-1, [chr(offset)]) def genop_force_token(self, op, arglocs, resloc): - self.mc.LEA(resloc, mem(ebp, FORCE_INDEX_OFS)) + # RegAlloc.consider_force_token ensures this: + assert isinstance(resloc, RegLoc) + self.mc.LEA_rb(resloc.value, FORCE_INDEX_OFS) def not_implemented_op_discard(self, op, arglocs): msg = "not implemented operation: %s" % op.getopname() @@ -1495,17 +1788,16 @@ return loop_token._x86_arglocs def closing_jump(self, loop_token): - self.mc.JMP(rel32(loop_token._x86_loop_code)) + self.mc.JMP(imm(loop_token._x86_loop_code)) def malloc_cond_fixedsize(self, nursery_free_adr, nursery_top_adr, size, tid): - # don't use self.mc - mc = self._start_block() - mc.MOV(eax, heap(nursery_free_adr)) - mc.LEA(edx, addr_add(eax, imm(size))) - mc.CMP(edx, heap(nursery_top_adr)) - mc.JNA(rel8_patched_later) - jmp_adr = mc.get_relative_pos() + self.mc.ensure_bytes_available(256) + self.mc.MOV(eax, heap(nursery_free_adr)) + self.mc.LEA_rm(edx.value, (eax.value, size)) + self.mc.CMP(edx, heap(nursery_top_adr)) + self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later + jmp_adr = self.mc.get_relative_pos() # See comments in _build_malloc_fixedsize_slowpath for the # details of the two helper functions that we are calling below. @@ -1521,17 +1813,16 @@ # reserve room for the argument to the real malloc and the # 8 saved XMM regs self._regalloc.reserve_param(1+16) - mc.CALL(rel32(slowpath_addr1)) + self.mc.CALL(imm(slowpath_addr1)) self.mark_gc_roots() slowpath_addr2 = self.malloc_fixedsize_slowpath2 - mc.CALL(rel32(slowpath_addr2)) + self.mc.CALL(imm(slowpath_addr2)) - offset = mc.get_relative_pos() - jmp_adr + offset = self.mc.get_relative_pos() - jmp_adr assert 0 < offset <= 127 - mc.overwrite(jmp_adr-1, [chr(offset)]) - mc.MOV(addr_add(eax, imm(0)), imm(tid)) - mc.MOV(heap(nursery_free_adr), edx) - self._stop_block() + self.mc.overwrite(jmp_adr-1, [chr(offset)]) + self.mc.MOV_mi((eax.value, 0), tid) + self.mc.MOV(heap(nursery_free_adr), edx) genop_discard_list = [Assembler386.not_implemented_op_discard] * rop._LAST genop_list = [Assembler386.not_implemented_op] * rop._LAST @@ -1551,32 +1842,20 @@ num = getattr(rop, opname.upper()) genop_list[num] = value -def new_addr_add(heap, mem, memsib): - def addr_add(reg_or_imm1, reg_or_imm2, offset=0, scale=0): - if isinstance(reg_or_imm1, IMM32): - if isinstance(reg_or_imm2, IMM32): - return heap(reg_or_imm1.value + offset + - (reg_or_imm2.value << scale)) - else: - return memsib(None, reg_or_imm2, scale, reg_or_imm1.value + offset) - else: - if isinstance(reg_or_imm2, IMM32): - return mem(reg_or_imm1, offset + (reg_or_imm2.value << scale)) - else: - return memsib(reg_or_imm1, reg_or_imm2, scale, offset) - return addr_add - -addr8_add = new_addr_add(heap8, mem8, memSIB8) -addr_add = new_addr_add(heap, mem, memSIB) -addr64_add = new_addr_add(heap64, mem64, memSIB64) - -def addr_add_const(reg_or_imm1, offset): - if isinstance(reg_or_imm1, IMM32): - return heap(reg_or_imm1.value + offset) - else: - return mem(reg_or_imm1, offset) - def round_up_to_4(size): if size < 4: return 4 return size + +# XXX: ri386 migration shims: +def addr_add(reg_or_imm1, reg_or_imm2, offset=0, scale=0): + return AddressLoc(reg_or_imm1, reg_or_imm2, scale, offset) + +def addr_add_const(reg_or_imm1, offset): + return AddressLoc(reg_or_imm1, ImmedLoc(0), 0, offset) + +def mem(loc, offset): + return AddressLoc(loc, ImmedLoc(0), 0, offset) + +def heap(addr): + return AddressLoc(ImmedLoc(addr), ImmedLoc(0), 0, 0) Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/codebuf.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/codebuf.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/codebuf.py Wed Aug 4 16:53:45 2010 @@ -2,12 +2,20 @@ import os, sys from pypy.rpython.lltypesystem import lltype, rffi from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.jit.backend.x86.ri386 import I386CodeBuilder +from pypy.jit.backend.x86.rx86 import X86_32_CodeBuilder, X86_64_CodeBuilder +from pypy.jit.backend.x86.regloc import LocationCodeBuilder from pypy.rlib.rmmap import PTR, alloc, free from pypy.rlib.debug import make_sure_not_resized +from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 +from pypy.rlib.objectmodel import we_are_translated +# XXX: Seems nasty to change the superclass of InMemoryCodeBuilder like this +if IS_X86_32: + codebuilder_cls = X86_32_CodeBuilder +elif IS_X86_64: + codebuilder_cls = X86_64_CodeBuilder -class InMemoryCodeBuilder(I386CodeBuilder): +class InMemoryCodeBuilder(codebuilder_cls, LocationCodeBuilder): _last_dump_start = 0 def __init__(self, start, end): @@ -31,13 +39,15 @@ def write(self, listofchars): self._pos = self.overwrite(self._pos, listofchars) - def writechr(self, n): - # purely for performance: don't make the one-element list [chr(n)] + def writechar(self, char): pos = self._pos assert pos + 1 <= self._size - self._data[pos] = chr(n) + self._data[pos] = char self._pos = pos + 1 + def writechr(self, n): + self.writechar(chr(n)) + def get_relative_pos(self): return self._pos @@ -50,11 +60,6 @@ self._pos = pos self._last_dump_start = pos - def execute(self, arg1, arg2): - # XXX old testing stuff - fnptr = rffi.cast(lltype.Ptr(BINARYFN), self._data) - return fnptr(arg1, arg2) - def done(self): # normally, no special action is needed here if machine_code_dumper.enabled: @@ -77,9 +82,6 @@ valgrind.discard_translations(self._data, self._size) -BINARYFN = lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed) - - class MachineCodeDumper: enabled = True log_fd = -1 @@ -107,7 +109,10 @@ return False # log the executable name from pypy.jit.backend.hlinfo import highleveljitinfo - os.write(self.log_fd, 'BACKEND i386\n') + if IS_X86_32: + os.write(self.log_fd, 'BACKEND x86\n') + elif IS_X86_64: + os.write(self.log_fd, 'BACKEND x86_64\n') if highleveljitinfo.sys_executable: os.write(self.log_fd, 'SYS_EXECUTABLE %s\n' % ( highleveljitinfo.sys_executable,)) @@ -137,6 +142,12 @@ def __init__(self, map_size): data = alloc(map_size) + if IS_X86_64 and not we_are_translated(): + # Hack to make sure that mcs are not within 32-bits of one + # another for testing purposes + from pypy.rlib.rmmap import hint + hint.pos += 0xFFFFFFFF + self._init(data, map_size) def __del__(self): Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/jump.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/jump.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/jump.py Wed Aug 4 16:53:45 2010 @@ -1,23 +1,6 @@ import sys from pypy.tool.pairtype import extendabletype -from pypy.jit.backend.x86.ri386 import * - -class __extend__(OPERAND): - __metaclass__ = extendabletype - def _getregkey(self): - raise AssertionError("should only happen to registers and frame " - "positions") - -class __extend__(REG): - __metaclass__ = extendabletype - def _getregkey(self): - return ~self.op - -class __extend__(MODRM): - __metaclass__ = extendabletype - def _getregkey(self): - return self.position - +from pypy.jit.backend.x86.regloc import ImmedLoc, StackLoc def remap_frame_layout(assembler, src_locations, dst_locations, tmpreg): pending_dests = len(dst_locations) @@ -27,7 +10,7 @@ srccount[dst._getregkey()] = 0 for i in range(len(dst_locations)): src = src_locations[i] - if isinstance(src, IMM32): + if isinstance(src, ImmedLoc): continue key = src._getregkey() if key in srccount: @@ -46,7 +29,7 @@ srccount[key] = -1 # means "it's done" pending_dests -= 1 src = src_locations[i] - if not isinstance(src, IMM32): + if not isinstance(src, ImmedLoc): key = src._getregkey() if key in srccount: srccount[key] -= 1 @@ -80,7 +63,7 @@ assert pending_dests == 0 def _move(assembler, src, dst, tmpreg): - if isinstance(dst, MODRM) and isinstance(src, MODRM): + if dst.is_memory_reference() and src.is_memory_reference(): assembler.regalloc_mov(src, tmpreg) src = tmpreg assembler.regalloc_mov(src, dst) Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/regalloc.py Wed Aug 4 16:53:45 2010 @@ -5,7 +5,7 @@ from pypy.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr, ResOperation, BoxPtr, LoopToken, INT, REF, FLOAT) -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.regloc import * from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi, rstr from pypy.rlib.objectmodel import we_are_translated from pypy.rlib import rgc @@ -17,16 +17,7 @@ from pypy.jit.backend.llsupport.descr import BaseCallDescr, BaseSizeDescr from pypy.jit.backend.llsupport.regalloc import FrameManager, RegisterManager,\ TempBox - -WORD = 4 -FRAME_FIXED_SIZE = 5 # ebp + ebx + esi + edi + force_index = 5 words -FORCE_INDEX_OFS = -4*WORD - -width_of_type = { - INT : 1, - REF : 1, - FLOAT : 2, - } +from pypy.jit.backend.x86.arch import WORD, FRAME_FIXED_SIZE, IS_X86_32, IS_X86_64 class X86RegisterManager(RegisterManager): @@ -50,6 +41,12 @@ print "convert_to_imm: got a %s" % c raise AssertionError +class X86_64_RegisterManager(X86RegisterManager): + # r11 omitted because it's used as scratch + all_regs = [eax, ecx, edx, ebx, esi, edi, r8, r9, r10, r12, r13, r14, r15] + no_lower_byte_regs = [] + save_around_call_regs = [eax, ecx, edx, esi, edi, r8, r9, r10] + class FloatConstants(object): BASE_CONSTANT_SIZE = 1000 @@ -80,7 +77,6 @@ all_regs = [xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7] # we never need lower byte I hope save_around_call_regs = all_regs - reg_width = 2 def __init__(self, longevity, frame_manager=None, assembler=None): RegisterManager.__init__(self, longevity, frame_manager=frame_manager, @@ -94,27 +90,35 @@ def convert_to_imm(self, c): adr = self.float_constants.record_float(c.getfloat()) - return heap64(adr) + return AddressLoc(ImmedLoc(adr), ImmedLoc(0), 0, 0) def after_call(self, v): # the result is stored in st0, but we don't have this around, # so genop_call will move it to some frame location immediately # after the call - return self.frame_manager.loc(v, 2) + return self.frame_manager.loc(v) -class X86FrameManager(FrameManager): +class X86_64_XMMRegisterManager(X86XMMRegisterManager): + # xmm15 reserved for scratch use + all_regs = [xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14] + save_around_call_regs = all_regs + + def call_result_location(self, v): + return xmm0 + + def after_call(self, v): + # We use RegisterManager's implementation, since X86XMMRegisterManager + # places the result on the stack, which we don't need to do when the + # calling convention places the result in xmm0 + return RegisterManager.after_call(self, v) +class X86FrameManager(FrameManager): @staticmethod - def frame_pos(i, size): - if size == 1: - res = mem(ebp, get_ebp_ofs(i)) - elif size == 2: - res = mem64(ebp, get_ebp_ofs(i + 1)) - else: - print "Unimplemented size %d" % i - raise NotImplementedError("unimplemented size %d" % i) - res.position = i - return res + def frame_pos(i, box_type): + if IS_X86_32 and box_type == FLOAT: + return StackLoc(i, get_ebp_ofs(i+1), 2, box_type) + else: + return StackLoc(i, get_ebp_ofs(i), 1, box_type) class RegAlloc(object): exc = False @@ -135,11 +139,21 @@ # compute longevity of variables longevity = self._compute_vars_longevity(inputargs, operations) self.longevity = longevity - self.rm = X86RegisterManager(longevity, - frame_manager = self.fm, - assembler = self.assembler) - self.xrm = X86XMMRegisterManager(longevity, frame_manager = self.fm, - assembler = self.assembler) + # XXX + if cpu.WORD == 4: + gpr_reg_mgr_cls = X86RegisterManager + xmm_reg_mgr_cls = X86XMMRegisterManager + elif cpu.WORD == 8: + gpr_reg_mgr_cls = X86_64_RegisterManager + xmm_reg_mgr_cls = X86_64_XMMRegisterManager + else: + raise AssertionError("Word size should be 4 or 8") + + self.rm = gpr_reg_mgr_cls(longevity, + frame_manager = self.fm, + assembler = self.assembler) + self.xrm = xmm_reg_mgr_cls(longevity, frame_manager = self.fm, + assembler = self.assembler) def prepare_loop(self, inputargs, operations, looptoken): self._prepare(inputargs, operations) @@ -184,7 +198,7 @@ if reg: loc = reg else: - loc = self.fm.loc(arg, width_of_type[arg.type]) + loc = self.fm.loc(arg) if arg.type == FLOAT: floatlocs[i] = loc else: @@ -252,23 +266,23 @@ arg = inputargs[i] i += 1 if arg.type == FLOAT: - if isinstance(loc, REG): + if isinstance(loc, RegLoc): self.xrm.reg_bindings[arg] = loc used[loc] = None else: self.fm.frame_bindings[arg] = loc else: - if isinstance(loc, REG): + if isinstance(loc, RegLoc): self.rm.reg_bindings[arg] = loc used[loc] = None else: self.fm.frame_bindings[arg] = loc self.rm.free_regs = [] - for reg in X86RegisterManager.all_regs: + for reg in self.rm.all_regs: if reg not in used: self.rm.free_regs.append(reg) self.xrm.free_regs = [] - for reg in X86XMMRegisterManager.all_regs: + for reg in self.xrm.all_regs: if reg not in used: self.xrm.free_regs.append(reg) # note: we need to make a copy of inputargs because possibly_free_vars @@ -646,7 +660,7 @@ vable_index = jd.index_of_virtualizable if vable_index >= 0: self.rm._sync_var(op.args[vable_index]) - vable = self.fm.loc(op.args[vable_index], 1) + vable = self.fm.loc(op.args[vable_index]) else: vable = imm(0) self._call(op, [imm(size), vable] + @@ -670,7 +684,7 @@ # function, a GC write barrier, is known not to touch them. # See remember_young_pointer() in rpython/memory/gc/generation.py. for v, reg in self.rm.reg_bindings.items(): - if ((reg is eax or reg is ecx or reg is edx) + if (reg in self.rm.save_around_call_regs and self.rm.stays_alive(v)): arglocs.append(reg) self.PerformDiscard(op, arglocs) @@ -809,7 +823,7 @@ def consider_setfield_gc(self, op): ofs_loc, size_loc, ptr = self._unpack_fielddescr(op.descr) - assert isinstance(size_loc, IMM32) + assert isinstance(size_loc, ImmedLoc) if size_loc.value == 1: need_lower_byte = True else: @@ -950,7 +964,7 @@ shape = gcrootmap.get_basic_shape() for v, val in self.fm.frame_bindings.items(): if (isinstance(v, BoxPtr) and self.rm.stays_alive(v)): - assert isinstance(val, MODRM) + assert isinstance(val, StackLoc) gcrootmap.add_ebp_offset(shape, get_ebp_ofs(val.position)) for v, reg in self.rm.reg_bindings.items(): if reg is eax: Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/runner.py Wed Aug 4 16:53:45 2010 @@ -4,11 +4,13 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.jit.metainterp import history, compile from pypy.jit.backend.x86.assembler import Assembler386 -from pypy.jit.backend.x86.regalloc import FORCE_INDEX_OFS +from pypy.jit.backend.x86.arch import FORCE_INDEX_OFS from pypy.jit.backend.x86.profagent import ProfileAgent from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU +from pypy.jit.backend.x86 import regloc +import sys -class CPU386(AbstractLLCPU): +class AbstractX86CPU(AbstractLLCPU): debug = True supports_floats = True @@ -44,6 +46,7 @@ self.profile_agent.startup() def finish_once(self): + self.assembler.finish_once() self.profile_agent.shutdown() def compile_loop(self, inputargs, operations, looptoken): @@ -131,10 +134,28 @@ assert fail_index == fail_index_2 return faildescr +class CPU386(AbstractX86CPU): + WORD = 4 + NUM_REGS = 8 + CALLEE_SAVE_REGISTERS = [regloc.ebx, regloc.esi, regloc.edi] + FRAME_FIXED_SIZE = len(CALLEE_SAVE_REGISTERS) + 2 + + def __init__(self, *args, **kwargs): + assert sys.maxint == (2**31 - 1) + super(CPU386, self).__init__(*args, **kwargs) class CPU386_NO_SSE2(CPU386): supports_floats = False +class CPU_X86_64(AbstractX86CPU): + WORD = 8 + NUM_REGS = 16 + CALLEE_SAVE_REGISTERS = [regloc.ebx, regloc.r12, regloc.r13, regloc.r14, regloc.r15] + FRAME_FIXED_SIZE = len(CALLEE_SAVE_REGISTERS) + 2 + + def __init__(self, *args, **kwargs): + assert sys.maxint == (2**63 - 1) + super(CPU_X86_64, self).__init__(*args, **kwargs) CPU = CPU386 Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/test/conftest.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/test/conftest.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/test/conftest.py Wed Aug 4 16:53:45 2010 @@ -3,6 +3,5 @@ cpu = detect_cpu.autodetect() def pytest_runtest_setup(item): - if cpu != 'x86': - py.test.skip("x86 directory skipped: cpu is %r" % (cpu,)) - + if cpu not in ('x86', 'x86_64'): + py.test.skip("x86/x86_64 tests skipped: cpu is %r" % (cpu,)) Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_assembler.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_assembler.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_assembler.py Wed Aug 4 16:53:45 2010 @@ -1,14 +1,19 @@ -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.regloc import * from pypy.jit.backend.x86.assembler import Assembler386, MachineCodeBlockWrapper from pypy.jit.backend.x86.regalloc import X86FrameManager, get_ebp_ofs -from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat +from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, INT, REF, FLOAT from pypy.rlib.rarithmetic import intmask from pypy.rpython.lltypesystem import lltype, llmemory, rffi +from pypy.jit.backend.x86.arch import WORD, IS_X86_32, IS_X86_64 +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.regalloc import X86RegisterManager, X86_64_RegisterManager, X86XMMRegisterManager, X86_64_XMMRegisterManager +ACTUAL_CPU = getcpuclass() class FakeCPU: rtyper = None supports_floats = True + NUM_REGS = ACTUAL_CPU.NUM_REGS class FakeMC: def __init__(self, base_address=0): @@ -25,7 +30,14 @@ self.content.append(("JMP", args)) def done(self): pass + def PUSH_r(self, reg): + pass + def POP_r(self, reg): + pass +class FakeAssembler: + def write_pending_failure_recoveries(self): + pass def test_write_failure_recovery_description(): assembler = Assembler386(FakeCPU()) @@ -33,12 +45,12 @@ failargs = [BoxInt(), BoxPtr(), BoxFloat()] * 3 failargs.insert(6, None) failargs.insert(7, None) - locs = [X86FrameManager.frame_pos(0, 1), - X86FrameManager.frame_pos(1, 1), - X86FrameManager.frame_pos(10, 2), - X86FrameManager.frame_pos(100, 1), - X86FrameManager.frame_pos(101, 1), - X86FrameManager.frame_pos(110, 2), + locs = [X86FrameManager.frame_pos(0, INT), + X86FrameManager.frame_pos(1, REF), + X86FrameManager.frame_pos(10, FLOAT), + X86FrameManager.frame_pos(100, INT), + X86FrameManager.frame_pos(101, REF), + X86FrameManager.frame_pos(110, FLOAT), None, None, ebx, @@ -46,17 +58,17 @@ xmm2] assert len(failargs) == len(locs) assembler.write_failure_recovery_description(mc, failargs, locs) - nums = [Assembler386.DESCR_INT + 4*(8+0), - Assembler386.DESCR_REF + 4*(8+1), - Assembler386.DESCR_FLOAT + 4*(8+10), - Assembler386.DESCR_INT + 4*(8+100), - Assembler386.DESCR_REF + 4*(8+101), - Assembler386.DESCR_FLOAT + 4*(8+110), + nums = [Assembler386.DESCR_INT + 4*(16+0), + Assembler386.DESCR_REF + 4*(16+1), + Assembler386.DESCR_FLOAT + 4*(16+10), + Assembler386.DESCR_INT + 4*(16+100), + Assembler386.DESCR_REF + 4*(16+101), + Assembler386.DESCR_FLOAT + 4*(16+110), Assembler386.CODE_HOLE, Assembler386.CODE_HOLE, - Assembler386.DESCR_INT + 4*ebx.op, - Assembler386.DESCR_REF + 4*esi.op, - Assembler386.DESCR_FLOAT + 4*xmm2.op] + Assembler386.DESCR_INT + 4*ebx.value, + Assembler386.DESCR_REF + 4*esi.value, + Assembler386.DESCR_FLOAT + 4*xmm2.value] double_byte_nums = [] for num in nums[3:6]: double_byte_nums.append((num & 0x7F) | 0x80) @@ -94,6 +106,9 @@ return lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S)) def get_random_float(): + # Returns , , + # NB: on 64-bit, will be the entire float and + # will be random garbage from malloc! assert withfloats value = random.random() - 0.5 # make sure it fits into 64 bits @@ -101,9 +116,16 @@ rffi.cast(rffi.DOUBLEP, tmp)[0] = value return rffi.cast(rffi.DOUBLEP, tmp)[0], tmp[0], tmp[1] + if IS_X86_32: + main_registers = X86RegisterManager.all_regs + xmm_registers = X86XMMRegisterManager.all_regs + elif IS_X86_64: + main_registers = X86_64_RegisterManager.all_regs + xmm_registers = X86_64_XMMRegisterManager.all_regs + # memory locations: 26 integers, 26 pointers, 26 floats # main registers: half of them as signed and the other half as ptrs - # xmm registers: all floats, from xmm0 to xmm7 + # xmm registers: all floats, from xmm0 to xmm(7|15) # holes: 8 locations = [] baseloc = 4 @@ -117,18 +139,17 @@ content = ([('int', locations.pop()) for _ in range(26)] + [('ptr', locations.pop()) for _ in range(26)] + [(['int', 'ptr'][random.randrange(0, 2)], reg) - for reg in [eax, ecx, edx, ebx, esi, edi]]) + for reg in main_registers]) if withfloats: content += ([('float', locations.pop()) for _ in range(26)] + - [('float', reg) for reg in [xmm0, xmm1, xmm2, xmm3, - xmm4, xmm5, xmm6, xmm7]]) + [('float', reg) for reg in xmm_registers]) for i in range(8): content.append(('hole', None)) random.shuffle(content) # prepare the expected target arrays, the descr_bytecode, # the 'registers' and the 'stack' arrays according to 'content' - xmmregisters = lltype.malloc(rffi.LONGP.TO, 16+9, flavor='raw') + xmmregisters = lltype.malloc(rffi.LONGP.TO, 16+ACTUAL_CPU.NUM_REGS+1, flavor='raw') registers = rffi.ptradd(xmmregisters, 16) stacklen = baseloc + 10 stack = lltype.malloc(rffi.LONGP.TO, stacklen, flavor='raw') @@ -140,8 +161,8 @@ assert loc >= 0 ofs = get_ebp_ofs(loc) assert ofs < 0 - assert (ofs % 4) == 0 - stack[stacklen + ofs//4] = value + assert (ofs % WORD) == 0 + stack[stacklen + ofs//WORD] = value descr_bytecode = [] for i, (kind, loc) in enumerate(content): @@ -152,12 +173,18 @@ value, lo, hi = get_random_float() expected_floats[i] = value kind = Assembler386.DESCR_FLOAT - if isinstance(loc, REG): - xmmregisters[2*loc.op] = lo - xmmregisters[2*loc.op+1] = hi + if isinstance(loc, RegLoc): + if WORD == 4: + xmmregisters[2*loc.value] = lo + xmmregisters[2*loc.value+1] = hi + elif WORD == 8: + xmmregisters[loc.value] = lo else: - write_in_stack(loc, hi) - write_in_stack(loc+1, lo) + if WORD == 4: + write_in_stack(loc, hi) + write_in_stack(loc+1, lo) + elif WORD == 8: + write_in_stack(loc, lo) else: if kind == 'int': value = get_random_int() @@ -170,15 +197,15 @@ value = rffi.cast(rffi.LONG, value) else: assert 0, kind - if isinstance(loc, REG): - registers[loc.op] = value + if isinstance(loc, RegLoc): + registers[loc.value] = value else: write_in_stack(loc, value) - if isinstance(loc, REG): - num = kind + 4*loc.op + if isinstance(loc, RegLoc): + num = kind + 4*loc.value else: - num = kind + 4*(8+loc) + num = kind + Assembler386.CODE_FROMSTACK + (4*loc) while num >= 0x80: descr_bytecode.append((num & 0x7F) | 0x80) num >>= 7 @@ -195,8 +222,8 @@ for i in range(len(descr_bytecode)): assert 0 <= descr_bytecode[i] <= 255 descr_bytes[i] = rffi.cast(rffi.UCHAR, descr_bytecode[i]) - registers[8] = rffi.cast(rffi.LONG, descr_bytes) - registers[ebp.op] = rffi.cast(rffi.LONG, stack) + 4*stacklen + registers[ACTUAL_CPU.NUM_REGS] = rffi.cast(rffi.LONG, descr_bytes) + registers[ebp.value] = rffi.cast(rffi.LONG, stack) + WORD*stacklen # run! assembler = Assembler386(FakeCPU()) @@ -237,7 +264,8 @@ def test_mc_wrapper_profile_agent(): agent = FakeProfileAgent() - mc = FakeMCWrapper(100, agent) + assembler = FakeAssembler() + mc = FakeMCWrapper(assembler, 100, agent) mc.start_function("abc") mc.writechr("x") mc.writechr("x") Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_basic.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_basic.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_basic.py Wed Aug 4 16:53:45 2010 @@ -1,5 +1,5 @@ import py -from pypy.jit.backend.x86.runner import CPU386 +from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.metainterp.warmspot import ll_meta_interp from pypy.jit.metainterp.test import test_basic from pypy.jit.codewriter.policy import StopAtXPolicy @@ -7,7 +7,7 @@ class Jit386Mixin(test_basic.LLJitMixin): type_system = 'lltype' - CPUClass = CPU386 + CPUClass = getcpuclass() def check_jumps(self, maxcount): pass Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_gc_integration.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_gc_integration.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_gc_integration.py Wed Aug 4 16:53:45 2010 @@ -9,13 +9,13 @@ from pypy.jit.codewriter import heaptracker from pypy.jit.backend.llsupport.descr import GcCache from pypy.jit.backend.llsupport.gc import GcLLDescription -from pypy.jit.backend.x86.runner import CPU -from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, FRAME_FIXED_SIZE +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.regalloc import RegAlloc +from pypy.jit.backend.x86.arch import WORD, FRAME_FIXED_SIZE from pypy.jit.metainterp.test.oparser import parse from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.annlowlevel import llhelper from pypy.rpython.lltypesystem import rclass, rstr -from pypy.jit.backend.x86.ri386 import * from pypy.jit.backend.llsupport.gc import GcLLDescr_framework, GcRefList, GcPtrFieldDescr from pypy.jit.backend.x86.test.test_regalloc import MockAssembler @@ -23,6 +23,8 @@ from pypy.jit.backend.x86.regalloc import X86RegisterManager, X86FrameManager,\ X86XMMRegisterManager +CPU = getcpuclass() + class MockGcRootMap(object): def get_basic_shape(self): return ['shape'] @@ -51,7 +53,7 @@ def initialize(self): self.gcrefs = GcRefList() self.gcrefs.initialize() - self.single_gcref_descr = GcPtrFieldDescr(0) + self.single_gcref_descr = GcPtrFieldDescr('', 0) rewrite_assembler = GcLLDescr_framework.rewrite_assembler.im_func @@ -84,7 +86,7 @@ mark = regalloc.get_mark_gc_roots(cpu.gc_ll_descr.gcrootmap) assert mark[0] == 'compressed' base = -WORD * FRAME_FIXED_SIZE - expected = ['ebx', 'esi', 'edi', base, base-4, base-8] + expected = ['ebx', 'esi', 'edi', base, base-WORD, base-WORD*2] assert dict.fromkeys(mark[1:]) == dict.fromkeys(expected) class TestRegallocGcIntegration(BaseTestRegalloc): @@ -175,7 +177,7 @@ self.addrs[1] = self.addrs[0] + 64 # 64 bytes def malloc_slowpath(size): - assert size == 8 + assert size == WORD*2 nadr = rffi.cast(lltype.Signed, self.nursery) self.addrs[0] = nadr + size return nadr @@ -199,7 +201,7 @@ return rffi.cast(lltype.Signed, self.addrs) def get_nursery_top_addr(self): - return rffi.cast(lltype.Signed, self.addrs) + 4 + return rffi.cast(lltype.Signed, self.addrs) + WORD def get_malloc_fixedsize_slowpath_addr(self): fptr = llhelper(lltype.Ptr(self.MALLOC_SLOWPATH), self.malloc_slowpath) @@ -213,7 +215,7 @@ def setup_method(self, method): cpu = CPU(None, None) - cpu.vtable_offset = 4 + cpu.vtable_offset = WORD cpu.gc_ll_descr = GCDescrFastpathMalloc() NODE = lltype.Struct('node', ('tid', lltype.Signed), @@ -249,7 +251,7 @@ assert gc_ll_descr.nursery[0] == self.nodedescr.tid assert gc_ll_descr.nursery[1] == 42 nurs_adr = rffi.cast(lltype.Signed, gc_ll_descr.nursery) - assert gc_ll_descr.addrs[0] == nurs_adr + 8 + assert gc_ll_descr.addrs[0] == nurs_adr + (WORD*2) def test_malloc_slowpath(self): ops = ''' @@ -269,7 +271,7 @@ # this should call slow path once gc_ll_descr = self.cpu.gc_ll_descr nadr = rffi.cast(lltype.Signed, gc_ll_descr.nursery) - assert gc_ll_descr.addrs[0] == nadr + 8 + assert gc_ll_descr.addrs[0] == nadr + (WORD*2) def test_new_with_vtable(self): ops = ''' @@ -284,4 +286,4 @@ assert gc_ll_descr.nursery[0] == self.descrsize.tid assert gc_ll_descr.nursery[1] == self.vtable_int nurs_adr = rffi.cast(lltype.Signed, gc_ll_descr.nursery) - assert gc_ll_descr.addrs[0] == nurs_adr + 12 + assert gc_ll_descr.addrs[0] == nurs_adr + (WORD*3) Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_jump.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_jump.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_jump.py Wed Aug 4 16:53:45 2010 @@ -1,6 +1,7 @@ -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.regloc import * from pypy.jit.backend.x86.regalloc import X86FrameManager from pypy.jit.backend.x86.jump import remap_frame_layout +from pypy.jit.metainterp.history import INT frame_pos = X86FrameManager.frame_pos @@ -25,7 +26,7 @@ continue assert len(op1) == len(op2) for x, y in zip(op1, op2): - if isinstance(x, MODRM) and isinstance(y, MODRM): + if isinstance(x, StackLoc) and isinstance(y, MODRM): assert x.byte == y.byte assert x.extradata == y.extradata else: @@ -41,9 +42,9 @@ remap_frame_layout(assembler, [eax, ebx, ecx, edx, esi, edi], [eax, ebx, ecx, edx, esi, edi], '?') assert assembler.ops == [] - s8 = frame_pos(1, 1) - s12 = frame_pos(31, 1) - s20 = frame_pos(6, 1) + s8 = frame_pos(1, INT) + s12 = frame_pos(31, INT) + s20 = frame_pos(6, INT) remap_frame_layout(assembler, [eax, ebx, ecx, s20, s8, edx, s12, esi, edi], [eax, ebx, ecx, s20, s8, edx, s12, esi, edi], '?') @@ -58,10 +59,10 @@ def test_simple_framelocs(): assembler = MockAssembler() - s8 = frame_pos(0, 1) - s12 = frame_pos(13, 1) - s20 = frame_pos(20, 1) - s24 = frame_pos(221, 1) + s8 = frame_pos(0, INT) + s12 = frame_pos(13, INT) + s20 = frame_pos(20, INT) + s24 = frame_pos(221, INT) remap_frame_layout(assembler, [s8, eax, s12], [s20, s24, edi], edx) assert assembler.ops == [('mov', s8, edx), ('mov', edx, s20), @@ -70,10 +71,10 @@ def test_reordering(): assembler = MockAssembler() - s8 = frame_pos(8, 1) - s12 = frame_pos(12, 1) - s20 = frame_pos(19, 1) - s24 = frame_pos(1, 1) + s8 = frame_pos(8, INT) + s12 = frame_pos(12, INT) + s20 = frame_pos(19, INT) + s24 = frame_pos(1, INT) remap_frame_layout(assembler, [eax, s8, s20, ebx], [s8, ebx, eax, edi], '?') assert assembler.got([('mov', ebx, edi), @@ -83,10 +84,10 @@ def test_cycle(): assembler = MockAssembler() - s8 = frame_pos(8, 1) - s12 = frame_pos(12, 1) - s20 = frame_pos(19, 1) - s24 = frame_pos(1, 1) + s8 = frame_pos(8, INT) + s12 = frame_pos(12, INT) + s20 = frame_pos(19, INT) + s24 = frame_pos(1, INT) remap_frame_layout(assembler, [eax, s8, s20, ebx], [s8, ebx, eax, s20], '?') assert assembler.got([('push', s8), @@ -97,12 +98,12 @@ def test_cycle_2(): assembler = MockAssembler() - s8 = frame_pos(8, 1) - s12 = frame_pos(12, 1) - s20 = frame_pos(19, 1) - s24 = frame_pos(1, 1) - s2 = frame_pos(2, 1) - s3 = frame_pos(3, 1) + s8 = frame_pos(8, INT) + s12 = frame_pos(12, INT) + s20 = frame_pos(19, INT) + s24 = frame_pos(1, INT) + s2 = frame_pos(2, INT) + s3 = frame_pos(3, INT) remap_frame_layout(assembler, [eax, s8, edi, s20, eax, s20, s24, esi, s2, s3], [s8, s20, edi, eax, edx, s24, ebx, s12, s3, s2], @@ -127,14 +128,14 @@ remap_frame_layout(assembler, [c3], [eax], '?') assert assembler.ops == [('mov', c3, eax)] assembler = MockAssembler() - s12 = frame_pos(12, 1) + s12 = frame_pos(12, INT) remap_frame_layout(assembler, [c3], [s12], '?') assert assembler.ops == [('mov', c3, s12)] def test_constants_and_cycle(): assembler = MockAssembler() c3 = imm(3) - s12 = frame_pos(13, 1) + s12 = frame_pos(13, INT) remap_frame_layout(assembler, [ebx, c3, s12], [s12, eax, ebx], edi) assert assembler.ops == [('mov', c3, eax), Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_recompilation.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_recompilation.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_recompilation.py Wed Aug 4 16:53:45 2010 @@ -1,6 +1,5 @@ - -from pypy.jit.backend.x86.runner import CPU from pypy.jit.backend.x86.test.test_regalloc import BaseTestRegalloc +from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 class TestRecompilation(BaseTestRegalloc): def test_compile_bridge_not_deeper(self): @@ -51,7 +50,9 @@ descr = loop.operations[2].descr new = descr._x86_bridge_frame_depth assert descr._x86_bridge_param_depth == 0 - assert new > previous + # XXX: Maybe add enough ops to force stack on 64-bit as well? + if IS_X86_32: + assert new > previous self.cpu.set_future_value_int(0, 0) fail = self.run(loop) assert fail.identifier == 2 @@ -111,7 +112,9 @@ guard_op = loop.operations[5] loop_frame_depth = loop.token._x86_frame_depth assert loop.token._x86_param_depth == 0 - assert guard_op.descr._x86_bridge_frame_depth > loop_frame_depth + # XXX: Maybe add enough ops to force stack on 64-bit as well? + if IS_X86_32: + assert guard_op.descr._x86_bridge_frame_depth > loop_frame_depth assert guard_op.descr._x86_bridge_param_depth == 0 self.cpu.set_future_value_int(0, 0) self.cpu.set_future_value_int(1, 0) Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_regalloc.py Wed Aug 4 16:53:45 2010 @@ -7,15 +7,17 @@ BoxPtr, ConstPtr, LoopToken, BasicFailDescr from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.backend.llsupport.descr import GcCache -from pypy.jit.backend.x86.runner import CPU -from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, X86RegisterManager,\ +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.regalloc import RegAlloc, X86RegisterManager,\ FloatConstants +from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 from pypy.jit.metainterp.test.oparser import parse from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.annlowlevel import llhelper from pypy.rpython.lltypesystem import rclass, rstr -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.rx86 import * +CPU = getcpuclass() class MockGcDescr(GcCache): def get_funcptr_for_new(self): return 123 @@ -92,13 +94,20 @@ def f2(x, y): return x*y + def f10(*args): + assert len(args) == 10 + return sum(args) + F1PTR = lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Signed)) F2PTR = lltype.Ptr(lltype.FuncType([lltype.Signed]*2, lltype.Signed)) + F10PTR = lltype.Ptr(lltype.FuncType([lltype.Signed]*10, lltype.Signed)) f1ptr = llhelper(F1PTR, f1) f2ptr = llhelper(F2PTR, f2) + f10ptr = llhelper(F10PTR, f10) f1_calldescr = cpu.calldescrof(F1PTR.TO, F1PTR.TO.ARGS, F1PTR.TO.RESULT) f2_calldescr = cpu.calldescrof(F2PTR.TO, F2PTR.TO.ARGS, F2PTR.TO.RESULT) + f10_calldescr = cpu.calldescrof(F10PTR.TO, F10PTR.TO.ARGS, F10PTR.TO.RESULT) namespace = locals().copy() type_system = 'lltype' @@ -541,6 +550,12 @@ assert self.getints(9) == [0, 1, 1, 1, 1, 1, 1, 1, 1] class TestRegAllocCallAndStackDepth(BaseTestRegalloc): + def expected_param_depth(self, num_args): + # Assumes the arguments are all non-float + if IS_X86_32: + return num_args + elif IS_X86_64: + return max(num_args - 6, 0) def test_one_call(self): ops = ''' @@ -550,7 +565,7 @@ ''' loop = self.interpret(ops, [4, 7, 9, 9 ,9, 9, 9, 9, 9, 9, 9]) assert self.getints(11) == [5, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9] - assert loop.token._x86_param_depth == 1 + assert loop.token._x86_param_depth == self.expected_param_depth(1) def test_two_calls(self): ops = ''' @@ -561,8 +576,21 @@ ''' loop = self.interpret(ops, [4, 7, 9, 9 ,9, 9, 9, 9, 9, 9, 9]) assert self.getints(11) == [5*7, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9] - assert loop.token._x86_param_depth == 2 - + assert loop.token._x86_param_depth == self.expected_param_depth(2) + + def test_call_many_arguments(self): + # NB: The first and last arguments in the call are constants. This + # is primarily for x86-64, to ensure that loading a constant to an + # argument register or to the stack works correctly + ops = ''' + [i0, i1, i2, i3, i4, i5, i6, i7] + i8 = call(ConstClass(f10ptr), 1, i0, i1, i2, i3, i4, i5, i6, i7, 10, descr=f10_calldescr) + finish(i8) + ''' + loop = self.interpret(ops, [2, 3, 4, 5, 6, 7, 8, 9]) + assert self.getint(0) == 55 + assert loop.token._x86_param_depth == self.expected_param_depth(10) + def test_bridge_calls_1(self): ops = ''' [i0, i1] @@ -579,7 +607,7 @@ ''' bridge = self.attach_bridge(ops, loop, -2) - assert loop.operations[-2].descr._x86_bridge_param_depth == 2 + assert loop.operations[-2].descr._x86_bridge_param_depth == self.expected_param_depth(2) self.cpu.set_future_value_int(0, 4) self.cpu.set_future_value_int(1, 7) @@ -602,7 +630,7 @@ ''' bridge = self.attach_bridge(ops, loop, -2) - assert loop.operations[-2].descr._x86_bridge_param_depth == 2 + assert loop.operations[-2].descr._x86_bridge_param_depth == self.expected_param_depth(2) self.cpu.set_future_value_int(0, 4) self.cpu.set_future_value_int(1, 7) Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_regalloc2.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_regalloc2.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_regalloc2.py Wed Aug 4 16:53:45 2010 @@ -2,7 +2,9 @@ from pypy.jit.metainterp.history import ResOperation, BoxInt, ConstInt,\ BoxPtr, ConstPtr, BasicFailDescr, LoopToken from pypy.jit.metainterp.resoperation import rop -from pypy.jit.backend.x86.runner import CPU +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.arch import WORD +CPU = getcpuclass() def test_bug_rshift(): v1 = BoxInt() @@ -281,5 +283,8 @@ assert cpu.get_latest_value_int(16) == -57344 assert cpu.get_latest_value_int(17) == 1 assert cpu.get_latest_value_int(18) == -1 - assert cpu.get_latest_value_int(19) == -2147483648 + if WORD == 4: + assert cpu.get_latest_value_int(19) == -2147483648 + elif WORD == 8: + assert cpu.get_latest_value_int(19) == 19327352832 assert cpu.get_latest_value_int(20) == -49 Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_runner.py Wed Aug 4 16:53:45 2010 @@ -2,15 +2,20 @@ from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rstr, rclass from pypy.jit.metainterp.history import ResOperation, LoopToken from pypy.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstPtr, - Box, BasicFailDescr) -from pypy.jit.backend.x86.runner import CPU -from pypy.jit.backend.x86.regalloc import WORD + Box, BoxFloat, BasicFailDescr) +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.arch import WORD from pypy.jit.backend.llsupport import symbolic from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.executor import execute from pypy.jit.backend.test.runner_test import LLtypeBackendTest +from pypy.jit.metainterp.test.oparser import parse +from pypy.tool.udir import udir import ctypes import sys +import os + +CPU = getcpuclass() class FakeStats(object): pass @@ -56,7 +61,7 @@ assert u.chars[3] == u'd' @staticmethod - def _resbuf(res, item_tp=ctypes.c_int): + def _resbuf(res, item_tp=ctypes.c_long): return ctypes.cast(res.value._obj.intval, ctypes.POINTER(item_tp)) def test_allocations(self): @@ -71,8 +76,11 @@ return ctypes.cast(buf, ctypes.c_void_p).value func = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int)(f) addr = ctypes.cast(func, ctypes.c_void_p).value + # ctypes produces an unsigned value. We need it to be signed for, eg, + # relative addressing to work properly. + addr = rffi.cast(lltype.Signed, addr) - self.cpu.assembler.make_sure_mc_exists() + self.cpu.assembler.setup() self.cpu.assembler.malloc_func_addr = addr ofs = symbolic.get_field_token(rstr.STR, 'chars', False)[0] @@ -357,7 +365,9 @@ self.cpu.compile_bridge(faildescr1, [i1b], bridge) name, address, size = agent.functions[1] assert name == "Bridge # 0: bye" - assert address == loopaddress + loopsize + # Would be exactly ==, but there are some guard failure recovery + # stubs in-between + assert address >= loopaddress + loopsize assert size >= 10 # randomish number self.cpu.set_future_value_int(0, 2) @@ -383,10 +393,100 @@ ops.append(ResOperation(rop.FINISH, [v], None, descr=BasicFailDescr())) looptoken = LoopToken() - self.cpu.assembler.make_sure_mc_exists() + self.cpu.assembler.setup() old_mc_mc = self.cpu.assembler.mc._mc self.cpu.compile_loop([base_v], ops, looptoken) assert self.cpu.assembler.mc._mc != old_mc_mc # overflowed self.cpu.set_future_value_int(0, base_v.value) self.cpu.execute_token(looptoken) assert self.cpu.get_latest_value_int(0) == 1024 + + def test_overflow_guard_float_cmp(self): + # The float comparisons on x86 tend to use small relative jumps, + # which may run into trouble if they fall on the edge of a + # MachineCodeBlock change. + a = BoxFloat(1.0) + b = BoxFloat(2.0) + failed = BoxInt(41) + finished = BoxInt(42) + + # We select guards that will always succeed, so that execution will + # continue through the entire set of comparisions + ops_to_test = ( + (rop.FLOAT_LT, [a, b], rop.GUARD_TRUE), + (rop.FLOAT_LT, [b, a], rop.GUARD_FALSE), + + (rop.FLOAT_LE, [a, a], rop.GUARD_TRUE), + (rop.FLOAT_LE, [a, b], rop.GUARD_TRUE), + (rop.FLOAT_LE, [b, a], rop.GUARD_FALSE), + + (rop.FLOAT_EQ, [a, a], rop.GUARD_TRUE), + (rop.FLOAT_EQ, [a, b], rop.GUARD_FALSE), + + (rop.FLOAT_NE, [a, b], rop.GUARD_TRUE), + (rop.FLOAT_NE, [a, a], rop.GUARD_FALSE), + + (rop.FLOAT_GT, [b, a], rop.GUARD_TRUE), + (rop.FLOAT_GT, [a, b], rop.GUARD_FALSE), + + (rop.FLOAT_GE, [a, a], rop.GUARD_TRUE), + (rop.FLOAT_GE, [b, a], rop.GUARD_TRUE), + (rop.FLOAT_GE, [a, b], rop.GUARD_FALSE), + ) + + for float_op, args, guard_op in ops_to_test: + ops = [] + + for i in range(200): + cmp_result = BoxInt() + ops.append(ResOperation(float_op, args, cmp_result)) + ops.append(ResOperation(guard_op, [cmp_result], None, descr=BasicFailDescr())) + ops[-1].fail_args = [failed] + + ops.append(ResOperation(rop.FINISH, [finished], None, descr=BasicFailDescr())) + + looptoken = LoopToken() + self.cpu.compile_loop([a, b, failed, finished], ops, looptoken) + self.cpu.set_future_value_float(0, a.value) + self.cpu.set_future_value_float(1, b.value) + self.cpu.set_future_value_int(2, failed.value) + self.cpu.set_future_value_int(3, finished.value) + self.cpu.execute_token(looptoken) + + # Really just a sanity check. We're actually interested in + # whether the test segfaults. + assert self.cpu.get_latest_value_int(0) == finished.value + + +class TestDebuggingAssembler(object): + def setup_method(self, meth): + self.pypylog = os.environ.get('PYPYLOG', None) + self.logfile = str(udir.join('x86_runner.log')) + os.environ['PYPYLOG'] = "mumble:" + self.logfile + self.cpu = CPU(rtyper=None, stats=FakeStats()) + + def teardown_method(self, meth): + if self.pypylog is not None: + os.environ['PYPYLOG'] = self.pypylog + + def test_debugger_on(self): + loop = """ + [i0] + debug_merge_point('xyz') + i1 = int_add(i0, 1) + i2 = int_ge(i1, 10) + guard_false(i2) [] + jump(i1) + """ + ops = parse(loop) + self.cpu.assembler.set_debug(True) + self.cpu.compile_loop(ops.inputargs, ops.operations, ops.token) + self.cpu.set_future_value_int(0, 0) + self.cpu.execute_token(ops.token) + # check debugging info + name, arr = self.cpu.assembler.loop_run_counters[0] + assert name == 'xyz' + assert arr[0] == 10 + self.cpu.finish_once() + lines = py.path.local(self.logfile + ".count").readlines() + assert lines[0] == 'xyz:10\n' Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_symbolic_x86.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_symbolic_x86.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_symbolic_x86.py Wed Aug 4 16:53:45 2010 @@ -1,6 +1,7 @@ import py from pypy.jit.backend.llsupport.symbolic import * from pypy.rpython.lltypesystem import lltype, rffi +from pypy.jit.backend.x86.arch import WORD # This test file is here and not in llsupport/test/ because it checks # that we get correct numbers for a 32-bit machine. @@ -19,32 +20,32 @@ ofs_z, size_z = get_field_token(S, 'z', False) # ofs_x might be 0 or not, depending on how we count the headers # but the rest should be as expected for a 386 machine - assert size_x == size_y == size_z == 4 + assert size_x == size_y == size_z == WORD assert ofs_x >= 0 - assert ofs_y == ofs_x + 4 - assert ofs_z == ofs_x + 8 + assert ofs_y == ofs_x + WORD + assert ofs_z == ofs_x + (WORD*2) def test_struct_size(): ofs_z, size_z = get_field_token(S, 'z', False) totalsize = get_size(S, False) - assert totalsize == ofs_z + 4 + assert totalsize == ofs_z + WORD def test_primitive_size(): - assert get_size(lltype.Signed, False) == 4 + assert get_size(lltype.Signed, False) == WORD assert get_size(lltype.Char, False) == 1 - assert get_size(lltype.Ptr(S), False) == 4 + assert get_size(lltype.Ptr(S), False) == WORD def test_array_token(): A = lltype.GcArray(lltype.Char) basesize, itemsize, ofs_length = get_array_token(A, False) - assert basesize >= 4 # at least the 'length', maybe some gc headers + assert basesize >= WORD # at least the 'length', maybe some gc headers assert itemsize == 1 - assert ofs_length == basesize - 4 + assert ofs_length == basesize - WORD A = lltype.GcArray(lltype.Signed) basesize, itemsize, ofs_length = get_array_token(A, False) - assert basesize >= 4 # at least the 'length', maybe some gc headers - assert itemsize == 4 - assert ofs_length == basesize - 4 + assert basesize >= WORD # at least the 'length', maybe some gc headers + assert itemsize == WORD + assert ofs_length == basesize - WORD def test_varsized_struct_size(): S1 = lltype.GcStruct('S1', ('parent', S), @@ -54,9 +55,9 @@ ofs_extra, size_extra = get_field_token(S1, 'extra', False) basesize, itemsize, ofs_length = get_array_token(S1, False) assert size_parent == ofs_extra - assert size_extra == 4 - assert ofs_length == ofs_extra + 4 - assert basesize == ofs_length + 4 + assert size_extra == WORD + assert ofs_length == ofs_extra + WORD + assert basesize == ofs_length + WORD assert itemsize == 1 def test_string(): Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_zll_random.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_zll_random.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_zll_random.py Wed Aug 4 16:53:45 2010 @@ -1,9 +1,11 @@ from pypy.jit.backend.test.test_random import check_random_function, Random from pypy.jit.backend.test.test_ll_random import LLtypeOperationBuilder -from pypy.jit.backend.x86.runner import CPU386 +from pypy.jit.backend.detect_cpu import getcpuclass + +CPU = getcpuclass() def test_stress(): - cpu = CPU386(None, None) + cpu = CPU(None, None) r = Random() for i in range(1000): check_random_function(cpu, LLtypeOperationBuilder, r, i, 1000) Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_zrpy_gc.py Wed Aug 4 16:53:45 2010 @@ -17,6 +17,8 @@ from pypy.jit.backend.llsupport.gc import GcRefList, GcRootMap_asmgcc from pypy.jit.backend.llsupport.gc import GcLLDescr_framework from pypy.tool.udir import udir +from pypy.jit.backend.x86.arch import IS_X86_64 +import py.test class X(object): def __init__(self, x=0): @@ -126,6 +128,10 @@ class TestCompileHybrid(object): def setup_class(cls): + if IS_X86_64: + # No hybrid GC on 64-bit for the time being + py.test.skip() + funcs = [] name_to_func = {} for fullname in dir(cls): Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_ztranslation.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_ztranslation.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_ztranslation.py Wed Aug 4 16:53:45 2010 @@ -3,13 +3,14 @@ from pypy.rlib.jit import JitDriver, OPTIMIZER_FULL, unroll_parameters from pypy.rlib.jit import PARAMETERS, dont_look_inside from pypy.jit.metainterp.jitprof import Profiler -from pypy.jit.backend.x86.runner import CPU386 +from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.backend.test.support import CCompiledMixin from pypy.jit.codewriter.policy import StopAtXPolicy from pypy.translator.translator import TranslationContext +from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 class TestTranslationX86(CCompiledMixin): - CPUClass = CPU386 + CPUClass = getcpuclass() def _check_cbuilder(self, cbuilder): # We assume here that we have sse2. If not, the CPUClass @@ -114,7 +115,7 @@ class TestTranslationRemoveTypePtrX86(CCompiledMixin): - CPUClass = CPU386 + CPUClass = getcpuclass() def _get_TranslationContext(self): t = TranslationContext() @@ -125,6 +126,10 @@ return t def test_external_exception_handling_translates(self): + # FIXME + if IS_X86_64: + import py.test; py.test.skip() + jitdriver = JitDriver(greens = [], reds = ['n', 'total']) class ImDone(Exception): Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/tool/viewcode.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/tool/viewcode.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/tool/viewcode.py Wed Aug 4 16:53:45 2010 @@ -31,16 +31,22 @@ if sys.platform == "win32": XXX # lots more in Psyco -def machine_code_dump(data, originaddr): - # the disassembler to use. 'objdump' writes GNU-style instructions. - # 'ndisasm' would use Intel syntax, but you need to fix the output parsing. - objdump = ('objdump -M intel -b binary -m i386 ' +def machine_code_dump(data, originaddr, backend_name): + objdump_backend_option = { + 'x86': 'i386', + 'x86_64': 'x86-64', + } + objdump = ('objdump -M intel,%(backend)s -b binary -m i386 ' '--adjust-vma=%(origin)d -D %(file)s') # f = open(tmpfile, 'wb') f.write(data) f.close() - g = os.popen(objdump % {'file': tmpfile, 'origin': originaddr}, 'r') + g = os.popen(objdump % { + 'file': tmpfile, + 'origin': originaddr, + 'backend': objdump_backend_option[backend_name], + }, 'r') result = g.readlines() g.close() return result[6:] # drop some objdump cruft @@ -126,7 +132,7 @@ def disassemble(self): if not hasattr(self, 'text'): - lines = machine_code_dump(self.data, self.addr) + lines = machine_code_dump(self.data, self.addr, self.world.backend_name) # instead of adding symbol names in the dumps we could # also make the 0xNNNNNNNN addresses be red and show the # symbol name when the mouse is over them @@ -171,10 +177,13 @@ self.jumps = {} self.symbols = {} self.logentries = {} + self.backend_name = None def parse(self, f, textonly=True): for line in f: - if line.startswith('CODE_DUMP '): + if line.startswith('BACKEND '): + self.backend_name = line.split(' ')[1].strip() + elif line.startswith('CODE_DUMP '): pieces = line.split() assert pieces[1].startswith('@') assert pieces[2].startswith('+') Modified: pypy/branch/interplevel-array/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/interplevel-array/pypy/jit/metainterp/optimizeopt.py Wed Aug 4 16:53:45 2010 @@ -18,6 +18,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.jit.metainterp.history import AbstractDescr, make_hashable_int + def optimize_loop_1(metainterp_sd, loop): """Optimize loop.operations to make it match the input of loop.specnodes and to remove internal overheadish operations. Note that loop.specnodes @@ -1039,6 +1040,25 @@ self.make_equal_to(op.result, v1) else: self.optimize_default(op) + + def optimize_INT_SUB(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + if v2.is_constant() and v2.box.getint() == 0: + self.make_equal_to(op.result, v1) + else: + return self.optimize_default(op) + + def optimize_INT_ADD(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + # If one side of the op is 0 the result is the other side. + if v1.is_constant() and v1.box.getint() == 0: + self.make_equal_to(op.result, v2) + elif v2.is_constant() and v2.box.getint() == 0: + self.make_equal_to(op.result, v1) + else: + self.optimize_default(op) optimize_ops = _findall(Optimizer, 'optimize_') Modified: pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_loop.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_loop.py (original) +++ pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_loop.py Wed Aug 4 16:53:45 2010 @@ -9,6 +9,10 @@ class LoopTest(object): optimizer = OPTIMIZER_SIMPLE + automatic_promotion_result = { + 'int_add' : 6, 'int_gt' : 1, 'guard_false' : 1, 'jump' : 1, + 'guard_value' : 3 + } def meta_interp(self, f, args, policy=None): return ll_meta_interp(f, args, optimizer=self.optimizer, @@ -477,9 +481,9 @@ res = self.meta_interp(main_interpreter_loop, [1]) assert res == main_interpreter_loop(1) self.check_loop_count(1) - # XXX maybe later optimize guard_value away - self.check_loops({'int_add' : 6, 'int_gt' : 1, - 'guard_false' : 1, 'jump' : 1, 'guard_value' : 3}) + # These loops do different numbers of ops based on which optimizer we + # are testing with. + self.check_loops(self.automatic_promotion_result) def test_can_enter_jit_outside_main_loop(self): myjitdriver = JitDriver(greens=[], reds=['i', 'j', 'a']) Modified: pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_loop_spec.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_loop_spec.py (original) +++ pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_loop_spec.py Wed Aug 4 16:53:45 2010 @@ -5,6 +5,10 @@ class LoopSpecTest(test_loop.LoopTest): optimizer = OPTIMIZER_FULL + automatic_promotion_result = { + 'int_add' : 3, 'int_gt' : 1, 'guard_false' : 1, 'jump' : 1, + 'guard_value' : 1 + } # ====> test_loop.py Modified: pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_optimizeopt.py Wed Aug 4 16:53:45 2010 @@ -2113,8 +2113,41 @@ jump(i1, i0) """ self.optimize_loop(ops, 'Not, Not', expected) - - + + def test_fold_partially_constant_ops(self): + ops = """ + [i0] + i1 = int_sub(i0, 0) + jump(i1) + """ + expected = """ + [i0] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + ops = """ + [i0] + i1 = int_add(i0, 0) + jump(i1) + """ + expected = """ + [i0] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + ops = """ + [i0] + i1 = int_add(0, i0) + jump(i1) + """ + expected = """ + [i0] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + # ---------- def make_fail_descr(self): Modified: pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_recursive.py Wed Aug 4 16:53:45 2010 @@ -523,7 +523,7 @@ def test_trace_from_start(self): def p(pc, code): code = hlstr(code) - return "%s %d %s" % (code, pc, code[pc]) + return "'%s' at %d: %s" % (code, pc, code[pc]) def c(pc, code): return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], @@ -537,9 +537,9 @@ op = code[pc] if op == "+": n += 7 - if op == "-": + elif op == "-": n -= 1 - if op == "c": + elif op == "c": n = f('---', n) elif op == "l": if n > 0: @@ -556,6 +556,7 @@ result = 0 for i in range(m): result += f('+-cl--', i) + g(50) self.meta_interp(g, [50], backendopt=True) self.check_tree_loop_count(3) self.check_history(int_add=1) Modified: pypy/branch/interplevel-array/pypy/module/__builtin__/compiling.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/__builtin__/compiling.py (original) +++ pypy/branch/interplevel-array/pypy/module/__builtin__/compiling.py Wed Aug 4 16:53:45 2010 @@ -38,7 +38,7 @@ str_ = space.str_w(w_source) ec = space.getexecutioncontext() - if flags & ~(ec.compiler.compiler_flags | consts.PyCF_AST_ONLY | + if flags & ~(ec.compiler.compiler_flags | consts.PyCF_ONLY_AST | consts.PyCF_DONT_IMPLY_DEDENT | consts.PyCF_SOURCE_IS_UTF8): raise OperationError(space.w_ValueError, space.wrap("compile() unrecognized flags")) @@ -53,7 +53,7 @@ "or 'eval' or 'single'")) if ast_node is None: - if flags & consts.PyCF_AST_ONLY: + if flags & consts.PyCF_ONLY_AST: mod = ec.compiler.compile_to_ast(str_, filename, mode, flags) return space.wrap(mod) else: Modified: pypy/branch/interplevel-array/pypy/module/_ast/__init__.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/_ast/__init__.py (original) +++ pypy/branch/interplevel-array/pypy/module/_ast/__init__.py Wed Aug 4 16:53:45 2010 @@ -5,7 +5,7 @@ class Module(MixedModule): interpleveldefs = { - "PyCF_AST_ONLY" : "space.wrap(%s)" % consts.PyCF_AST_ONLY + "PyCF_ONLY_AST" : "space.wrap(%s)" % consts.PyCF_ONLY_AST } appleveldefs = {} Modified: pypy/branch/interplevel-array/pypy/module/_ast/test/test_ast.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/_ast/test/test_ast.py (original) +++ pypy/branch/interplevel-array/pypy/module/_ast/test/test_ast.py Wed Aug 4 16:53:45 2010 @@ -10,7 +10,7 @@ cls.w_get_ast = cls.space.appexec([], """(): def get_ast(source, mode="exec"): import _ast as ast - mod = compile(source, "", mode, ast.PyCF_AST_ONLY) + mod = compile(source, "", mode, ast.PyCF_ONLY_AST) assert isinstance(mod, ast.mod) return mod return get_ast""") Modified: pypy/branch/interplevel-array/pypy/module/_file/interp_file.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/_file/interp_file.py (original) +++ pypy/branch/interplevel-array/pypy/module/_file/interp_file.py Wed Aug 4 16:53:45 2010 @@ -4,6 +4,7 @@ from pypy.rlib.rarithmetic import r_longlong from pypy.module._file.interp_stream import W_AbstractStream from pypy.module._file.interp_stream import StreamErrors, wrap_streamerror +from pypy.module.posix.interp_posix import dispatch_filename from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import ObjSpace, W_Root, Arguments from pypy.interpreter.typedef import TypeDef, GetSetProperty @@ -81,11 +82,11 @@ # file lock. They don't convert StreamErrors to OperationErrors, too. def direct___init__(self, w_name, mode='r', buffering=-1): - name = self.space.str_w(w_name) self.direct_close() self.w_name = w_name self.check_mode_ok(mode) - stream = streamio.open_file_as_stream(name, mode, buffering) + stream = dispatch_filename(streamio.open_file_as_stream)( + self.space, w_name, mode, buffering) fd = stream.try_to_find_file_descriptor() self.fdopenstream(stream, fd, mode) Modified: pypy/branch/interplevel-array/pypy/module/_file/test/test_file.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/_file/test/test_file.py (original) +++ pypy/branch/interplevel-array/pypy/module/_file/test/test_file.py Wed Aug 4 16:53:45 2010 @@ -125,6 +125,15 @@ assert type(res) is str f.close() + def test_unicode_filename(self): + import sys + try: + u'\xe9'.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + skip("encoding not good enough") + f = self.file(self.temppath + u'\xe9', "w") + f.close() + def test_oserror_has_filename(self): try: f = self.file("file that is clearly not there") Modified: pypy/branch/interplevel-array/pypy/module/_rawffi/test/test__rawffi.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/_rawffi/test/test__rawffi.py (original) +++ pypy/branch/interplevel-array/pypy/module/_rawffi/test/test__rawffi.py Wed Aug 4 16:53:45 2010 @@ -677,7 +677,12 @@ a = A(1) a[0] = -1234 a.free() - + + def test_long_with_fromaddress(self): + import _rawffi + addr = -1 + raises(ValueError, _rawffi.Array('u').fromaddress, addr, 100) + def test_passing_raw_pointers(self): import _rawffi lib = _rawffi.CDLL(self.lib_name) Modified: pypy/branch/interplevel-array/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/cpyext/methodobject.py (original) +++ pypy/branch/interplevel-array/pypy/module/cpyext/methodobject.py Wed Aug 4 16:53:45 2010 @@ -100,7 +100,11 @@ return generic_cpy_call(space, self.ml.c_ml_meth, w_self, w_arg) def get_doc(space, self): - return space.wrap(rffi.charp2str(self.ml.c_ml_doc)) + doc = self.ml.c_ml_doc + if doc: + return space.wrap(rffi.charp2str(doc)) + else: + return space.w_None class W_PyCMethodObject(W_PyCFunctionObject): Modified: pypy/branch/interplevel-array/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/interplevel-array/pypy/module/posix/interp_posix.py Wed Aug 4 16:53:45 2010 @@ -1,8 +1,9 @@ from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped from pypy.rlib import rposix +from pypy.rlib.objectmodel import specialize from pypy.rlib.rarithmetic import r_longlong from pypy.rlib.unroll import unrolling_iterable -from pypy.interpreter.error import OperationError, wrap_oserror +from pypy.interpreter.error import OperationError, wrap_oserror, wrap_oserror2 from pypy.rpython.module.ll_os import RegisterOs from pypy.rpython.module import ll_os_stat from pypy.rpython.lltypesystem import rffi, lltype @@ -12,15 +13,78 @@ import os, sys _WIN = sys.platform == 'win32' -def open(space, fname, flag, mode=0777): +class FileEncoder: + def __init__(self, space, w_obj): + self.space = space + self.w_obj = w_obj + + def as_bytes(self): + from pypy.module.sys.interp_encoding import getfilesystemencoding + space = self.space + w_bytes = space.call_method(self.w_obj, 'encode', + getfilesystemencoding(space)) + return space.str_w(w_bytes) + + def as_unicode(self): + return self.space.unicode_w(self.w_obj) + +class FileDecoder: + def __init__(self, space, w_obj): + self.space = space + self.w_obj = w_obj + + def as_bytes(self): + return self.space.str_w(self.w_obj) + + def as_unicode(self): + from pypy.module.sys.interp_encoding import getfilesystemencoding + space = self.space + w_unicode = space.call_method(self.w_obj, 'decode', + getfilesystemencoding(space)) + return space.unicode_w(w_unicode) + + at specialize.memo() +def dispatch_filename(func, tag=0): + def dispatch(space, w_fname, *args): + if space.isinstance_w(w_fname, space.w_unicode): + fname = FileEncoder(space, w_fname) + return func(fname, *args) + else: + fname = space.str_w(w_fname) + return func(fname, *args) + return dispatch + + at specialize.memo() +def dispatch_filename_2(func): + def dispatch(space, w_fname1, w_fname2, *args): + if space.isinstance_w(w_fname1, space.w_unicode): + fname1 = FileEncoder(space, w_fname1) + if space.isinstance_w(w_fname2, space.w_unicode): + fname2 = FileEncoder(space, w_fname2) + return func(fname1, fname2, *args) + else: + fname2 = FileDecoder(space, w_fname2) + return func(fname1, fname2, *args) + else: + fname1 = FileDecoder(space, w_fname1) + if space.isinstance_w(w_fname2, space.w_unicode): + fname2 = FileEncoder(space, w_fname2) + return func(fname1, fname2, *args) + else: + fname2 = FileDecoder(space, w_fname2) + return func(fname1, fname2, *args) + return dispatch + +def open(space, w_fname, flag, mode=0777): """Open a file (for low level IO). Return a file descriptor (a small integer).""" - try: - fd = os.open(fname, flag, mode) + try: + fd = dispatch_filename(rposix.open)( + space, w_fname, flag, mode) except OSError, e: - raise wrap_oserror(space, e, fname) + raise wrap_oserror2(space, e, w_fname) return space.wrap(fd) -open.unwrap_spec = [ObjSpace, 'path', "c_int", "c_int"] +open.unwrap_spec = [ObjSpace, W_Root, "c_int", "c_int"] def lseek(space, fd, pos, how): """Set the current position of a file descriptor. Return the new position. @@ -159,7 +223,7 @@ return build_stat_result(space, st) fstat.unwrap_spec = [ObjSpace, "c_int"] -def stat(space, path): +def stat(space, w_path): """Perform a stat system call on the given path. Return an object with (at least) the following attributes: st_mode @@ -175,22 +239,22 @@ """ try: - st = os.stat(path) + st = dispatch_filename(rposix.stat)(space, w_path) except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) else: return build_stat_result(space, st) -stat.unwrap_spec = [ObjSpace, 'path'] +stat.unwrap_spec = [ObjSpace, W_Root] -def lstat(space, path): +def lstat(space, w_path): "Like stat(path), but do no follow symbolic links." try: - st = os.lstat(path) + st = dispatch_filename(rposix.lstat)(space, w_path) except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) else: return build_stat_result(space, st) -lstat.unwrap_spec = [ObjSpace, 'path'] +lstat.unwrap_spec = [ObjSpace, W_Root] class StatState(object): def __init__(self, space): @@ -231,7 +295,7 @@ raise wrap_oserror(space, e) dup2.unwrap_spec = [ObjSpace, "c_int", "c_int"] -def access(space, path, mode): +def access(space, w_path, mode): """ access(path, mode) -> 1 if granted, 0 otherwise @@ -242,12 +306,12 @@ existence, or the inclusive-OR of R_OK, W_OK, and X_OK. """ try: - ok = os.access(path, mode) - except OSError, e: - raise wrap_oserror(space, e, path) + ok = dispatch_filename(rposix.access)(space, w_path, mode) + except OSError, e: + raise wrap_oserror2(space, e, w_path) else: return space.wrap(ok) -access.unwrap_spec = [ObjSpace, str, "c_int"] +access.unwrap_spec = [ObjSpace, W_Root, "c_int"] def times(space): @@ -278,32 +342,38 @@ return space.wrap(rc) system.unwrap_spec = [ObjSpace, str] -def unlink(space, path): +def unlink(space, w_path): """Remove a file (same as remove(path)).""" try: - os.unlink(path) - except OSError, e: - raise wrap_oserror(space, e, path) -unlink.unwrap_spec = [ObjSpace, 'path'] + dispatch_filename(rposix.unlink)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +unlink.unwrap_spec = [ObjSpace, W_Root] -def remove(space, path): +def remove(space, w_path): """Remove a file (same as unlink(path)).""" try: - os.unlink(path) - except OSError, e: - raise wrap_oserror(space, e, path) -remove.unwrap_spec = [ObjSpace, 'path'] + dispatch_filename(rposix.unlink)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +remove.unwrap_spec = [ObjSpace, W_Root] -def _getfullpathname(space, path): +def _getfullpathname(space, w_path): """helper for ntpath.abspath """ - posix = __import__(os.name) # nt specific try: - fullpath = posix._getfullpathname(path) + if space.isinstance_w(w_path, space.w_unicode): + path = FileEncoder(space, w_path) + fullpath = rposix._getfullpathname(path) + w_fullpath = space.wrap(fullpath) + else: + path = space.str_w(w_path) + fullpath = rposix._getfullpathname(path) + w_fullpath = space.wrap(fullpath) except OSError, e: - raise wrap_oserror(space, e, path) - else: - return space.wrap(fullpath) -_getfullpathname.unwrap_spec = [ObjSpace, str] + raise wrap_oserror2(space, e, w_path) + else: + return w_fullpath +_getfullpathname.unwrap_spec = [ObjSpace, W_Root] def getcwd(space): """Return the current working directory.""" @@ -315,35 +385,46 @@ return space.wrap(cur) getcwd.unwrap_spec = [ObjSpace] -def getcwdu(space): - """Return the current working directory as a unicode string.""" - # XXX ascii encoding for now - return space.call_method(getcwd(space), 'decode') +if sys.platform == 'win32': + def getcwdu(space): + """Return the current working directory as a unicode string.""" + try: + cur = os.getcwdu() + except OSError, e: + raise wrap_oserror(space, e) + else: + return space.wrap(cur) +else: + def getcwdu(space): + """Return the current working directory as a unicode string.""" + filesystemencoding = space.sys.filesystemencoding + return space.call_method(getcwd(space), 'decode', + space.wrap(filesystemencoding)) getcwdu.unwrap_spec = [ObjSpace] -def chdir(space, path): +def chdir(space, w_path): """Change the current working directory to the specified path.""" try: - os.chdir(path) - except OSError, e: - raise wrap_oserror(space, e, path) -chdir.unwrap_spec = [ObjSpace, str] + dispatch_filename(rposix.chdir)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +chdir.unwrap_spec = [ObjSpace, W_Root] -def mkdir(space, path, mode=0777): +def mkdir(space, w_path, mode=0777): """Create a directory.""" try: - os.mkdir(path, mode) - except OSError, e: - raise wrap_oserror(space, e, path) -mkdir.unwrap_spec = [ObjSpace, str, "c_int"] + dispatch_filename(rposix.mkdir)(space, w_path, mode) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +mkdir.unwrap_spec = [ObjSpace, W_Root, "c_int"] -def rmdir(space, path): +def rmdir(space, w_path): """Remove a directory.""" try: - os.rmdir(path) - except OSError, e: - raise wrap_oserror(space, e, path) -rmdir.unwrap_spec = [ObjSpace, str] + dispatch_filename(rposix.rmdir)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +rmdir.unwrap_spec = [ObjSpace, W_Root] def strerror(space, errno): """Translate an error code to a message string.""" @@ -410,7 +491,7 @@ unsetenv.unwrap_spec = [ObjSpace, str] -def listdir(space, dirname): +def listdir(space, w_dirname): """Return a list containing the names of the entries in the directory. \tpath: path of directory to list @@ -418,12 +499,18 @@ The list is in arbitrary order. It does not include the special entries '.' and '..' even if they are present in the directory.""" try: - result = os.listdir(dirname) + if space.isinstance_w(w_dirname, space.w_unicode): + dirname = FileEncoder(space, w_dirname) + result = rposix.listdir(dirname) + result_w = [space.wrap(s) for s in result] + else: + dirname = space.str_w(w_dirname) + result = rposix.listdir(dirname) + result_w = [space.wrap(s) for s in result] except OSError, e: - raise wrap_oserror(space, e, dirname) - result_w = [space.wrap(s) for s in result] + raise wrap_oserror2(space, e, w_dirname) return space.newlist(result_w) -listdir.unwrap_spec = [ObjSpace, str] +listdir.unwrap_spec = [ObjSpace, W_Root] def pipe(space): "Create a pipe. Returns (read_end, write_end)." @@ -434,21 +521,21 @@ return space.newtuple([space.wrap(fd1), space.wrap(fd2)]) pipe.unwrap_spec = [ObjSpace] -def chmod(space, path, mode): +def chmod(space, w_path, mode): "Change the access permissions of a file." - try: - os.chmod(path, mode) - except OSError, e: - raise wrap_oserror(space, e, path) -chmod.unwrap_spec = [ObjSpace, str, "c_int"] + try: + dispatch_filename(rposix.chmod)(space, w_path, mode) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +chmod.unwrap_spec = [ObjSpace, W_Root, "c_int"] -def rename(space, old, new): +def rename(space, w_old, w_new): "Rename a file or directory." - try: - os.rename(old, new) - except OSError, e: + try: + dispatch_filename_2(rposix.rename)(space, w_old, w_new) + except OSError, e: raise wrap_oserror(space, e) -rename.unwrap_spec = [ObjSpace, str, str] +rename.unwrap_spec = [ObjSpace, W_Root, W_Root] def umask(space, mask): "Set the current numeric umask and return the previous umask." @@ -576,7 +663,7 @@ raise wrap_oserror(space, e) execve.unwrap_spec = [ObjSpace, str, W_Root, W_Root] -def utime(space, path, w_tuple): +def utime(space, w_path, w_tuple): """ utime(path, (atime, mtime)) utime(path, None) @@ -585,10 +672,10 @@ """ if space.is_w(w_tuple, space.w_None): try: - os.utime(path, None) + dispatch_filename(rposix.utime, 1)(space, w_path, None) return except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) try: msg = "utime() arg 2 must be a tuple (atime, mtime) or None" args_w = space.fixedview(w_tuple) @@ -596,14 +683,14 @@ raise OperationError(space.w_TypeError, space.wrap(msg)) actime = space.float_w(args_w[0]) modtime = space.float_w(args_w[1]) - os.utime(path, (actime, modtime)) + dispatch_filename(rposix.utime, 2)(space, w_path, (actime, modtime)) except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) except OperationError, e: if not e.match(space, space.w_TypeError): raise raise OperationError(space.w_TypeError, space.wrap(msg)) -utime.unwrap_spec = [ObjSpace, str, W_Root] +utime.unwrap_spec = [ObjSpace, W_Root, W_Root] def setsid(space): """setsid() -> pid Modified: pypy/branch/interplevel-array/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/posix/test/test_posix2.py (original) +++ pypy/branch/interplevel-array/pypy/module/posix/test/test_posix2.py Wed Aug 4 16:53:45 2010 @@ -32,6 +32,9 @@ # even when running on top of CPython 2.4. os.stat_float_times(True) + # Initialize sys.filesystemencoding + space.call_method(space.getbuiltinmodule('sys'), 'getfilesystemencoding') + def need_sparse_files(): if sys.platform == 'darwin': py.test.skip("no sparse files on default Mac OS X file system") @@ -706,6 +709,28 @@ except OSError: pass +class AppTestUnicodeFilename: + def setup_class(cls): + ufilename = (unicode(udir.join('test_unicode_filename_')) + + u'\u65e5\u672c.txt') # "Japan" + try: + f = file(ufilename, 'w') + except UnicodeEncodeError: + py.test.skip("encoding not good enough") + f.write("test") + f.close() + cls.space = space + cls.w_filename = space.wrap(ufilename) + cls.w_posix = space.appexec([], GET_POSIX) + + def test_open(self): + fd = self.posix.open(self.filename, self.posix.O_RDONLY) + try: + content = self.posix.read(fd, 50) + finally: + self.posix.close(fd) + assert content == "test" + class TestPexpect(object): # XXX replace with AppExpectTest class as soon as possible Modified: pypy/branch/interplevel-array/pypy/module/signal/interp_signal.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/signal/interp_signal.py (original) +++ pypy/branch/interplevel-array/pypy/module/signal/interp_signal.py Wed Aug 4 16:53:45 2010 @@ -7,6 +7,7 @@ from pypy.translator.tool.cbuild import ExternalCompilationInfo import py from pypy.tool import autopath +from pypy.rlib import jit def setup(): for key, value in cpy_signal.__dict__.items(): @@ -159,10 +160,12 @@ return space.wrap(SIG_DFL) getsignal.unwrap_spec = [ObjSpace, int] + at jit.dont_look_inside def alarm(space, timeout): return space.wrap(c_alarm(timeout)) alarm.unwrap_spec = [ObjSpace, int] + at jit.dont_look_inside def pause(space): c_pause() return space.w_None @@ -173,6 +176,7 @@ raise OperationError(space.w_ValueError, space.wrap("signal number out of range")) + at jit.dont_look_inside def signal(space, signum, w_handler): """ signal(sig, action) -> action Modified: pypy/branch/interplevel-array/pypy/objspace/std/callmethod.py ============================================================================== --- pypy/branch/interplevel-array/pypy/objspace/std/callmethod.py (original) +++ pypy/branch/interplevel-array/pypy/objspace/std/callmethod.py Wed Aug 4 16:53:45 2010 @@ -12,7 +12,7 @@ from pypy.interpreter import function from pypy.objspace.descroperation import object_getattribute -from pypy.rlib import rstack # for resume points +from pypy.rlib import jit, rstack # for resume points # This module exports two extra methods for StdObjSpaceFrame implementing # the LOOKUP_METHOD and CALL_METHOD opcodes in an efficient way, as well @@ -56,16 +56,41 @@ f.pushvalue(w_value) f.pushvalue(None) -def CALL_METHOD(f, nargs, *ignored): - # 'nargs' is the argument count excluding the implicit 'self' - w_self = f.peekvalue(nargs) - w_callable = f.peekvalue(nargs + 1) - n = nargs + (w_self is not None) - try: - w_result = f.space.call_valuestack(w_callable, n, f) - rstack.resume_point("CALL_METHOD", f, nargs, returns=w_result) - finally: - f.dropvalues(nargs + 2) + at jit.unroll_safe +def CALL_METHOD(f, oparg, *ignored): + # opargs contains the arg, and kwarg count, excluding the implicit 'self' + n_args = oparg & 0xff + n_kwargs = (oparg >> 8) & 0xff + w_self = f.peekvalue(n_args + (2 * n_kwargs)) + w_callable = f.peekvalue(n_args + (2 * n_kwargs) + 1) + n = n_args + (w_self is not None) + + if not n_kwargs: + try: + w_result = f.space.call_valuestack(w_callable, n, f) + finally: + f.dropvalues(n_args + 2) + else: + keywords = [None] * n_kwargs + keywords_w = [None] * n_kwargs + while True: + n_kwargs -= 1 + if n_kwargs < 0: + break + w_value = f.popvalue() + w_key = f.popvalue() + key = f.space.str_w(w_key) + keywords[n_kwargs] = key + keywords_w[n_kwargs] = w_value + + arguments = f.popvalues(n) + args = f.argument_factory(arguments, keywords, keywords_w, None, None) + + try: + w_result = f.space.call_args(w_callable, args) + finally: + f.dropvalues(1 + (w_self is None)) + rstack.resume_point("CALL_METHOD", f, returns=w_result) f.pushvalue(w_result) Modified: pypy/branch/interplevel-array/pypy/objspace/std/test/test_callmethod.py ============================================================================== --- pypy/branch/interplevel-array/pypy/objspace/std/test/test_callmethod.py (original) +++ pypy/branch/interplevel-array/pypy/objspace/std/test/test_callmethod.py Wed Aug 4 16:53:45 2010 @@ -106,6 +106,15 @@ else: raise Exception("did not raise?") """ + + def test_kwargs(self): + exec """if 1: + class C(object): + def f(self, a): + return a + 2 + + assert C().f(a=3) == 5 + """ class AppTestCallMethodWithGetattributeShortcut(AppTestCallMethod): Modified: pypy/branch/interplevel-array/pypy/rlib/objectmodel.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rlib/objectmodel.py (original) +++ pypy/branch/interplevel-array/pypy/rlib/objectmodel.py Wed Aug 4 16:53:45 2010 @@ -19,32 +19,80 @@ # def f(... # -class _AttachSpecialization(object): +class _Specialize(object): + def memo(self): + """ Specialize functions based on argument values. All arguments has + to be constant at the compile time. The whole function call is replaced + by a call result then. + """ + def decorated_func(func): + func._annspecialcase_ = 'specialize:memo' + return func + return decorated_func - def __init__(self, tag): - self.tag = tag + def arg(self, *args): + """ Specialize function based on values of given positions of arguments. + They must be compile-time constants in order to work. + + There will be a copy of provided function for each combination + of given arguments on positions in args (that can lead to + exponential behavior!). + """ + def decorated_func(func): + func._annspecialcase_ = 'specialize:arg' + self._wrap(args) + return func - def __call__(self, *args): - if not args: - args = "" - else: - args = "("+','.join([repr(arg) for arg in args]) +")" - specialcase = "specialize:%s%s" % (self.tag, args) - - def specialize_decorator(func): - "NOT_RPYTHON" - func._annspecialcase_ = specialcase + return decorated_func + + def argtype(self, *args): + """ Specialize function based on types of arguments on given positions. + + There will be a copy of provided function for each combination + of given arguments on positions in args (that can lead to + exponential behavior!). + """ + def decorated_func(func): + func._annspecialcase_ = 'specialize:argtype' + self._wrap(args) return func - return specialize_decorator - -class _Specialize(object): + return decorated_func - def __getattr__(self, name): - return _AttachSpecialization(name) + def ll(self): + """ This is version of argtypes that cares about low-level types + (so it'll get additional copies for two different types of pointers + for example). Same warnings about exponential behavior apply. + """ + def decorated_func(func): + func._annspecialcase_ = 'specialize:ll' + return func + + return decorated_func + + def ll_and_arg(self, arg): + """ XXX what does that do? + """ + def decorated_func(func): + func._annspecialcase_ = 'specialize:ll_and_arg(%d)' % arg + return func + + return decorated_func + + def _wrap(self, args): + return "("+','.join([repr(arg) for arg in args]) +")" specialize = _Specialize() +def enforceargs(*args): + """ Decorate a function with forcing of RPython-level types on arguments. + None means no enforcing. + + XXX shouldn't we also add asserts in function body? + """ + def decorator(f): + f._annenforceargs_ = args + return f + return decorator + # ____________________________________________________________ class Symbolic(object): Modified: pypy/branch/interplevel-array/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rlib/rarithmetic.py (original) +++ pypy/branch/interplevel-array/pypy/rlib/rarithmetic.py Wed Aug 4 16:53:45 2010 @@ -33,7 +33,7 @@ """ -import math +import sys, math from pypy.rpython import extregistry from pypy.rlib import objectmodel @@ -113,14 +113,23 @@ "NOT_RPYTHON" return _local_ovfcheck(int(long(a) << b)) -def ovfcheck_float_to_int(x): - try: - result = int(int(x)) # -2147483648.0 => -2147483648L => -2147483648 - except (OverflowError, ValueError): # ValueError for int(nan) on Py>=2.6 +# Strange things happening for float to int on 64 bit: +# int(float(i)) != i because of rounding issues. +# These are the minimum and maximum float value that can +# successfully be casted to an int. +if sys.maxint == 2147483647: + def ovfcheck_float_to_int(x): + if -2147483649.0 < x < 2147483648.0: + return int(x) raise OverflowError - if not objectmodel.we_are_translated() and type(result) is not int: +else: + # The following values are not quite +/-sys.maxint. + # Note the "<= x <" here, as opposed to "< x <" above. + # This is justified by test_typed in translator/c/test. + def ovfcheck_float_to_int(x): + if -9223372036854776832.0 <= x < 9223372036854775296.0: + return int(x) raise OverflowError - return result def compute_restype(self_type, other_type): if self_type is other_type: Modified: pypy/branch/interplevel-array/pypy/rlib/rdynload.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rlib/rdynload.py (original) +++ pypy/branch/interplevel-array/pypy/rlib/rdynload.py Wed Aug 4 16:53:45 2010 @@ -18,8 +18,6 @@ if _WIN32: from pypy.rlib import rwin32 - -if _WIN32: includes = ['windows.h'] else: includes = ['dlfcn.h'] Modified: pypy/branch/interplevel-array/pypy/rlib/rposix.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rlib/rposix.py (original) +++ pypy/branch/interplevel-array/pypy/rlib/rposix.py Wed Aug 4 16:53:45 2010 @@ -3,6 +3,7 @@ from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rlib.rarithmetic import intmask +from pypy.rlib.objectmodel import specialize class CConstantErrno(CConstant): # these accessors are used when calling get_errno() or set_errno() @@ -42,3 +43,102 @@ os.close(fd) except OSError: pass + +#___________________________________________________________________ +# Wrappers around posix functions, that accept either strings, or +# instances with a "as_bytes()" method. +# - pypy.modules.posix.interp_posix passes an object containing a unicode path +# which can encode itself with sys.filesystemencoding. +# - but pypy.rpython.module.ll_os.py on Windows will replace these functions +# with other wrappers that directly handle unicode strings. + at specialize.argtype(0) +def open(path, flags, mode): + if isinstance(path, str): + return os.open(path, flags, mode) + else: + return os.open(path.as_bytes(), flags, mode) + + at specialize.argtype(0) +def stat(path): + if isinstance(path, str): + return os.stat(path) + else: + return os.stat(path.as_bytes()) + + at specialize.argtype(0) +def lstat(path): + if isinstance(path, str): + return os.lstat(path) + else: + return os.lstat(path.as_bytes()) + + at specialize.argtype(0) +def unlink(path): + if isinstance(path, str): + return os.unlink(path) + else: + return os.unlink(path.as_bytes()) + + at specialize.argtype(0, 1) +def rename(path1, path2): + if isinstance(path1, str): + return os.rename(path1, path2) + else: + return os.rename(path1.as_bytes(), path2.as_bytes()) + + at specialize.argtype(0) +def listdir(dirname): + if isinstance(dirname, str): + return os.listdir(dirname) + else: + return os.listdir(dirname.as_bytes()) + + at specialize.argtype(0) +def access(path, mode): + if isinstance(path, str): + return os.access(path, mode) + else: + return os.access(path.as_bytes(), mode) + + at specialize.argtype(0) +def chmod(path, mode): + if isinstance(path, str): + return os.chmod(path, mode) + else: + return os.chmod(path.as_bytes(), mode) + + at specialize.argtype(0, 1) +def utime(path, times): + if isinstance(path, str): + return os.utime(path, times) + else: + return os.utime(path.as_bytes(), times) + + at specialize.argtype(0) +def chdir(path): + if isinstance(path, str): + return os.chdir(path) + else: + return os.chdir(path.as_bytes()) + + at specialize.argtype(0) +def mkdir(path, mode=0777): + if isinstance(path, str): + return os.mkdir(path, mode) + else: + return os.mkdir(path.as_bytes(), mode) + + at specialize.argtype(0) +def rmdir(path): + if isinstance(path, str): + return os.rmdir(path) + else: + return os.rmdir(path.as_bytes()) + +if os.name == 'nt': + import nt + def _getfullpathname(path): + if isinstance(path, str): + return nt._getfullpathname(path) + else: + return nt._getfullpathname(path.as_bytes()) Modified: pypy/branch/interplevel-array/pypy/rlib/rwin32.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rlib/rwin32.py (original) +++ pypy/branch/interplevel-array/pypy/rlib/rwin32.py Wed Aug 4 16:53:45 2010 @@ -31,6 +31,7 @@ DWORD = rffi_platform.SimpleType("DWORD", rffi.UINT) BOOL = rffi_platform.SimpleType("BOOL", rffi.LONG) BYTE = rffi_platform.SimpleType("BYTE", rffi.UCHAR) + WCHAR = rffi_platform.SimpleType("WCHAR", rffi.UCHAR) INT = rffi_platform.SimpleType("INT", rffi.INT) LONG = rffi_platform.SimpleType("LONG", rffi.LONG) PLONG = rffi_platform.SimpleType("PLONG", rffi.LONGP) @@ -38,6 +39,8 @@ LPCVOID = rffi_platform.SimpleType("LPCVOID", rffi.VOIDP) LPSTR = rffi_platform.SimpleType("LPSTR", rffi.CCHARP) LPCSTR = rffi_platform.SimpleType("LPCSTR", rffi.CCHARP) + LPWSTR = rffi_platform.SimpleType("LPWSTR", rffi.CWCHARP) + LPCWSTR = rffi_platform.SimpleType("LPCWSTR", rffi.CWCHARP) LPDWORD = rffi_platform.SimpleType("LPDWORD", rffi.INTP) SIZE_T = rffi_platform.SimpleType("SIZE_T", rffi.SIZE_T) ULONG_PTR = rffi_platform.SimpleType("ULONG_PTR", rffi.ULONG) @@ -87,6 +90,10 @@ GetLastError = winexternal('GetLastError', [], DWORD) SetLastError = winexternal('SetLastError', [DWORD], lltype.Void) + # In tests, the first call to GetLastError is always wrong, because error + # is hidden by operations in ll2ctypes. Call it now. + GetLastError() + LoadLibrary = winexternal('LoadLibraryA', [rffi.CCHARP], rffi.VOIDP) GetProcAddress = winexternal('GetProcAddress', [rffi.VOIDP, rffi.CCHARP], @@ -129,13 +136,29 @@ } return 0; }''') - exename = static_platform.compile( - [cfile], ExternalCompilationInfo(), - outputfilename = "dosmaperr", - standalone=True) - output = os.popen(str(exename)) - errors = dict(map(int, line.split()) - for line in output) + try: + exename = static_platform.compile( + [cfile], ExternalCompilationInfo(), + outputfilename = "dosmaperr", + standalone=True) + except WindowsError: + # Fallback for the mingw32 compiler + errors = { + 2: 2, 3: 2, 4: 24, 5: 13, 6: 9, 7: 12, 8: 12, 9: 12, 10: 7, + 11: 8, 15: 2, 16: 13, 17: 18, 18: 2, 19: 13, 20: 13, 21: 13, + 22: 13, 23: 13, 24: 13, 25: 13, 26: 13, 27: 13, 28: 13, + 29: 13, 30: 13, 31: 13, 32: 13, 33: 13, 34: 13, 35: 13, + 36: 13, 53: 2, 65: 13, 67: 2, 80: 17, 82: 13, 83: 13, 89: 11, + 108: 13, 109: 32, 112: 28, 114: 9, 128: 10, 129: 10, 130: 9, + 132: 13, 145: 41, 158: 13, 161: 2, 164: 11, 167: 13, 183: 17, + 188: 8, 189: 8, 190: 8, 191: 8, 192: 8, 193: 8, 194: 8, + 195: 8, 196: 8, 197: 8, 198: 8, 199: 8, 200: 8, 201: 8, + 202: 8, 206: 2, 215: 11, 1816: 12, + } + else: + output = os.popen(str(exename)) + errors = dict(map(int, line.split()) + for line in output) return errors, errno.EINVAL # A bit like strerror... Modified: pypy/branch/interplevel-array/pypy/rlib/streamio.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rlib/streamio.py (original) +++ pypy/branch/interplevel-array/pypy/rlib/streamio.py Wed Aug 4 16:53:45 2010 @@ -38,7 +38,9 @@ # import os, sys +from pypy.rlib.objectmodel import specialize from pypy.rlib.rarithmetic import r_longlong, intmask +from pypy.rlib import rposix from os import O_RDONLY, O_WRONLY, O_RDWR, O_CREAT, O_TRUNC O_BINARY = getattr(os, "O_BINARY", 0) @@ -71,6 +73,7 @@ return s.join(string.split(c)) + at specialize.argtype(0) def open_file_as_stream(path, mode="r", buffering=-1): os_flags, universal, reading, writing, basemode, binary = decode_mode(mode) stream = open_path_helper(path, os_flags, basemode == "a") @@ -89,9 +92,10 @@ return construct_stream_tower(stream, buffering, universal, reading, writing, binary) + at specialize.argtype(0) def open_path_helper(path, os_flags, append): # XXX for now always return DiskFile - fd = os.open(path, os_flags, 0666) + fd = rposix.open(path, os_flags, 0666) if append: try: os.lseek(fd, 0, 2) Modified: pypy/branch/interplevel-array/pypy/rlib/test/test_objectmodel.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rlib/test/test_objectmodel.py (original) +++ pypy/branch/interplevel-array/pypy/rlib/test/test_objectmodel.py Wed Aug 4 16:53:45 2010 @@ -404,6 +404,13 @@ assert f._annspecialcase_ == 'specialize:arg(1)' +def test_enforceargs_decorator(): + @enforceargs(int, str, None) + def f(a, b, c): + pass + + assert f._annenforceargs_ == (int, str, None) + def getgraph(f, argtypes): from pypy.translator.translator import TranslationContext, graphof from pypy.translator.backendopt.all import backend_optimizations Modified: pypy/branch/interplevel-array/pypy/rpython/extfunc.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rpython/extfunc.py (original) +++ pypy/branch/interplevel-array/pypy/rpython/extfunc.py Wed Aug 4 16:53:45 2010 @@ -52,7 +52,10 @@ return super(ExtRegistryEntry, self).__getattr__(attr) raise exc, exc_inst, tb -def registering(func): +def registering(func, condition=True): + if not condition: + return lambda method: None + def decorator(method): method._registering_func = func return method @@ -63,11 +66,9 @@ func = getattr(ns, name) except AttributeError: condition = False + func = None - if condition: - return registering(func) - else: - return lambda method: None + return registering(func, condition=condition) class LazyRegisteringMeta(type): def __new__(self, _name, _type, _vars): @@ -167,8 +168,6 @@ return signature_args def compute_result_annotation(self, *args_s): - if hasattr(self, 'ann_hook'): - self.ann_hook() self.normalize_args(*args_s) # check arguments return self.signature_result @@ -235,7 +234,6 @@ def register_external(function, args, result=None, export_name=None, llimpl=None, ooimpl=None, llfakeimpl=None, oofakeimpl=None, - annotation_hook=None, sandboxsafe=False): """ function: the RPython function that will be rendered as an external function (e.g.: math.floor) @@ -244,7 +242,6 @@ export_name: the name of the function as it will be seen by the backends llimpl, ooimpl: optional; if provided, these RPython functions are called instead of the target function llfakeimpl, oofakeimpl: optional; if provided, they are called by the llinterpreter - annotationhook: optional; a callable that is called during annotation, useful for genc hacks sandboxsafe: use True if the function performs no I/O (safe for --sandbox) """ @@ -271,8 +268,6 @@ lltypefakeimpl = staticmethod(llfakeimpl) if oofakeimpl: ootypefakeimpl = staticmethod(oofakeimpl) - if annotation_hook: - ann_hook = staticmethod(annotation_hook) if export_name: FunEntry.__name__ = export_name Modified: pypy/branch/interplevel-array/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/interplevel-array/pypy/rpython/lltypesystem/rffi.py Wed Aug 4 16:53:45 2010 @@ -176,6 +176,15 @@ # XXX leaks if a str2charp() fails with MemoryError # and was not the first in this function freeme = arg + elif TARGET == CWCHARP: + if arg is None: + arg = lltype.nullptr(CWCHARP.TO) # None => (wchar_t*)NULL + freeme = arg + elif isinstance(arg, unicode): + arg = unicode2wcharp(arg) + # XXX leaks if a unicode2wcharp() fails with MemoryError + # and was not the first in this function + freeme = arg elif _isfunctype(TARGET) and not _isllptr(arg): # XXX pass additional arguments if invoke_around_handlers: Modified: pypy/branch/interplevel-array/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/interplevel-array/pypy/rpython/lltypesystem/rstr.py Wed Aug 4 16:53:45 2010 @@ -2,7 +2,7 @@ from pypy.tool.pairtype import pairtype from pypy.rpython.error import TyperError from pypy.rlib.objectmodel import malloc_zero_filled, we_are_translated -from pypy.rlib.objectmodel import _hash_string +from pypy.rlib.objectmodel import _hash_string, enforceargs from pypy.rlib.debug import ll_assert from pypy.rlib.jit import purefunction from pypy.rpython.robject import PyObjRepr, pyobj_repr @@ -56,6 +56,7 @@ llmemory.itemoffsetof(TP.chars, 0) + llmemory.sizeof(CHAR_TP) * item) + @enforceargs(None, None, int, int, int) def copy_string_contents(src, dst, srcstart, dststart, length): assert srcstart >= 0 assert dststart >= 0 @@ -674,6 +675,7 @@ res_index += item_len i += 1 return result + ll_join_strs._annenforceargs_ = [int, None] def ll_join_chars(length, chars): # no need to optimize this, will be replaced by string builder Modified: pypy/branch/interplevel-array/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/interplevel-array/pypy/rpython/module/ll_os.py Wed Aug 4 16:53:45 2010 @@ -6,12 +6,15 @@ # might be found in doc/rffi.txt import os, sys, errno +import py from pypy.rpython.module.support import ll_strcpy, OOSupport -from pypy.tool.sourcetools import func_with_new_name +from pypy.tool.sourcetools import func_with_new_name, func_renamer from pypy.rlib.rarithmetic import r_longlong -from pypy.rpython.extfunc import BaseLazyRegistering +from pypy.rpython.extfunc import ( + BaseLazyRegistering, lazy_register, register_external) from pypy.rpython.extfunc import registering, registering_if, extdef -from pypy.annotation.model import SomeInteger, SomeString, SomeTuple, SomeFloat +from pypy.annotation.model import ( + SomeInteger, SomeString, SomeTuple, SomeFloat, SomeUnicodeString) from pypy.annotation.model import s_ImpossibleValue, s_None, s_Bool from pypy.rpython.lltypesystem import rffi from pypy.rpython.lltypesystem import lltype @@ -26,7 +29,99 @@ from pypy.rpython.lltypesystem.rstr import STR from pypy.rpython.annlowlevel import llstr from pypy.rlib import rgc -from pypy.rlib.objectmodel import keepalive_until_here +from pypy.rlib.objectmodel import keepalive_until_here, specialize + +def monkeypatch_rposix(posixfunc, unicodefunc, signature): + func_name = posixfunc.__name__ + + if hasattr(signature, '_default_signature_'): + signature = signature._default_signature_ + arglist = ['arg%d' % (i,) for i in range(len(signature))] + transformed_arglist = arglist[:] + for i, arg in enumerate(signature): + if arg is unicode: + transformed_arglist[i] = transformed_arglist[i] + '.as_unicode()' + + args = ', '.join(arglist) + transformed_args = ', '.join(transformed_arglist) + main_arg = 'arg%d' % (signature.index(unicode),) + + source = py.code.Source(""" + def %(func_name)s(%(args)s): + if isinstance(%(main_arg)s, str): + return posixfunc(%(args)s) + else: + return unicodefunc(%(transformed_args)s) + """ % locals()) + miniglobals = {'posixfunc' : posixfunc, + 'unicodefunc': unicodefunc, + '__name__': __name__, # for module name propagation + } + exec source.compile() in miniglobals + new_func = miniglobals[func_name] + specialized_args = [i for i in range(len(signature)) + if signature[i] in (unicode, None)] + new_func = specialize.argtype(*specialized_args)(new_func) + + # Monkeypatch the function in pypy.rlib.rposix + setattr(rposix, func_name, new_func) + +class StringTraits: + str = str + CHAR = rffi.CHAR + CCHARP = rffi.CCHARP + charp2str = staticmethod(rffi.charp2str) + str2charp = staticmethod(rffi.str2charp) + free_charp = staticmethod(rffi.free_charp) + + @staticmethod + def posix_function_name(name): + return underscore_on_windows + name + + @staticmethod + def ll_os_name(name): + return 'll_os.ll_os_' + name + +class UnicodeTraits: + str = unicode + CHAR = rffi.WCHAR_T + CCHARP = rffi.CWCHARP + charp2str = staticmethod(rffi.wcharp2unicode) + str2charp = staticmethod(rffi.unicode2wcharp) + free_charp = staticmethod(rffi.free_wcharp) + + @staticmethod + def posix_function_name(name): + return underscore_on_windows + 'w' + name + + @staticmethod + def ll_os_name(name): + return 'll_os.ll_os_w' + name + +def registering_str_unicode(posixfunc, condition=True): + if not condition: + return registering(None, condition=False) + + func_name = posixfunc.__name__ + + def register_posixfunc(self, method): + val = method(self, StringTraits()) + register_external(posixfunc, *val.def_args, **val.def_kwds) + + if sys.platform == 'win32': + val = method(self, UnicodeTraits()) + @func_renamer(func_name + "_unicode") + def unicodefunc(*args): + return posixfunc(*args) + register_external(unicodefunc, *val.def_args, **val.def_kwds) + signature = val.def_args[0] + monkeypatch_rposix(posixfunc, unicodefunc, signature) + + def decorator(method): + decorated = lambda self: register_posixfunc(self, method) + decorated._registering_func = posixfunc + return decorated + return decorator posix = __import__(os.name) @@ -282,8 +377,8 @@ return extdef([int, int], s_None, llimpl=dup2_llimpl, export_name="ll_os.ll_os_dup2") - @registering(os.utime) - def register_os_utime(self): + @registering_str_unicode(os.utime) + def register_os_utime(self, traits): UTIMBUFP = lltype.Ptr(self.UTIMBUF) os_utime = self.llexternal('utime', [rffi.CCHARP, UTIMBUFP], rffi.INT) @@ -336,6 +431,9 @@ # tp is known to be None, and one version where it is known # to be a tuple of 2 floats. if not _WIN32: + assert traits.str is str + + @specialize.argtype(1) def os_utime_llimpl(path, tp): if tp is None: error = os_utime(path, lltype.nullptr(UTIMBUFP.TO)) @@ -346,85 +444,13 @@ if error == -1: raise OSError(rposix.get_errno(), "os_utime failed") else: - from pypy.rlib import rwin32 - from pypy.rpython.module.ll_os_stat import time_t_to_FILE_TIME - - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h'], - ) - - FILE_WRITE_ATTRIBUTES = platform.ConstantInteger( - 'FILE_WRITE_ATTRIBUTES') - OPEN_EXISTING = platform.ConstantInteger( - 'OPEN_EXISTING') - FILE_FLAG_BACKUP_SEMANTICS = platform.ConstantInteger( - 'FILE_FLAG_BACKUP_SEMANTICS') - globals().update(platform.configure(CConfig)) - - CreateFile = rffi.llexternal( - 'CreateFileA', - [rwin32.LPCSTR, rwin32.DWORD, rwin32.DWORD, - rwin32.LPSECURITY_ATTRIBUTES, rwin32.DWORD, rwin32.DWORD, - rwin32.HANDLE], - rwin32.HANDLE, - calling_conv='win') - - GetSystemTime = rffi.llexternal( - 'GetSystemTime', - [lltype.Ptr(rwin32.SYSTEMTIME)], - lltype.Void, - calling_conv='win') - - SystemTimeToFileTime = rffi.llexternal( - 'SystemTimeToFileTime', - [lltype.Ptr(rwin32.SYSTEMTIME), - lltype.Ptr(rwin32.FILETIME)], - rwin32.BOOL, - calling_conv='win') - - SetFileTime = rffi.llexternal( - 'SetFileTime', - [rwin32.HANDLE, - lltype.Ptr(rwin32.FILETIME), - lltype.Ptr(rwin32.FILETIME), - lltype.Ptr(rwin32.FILETIME)], - rwin32.BOOL, - calling_conv = 'win') + from pypy.rpython.module.ll_win32file import make_utime_impl + os_utime_llimpl = make_utime_impl(traits) - def os_utime_llimpl(path, tp): - hFile = CreateFile(path, - FILE_WRITE_ATTRIBUTES, 0, - None, OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS, 0) - if hFile == rwin32.INVALID_HANDLE_VALUE: - raise rwin32.lastWindowsError() - ctime = lltype.nullptr(rwin32.FILETIME) - atime = lltype.malloc(rwin32.FILETIME, flavor='raw') - mtime = lltype.malloc(rwin32.FILETIME, flavor='raw') - try: - if tp is None: - now = lltype.malloc(rwin32.SYSTEMTIME, flavor='raw') - try: - GetSystemTime(now) - if (not SystemTimeToFileTime(now, atime) or - not SystemTimeToFileTime(now, mtime)): - raise rwin32.lastWindowsError() - finally: - lltype.free(now, flavor='raw') - else: - actime, modtime = tp - time_t_to_FILE_TIME(actime, atime) - time_t_to_FILE_TIME(modtime, mtime) - if not SetFileTime(hFile, ctime, atime, mtime): - raise rwin32.lastWindowsError() - finally: - rwin32.CloseHandle(hFile) - lltype.free(atime, flavor='raw') - lltype.free(mtime, flavor='raw') - os_utime_llimpl._annspecialcase_ = 'specialize:argtype(1)' - - s_string = SomeString() + if traits.str is str: + s_string = SomeString() + else: + s_string = SomeUnicodeString() s_tuple_of_2_floats = SomeTuple([SomeFloat(), SomeFloat()]) def os_utime_normalize_args(s_path, s_times): @@ -445,12 +471,12 @@ else: raise Exception("os.utime() arg 2 must be None or a tuple of " "2 floats, got %s" % (s_times,)) + os_utime_normalize_args._default_signature_ = [traits.str, None] return extdef(os_utime_normalize_args, s_None, "ll_os.ll_os_utime", llimpl=os_utime_llimpl) - @registering(os.times) def register_os_times(self): if sys.platform.startswith('win'): @@ -687,22 +713,21 @@ def register_os_setsid(self): return self.extdef_for_os_function_returning_int('setsid') - @registering(os.open) - def register_os_open(self): - os_open = self.llexternal(underscore_on_windows+'open', - [rffi.CCHARP, rffi.INT, rffi.MODE_T], + @registering_str_unicode(os.open) + def register_os_open(self, traits): + os_open = self.llexternal(traits.posix_function_name('open'), + [traits.CCHARP, rffi.INT, rffi.MODE_T], rffi.INT) - def os_open_llimpl(path, flags, mode): result = rffi.cast(rffi.LONG, os_open(path, flags, mode)) if result == -1: raise OSError(rposix.get_errno(), "os_open failed") return result - def os_open_oofakeimpl(o_path, flags, mode): - return os.open(o_path._str, flags, mode) + def os_open_oofakeimpl(path, flags, mode): + return os.open(OOSupport.from_rstr(path), flags, mode) - return extdef([str, int, int], int, "ll_os.ll_os_open", + return extdef([traits.str, int, int], int, traits.ll_os_name('open'), llimpl=os_open_llimpl, oofakeimpl=os_open_oofakeimpl) # ------------------------------- os.read ------------------------------- @@ -862,10 +887,10 @@ llimpl=fdatasync_llimpl, export_name="ll_os.ll_os_fdatasync") - @registering(os.access) - def register_os_access(self): - os_access = self.llexternal(underscore_on_windows + 'access', - [rffi.CCHARP, rffi.INT], + @registering_str_unicode(os.access) + def register_os_access(self, traits): + os_access = self.llexternal(traits.posix_function_name('access'), + [traits.CCHARP, rffi.INT], rffi.INT) if sys.platform.startswith('win'): @@ -882,44 +907,22 @@ def os_access_oofakeimpl(path, mode): return os.access(OOSupport.from_rstr(path), mode) - return extdef([str, int], s_Bool, llimpl=access_llimpl, - export_name="ll_os.ll_os_access", + return extdef([traits.str, int], s_Bool, llimpl=access_llimpl, + export_name=traits.ll_os_name("access"), oofakeimpl=os_access_oofakeimpl) - @registering_if(posix, '_getfullpathname') - def register_posix__getfullpathname(self): - from pypy.rlib import rwin32 + @registering_str_unicode(getattr(posix, '_getfullpathname', None), + condition=sys.platform=='win32') + def register_posix__getfullpathname(self, traits): # this nt function is not exposed via os, but needed # to get a correct implementation of os.abspath - # XXX why do we ignore WINAPI conventions everywhere? - LPSTRP = rffi.CArrayPtr(rwin32.LPSTR) - # XXX unicode? - GetFullPathName = self.llexternal( - 'GetFullPathNameA', - [rwin32.LPCSTR, - rwin32.DWORD, - rwin32.LPSTR, - rffi.CArrayPtr(rwin32.LPSTR)], - rwin32.DWORD) - - def _getfullpathname_llimpl(lpFileName): - nBufferLength = rwin32.MAX_PATH + 1 - lpBuffer = lltype.malloc(rwin32.LPSTR.TO, nBufferLength, flavor='raw') - try: - res = GetFullPathName( - lpFileName, rffi.cast(rwin32.DWORD, nBufferLength), - lpBuffer, lltype.nullptr(LPSTRP.TO)) - if res == 0: - raise rwin32.lastWindowsError("_getfullpathname failed") - result = rffi.charp2str(lpBuffer) - return result - finally: - lltype.free(lpBuffer, flavor='raw') + from pypy.rpython.module.ll_win32file import make_getfullpathname_impl + getfullpathname_llimpl = make_getfullpathname_impl(traits) - return extdef([str], # a single argument which is a str - str, # returns a string - "ll_os.posix__getfullpathname", - llimpl=_getfullpathname_llimpl) + return extdef([traits.str], # a single argument which is a str + traits.str, # returns a string + traits.ll_os_name('_getfullpathname'), + llimpl=getfullpathname_llimpl) @registering(os.getcwd) def register_os_getcwd(self): @@ -953,71 +956,42 @@ "ll_os.ll_os_getcwd", llimpl=os_getcwd_llimpl, oofakeimpl=os_getcwd_oofakeimpl) - @registering(os.listdir) - def register_os_listdir(self): - # we need a different approach on Windows and on Posix - if sys.platform.startswith('win'): - from pypy.rlib import rwin32 - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h'] - ) - WIN32_FIND_DATA = platform.Struct('struct _WIN32_FIND_DATAA', - [('cFileName', lltype.FixedSizeArray(rffi.CHAR, 1))]) - ERROR_FILE_NOT_FOUND = platform.ConstantInteger( - 'ERROR_FILE_NOT_FOUND') - ERROR_NO_MORE_FILES = platform.ConstantInteger( - 'ERROR_NO_MORE_FILES') + @registering(os.getcwdu, condition=sys.platform=='win32') + def register_os_getcwdu(self): + os_wgetcwd = self.llexternal(underscore_on_windows + 'wgetcwd', + [rffi.CWCHARP, rffi.SIZE_T], + rffi.CWCHARP) - config = platform.configure(CConfig) - WIN32_FIND_DATA = config['WIN32_FIND_DATA'] - ERROR_FILE_NOT_FOUND = config['ERROR_FILE_NOT_FOUND'] - ERROR_NO_MORE_FILES = config['ERROR_NO_MORE_FILES'] - LPWIN32_FIND_DATA = lltype.Ptr(WIN32_FIND_DATA) - - FindFirstFile = self.llexternal('FindFirstFile', - [rwin32.LPCSTR, LPWIN32_FIND_DATA], - rwin32.HANDLE) - FindNextFile = self.llexternal('FindNextFile', - [rwin32.HANDLE, LPWIN32_FIND_DATA], - rwin32.BOOL) - FindClose = self.llexternal('FindClose', - [rwin32.HANDLE], - rwin32.BOOL) + def os_getcwd_llimpl(): + bufsize = 256 + while True: + buf = lltype.malloc(rffi.CWCHARP.TO, bufsize, flavor='raw') + res = os_wgetcwd(buf, rffi.cast(rffi.SIZE_T, bufsize)) + if res: + break # ok + error = rposix.get_errno() + lltype.free(buf, flavor='raw') + if error != errno.ERANGE: + raise OSError(error, "getcwd failed") + # else try again with a larger buffer, up to some sane limit + bufsize *= 4 + if bufsize > 1024*1024: # xxx hard-coded upper limit + raise OSError(error, "getcwd result too large") + result = rffi.wcharp2unicode(res) + lltype.free(buf, flavor='raw') + return result - def os_listdir_llimpl(path): - if path and path[-1] not in ('/', '\\', ':'): - path += '/' - path += '*.*' - filedata = lltype.malloc(WIN32_FIND_DATA, flavor='raw') - try: - result = [] - hFindFile = FindFirstFile(path, filedata) - if hFindFile == rwin32.INVALID_HANDLE_VALUE: - error = rwin32.GetLastError() - if error == ERROR_FILE_NOT_FOUND: - return result - else: - raise WindowsError(error, "FindFirstFile failed") - while True: - name = rffi.charp2str(rffi.cast(rffi.CCHARP, - filedata.c_cFileName)) - if name != "." and name != "..": # skip these - result.append(name) - if not FindNextFile(hFindFile, filedata): - break - # FindNextFile sets error to ERROR_NO_MORE_FILES if - # it got to the end of the directory - error = rwin32.GetLastError() - FindClose(hFindFile) - if error == ERROR_NO_MORE_FILES: - return result - else: - raise WindowsError(error, "FindNextFile failed") - finally: - lltype.free(filedata, flavor='raw') + return extdef([], unicode, + "ll_os.ll_os_wgetcwd", llimpl=os_getcwd_llimpl) + @registering_str_unicode(os.listdir) + def register_os_listdir(self, traits): + # we need a different approach on Windows and on Posix + if sys.platform.startswith('win'): + from pypy.rpython.module.ll_win32file import make_listdir_impl + os_listdir_llimpl = make_listdir_impl(traits) else: + assert traits.str is str compilation_info = ExternalCompilationInfo( includes = ['sys/types.h', 'dirent.h'] ) @@ -1057,9 +1031,9 @@ raise OSError(error, "os_readdir failed") return result - return extdef([str], # a single argument which is a str - [str], # returns a list of strings - "ll_os.ll_os_listdir", + return extdef([traits.str], # a single argument which is a str + [traits.str], # returns a list of strings + traits.ll_os_name('listdir'), llimpl=os_listdir_llimpl) @registering(os.pipe) @@ -1234,38 +1208,40 @@ return extdef([str], int, llimpl=system_llimpl, export_name="ll_os.ll_os_system") - @registering(os.unlink) - def register_os_unlink(self): - os_unlink = self.llexternal(underscore_on_windows+'unlink', [rffi.CCHARP], rffi.INT) + @registering_str_unicode(os.unlink) + def register_os_unlink(self, traits): + os_unlink = self.llexternal(traits.posix_function_name('unlink'), + [traits.CCHARP], rffi.INT) def unlink_llimpl(pathname): res = rffi.cast(lltype.Signed, os_unlink(pathname)) if res < 0: raise OSError(rposix.get_errno(), "os_unlink failed") - return extdef([str], s_None, llimpl=unlink_llimpl, - export_name="ll_os.ll_os_unlink") + return extdef([traits.str], s_None, llimpl=unlink_llimpl, + export_name=traits.ll_os_name('unlink')) - @registering(os.chdir) - def register_os_chdir(self): - os_chdir = self.llexternal(underscore_on_windows+'chdir', [rffi.CCHARP], rffi.INT) + @registering_str_unicode(os.chdir) + def register_os_chdir(self, traits): + os_chdir = self.llexternal(traits.posix_function_name('chdir'), + [traits.CCHARP], rffi.INT) def chdir_llimpl(path): res = rffi.cast(lltype.Signed, os_chdir(path)) if res < 0: raise OSError(rposix.get_errno(), "os_chdir failed") - return extdef([str], s_None, llimpl=chdir_llimpl, - export_name="ll_os.ll_os_chdir") + return extdef([traits.str], s_None, llimpl=chdir_llimpl, + export_name=traits.ll_os_name('chdir')) - @registering(os.mkdir) - def register_os_mkdir(self): + @registering_str_unicode(os.mkdir) + def register_os_mkdir(self, traits): if os.name == 'nt': ARG2 = [] # no 'mode' argument on Windows - just ignored else: ARG2 = [rffi.MODE_T] - os_mkdir = self.llexternal(underscore_on_windows+'mkdir', - [rffi.CCHARP]+ARG2, rffi.INT) + os_mkdir = self.llexternal(traits.posix_function_name('mkdir'), + [traits.CCHARP] + ARG2, rffi.INT) IGNORE_MODE = len(ARG2) == 0 def mkdir_llimpl(pathname, mode): @@ -1277,46 +1253,47 @@ if res < 0: raise OSError(rposix.get_errno(), "os_mkdir failed") - return extdef([str, int], s_None, llimpl=mkdir_llimpl, - export_name="ll_os.ll_os_mkdir") + return extdef([traits.str, int], s_None, llimpl=mkdir_llimpl, + export_name=traits.ll_os_name('mkdir')) - @registering(os.rmdir) - def register_os_rmdir(self): - os_rmdir = self.llexternal(underscore_on_windows+'rmdir', [rffi.CCHARP], rffi.INT) + @registering_str_unicode(os.rmdir) + def register_os_rmdir(self, traits): + os_rmdir = self.llexternal(traits.posix_function_name('rmdir'), + [traits.CCHARP], rffi.INT) def rmdir_llimpl(pathname): res = rffi.cast(lltype.Signed, os_rmdir(pathname)) if res < 0: raise OSError(rposix.get_errno(), "os_rmdir failed") - return extdef([str], s_None, llimpl=rmdir_llimpl, - export_name="ll_os.ll_os_rmdir") + return extdef([traits.str], s_None, llimpl=rmdir_llimpl, + export_name=traits.ll_os_name('rmdir')) - @registering(os.chmod) - def register_os_chmod(self): - os_chmod = self.llexternal(underscore_on_windows+'chmod', [rffi.CCHARP, rffi.MODE_T], - rffi.INT) + @registering_str_unicode(os.chmod) + def register_os_chmod(self, traits): + os_chmod = self.llexternal(traits.posix_function_name('chmod'), + [traits.CCHARP, rffi.MODE_T], rffi.INT) def chmod_llimpl(path, mode): res = rffi.cast(lltype.Signed, os_chmod(path, rffi.cast(rffi.MODE_T, mode))) if res < 0: raise OSError(rposix.get_errno(), "os_chmod failed") - return extdef([str, int], s_None, llimpl=chmod_llimpl, - export_name="ll_os.ll_os_chmod") + return extdef([traits.str, int], s_None, llimpl=chmod_llimpl, + export_name=traits.ll_os_name('chmod')) - @registering(os.rename) - def register_os_rename(self): - os_rename = self.llexternal('rename', [rffi.CCHARP, rffi.CCHARP], - rffi.INT) + @registering_str_unicode(os.rename) + def register_os_rename(self, traits): + os_rename = self.llexternal(traits.posix_function_name('rename'), + [traits.CCHARP, traits.CCHARP], rffi.INT) def rename_llimpl(oldpath, newpath): res = rffi.cast(lltype.Signed, os_rename(oldpath, newpath)) if res < 0: raise OSError(rposix.get_errno(), "os_rename failed") - return extdef([str, str], s_None, llimpl=rename_llimpl, - export_name="ll_os.ll_os_rename") + return extdef([traits.str, traits.str], s_None, llimpl=rename_llimpl, + export_name=traits.ll_os_name('rename')) @registering(os.umask) def register_os_umask(self): @@ -1425,17 +1402,17 @@ @registering(os.fstat) def register_os_fstat(self): from pypy.rpython.module import ll_os_stat - ll_os_stat.register_stat_variant('fstat') + return ll_os_stat.register_stat_variant('fstat', StringTraits()) - @registering(os.stat) - def register_os_stat(self): + @registering_str_unicode(os.stat) + def register_os_stat(self, traits): from pypy.rpython.module import ll_os_stat - ll_os_stat.register_stat_variant('stat') + return ll_os_stat.register_stat_variant('stat', traits) - @registering(os.lstat) - def register_os_lstat(self): + @registering_str_unicode(os.lstat) + def register_os_lstat(self, traits): from pypy.rpython.module import ll_os_stat - ll_os_stat.register_stat_variant('lstat') + return ll_os_stat.register_stat_variant('lstat', traits) # ------------------------------- os.W* --------------------------------- Modified: pypy/branch/interplevel-array/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/branch/interplevel-array/pypy/rpython/module/ll_os_stat.py Wed Aug 4 16:53:45 2010 @@ -5,13 +5,14 @@ import os, sys from pypy.annotation import model as annmodel from pypy.tool.pairtype import pairtype -from pypy.tool.sourcetools import func_with_new_name +from pypy.tool.sourcetools import func_with_new_name, func_renamer from pypy.rpython import extregistry -from pypy.rpython.extfunc import register_external +from pypy.rpython.extfunc import register_external, extdef from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.tool import rffi_platform as platform from pypy.rpython.lltypesystem.rtupletype import TUPLE_TYPE from pypy.rlib import rposix +from pypy.rlib.objectmodel import specialize from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rpython.annlowlevel import hlstr @@ -211,13 +212,27 @@ return make_stat_result(result) -def register_stat_variant(name): - if sys.platform.startswith('win'): - _functions = {'stat': '_stati64', - 'fstat': '_fstati64', - 'lstat': '_stati64'} # no lstat on Windows - c_func_name = _functions[name] - elif sys.platform.startswith('linux'): +def register_stat_variant(name, traits): + if name != 'fstat': + arg_is_path = True + s_arg = traits.str + ARG1 = traits.CCHARP + else: + arg_is_path = False + s_arg = int + ARG1 = rffi.INT + + if sys.platform == 'win32': + # See Win32 implementation below + posix_stat_llimpl = make_win32_stat_impl(name, traits) + + return extdef( + [s_arg], s_StatResult, traits.ll_os_name(name), + llimpl=posix_stat_llimpl) + + assert traits.str is str + + if sys.platform.startswith('linux'): # because we always use _FILE_OFFSET_BITS 64 - this helps things work that are not a c compiler _functions = {'stat': 'stat64', 'fstat': 'fstat64', @@ -226,22 +241,26 @@ else: c_func_name = name - arg_is_path = (name != 'fstat') + posix_mystat = rffi.llexternal(c_func_name, + [ARG1, STAT_STRUCT], rffi.INT, + compilation_info=compilation_info) + @func_renamer('os_%s_llimpl' % (name,)) def posix_stat_llimpl(arg): stresult = lltype.malloc(STAT_STRUCT.TO, flavor='raw') try: if arg_is_path: - arg = rffi.str2charp(arg) + arg = traits.str2charp(arg) error = rffi.cast(rffi.LONG, posix_mystat(arg, stresult)) if arg_is_path: - rffi.free_charp(arg) + traits.free_charp(arg) if error != 0: raise OSError(rposix.get_errno(), "os_?stat failed") return build_stat_result(stresult) finally: lltype.free(stresult, flavor='raw') + @func_renamer('os_%s_fake' % (name,)) def posix_fakeimpl(arg): if s_arg == str: arg = hlstr(arg) @@ -259,40 +278,17 @@ setattr(ll_tup, 'item%d' % i, val) return ll_tup - if arg_is_path: - s_arg = str - ARG1 = rffi.CCHARP - else: - s_arg = int - ARG1 = rffi.INT + return extdef( + [s_arg], s_StatResult, "ll_os.ll_os_%s" % (name,), + llimpl=posix_stat_llimpl, llfakeimpl=posix_fakeimpl) - if sys.platform != 'win32': - posix_mystat = rffi.llexternal(c_func_name, - [ARG1, STAT_STRUCT], rffi.INT, - compilation_info=compilation_info) - - register_external( - getattr(os, name), [s_arg], s_StatResult, - "ll_os.ll_os_%s" % (name,), - llimpl=func_with_new_name(posix_stat_llimpl, - 'os_%s_llimpl' % (name,)), - llfakeimpl=func_with_new_name(posix_fakeimpl, - 'os_%s_fake' % (name,)), - ) - else: - # See Win32 implementation below - register_external( - getattr(os, name), [s_arg], s_StatResult, - "ll_os.ll_os_%s" % (name,), - llimpl=func_with_new_name(globals()['win32_%s_llimpl' % (name,)], - 'os_%s_llimpl' % (name,)), - ) +def make_win32_stat_impl(name, traits): + from pypy.rlib import rwin32 + from pypy.rpython.module.ll_win32file import make_win32_traits + win32traits = make_win32_traits(traits) -# ____________________________________________________________ -if sys.platform == 'win32': # The CRT of Windows has a number of flaws wrt. its stat() implementation: - # - for when we implement subsecond resolution in RPython, time stamps - # would be restricted to second resolution + # - time stamps are restricted to second resolution # - file modification times suffer from forth-and-back conversions between # UTC and local time # Therefore, we implement our own stat, based on the Win32 API directly. @@ -302,122 +298,18 @@ assert len(STAT_FIELDS) == 10 # no extra fields on Windows - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h', 'winbase.h', 'sys/stat.h'], - ) - - GetFileExInfoStandard = platform.ConstantInteger( - 'GetFileExInfoStandard') - FILE_ATTRIBUTE_DIRECTORY = platform.ConstantInteger( - 'FILE_ATTRIBUTE_DIRECTORY') - FILE_ATTRIBUTE_READONLY = platform.ConstantInteger( - 'FILE_ATTRIBUTE_READONLY') - ERROR_SHARING_VIOLATION = platform.ConstantInteger( - 'ERROR_SHARING_VIOLATION') - _S_IFDIR = platform.ConstantInteger('_S_IFDIR') - _S_IFREG = platform.ConstantInteger('_S_IFREG') - _S_IFCHR = platform.ConstantInteger('_S_IFCHR') - _S_IFIFO = platform.ConstantInteger('_S_IFIFO') - FILE_TYPE_UNKNOWN = platform.ConstantInteger('FILE_TYPE_UNKNOWN') - FILE_TYPE_CHAR = platform.ConstantInteger('FILE_TYPE_CHAR') - FILE_TYPE_PIPE = platform.ConstantInteger('FILE_TYPE_PIPE') - - WIN32_FILE_ATTRIBUTE_DATA = platform.Struct( - 'WIN32_FILE_ATTRIBUTE_DATA', - [('dwFileAttributes', rwin32.DWORD), - ('nFileSizeHigh', rwin32.DWORD), - ('nFileSizeLow', rwin32.DWORD), - ('ftCreationTime', rwin32.FILETIME), - ('ftLastAccessTime', rwin32.FILETIME), - ('ftLastWriteTime', rwin32.FILETIME)]) - - BY_HANDLE_FILE_INFORMATION = platform.Struct( - 'BY_HANDLE_FILE_INFORMATION', - [('dwFileAttributes', rwin32.DWORD), - ('nFileSizeHigh', rwin32.DWORD), - ('nFileSizeLow', rwin32.DWORD), - ('nNumberOfLinks', rwin32.DWORD), - ('nFileIndexHigh', rwin32.DWORD), - ('nFileIndexLow', rwin32.DWORD), - ('ftCreationTime', rwin32.FILETIME), - ('ftLastAccessTime', rwin32.FILETIME), - ('ftLastWriteTime', rwin32.FILETIME)]) - - WIN32_FIND_DATA = platform.Struct( - 'WIN32_FIND_DATAA', - # Only interesting fields - [('dwFileAttributes', rwin32.DWORD), - ('nFileSizeHigh', rwin32.DWORD), - ('nFileSizeLow', rwin32.DWORD), - ('ftCreationTime', rwin32.FILETIME), - ('ftLastAccessTime', rwin32.FILETIME), - ('ftLastWriteTime', rwin32.FILETIME)]) - - globals().update(platform.configure(CConfig)) - GET_FILEEX_INFO_LEVELS = rffi.ULONG # an enumeration - - GetFileAttributesEx = rffi.llexternal( - 'GetFileAttributesExA', - [rffi.CCHARP, GET_FILEEX_INFO_LEVELS, - lltype.Ptr(WIN32_FILE_ATTRIBUTE_DATA)], - rwin32.BOOL, - calling_conv='win') - - GetFileInformationByHandle = rffi.llexternal( - 'GetFileInformationByHandle', - [rwin32.HANDLE, lltype.Ptr(BY_HANDLE_FILE_INFORMATION)], - rwin32.BOOL, - calling_conv='win') - - GetFileType = rffi.llexternal( - 'GetFileType', - [rwin32.HANDLE], - rwin32.DWORD, - calling_conv='win') - - FindFirstFile = rffi.llexternal( - 'FindFirstFileA', - [rffi.CCHARP, lltype.Ptr(WIN32_FIND_DATA)], - rwin32.HANDLE, - calling_conv='win') - - FindClose = rffi.llexternal( - 'FindClose', - [rwin32.HANDLE], - rwin32.BOOL, - calling_conv='win') - def attributes_to_mode(attributes): m = 0 - if attributes & FILE_ATTRIBUTE_DIRECTORY: - m |= _S_IFDIR | 0111 # IFEXEC for user,group,other + if attributes & win32traits.FILE_ATTRIBUTE_DIRECTORY: + m |= win32traits._S_IFDIR | 0111 # IFEXEC for user,group,other else: - m |= _S_IFREG - if attributes & FILE_ATTRIBUTE_READONLY: + m |= win32traits._S_IFREG + if attributes & win32traits.FILE_ATTRIBUTE_READONLY: m |= 0444 else: m |= 0666 return m - def make_longlong(high, low): - return (lltype.r_longlong(high) << 32) + lltype.r_longlong(low) - - # Seconds between 1.1.1601 and 1.1.1970 - secs_between_epochs = lltype.r_longlong(11644473600) - - def FILE_TIME_to_time_t_nsec(filetime): - ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime) - # FILETIME is in units of 100 nsec - nsec = (ft % 10000000) * 100 - time = (ft / 10000000) - secs_between_epochs - return time, nsec - - def time_t_to_FILE_TIME(time, filetime): - ft = lltype.r_longlong((time + secs_between_epochs) * 10000000) - filetime.c_dwHighDateTime = lltype.r_uint(ft >> 32) - filetime.c_dwLowDateTime = lltype.r_uint(ft & ((1 << 32) - 1)) - def attribute_data_to_stat(info): st_mode = attributes_to_mode(info.c_dwFileAttributes) st_size = make_longlong(info.c_nFileSizeHigh, info.c_nFileSizeLow) @@ -456,65 +348,94 @@ return make_stat_result(result) def attributes_from_dir(l_path, data): - filedata = lltype.malloc(WIN32_FIND_DATA, flavor='raw') - hFindFile = FindFirstFile(l_path, filedata) - if hFindFile == rwin32.INVALID_HANDLE_VALUE: - return 0 - FindClose(hFindFile) - data.c_dwFileAttributes = filedata.c_dwFileAttributes - rffi.structcopy(data.c_ftCreationTime, filedata.c_ftCreationTime) - rffi.structcopy(data.c_ftLastAccessTime, filedata.c_ftLastAccessTime) - rffi.structcopy(data.c_ftLastWriteTime, filedata.c_ftLastWriteTime) - data.c_nFileSizeHigh = filedata.c_nFileSizeHigh - data.c_nFileSizeLow = filedata.c_nFileSizeLow - return 1 + filedata = lltype.malloc(win32traits.WIN32_FIND_DATA, flavor='raw') + try: + hFindFile = win32traits.FindFirstFile(l_path, filedata) + if hFindFile == rwin32.INVALID_HANDLE_VALUE: + return 0 + win32traits.FindClose(hFindFile) + data.c_dwFileAttributes = filedata.c_dwFileAttributes + rffi.structcopy(data.c_ftCreationTime, filedata.c_ftCreationTime) + rffi.structcopy(data.c_ftLastAccessTime, filedata.c_ftLastAccessTime) + rffi.structcopy(data.c_ftLastWriteTime, filedata.c_ftLastWriteTime) + data.c_nFileSizeHigh = filedata.c_nFileSizeHigh + data.c_nFileSizeLow = filedata.c_nFileSizeLow + return 1 + finally: + lltype.free(filedata, flavor='raw') def win32_stat_llimpl(path): - data = lltype.malloc(WIN32_FILE_ATTRIBUTE_DATA, flavor='raw') + data = lltype.malloc(win32traits.WIN32_FILE_ATTRIBUTE_DATA, flavor='raw') try: - l_path = rffi.str2charp(path) - res = GetFileAttributesEx(l_path, GetFileExInfoStandard, data) + l_path = traits.str2charp(path) + res = win32traits.GetFileAttributesEx(l_path, win32traits.GetFileExInfoStandard, data) errcode = rwin32.GetLastError() if res == 0: - if errcode == ERROR_SHARING_VIOLATION: + if errcode == win32traits.ERROR_SHARING_VIOLATION: res = attributes_from_dir(l_path, data) errcode = rwin32.GetLastError() - rffi.free_charp(l_path) + traits.free_charp(l_path) if res == 0: raise WindowsError(errcode, "os_stat failed") return attribute_data_to_stat(data) finally: lltype.free(data, flavor='raw') - win32_lstat_llimpl = win32_stat_llimpl def win32_fstat_llimpl(fd): handle = rwin32._get_osfhandle(fd) - filetype = GetFileType(handle) - if filetype == FILE_TYPE_CHAR: + filetype = win32traits.GetFileType(handle) + if filetype == win32traits.FILE_TYPE_CHAR: # console or LPT device - return make_stat_result((_S_IFCHR, + return make_stat_result((win32traits._S_IFCHR, 0, 0, 0, 0, 0, 0, 0, 0, 0)) - elif filetype == FILE_TYPE_PIPE: + elif filetype == win32traits.FILE_TYPE_PIPE: # socket or named pipe - return make_stat_result((_S_IFIFO, + return make_stat_result((win32traits._S_IFIFO, 0, 0, 0, 0, 0, 0, 0, 0, 0)) - elif filetype == FILE_TYPE_UNKNOWN: + elif filetype == win32traits.FILE_TYPE_UNKNOWN: error = rwin32.GetLastError() if error != 0: raise WindowsError(error, "os_fstat failed") # else: unknown but valid file # normal disk file (FILE_TYPE_DISK) - info = lltype.malloc(BY_HANDLE_FILE_INFORMATION, flavor='raw', - zero=True) + info = lltype.malloc(win32traits.BY_HANDLE_FILE_INFORMATION, + flavor='raw', zero=True) try: - res = GetFileInformationByHandle(handle, info) + res = win32traits.GetFileInformationByHandle(handle, info) if res == 0: raise WindowsError(rwin32.GetLastError(), "os_fstat failed") return by_handle_info_to_stat(info) finally: lltype.free(info, flavor='raw') + if name == 'fstat': + return win32_fstat_llimpl + else: + return win32_stat_llimpl + + +#__________________________________________________ +# Helper functions for win32 + +def make_longlong(high, low): + return (lltype.r_longlong(high) << 32) + lltype.r_longlong(low) + +# Seconds between 1.1.1601 and 1.1.1970 +secs_between_epochs = lltype.r_longlong(11644473600) + +def FILE_TIME_to_time_t_nsec(filetime): + ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime) + # FILETIME is in units of 100 nsec + nsec = (ft % 10000000) * 100 + time = (ft / 10000000) - secs_between_epochs + return time, nsec + +def time_t_to_FILE_TIME(time, filetime): + ft = lltype.r_longlong((time + secs_between_epochs) * 10000000) + filetime.c_dwHighDateTime = lltype.r_uint(ft >> 32) + filetime.c_dwLowDateTime = lltype.r_uint(ft & ((1 << 32) - 1)) + Modified: pypy/branch/interplevel-array/pypy/rpython/module/test/test_ll_os_stat.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rpython/module/test/test_ll_os_stat.py (original) +++ pypy/branch/interplevel-array/pypy/rpython/module/test/test_ll_os_stat.py Wed Aug 4 16:53:45 2010 @@ -1,4 +1,4 @@ -from pypy.rpython.module import ll_os_stat +from pypy.rpython.module import ll_os_stat, ll_os import sys, os import py @@ -8,14 +8,18 @@ py.test.skip("win32 specific tests") def test_stat(self): - stat = ll_os_stat.win32_stat_llimpl + stat = ll_os_stat.make_win32_stat_impl('stat', ll_os.StringTraits()) + wstat = ll_os_stat.make_win32_stat_impl('stat', ll_os.UnicodeTraits()) def check(f): - assert stat(f).st_mtime == os.stat(f).st_mtime + expected = os.stat(f).st_mtime + assert stat(f).st_mtime == expected + assert wstat(unicode(f)).st_mtime == expected check('c:/') check('c:/temp') check('c:/pagefile.sys') def test_fstat(self): - stat = ll_os_stat.win32_fstat_llimpl(0) # stdout + fstat = ll_os_stat.make_win32_stat_impl('fstat', ll_os.StringTraits()) + stat = fstat(0) # stdout assert stat.st_mode != 0 Modified: pypy/branch/interplevel-array/pypy/rpython/rstr.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rpython/rstr.py (original) +++ pypy/branch/interplevel-array/pypy/rpython/rstr.py Wed Aug 4 16:53:45 2010 @@ -288,7 +288,11 @@ if not hop.args_s[1].is_constant(): raise TyperError("encoding must be constant") encoding = hop.args_s[1].const - v_self = hop.inputarg(self.repr, 0) + if encoding == "ascii": + expect = self.lowleveltype # can be a UniChar + else: + expect = self.repr # must be a regular unicode string + v_self = hop.inputarg(expect, 0) hop.exception_is_here() if encoding == "ascii": return hop.gendirectcall(self.ll_str, v_self) @@ -415,7 +419,17 @@ sourcevars.append((v_item, r_arg)) return r_str.ll.do_stringformat(hop, sourcevars) - + + +class __extend__(AbstractCharRepr): + def ll_str(self, ch): + return self.ll.ll_chr2str(ch) + +class __extend__(AbstractUniCharRepr): + def ll_str(self, ch): + # xxx suboptimal, maybe + return str(unicode(ch)) + class __extend__(AbstractCharRepr, AbstractUniCharRepr): @@ -433,9 +447,6 @@ get_ll_fasthash_function = get_ll_hash_function - def ll_str(self, ch): - return self.ll.ll_chr2str(ch) - def rtype_len(_, hop): return hop.inputconst(Signed, 1) Modified: pypy/branch/interplevel-array/pypy/rpython/test/test_extfunc.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rpython/test/test_extfunc.py (original) +++ pypy/branch/interplevel-array/pypy/rpython/test/test_extfunc.py Wed Aug 4 16:53:45 2010 @@ -6,150 +6,164 @@ from pypy.annotation.policy import AnnotatorPolicy from pypy.rpython.test.test_llinterp import interpret -def b(x): - return eval("x+40") +class TestExtFuncEntry: -class BTestFuncEntry(ExtFuncEntry): - _about_ = b - name = 'b' - signature_args = [annmodel.SomeInteger()] - signature_result = annmodel.SomeInteger() - -def test_annotation_b(): - def f(): - return b(1) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeInteger) - -def test_rtyping_b(): - def f(): - return b(2) - - res = interpret(f, []) - assert res == 42 - -def c(y, x): - yyy - -class CTestFuncEntry(ExtFuncEntry): - _about_ = c - name = 'ccc' - signature_args = [annmodel.SomeInteger()] * 2 - signature_result = annmodel.SomeInteger() - - def lltypeimpl(y, x): - return y + x - lltypeimpl = staticmethod(lltypeimpl) - -def test_interp_c(): - def f(): - return c(3, 4) - - res = interpret(f, []) - assert res == 7 - -def d(y): - return eval("y()") - -class DTestFuncEntry(ExtFuncEntry): - _about_ = d - name = 'd' - signature_args = [annmodel.SomeGenericCallable(args=[], result= - annmodel.SomeFloat())] - signature_result = annmodel.SomeFloat() - -def test_callback(): - def callback(): - return 2.5 - - def f(): - return d(callback) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeFloat) - assert a.translator._graphof(callback) - -def dd(): - pass - -register_external(dd, [int], int) - -def test_register_external_signature(): - def f(): - return dd(3) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeInteger) - - -def function_with_tuple_arg(): - """ - Dummy function which is declared via register_external to take a tuple as - an argument so that register_external's behavior for tuple-taking functions - can be verified. - """ -register_external(function_with_tuple_arg, [(int,)], int) - -def test_register_external_tuple_args(): - """ - Verify the annotation of a registered external function which takes a tuple - argument. - """ - def f(): - return function_with_tuple_arg((1,)) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - - # Not a very good assertion, but at least it means _something_ happened. - assert isinstance(s, annmodel.SomeInteger) - -def function_with_list(): - pass -register_external(function_with_list, [[int]], int) - -def function_returning_list(): - pass -register_external(function_returning_list, [], [int]) - -def test_register_external_return_goes_back(): - """ - Check whether it works to pass the same list from one external - fun to another - [bookkeeper and list joining issues] - """ - def f(): - return function_with_list(function_returning_list()) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeInteger) - -def function_withspecialcase(arg): - return repr(arg) -register_external(function_withspecialcase, args=None, result=str) - -def test_register_external_specialcase(): - def f(): - x = function_withspecialcase - return x(33) + x("aaa") + x([]) + "\n" - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeString) + def test_basic(self): + """ + A ExtFuncEntry provides an annotation for a function, no need to flow + its graph. + """ + def b(x): + "NOT_RPYTHON" + return eval("x+40") + + class BTestFuncEntry(ExtFuncEntry): + _about_ = b + name = 'b' + signature_args = [annmodel.SomeInteger()] + signature_result = annmodel.SomeInteger() + + def f(): + return b(2) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeInteger) + + res = interpret(f, []) + assert res == 42 + + def test_lltypeimpl(self): + """ + interpret() calls lltypeimpl instead of of the function/ + """ + def c(y, x): + yyy + + class CTestFuncEntry(ExtFuncEntry): + _about_ = c + name = 'ccc' + signature_args = [annmodel.SomeInteger()] * 2 + signature_result = annmodel.SomeInteger() + + def lltypeimpl(y, x): + return y + x + lltypeimpl = staticmethod(lltypeimpl) + + def f(): + return c(3, 4) + + res = interpret(f, []) + assert res == 7 + + def test_callback(self): + """ + Verify annotation when a callback function is in the arguments list. + """ + def d(y): + return eval("y()") + + class DTestFuncEntry(ExtFuncEntry): + _about_ = d + name = 'd' + signature_args = [annmodel.SomeGenericCallable(args=[], result= + annmodel.SomeFloat())] + signature_result = annmodel.SomeFloat() + + def callback(): + return 2.5 + + def f(): + return d(callback) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeFloat) + assert a.translator._graphof(callback) + + def test_register_external_signature(self): + """ + Test the standard interface for external functions. + """ + def dd(): + pass + register_external(dd, [int], int) + + def f(): + return dd(3) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeInteger) + + def test_register_external_tuple_args(self): + """ + Verify the annotation of a registered external function which takes a + tuple argument. + """ + + def function_with_tuple_arg(): + """ + Dummy function which is declared via register_external to take a + tuple as an argument so that register_external's behavior for + tuple-taking functions can be verified. + """ + register_external(function_with_tuple_arg, [(int,)], int) + + def f(): + return function_with_tuple_arg((1,)) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + + # Not a very good assertion, but at least it means _something_ happened. + assert isinstance(s, annmodel.SomeInteger) + + def test_register_external_return_goes_back(self): + """ + Check whether it works to pass the same list from one external + fun to another + [bookkeeper and list joining issues] + """ + def function_with_list(): + pass + register_external(function_with_list, [[int]], int) + + def function_returning_list(): + pass + register_external(function_returning_list, [], [int]) + + def f(): + return function_with_list(function_returning_list()) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeInteger) + + def test_register_external_specialcase(self): + """ + When args=None, the external function accepts any arguments unmodified. + """ + def function_withspecialcase(arg): + return repr(arg) + register_external(function_withspecialcase, args=None, result=str) + + def f(): + x = function_withspecialcase + return x(33) + x("aaa") + x([]) + "\n" + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeString) Modified: pypy/branch/interplevel-array/pypy/rpython/test/test_rstr.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rpython/test/test_rstr.py (original) +++ pypy/branch/interplevel-array/pypy/rpython/test/test_rstr.py Wed Aug 4 16:53:45 2010 @@ -863,6 +863,24 @@ res = self.interpret(f, [1]) assert self.ll_to_string(res) == "hello" + def test_str_unichar(self): + def f(i): + c = u"abc" + return str(c[i])[0] + assert self.interpret(f, [1]) == "b" + + def test_encode_char(self): + def f(i): + c = u"abc" + return c[i].encode("ascii") + assert self.ll_to_string(self.interpret(f, [0])) == "a" + + def test_encode_char_latin1(self): + def f(i): + c = u"abc" + return c[i].encode("latin-1") + assert self.ll_to_string(self.interpret(f, [0])) == "a" + def FIXME_test_str_to_pystringobj(): def f(n): if n >= 0: Modified: pypy/branch/interplevel-array/pypy/translator/c/test/test_typed.py ============================================================================== --- pypy/branch/interplevel-array/pypy/translator/c/test/test_typed.py (original) +++ pypy/branch/interplevel-array/pypy/translator/c/test/test_typed.py Wed Aug 4 16:53:45 2010 @@ -789,3 +789,37 @@ return u'hello' + unichr(i) f = self.getcompiled(func, [int]) assert f(0x1234) == u'hello\u1234' + + def test_ovfcheck_float_to_int(self): + from pypy.rlib.rarithmetic import ovfcheck_float_to_int + + def func(fl): + try: + return ovfcheck_float_to_int(fl) + except OverflowError: + return -666 + f = self.getcompiled(func, [float]) + assert f(-123.0) == -123 + + for frac in [0.0, 0.01, 0.99]: + # strange things happening for float to int on 64 bit: + # int(float(i)) != i because of rounding issues + x = sys.maxint + while int(x + frac) > sys.maxint: + x -= 1 + assert f(x + frac) == int(x + frac) + + x = sys.maxint + while int(x - frac) <= sys.maxint: + x += 1 + assert f(x - frac) == -666 + + x = -sys.maxint-1 + while int(x - frac) < -sys.maxint-1: + x += 1 + assert f(x - frac) == int(x - frac) + + x = -sys.maxint-1 + while int(x + frac) >= -sys.maxint-1: + x -= 1 + assert f(x + frac) == -666 Modified: pypy/branch/interplevel-array/pypy/translator/goal/app_main.py ============================================================================== --- pypy/branch/interplevel-array/pypy/translator/goal/app_main.py (original) +++ pypy/branch/interplevel-array/pypy/translator/goal/app_main.py Wed Aug 4 16:53:45 2010 @@ -223,7 +223,6 @@ path = os.getenv('PYTHONPATH') if path: newpath = path.split(os.pathsep) + newpath - newpath.insert(0, '') # remove duplicates _seen = {} del sys.path[:] @@ -327,6 +326,10 @@ except: print >> sys.stderr, "'import site' failed" + # update sys.path *after* loading site.py, in case there is a + # "site.py" file in the script's directory. + sys.path.insert(0, '') + if warnoptions: sys.warnoptions.append(warnoptions) from warnings import _processoptions Modified: pypy/branch/interplevel-array/pypy/translator/goal/test2/test_app_main.py ============================================================================== --- pypy/branch/interplevel-array/pypy/translator/goal/test2/test_app_main.py (original) +++ pypy/branch/interplevel-array/pypy/translator/goal/test2/test_app_main.py Wed Aug 4 16:53:45 2010 @@ -5,6 +5,7 @@ import sys, os, re import autopath from pypy.tool.udir import udir +from contextlib import contextmanager banner = sys.version.splitlines()[0] @@ -326,8 +327,9 @@ class TestNonInteractive: def run(self, cmdline, senddata='', expect_prompt=False, - expect_banner=False): - cmdline = '%s "%s" %s' % (sys.executable, app_main, cmdline) + expect_banner=False, python_flags=''): + cmdline = '%s %s "%s" %s' % (sys.executable, python_flags, + app_main, cmdline) print 'POPEN:', cmdline child_in, child_out_err = os.popen4(cmdline) child_in.write(senddata) @@ -449,6 +451,43 @@ assert data == '\x00(STDOUT)\n\x00' # from stdout child_out_err.close() + def test_proper_sys_path(self, tmpdir): + + @contextmanager + def chdir_and_unset_pythonpath(new_cwd): + old_cwd = new_cwd.chdir() + old_pythonpath = os.getenv('PYTHONPATH') + os.unsetenv('PYTHONPATH') + try: + yield + finally: + old_cwd.chdir() + os.putenv('PYTHONPATH', old_pythonpath) + + tmpdir.join('site.py').write('print "SHOULD NOT RUN"') + runme_py = tmpdir.join('runme.py') + runme_py.write('print "some text"') + + cmdline = str(runme_py) + + with chdir_and_unset_pythonpath(tmpdir): + data = self.run(cmdline, python_flags='-S') + + assert data == "some text\n" + + runme2_py = tmpdir.mkdir('otherpath').join('runme2.py') + runme2_py.write('print "some new text"\n' + 'import sys\n' + 'print sys.path\n') + + cmdline2 = str(runme2_py) + + with chdir_and_unset_pythonpath(tmpdir): + data = self.run(cmdline2, python_flags='-S') + + assert data.startswith("some new text\n") + assert repr(str(tmpdir.join('otherpath'))) in data + class AppTestAppMain: Modified: pypy/branch/interplevel-array/pypy/translator/platform/__init__.py ============================================================================== --- pypy/branch/interplevel-array/pypy/translator/platform/__init__.py (original) +++ pypy/branch/interplevel-array/pypy/translator/platform/__init__.py Wed Aug 4 16:53:45 2010 @@ -161,7 +161,7 @@ return (library_dirs + self.link_flags + export_flags + link_files + list(eci.link_extra) + libraries) - def _exportsymbols_link_flags(self, eci): + def _exportsymbols_link_flags(self, eci, relto=None): if eci.export_symbols: raise ValueError("This platform does not support export symbols") return [] Modified: pypy/branch/interplevel-array/pypy/translator/platform/darwin.py ============================================================================== --- pypy/branch/interplevel-array/pypy/translator/platform/darwin.py (original) +++ pypy/branch/interplevel-array/pypy/translator/platform/darwin.py Wed Aug 4 16:53:45 2010 @@ -56,7 +56,7 @@ include_dirs = self._includedirs(eci.include_dirs) return (args + frameworks + include_dirs) - def _exportsymbols_link_flags(self, eci): + def _exportsymbols_link_flags(self, eci, relto=None): if not eci.export_symbols: return [] @@ -65,6 +65,9 @@ for sym in eci.export_symbols: f.write("_%s\n" % (sym,)) f.close() + + if relto: + response_file = relto.bestrelpath(response_file) return ["-Wl,-exported_symbols_list,%s" % (response_file,)] class Darwin_i386(Darwin): Modified: pypy/branch/interplevel-array/pypy/translator/platform/posix.py ============================================================================== --- pypy/branch/interplevel-array/pypy/translator/platform/posix.py (original) +++ pypy/branch/interplevel-array/pypy/translator/platform/posix.py Wed Aug 4 16:53:45 2010 @@ -42,7 +42,7 @@ def _link_args_from_eci(self, eci, standalone): return Platform._link_args_from_eci(self, eci, standalone) - def _exportsymbols_link_flags(self, eci): + def _exportsymbols_link_flags(self, eci, relto=None): if not eci.export_symbols: return [] @@ -53,6 +53,9 @@ f.write("%s;\n" % (sym,)) f.write("};") f.close() + + if relto: + response_file = relto.bestrelpath(response_file) return ["-Wl,--export-dynamic,--version-script=%s" % (response_file,)] def _link(self, cc, ofiles, link_args, standalone, exe_name): @@ -93,7 +96,7 @@ if shared: linkflags = self._args_for_shared(linkflags) - linkflags += self._exportsymbols_link_flags(eci) + linkflags += self._exportsymbols_link_flags(eci, relto=path) if shared: libname = exe_name.new(ext='').basename Modified: pypy/branch/interplevel-array/pypy/translator/platform/windows.py ============================================================================== --- pypy/branch/interplevel-array/pypy/translator/platform/windows.py (original) +++ pypy/branch/interplevel-array/pypy/translator/platform/windows.py Wed Aug 4 16:53:45 2010 @@ -141,7 +141,7 @@ # Windows needs to resolve all symbols even for DLLs return super(MsvcPlatform, self)._link_args_from_eci(eci, standalone=True) - def _exportsymbols_link_flags(self, eci): + def _exportsymbols_link_flags(self, eci, relto=None): if not eci.export_symbols: return [] @@ -150,6 +150,9 @@ for sym in eci.export_symbols: f.write("/EXPORT:%s\n" % (sym,)) f.close() + + if relto: + response_file = relto.bestrelpath(response_file) return ["@%s" % (response_file,)] def _compile_c_file(self, cc, cfile, compile_args): @@ -219,7 +222,7 @@ if shared: linkflags = self._args_for_shared(linkflags) + [ '/EXPORT:$(PYPY_MAIN_FUNCTION)'] - linkflags += self._exportsymbols_link_flags(eci) + linkflags += self._exportsymbols_link_flags(eci, relto=path) if shared: so_name = exe_name.new(purebasename='lib' + exe_name.purebasename, From hakanardo at codespeak.net Wed Aug 4 17:01:33 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Wed, 4 Aug 2010 17:01:33 +0200 (CEST) Subject: [pypy-svn] r76470 - pypy/branch/interplevel-array/pypy/jit/backend/x86 Message-ID: <20100804150133.52106282BAD@codespeak.net> Author: hakanardo Date: Wed Aug 4 17:01:31 2010 New Revision: 76470 Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/assembler.py Log: 16 bit arrayitem Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/assembler.py Wed Aug 4 17:01:31 2010 @@ -1064,6 +1064,9 @@ if scale.value == 0: self.mc.MOVZX8(resloc, addr_add(base_loc, ofs_loc, ofs.value, scale.value)) + elif scale.value == 1: + self.mc.MOVZX16(resloc, addr_add(base_loc, ofs_loc, ofs.value, + scale.value)) elif (1 << scale.value) == WORD: self.mc.MOV(resloc, addr_add(base_loc, ofs_loc, ofs.value, scale.value)) @@ -1101,6 +1104,8 @@ else: if (1 << scale_loc.value) == WORD: self.mc.MOV(dest_addr, value_loc) + elif scale_loc.value == 1: + self.mc.MOV16(dest_addr, value_loc) elif scale_loc.value == 0: self.mc.MOV8(dest_addr, value_loc.lowest8bits()) else: From fijal at codespeak.net Wed Aug 4 17:10:40 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 4 Aug 2010 17:10:40 +0200 (CEST) Subject: [pypy-svn] r76471 - pypy/trunk/pypy/objspace/std Message-ID: <20100804151040.0BC85282BAD@codespeak.net> Author: fijal Date: Wed Aug 4 17:10:39 2010 New Revision: 76471 Modified: pypy/trunk/pypy/objspace/std/callmethod.py Log: This has to match things in module/stackless/interp_coroutine. Let's say it does Modified: pypy/trunk/pypy/objspace/std/callmethod.py ============================================================================== --- pypy/trunk/pypy/objspace/std/callmethod.py (original) +++ pypy/trunk/pypy/objspace/std/callmethod.py Wed Aug 4 17:10:39 2010 @@ -90,7 +90,7 @@ w_result = f.space.call_args(w_callable, args) finally: f.dropvalues(1 + (w_self is None)) - rstack.resume_point("CALL_METHOD", f, returns=w_result) + rstack.resume_point("CALL_METHOD", f, nargs, returns=w_result) f.pushvalue(w_result) From fijal at codespeak.net Wed Aug 4 17:14:44 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 4 Aug 2010 17:14:44 +0200 (CEST) Subject: [pypy-svn] r76472 - pypy/trunk/pypy/objspace/std Message-ID: <20100804151444.62523282BAD@codespeak.net> Author: fijal Date: Wed Aug 4 17:14:42 2010 New Revision: 76472 Modified: pypy/trunk/pypy/objspace/std/callmethod.py Log: I'll always run tests before checking in .... Modified: pypy/trunk/pypy/objspace/std/callmethod.py ============================================================================== --- pypy/trunk/pypy/objspace/std/callmethod.py (original) +++ pypy/trunk/pypy/objspace/std/callmethod.py Wed Aug 4 17:14:42 2010 @@ -90,7 +90,7 @@ w_result = f.space.call_args(w_callable, args) finally: f.dropvalues(1 + (w_self is None)) - rstack.resume_point("CALL_METHOD", f, nargs, returns=w_result) + rstack.resume_point("CALL_METHOD", f, n_args, returns=w_result) f.pushvalue(w_result) From fijal at codespeak.net Wed Aug 4 17:33:45 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 4 Aug 2010 17:33:45 +0200 (CEST) Subject: [pypy-svn] r76473 - pypy/trunk/pypy/jit/backend/x86/test Message-ID: <20100804153345.D98D5282BAD@codespeak.net> Author: fijal Date: Wed Aug 4 17:33:44 2010 New Revision: 76473 Modified: pypy/trunk/pypy/jit/backend/x86/test/test_assembler.py Log: Fix tests Modified: pypy/trunk/pypy/jit/backend/x86/test/test_assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_assembler.py Wed Aug 4 17:33:44 2010 @@ -15,6 +15,9 @@ supports_floats = True NUM_REGS = ACTUAL_CPU.NUM_REGS + def arraydescrof(self, ARR): + return 42 + class FakeMC: def __init__(self, base_address=0): self.content = [] From fijal at codespeak.net Wed Aug 4 17:34:07 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 4 Aug 2010 17:34:07 +0200 (CEST) Subject: [pypy-svn] r76474 - pypy/trunk/pypy/jit/backend/x86/tool Message-ID: <20100804153407.5F1E6282BAD@codespeak.net> Author: fijal Date: Wed Aug 4 17:34:05 2010 New Revision: 76474 Modified: pypy/trunk/pypy/jit/backend/x86/tool/viewcode.py Log: It seems to be needed for my particular platform Modified: pypy/trunk/pypy/jit/backend/x86/tool/viewcode.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/tool/viewcode.py (original) +++ pypy/trunk/pypy/jit/backend/x86/tool/viewcode.py Wed Aug 4 17:34:05 2010 @@ -35,6 +35,7 @@ objdump_backend_option = { 'x86': 'i386', 'x86_64': 'x86-64', + 'i386': 'i386', } objdump = ('objdump -M intel,%(backend)s -b binary -m i386 ' '--adjust-vma=%(origin)d -D %(file)s') From arigo at codespeak.net Wed Aug 4 18:55:06 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 4 Aug 2010 18:55:06 +0200 (CEST) Subject: [pypy-svn] r76475 - in pypy/trunk/pypy: module/_stackless objspace/std Message-ID: <20100804165506.A8527282BAD@codespeak.net> Author: arigo Date: Wed Aug 4 18:55:02 2010 New Revision: 76475 Modified: pypy/trunk/pypy/module/_stackless/interp_coroutine.py pypy/trunk/pypy/objspace/std/callmethod.py Log: Properly fix the rstack.resume_point()s. Note that there is no need for try:finally: in one of the two cases in callmethod.py, if we make sure to pop all arguments before the call, as pyopcode.py does. Modified: pypy/trunk/pypy/module/_stackless/interp_coroutine.py ============================================================================== --- pypy/trunk/pypy/module/_stackless/interp_coroutine.py (original) +++ pypy/trunk/pypy/module/_stackless/interp_coroutine.py Wed Aug 4 18:55:02 2010 @@ -265,10 +265,14 @@ instr += 1 oparg = ord(code[instr]) | ord(code[instr + 1]) << 8 nargs = oparg & 0xff + nkwds = (oparg >> 8) & 0xff if space.config.objspace.opcodes.CALL_METHOD and opcode == map['CALL_METHOD']: - chain = resume_state_create(chain, 'CALL_METHOD', frame, - nargs) - elif opcode == map['CALL_FUNCTION'] and (oparg >> 8) & 0xff == 0: + if nkwds == 0: # only positional arguments + chain = resume_state_create(chain, 'CALL_METHOD', frame, + nargs) + else: # includes keyword arguments + chain = resume_state_create(chain, 'CALL_METHOD_KW', frame) + elif opcode == map['CALL_FUNCTION'] and nkwds == 0: # Only positional arguments # case1: ("CALL_FUNCTION", f, nargs, returns=w_result) chain = resume_state_create(chain, 'CALL_FUNCTION', frame, Modified: pypy/trunk/pypy/objspace/std/callmethod.py ============================================================================== --- pypy/trunk/pypy/objspace/std/callmethod.py (original) +++ pypy/trunk/pypy/objspace/std/callmethod.py Wed Aug 4 18:55:02 2010 @@ -62,12 +62,13 @@ n_args = oparg & 0xff n_kwargs = (oparg >> 8) & 0xff w_self = f.peekvalue(n_args + (2 * n_kwargs)) - w_callable = f.peekvalue(n_args + (2 * n_kwargs) + 1) n = n_args + (w_self is not None) if not n_kwargs: + w_callable = f.peekvalue(n_args + (2 * n_kwargs) + 1) try: w_result = f.space.call_valuestack(w_callable, n, f) + rstack.resume_point("CALL_METHOD", f, n_args, returns=w_result) finally: f.dropvalues(n_args + 2) else: @@ -83,14 +84,13 @@ keywords[n_kwargs] = key keywords_w[n_kwargs] = w_value - arguments = f.popvalues(n) + arguments = f.popvalues(n) # includes w_self if it is not None args = f.argument_factory(arguments, keywords, keywords_w, None, None) - - try: - w_result = f.space.call_args(w_callable, args) - finally: - f.dropvalues(1 + (w_self is None)) - rstack.resume_point("CALL_METHOD", f, n_args, returns=w_result) + if w_self is None: + f.popvalue() # removes w_self, which is None + w_callable = f.popvalue() + w_result = f.space.call_args(w_callable, args) + rstack.resume_point("CALL_METHOD_KW", f, returns=w_result) f.pushvalue(w_result) From arigo at codespeak.net Wed Aug 4 19:23:55 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 4 Aug 2010 19:23:55 +0200 (CEST) Subject: [pypy-svn] r76477 - pypy/trunk/lib_pypy Message-ID: <20100804172355.B7DC8282BAD@codespeak.net> Author: arigo Date: Wed Aug 4 19:23:54 2010 New Revision: 76477 Removed: pypy/trunk/lib_pypy/greenlet.py Log: Kill greenlet.py, now that I changed the greenlet module to be importable on top of PyPy (by doing the same thing). From arigo at codespeak.net Wed Aug 4 19:31:46 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 4 Aug 2010 19:31:46 +0200 (CEST) Subject: [pypy-svn] r76478 - pypy/trunk/pypy/interpreter Message-ID: <20100804173146.A658E282BAD@codespeak.net> Author: arigo Date: Wed Aug 4 19:31:45 2010 New Revision: 76478 Modified: pypy/trunk/pypy/interpreter/pyopcode.py Log: Fix CALL_METHOD here too, to support r76412. Modified: pypy/trunk/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyopcode.py (original) +++ pypy/trunk/pypy/interpreter/pyopcode.py Wed Aug 4 19:31:45 2010 @@ -988,23 +988,9 @@ self.dropvalues(nargs) self.pushvalue(w_result) - def LOOKUP_METHOD(self, nameindex, next_instr): - # overridden by faster version in the standard object space. - space = self.space - w_obj = self.popvalue() - w_name = self.getname_w(nameindex) - w_value = space.getattr(w_obj, w_name) - self.pushvalue(w_value) - - def CALL_METHOD(self, nargs, next_instr): - # overridden by faster version in the standard object space. - # 'nargs' is the argument count excluding the implicit 'self' - w_callable = self.peekvalue(nargs) - try: - w_result = self.space.call_valuestack(w_callable, nargs, self) - finally: - self.dropvalues(nargs + 1) - self.pushvalue(w_result) + # overridden by faster version in the standard object space. + LOOKUP_METHOD = LOAD_ATTR + CALL_METHOD = CALL_FUNCTION def MISSING_OPCODE(self, oparg, next_instr): ofs = self.last_instr From fijal at codespeak.net Wed Aug 4 19:42:11 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 4 Aug 2010 19:42:11 +0200 (CEST) Subject: [pypy-svn] r76479 - pypy/branch/faster-compilation/pypy/interpreter Message-ID: <20100804174211.2AC99282BAD@codespeak.net> Author: fijal Date: Wed Aug 4 19:42:10 2010 New Revision: 76479 Modified: pypy/branch/faster-compilation/pypy/interpreter/pyopcode.py Log: Merge 76478 Modified: pypy/branch/faster-compilation/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/faster-compilation/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/faster-compilation/pypy/interpreter/pyopcode.py Wed Aug 4 19:42:10 2010 @@ -992,25 +992,9 @@ f.dropvalues(nargs) f.pushvalue(w_result) - def LOOKUP_METHOD(f, nameindex, *ignored): - # overridden by faster version in the standard object space. - space = f.space - w_obj = f.popvalue() - w_name = f.getname_w(nameindex) - w_value = space.getattr(w_obj, w_name) - f.pushvalue(w_value) - #f.pushvalue(None) - - def CALL_METHOD(f, nargs, *ignored): - # overridden by faster version in the standard object space. - # 'nargs' is the argument count excluding the implicit 'self' - w_callable = f.peekvalue(nargs) - try: - w_result = f.space.call_valuestack(w_callable, nargs, f) - finally: - f.dropvalues(nargs + 1) - f.pushvalue(w_result) - + # overridden by faster version in the standard object space. + LOOKUP_METHOD = LOAD_ATTR + CALL_METHOD = CALL_FUNCTION ## def EXTENDED_ARG(f, oparg, *ignored): ## opcode = f.nextop() ## oparg = oparg<<16 | f.nextarg() From wlav at codespeak.net Thu Aug 5 00:10:48 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Thu, 5 Aug 2010 00:10:48 +0200 (CEST) Subject: [pypy-svn] r76481 - in pypy/branch/reflex-support/pypy/module/cppyy: . include src test Message-ID: <20100804221048.7AF0D282BF8@codespeak.net> Author: wlav Date: Thu Aug 5 00:10:46 2010 New Revision: 76481 Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py pypy/branch/reflex-support/pypy/module/cppyy/converter.py pypy/branch/reflex-support/pypy/module/cppyy/executor.py pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py Log: Executors and converters for more integer types Modified: pypy/branch/reflex-support/pypy/module/cppyy/capi.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/capi.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/capi.py Thu Aug 5 00:10:46 2010 @@ -54,6 +54,10 @@ "cppyy_call_c", [C_TYPEHANDLE, rffi.INT, C_OBJECT, rffi.INT, rffi.VOIDPP], rffi.CHAR, compilation_info=eci) +c_call_h = rffi.llexternal( + "cppyy_call_h", + [C_TYPEHANDLE, rffi.INT, C_OBJECT, rffi.INT, rffi.VOIDPP], rffi.SHORT, + compilation_info=eci) c_call_l = rffi.llexternal( "cppyy_call_l", [C_TYPEHANDLE, rffi.INT, C_OBJECT, rffi.INT, rffi.VOIDPP], rffi.LONG, Modified: pypy/branch/reflex-support/pypy/module/cppyy/converter.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/converter.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/converter.py Thu Aug 5 00:10:46 2010 @@ -12,14 +12,18 @@ obj = space.interpclass_w(space.findattr(w_obj, space.wrap("_cppinstance"))) return lltype.direct_ptradd(obj.rawobject, offset) + def _is_abstract(self): + raise NotImplementedError( + "abstract base class (actual: %s)" % type(self).__name__) + def convert_argument(self, space, w_obj): - raise NotImplementedError("abstract base class") + self._is_abstract() def from_memory(self, space, w_obj, offset): - raise NotImplementedError("abstract base class") + self._is_abstract() def to_memory(self, space, w_obj, w_value, offset): - raise NotImplementedError("abstract base class") + self._is_abstract() def free_argument(self, arg): lltype.free(arg, flavor='raw') @@ -73,13 +77,36 @@ fieldptr = self._get_fieldptr(space, w_obj, offset) fieldptr[0] = self._from_space(space, w_value) -class IntConverter(TypeConverter): +class LongConverter(TypeConverter): def convert_argument(self, space, w_obj): arg = space.c_int_w(w_obj) x = lltype.malloc(rffi.LONGP.TO, 1, flavor='raw') x[0] = arg return rffi.cast(rffi.VOIDP, x) + def from_memory(self, space, w_obj, offset): + fieldptr = self._get_fieldptr(space, w_obj, offset) + longptr = rffi.cast(rffi.LONGP, fieldptr) + return space.wrap(longptr[0]) + + def to_memory(self, space, w_obj, w_value, offset): + fieldptr = self._get_fieldptr(space, w_obj, offset) + longptr = rffi.cast(rffi.LONGP, fieldptr) + longptr[0] = space.c_int_w(w_value) + +class ShortConverter(LongConverter): + def from_memory(self, space, w_obj, offset): + fieldptr = self._get_fieldptr(space, w_obj, offset) + intptr = rffi.cast(rffi.SHORTP, fieldptr) + return space.wrap(intptr[0]) + + def to_memory(self, space, w_obj, w_value, offset): + import struct + fieldptr = self._get_fieldptr(space, w_obj, offset) + pack = struct.pack('h', space.unwrap(w_value)) # unchecked + fieldptr[0] = pack[0] + fieldptr[1] = pack[1] + class FloatConverter(TypeConverter): def convert_argument(self, space, w_obj): arg = space.float_w(w_obj) @@ -173,7 +200,12 @@ _converters["bool"] = BoolConverter() _converters["char"] = CharConverter() _converters["unsigned char"] = CharConverter() -_converters["int"] = IntConverter() +_converters["short int"] = ShortConverter() +_converters["unsigned short int"] = ShortConverter() +_converters["int"] = LongConverter() +_converters["unsigned int"] = LongConverter() +_converters["long int"] = LongConverter() +_converters["unsigned long int"] = LongConverter() _converters["float"] = FloatConverter() _converters["double"] = DoubleConverter() _converters["const char*"] = CStringConverter() Modified: pypy/branch/reflex-support/pypy/module/cppyy/executor.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/executor.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/executor.py Thu Aug 5 00:10:46 2010 @@ -7,7 +7,8 @@ class FunctionExecutor(object): def execute(self, space, func, cppthis, num_args, args): - raise NotImplementedError("abstract base class") + raise NotImplementedError( + "abstract base class (actual: %s)" % type(self).__name__) class VoidExecutor(FunctionExecutor): @@ -15,6 +16,7 @@ capi.c_call_v(func.cpptype.handle, func.method_index, cppthis, num_args, args) return space.w_None + class BoolExecutor(FunctionExecutor): def execute(self, space, func, cppthis, num_args, args): result = capi.c_call_b(func.cpptype.handle, func.method_index, cppthis, num_args, args) @@ -25,6 +27,11 @@ result = capi.c_call_c(func.cpptype.handle, func.method_index, cppthis, num_args, args) return space.wrap(result) +class ShortExecutor(FunctionExecutor): + def execute(self, space, func, cppthis, num_args, args): + result = capi.c_call_h(func.cpptype.handle, func.method_index, cppthis, num_args, args) + return space.wrap(result) + class LongExecutor(FunctionExecutor): def execute(self, space, func, cppthis, num_args, args): result = capi.c_call_l(func.cpptype.handle, func.method_index, cppthis, num_args, args) @@ -81,8 +88,12 @@ _executors["bool"] = BoolExecutor() _executors["char"] = CharExecutor() _executors["unsigned char"] = CharExecutor() +_executors["short int"] = ShortExecutor() +_executors["unsigned short int"] = ShortExecutor() _executors["int"] = LongExecutor() +_executors["unsigned int"] = LongExecutor() _executors["long int"] = LongExecutor() +_executors["unsigned long int"] = LongExecutor() _executors["float"] = FloatExecutor() _executors["double"] = DoubleExecutor() _executors["char*"] = CStringExecutor() Modified: pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/include/reflexcwrapper.h Thu Aug 5 00:10:46 2010 @@ -18,6 +18,7 @@ void cppyy_call_v(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); int cppyy_call_b(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); char cppyy_call_c(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); + short cppyy_call_h(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); long cppyy_call_l(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); double cppyy_call_f(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); double cppyy_call_d(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]); Modified: pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/src/reflexcwrapper.cxx Thu Aug 5 00:10:46 2010 @@ -58,6 +58,11 @@ return cppyy_call_T(handle, method_index, self, numargs, args); } +short cppyy_call_h(cppyy_typehandle_t handle, int method_index, + cppyy_object_t self, int numargs, void* args[]) { + return cppyy_call_T(handle, method_index, self, numargs, args); +} + long cppyy_call_l(cppyy_typehandle_t handle, int method_index, cppyy_object_t self, int numargs, void* args[]) { return cppyy_call_T(handle, method_index, self, numargs, args); Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py Thu Aug 5 00:10:46 2010 @@ -72,7 +72,6 @@ raises(TypeError, 'c.set_uchar("string")') # TODO: raises(TypeError, 'c.set_uchar(-1)') - """ # integer types names = ['short', 'ushort', 'int', 'uint', 'long', 'ulong'] for i in range(len(names)): @@ -82,7 +81,6 @@ for i in range(len(names)): exec 'c.set_%s = %d' % (names[i],2*i) assert eval('c.m_%s' % names[i]) == i - """ # float types through functions c.set_float( 0.123 ); assert round(c.get_float() - 0.123, 5) == 0 From hakanardo at codespeak.net Thu Aug 5 20:02:02 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Thu, 5 Aug 2010 20:02:02 +0200 (CEST) Subject: [pypy-svn] r76486 - in pypy/branch/interplevel-array/pypy: jit/backend/x86 module/pypyjit/test objspace/std Message-ID: <20100805180202.EEFB8282BF3@codespeak.net> Author: hakanardo Date: Thu Aug 5 20:01:57 2010 New Revision: 76486 Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/assembler.py pypy/branch/interplevel-array/pypy/module/pypyjit/test/test_pypy_c.py pypy/branch/interplevel-array/pypy/objspace/std/itertype.py Log: pypy-c tests Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/assembler.py Thu Aug 5 20:01:57 2010 @@ -1071,7 +1071,7 @@ self.mc.MOV(resloc, addr_add(base_loc, ofs_loc, ofs.value, scale.value)) else: - print "[asmgen]setarrayitem unsupported size: %d" % scale.value + print "[asmgen]getarrayitem unsupported size: %d" % scale.value raise NotImplementedError() genop_getarrayitem_gc_pure = genop_getarrayitem_gc Modified: pypy/branch/interplevel-array/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/interplevel-array/pypy/module/pypyjit/test/test_pypy_c.py Thu Aug 5 20:01:57 2010 @@ -779,8 +779,13 @@ '''%(e1, e2), n, ([], res)) def test_array_sum(self): - #for tc in 'bhilBHILfd': - for tc in 'd': + for tc, maxops in zip('bhilBHILfd', (38,) * 6 + (40, 40, 41, 38)): + res = 19352859 + if tc in 'IL': + res = long(res) + elif tc in 'fd': + res = float(res) + self.run_source(''' from array import array @@ -791,20 +796,56 @@ l += img[i] i += 1 return l - ''' % tc, 38, ([], 19352859)) + ''' % tc, maxops, ([], res)) def test_array_sum_char(self): self.run_source(''' from array import array def main(): - img = array("c", ' ') * 640 * 480 + img = array("c", "Hello") * 130 * 480 l, i = 0, 0 while i < 640 * 480: l += ord(img[i]) i += 1 return l - ''', 60, ([], 9830400)) + ''', 60, ([], 30720000)) + + def test_array_sum_unicode(self): + self.run_source(''' + from array import array + + def main(): + img = array("u", u"Hello") * 130 * 480 + l, i = 0, 0 + while i < 640 * 480: + if img[i] == u"l": + l += 1 + i += 1 + return l + ''', 65, ([], 122880)) + + def test_array_intimg(self): + for tc, maxops in zip('ilILd', (67, 67, 69, 69, 61)): + res = 73574560 + if tc in 'IL': + res = long(res) + elif tc in 'fd': + res = float(res) + + self.run_source(''' + from array import array + + def main(tc): + img = array(tc, range(3)) * (350 * 480) + intimg = array(tc, (0,)) * (640 * 480) + l, i = 0, 640 + while i < 640 * 480: + l = l + img[i] + intimg[i] = (intimg[i-640] + l) + i += 1 + return intimg[i - 1] + ''', maxops, ([tc], res)) class AppTestJIT(PyPyCJITTests): def setup_class(cls): @@ -828,6 +869,7 @@ cls.counter = 0 cls.pypy_c = option.pypy_c + def has_info(pypy_c, option): g = os.popen('"%s" --info' % pypy_c, 'r') lines = g.readlines() Modified: pypy/branch/interplevel-array/pypy/objspace/std/itertype.py ============================================================================== --- pypy/branch/interplevel-array/pypy/objspace/std/itertype.py (original) +++ pypy/branch/interplevel-array/pypy/objspace/std/itertype.py Thu Aug 5 20:01:57 2010 @@ -11,7 +11,8 @@ """ # cpython does not support pickling iterators - raise OperationError(space.w_TypeError, space.w_None) + msg = 'Pickling for iterators dissabled as cpython does not support it' + raise OperationError(space.w_TypeError, space.wrap(msg)) from pypy.objspace.std.iterobject import W_AbstractSeqIterObject assert isinstance(w_self, W_AbstractSeqIterObject) From hakanardo at codespeak.net Thu Aug 5 21:35:23 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Thu, 5 Aug 2010 21:35:23 +0200 (CEST) Subject: [pypy-svn] r76488 - in pypy/branch/interplevel-array/pypy/module/array: . benchmark test Message-ID: <20100805193523.2C0C0282BE8@codespeak.net> Author: hakanardo Date: Thu Aug 5 21:35:21 2010 New Revision: 76488 Added: pypy/branch/interplevel-array/pypy/module/array/benchmark/ pypy/branch/interplevel-array/pypy/module/array/benchmark/Makefile (contents, props changed) - copied, changed from r76468, pypy/branch/interplevel-array/pypy/module/array/test/Makefile pypy/branch/interplevel-array/pypy/module/array/benchmark/README pypy/branch/interplevel-array/pypy/module/array/benchmark/intimg.c (props changed) - copied unchanged from r76468, pypy/branch/interplevel-array/pypy/module/array/test/intimg.c pypy/branch/interplevel-array/pypy/module/array/benchmark/intimgtst.c (props changed) - copied unchanged from r76468, pypy/branch/interplevel-array/pypy/module/array/test/tstintimg.c pypy/branch/interplevel-array/pypy/module/array/benchmark/intimgtst.py (props changed) - copied unchanged from r76468, pypy/branch/interplevel-array/pypy/module/array/test/inttst.py pypy/branch/interplevel-array/pypy/module/array/benchmark/loop.c (props changed) - copied unchanged from r76468, pypy/branch/interplevel-array/pypy/module/array/test/loop.c pypy/branch/interplevel-array/pypy/module/array/benchmark/loop.py pypy/branch/interplevel-array/pypy/module/array/benchmark/sum.c (props changed) - copied unchanged from r76468, pypy/branch/interplevel-array/pypy/module/array/test/sum.c pypy/branch/interplevel-array/pypy/module/array/benchmark/sumtst.c (props changed) - copied unchanged from r76468, pypy/branch/interplevel-array/pypy/module/array/test/sumtst.c pypy/branch/interplevel-array/pypy/module/array/benchmark/sumtst.py (props changed) - copied unchanged from r76468, pypy/branch/interplevel-array/pypy/module/array/test/sumtst.py Removed: pypy/branch/interplevel-array/pypy/module/array/app_array.py pypy/branch/interplevel-array/pypy/module/array/interp_array_multiclass.py pypy/branch/interplevel-array/pypy/module/array/interp_array_try1.py pypy/branch/interplevel-array/pypy/module/array/interp_array_wrapped.py pypy/branch/interplevel-array/pypy/module/array/interp_simple.py pypy/branch/interplevel-array/pypy/module/array/test/Makefile pypy/branch/interplevel-array/pypy/module/array/test/imtst.py pypy/branch/interplevel-array/pypy/module/array/test/intimg.c pypy/branch/interplevel-array/pypy/module/array/test/inttst.py pypy/branch/interplevel-array/pypy/module/array/test/loop.c pypy/branch/interplevel-array/pypy/module/array/test/sum.c pypy/branch/interplevel-array/pypy/module/array/test/sumtst.c pypy/branch/interplevel-array/pypy/module/array/test/sumtst.py pypy/branch/interplevel-array/pypy/module/array/test/tstintimg.c Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: clean house Copied: pypy/branch/interplevel-array/pypy/module/array/benchmark/Makefile (from r76468, pypy/branch/interplevel-array/pypy/module/array/test/Makefile) ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/Makefile (original) +++ pypy/branch/interplevel-array/pypy/module/array/benchmark/Makefile Thu Aug 5 21:35:21 2010 @@ -1,6 +1,6 @@ CFLAGS=-O3 -mtune=native -march=native -std=gnu99 -tst: tstintimg.o intimg.o +intimg: intimgtst.o intimg.o gcc -o $@ $^ loop: loop.o gcc -o $@ $^ Added: pypy/branch/interplevel-array/pypy/module/array/benchmark/README ============================================================================== --- (empty file) +++ pypy/branch/interplevel-array/pypy/module/array/benchmark/README Thu Aug 5 21:35:21 2010 @@ -0,0 +1,6 @@ +Some simple benchmarks implemented in C and in python. + +sumtst - Sums up the elements in an array +loop - Same loop as in sumtst but without array accesses +intimg - Calculates a integral image transform + Added: pypy/branch/interplevel-array/pypy/module/array/benchmark/loop.py ============================================================================== --- (empty file) +++ pypy/branch/interplevel-array/pypy/module/array/benchmark/loop.py Thu Aug 5 21:35:21 2010 @@ -0,0 +1,7 @@ +def h(): + s=0 + i=0 + while i<100000: + s+=i + i+=1 + return s Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Thu Aug 5 21:35:21 2010 @@ -1,5 +1,6 @@ from pypy.conftest import gettestobjspace import py +import py.test ## class AppTestSimpleArray: @@ -17,6 +18,8 @@ class BaseArrayTests: + + def test_ctor(self): assert len(self.array('c')) == 0 assert len(self.array('i')) == 0 @@ -179,12 +182,12 @@ ## def read(self,n): ## return self.c*min(n,self.s) def myfile(c, s): - f = open('/tmp/deleteme', 'w') + f = open(self.tempfile, 'w') f.write(c * s) f.close() - return open('/tmp/deleteme', 'r') + return open(self.tempfile, 'r') - f = open('/dev/zero', 'r') + f = myfile('\x00', 100) for t in 'bBhHiIlLfd': a = self.array(t) a.fromfile(f, 2) @@ -415,13 +418,13 @@ a = unpack(tc * len(values), s) assert a == values - f = open('/tmp/deleteme', 'w') + f = open(self.tempfile, 'w') self.array('c', ('h', 'i')).tofile(f) f.close() - assert open('/tmp/deleteme', 'r').readline() == 'hi' + assert open(self.tempfile, 'r').readline() == 'hi' a = self.array('c') - a.fromfile(open('/tmp/deleteme', 'r'), 2) + a.fromfile(open(self.tempfile, 'r'), 2) assert repr(a) == "array('c', 'hi')" raises(ValueError, self.array('i').tounicode) @@ -756,7 +759,7 @@ cls.array = array.array import struct cls.struct = struct - + cls.tempfile = str(py.test.ensuretemp('array').join('tmpfile')) class AppTestArray(BaseArrayTests): def setup_class(cls): @@ -773,6 +776,12 @@ import _rawffi return _rawffi """) + cls.w_tempfile = cls.space.wrap( + str(py.test.ensuretemp('array').join('tmpfile'))) + + + + def test_buffer_info(self): a = self.array('c', 'Hi!') From jcreigh at codespeak.net Thu Aug 5 21:48:53 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Thu, 5 Aug 2010 21:48:53 +0200 (CEST) Subject: [pypy-svn] r76490 - pypy/trunk/pypy/jit/backend/x86/test Message-ID: <20100805194853.09D3F282C00@codespeak.net> Author: jcreigh Date: Thu Aug 5 21:48:52 2010 New Revision: 76490 Modified: pypy/trunk/pypy/jit/backend/x86/test/test_runner.py Log: add test for calling function from JIT with ConstFloat arguments Modified: pypy/trunk/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_runner.py Thu Aug 5 21:48:52 2010 @@ -1,8 +1,9 @@ import py from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rstr, rclass +from pypy.rpython.annlowlevel import llhelper from pypy.jit.metainterp.history import ResOperation, LoopToken -from pypy.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstPtr, - Box, BoxFloat, BasicFailDescr) +from pypy.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstFloat, + ConstPtr, Box, BoxFloat, BasicFailDescr) from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.backend.x86.arch import WORD from pypy.jit.backend.llsupport import symbolic @@ -376,6 +377,19 @@ res = self.cpu.get_latest_value_int(0) assert res == 20 + def test_call_with_const_floats(self): + def func(f1, f2): + return f1 + f2 + + FUNC = self.FuncType([lltype.Float, lltype.Float], lltype.Float) + FPTR = self.Ptr(FUNC) + calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + func_ptr = llhelper(FPTR, func) + funcbox = self.get_funcbox(self.cpu, func_ptr) + res = self.execute_operation(rop.CALL, [funcbox, ConstFloat(1.5), ConstFloat(2.5)], 'float', descr=calldescr) + assert res.value == 4.0 + + class TestX86OverflowMC(TestX86): def setup_method(self, meth): From jcreigh at codespeak.net Thu Aug 5 21:51:51 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Thu, 5 Aug 2010 21:51:51 +0200 (CEST) Subject: [pypy-svn] r76491 - pypy/trunk/pypy/jit/backend/x86 Message-ID: <20100805195151.A4BB9282BE8@codespeak.net> Author: jcreigh Date: Thu Aug 5 21:51:50 2010 New Revision: 76491 Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/regalloc.py pypy/trunk/pypy/jit/backend/x86/regloc.py Log: fix for r76490, calling functions in JIT with const float arguments. (somewhat ugly) Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Thu Aug 5 21:51:50 2010 @@ -819,10 +819,8 @@ for i in range(start, len(arglocs)): loc = arglocs[i] - assert isinstance(loc, RegLoc) or isinstance(loc, ImmedLoc) or isinstance(loc, StackLoc) - # XXX: Should be much simplier to tell whether a location is a float! - if (isinstance(loc, RegLoc) and loc.is_xmm) or (isinstance(loc, StackLoc) and loc.type == FLOAT): + if (isinstance(loc, RegLoc) and loc.is_xmm) or (loc.is_memory_reference() and loc.type == FLOAT): if len(unused_xmm) > 0: xmm_src_locs.append(loc) xmm_dst_locs.append(unused_xmm.pop()) Modified: pypy/trunk/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regalloc.py Thu Aug 5 21:51:50 2010 @@ -53,6 +53,7 @@ def __init__(self): self.cur_array_free = 0 + self.const_id = 0 def _get_new_array(self): n = self.BASE_CONSTANT_SIZE @@ -68,7 +69,8 @@ n = self.cur_array_free - 1 arr[n] = floatval self.cur_array_free = n - return rffi.cast(lltype.Signed, arr) + n * 8 + self.const_id += 1 + return (self.const_id, rffi.cast(lltype.Signed, arr) + n * 8) class X86XMMRegisterManager(RegisterManager): @@ -89,8 +91,8 @@ self.float_constants = assembler._float_constants def convert_to_imm(self, c): - adr = self.float_constants.record_float(c.getfloat()) - return AddressLoc(ImmedLoc(adr), ImmedLoc(0), 0, 0) + const_id, adr = self.float_constants.record_float(c.getfloat()) + return ConstFloatLoc(adr, const_id) def after_call(self, v): # the result is stored in st0, but we don't have this around, Modified: pypy/trunk/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regloc.py Thu Aug 5 21:51:50 2010 @@ -5,6 +5,7 @@ from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.objectmodel import specialize from pypy.rlib.rarithmetic import intmask +from pypy.jit.metainterp.history import FLOAT # # This module adds support for "locations", which can be either in a Const, @@ -145,6 +146,28 @@ def value_m(self): return self.loc_m +class ConstFloatLoc(AssemblerLocation): + # XXX: We have to use this class instead of just AddressLoc because + # AddressLoc is "untyped" and also we to have need some sort of unique + # identifier that we can use in _getregkey (for jump.py) + + _immutable_ = True + + width = 8 + type = FLOAT + + def __init__(self, address, const_id): + self.value = address + self.const_id = const_id + + def _getregkey(self): + # XXX: 1000 is kind of magic: We just don't want to be confused + # with any registers + return 1000 + self.const_id + + def location_code(self): + return 'j' + REGLOCS = [RegLoc(i, is_xmm=False) for i in range(16)] XMMREGLOCS = [RegLoc(i, is_xmm=True) for i in range(16)] eax, ecx, edx, ebx, esp, ebp, esi, edi, r8, r9, r10, r11, r12, r13, r14, r15 = REGLOCS From hakanardo at codespeak.net Thu Aug 5 21:52:17 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Thu, 5 Aug 2010 21:52:17 +0200 (CEST) Subject: [pypy-svn] r76492 - pypy/branch/interplevel-array/pypy/module/array/test Message-ID: <20100805195217.E76BF282BE8@codespeak.net> Author: hakanardo Date: Thu Aug 5 21:52:16 2010 New Revision: 76492 Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Log: try except => raises Modified: pypy/branch/interplevel-array/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/test/test_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/test/test_array.py Thu Aug 5 21:52:16 2010 @@ -291,23 +291,9 @@ a[0:-1:2] = self.array('i', [7, 8]) assert a[0] == 7 and a[1] == 5 and a[2] == 8 and a[3] == 4 - try: - a[1:2:4] = self.array('i', [5, 6, 7]) - assert False - except ValueError: - pass - - try: - a[1:3] = self.array('I', [5, 6]) - assert False - except TypeError: - pass - - try: - a[1:3] = [5, 6] - assert False - except TypeError: - pass + raises(ValueError, "a[1:2:4] = self.array('i', [5, 6, 7])") + raises(TypeError, "a[1:3] = self.array('I', [5, 6])") + raises(TypeError, "a[1:3] = [5, 6]") a = self.array('i', [1, 2, 3]) assert a.__getslice__(1, 2) == a[1:2] @@ -593,19 +579,8 @@ a += self.array('i', (7,)) assert repr(a) == "array('i', [1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 7])" - try: - raised = False - a = self.array('i') + 2 - except TypeError: - raised = True - assert raised - - try: - raised = False - a = self.array('i') + self.array('b') - except TypeError: - raised = True - assert raised + raises(TypeError, "a = self.array('i') + 2") + raises(TypeError, "self.array('i') + self.array('b')") # Calling __add__ directly raises TypeError in cpython but # returns NotImplemented in pypy if placed within a From jcreigh at codespeak.net Thu Aug 5 22:25:20 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Thu, 5 Aug 2010 22:25:20 +0200 (CEST) Subject: [pypy-svn] r76493 - pypy/trunk/pypy/jit/backend/x86 Message-ID: <20100805202520.A9325282BE8@codespeak.net> Author: jcreigh Date: Thu Aug 5 22:25:18 2010 New Revision: 76493 Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/regalloc.py pypy/trunk/pypy/jit/backend/x86/regloc.py Log: revert r76491 (grr, doesn't translate) Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Thu Aug 5 22:25:18 2010 @@ -819,8 +819,10 @@ for i in range(start, len(arglocs)): loc = arglocs[i] + assert isinstance(loc, RegLoc) or isinstance(loc, ImmedLoc) or isinstance(loc, StackLoc) + # XXX: Should be much simplier to tell whether a location is a float! - if (isinstance(loc, RegLoc) and loc.is_xmm) or (loc.is_memory_reference() and loc.type == FLOAT): + if (isinstance(loc, RegLoc) and loc.is_xmm) or (isinstance(loc, StackLoc) and loc.type == FLOAT): if len(unused_xmm) > 0: xmm_src_locs.append(loc) xmm_dst_locs.append(unused_xmm.pop()) Modified: pypy/trunk/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regalloc.py Thu Aug 5 22:25:18 2010 @@ -53,7 +53,6 @@ def __init__(self): self.cur_array_free = 0 - self.const_id = 0 def _get_new_array(self): n = self.BASE_CONSTANT_SIZE @@ -69,8 +68,7 @@ n = self.cur_array_free - 1 arr[n] = floatval self.cur_array_free = n - self.const_id += 1 - return (self.const_id, rffi.cast(lltype.Signed, arr) + n * 8) + return rffi.cast(lltype.Signed, arr) + n * 8 class X86XMMRegisterManager(RegisterManager): @@ -91,8 +89,8 @@ self.float_constants = assembler._float_constants def convert_to_imm(self, c): - const_id, adr = self.float_constants.record_float(c.getfloat()) - return ConstFloatLoc(adr, const_id) + adr = self.float_constants.record_float(c.getfloat()) + return AddressLoc(ImmedLoc(adr), ImmedLoc(0), 0, 0) def after_call(self, v): # the result is stored in st0, but we don't have this around, Modified: pypy/trunk/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regloc.py Thu Aug 5 22:25:18 2010 @@ -5,7 +5,6 @@ from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.objectmodel import specialize from pypy.rlib.rarithmetic import intmask -from pypy.jit.metainterp.history import FLOAT # # This module adds support for "locations", which can be either in a Const, @@ -146,28 +145,6 @@ def value_m(self): return self.loc_m -class ConstFloatLoc(AssemblerLocation): - # XXX: We have to use this class instead of just AddressLoc because - # AddressLoc is "untyped" and also we to have need some sort of unique - # identifier that we can use in _getregkey (for jump.py) - - _immutable_ = True - - width = 8 - type = FLOAT - - def __init__(self, address, const_id): - self.value = address - self.const_id = const_id - - def _getregkey(self): - # XXX: 1000 is kind of magic: We just don't want to be confused - # with any registers - return 1000 + self.const_id - - def location_code(self): - return 'j' - REGLOCS = [RegLoc(i, is_xmm=False) for i in range(16)] XMMREGLOCS = [RegLoc(i, is_xmm=True) for i in range(16)] eax, ecx, edx, ebx, esp, ebp, esi, edi, r8, r9, r10, r11, r12, r13, r14, r15 = REGLOCS From getxsick at codespeak.net Fri Aug 6 00:11:43 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Fri, 6 Aug 2010 00:11:43 +0200 (CEST) Subject: [pypy-svn] r76494 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100805221143.7049D282BE8@codespeak.net> Author: getxsick Date: Fri Aug 6 00:11:41 2010 New Revision: 76494 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: add cache argument to CDLL.get Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Fri Aug 6 00:11:41 2010 @@ -19,8 +19,9 @@ self.name = name self.cpu = GLOBAL_CPU - def get(self, func, args_type, res_type='v', push_result=None): - return _Get(self.cpu, self.lib, func, args_type, res_type, push_result) + def get(self, func, args_type, res_type='v', push_result=None, cache=False): + return _Get(self.cpu, self.lib, func, args_type, res_type, + push_result, cache) class _LibHandler(object): def __init__(self, name): From getxsick at codespeak.net Fri Aug 6 00:12:42 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Fri, 6 Aug 2010 00:12:42 +0200 (CEST) Subject: [pypy-svn] r76495 - pypy/branch/fast-ctypes/pypy/rlib/test Message-ID: <20100805221242.06252282BE8@codespeak.net> Author: getxsick Date: Fri Aug 6 00:12:41 2010 New Revision: 76495 Modified: pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Log: use cache in tests Modified: pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Fri Aug 6 00:12:41 2010 @@ -65,34 +65,61 @@ def push_result(self, value): # mock function return value + def fromcache(self, f, args_type, res_type): + if not hasattr(self, 'cache'): + self.cache = {} + + arg_classes = ''.join(args_type) + key = (res_type, arg_classes) + try: + f.looptoken = self.cache[key] + except KeyError: + f.gen_looptaken() + self.cache[key] = f.looptoken + def test_missing_lib(self): py.test.raises(OSError, rjitffi.CDLL, 'xxxfoo888baryyy') def test_get(self): lib = rjitffi.CDLL(self.lib_name) - func = lib.get('add_integers', ['i', 'i'], 'i', self.push_result) + args_type = ['i', 'i'] + res_type = 'i' + + func = lib.get('add_integers', args_type, res_type, self.push_result, cache=True) + self.fromcache(func, args_type, res_type) func.push_int(1) func.push_int(2) assert func.call() == 3 - func = lib.get('add_integers', ['i', 'i'], 'i', self.push_result) + func = lib.get('add_integers', args_type, res_type, self.push_result, cache=True) + self.fromcache(func, args_type, res_type) func.push_int(-1) func.push_int(2) assert func.call() == 1 - func = lib.get('add_integers', ['i', 'i'], 'i', self.push_result) + func = lib.get('add_integers', args_type, res_type, self.push_result, cache=True) + self.fromcache(func, args_type, res_type) func.push_int(0) func.push_int(0) assert func.call() == 0 + args_type = ['i', 'i', 'i'] + res_type = 'i' + + func = lib.get('add_integers', args_type, res_type, self.push_result, cache=True) + self.fromcache(func, args_type, res_type) func = lib.get('max3', ['i', 'i', 'i'], 'i', self.push_result) func.push_int(2) func.push_int(8) func.push_int(3) assert func.call() == 8 - func = lib.get('add_floats', ['f', 'f'], 'f', self.push_result) + args_type = ['f', 'f'] + res_type = 'f' + + func = lib.get('add_floats', args_type, res_type, self.push_result, cache=True) + self.fromcache(func, args_type, res_type) func.push_float(1.2) func.push_float(1.5) assert func.call() == 2.7 @@ -100,15 +127,24 @@ def test_get_void(self): lib = rjitffi.CDLL(self.lib_name) - func = lib.get('fvoid', [], 'i', self.push_result) + args_type = [] + res_type = 'i' + + func = lib.get('fvoid', args_type, res_type, self.push_result, cache=True) + self.fromcache(func, args_type, res_type) assert func.call() == 1 - func = lib.get('return_void', ['i', 'i'], 'v', self.push_result) + args_type = ['i', 'i'] + res_type = 'v' + + func = lib.get('return_void', args_type, res_type, self.push_result, cache=True) + self.fromcache(func, args_type, res_type) func.push_int(1) func.push_int(2) assert func.call() is None - func = lib.get('return_void', ['i', 'i'], push_result=self.push_result) + func = lib.get('return_void', args_type, push_result=self.push_result, cache=True) + self.fromcache(func, args_type, res_type) func.push_int(1) func.push_int(2) assert func.call() is None @@ -116,7 +152,10 @@ def test_various_type_args(self): lib = rjitffi.CDLL(self.lib_name) - func = lib.get('add_intfloat', ['i', 'f'], 'i', self.push_result) + args_type = ['i', 'f'] + res_type = 'i' + func = lib.get('add_intfloat', args_type, res_type, self.push_result, cache=True) + self.fromcache(func, args_type, res_type) func.push_int(1) func.push_float(2.9) assert func.call() == 3 @@ -126,6 +165,14 @@ func.push_float(1.3) assert func.call() == 1 + def test_nocache(self): + lib = rjitffi.CDLL(self.lib_name) + + func = lib.get('add_integers', ['i', 'i'], 'i', self.push_result) + func.push_int(1) + func.push_int(0) + assert func.call() == 1 + def test_undefined_func(self): lib = rjitffi.CDLL(self.lib_name) # xxxfoo888baryyy - not existed function From getxsick at codespeak.net Fri Aug 6 01:08:42 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Fri, 6 Aug 2010 01:08:42 +0200 (CEST) Subject: [pypy-svn] r76496 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100805230842.AC70C282BE8@codespeak.net> Author: getxsick Date: Fri Aug 6 01:08:41 2010 New Revision: 76496 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: use bargs as a local list Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Fri Aug 6 01:08:41 2010 @@ -41,24 +41,23 @@ self.res_type = res_type self.push_result = push_result self.cpu = cpu - self.bargs = [] lib = lib.handler try: self.funcaddr = rffi.cast(lltype.Signed, rdynload.dlsym(lib, func)) except KeyError: raise ValueError("Cannot find symbol %s", func) - self.bargs.append(BoxInt()) + self.setup_stack() if not cache: self.gen_looptaken() - self.setup_stack() def gen_looptaken(self): + bargs = [BoxInt()] # func addr for arg in self.args_type: if arg == 'i': - self.bargs.append(BoxInt()) + bargs.append(BoxInt()) elif arg == 'f': - self.bargs.append(BoxFloat()) + bargs.append(BoxFloat()) else: raise ValueError(arg) @@ -74,11 +73,11 @@ calldescr = self.get_calldescr() self.looptoken = LoopToken() # make sure it's not resized before ResOperation - self.bargs = list(self.bargs) - oplist = [ResOperation(rop.CALL, self.bargs, bres, descr=calldescr), + bargs = list(bargs) + oplist = [ResOperation(rop.CALL, bargs, bres, descr=calldescr), ResOperation(rop.FINISH, [bres], None, descr=BasicFailDescr(0))] - self.cpu.compile_loop(self.bargs, oplist, self.looptoken) + self.cpu.compile_loop(bargs, oplist, self.looptoken) def get_calldescr(self): if self.res_type == 'i': From dan at codespeak.net Fri Aug 6 10:35:59 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Fri, 6 Aug 2010 10:35:59 +0200 (CEST) Subject: [pypy-svn] r76497 - in pypy/branch/micronumpy/pypy/module/micronumpy: . test Message-ID: <20100806083559.B4D8D282BE8@codespeak.net> Author: dan Date: Fri Aug 6 10:35:57 2010 New Revision: 76497 Modified: pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py Log: It's been too long since the last commit, things are broken. Mid-addition-of-slicing. Also pushing to save some code. Modified: pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py Fri Aug 6 10:35:57 2010 @@ -70,11 +70,9 @@ def setitem(self, data, index, value): array = rffi.cast(lltype.Ptr(arraytype), data) - #array[index] = rffi.cast(ll_type, value) - array[index] = value # XXX: let's see if this works + array[index] = value def alloc(self, count): - #return lltype.malloc(arraytype, count, flavor='raw') mem = lltype.malloc(arraytype, count, flavor='raw') return rffi.cast(storage_type, mem) @@ -85,6 +83,12 @@ assert _w_descriptors[self.typeid].dtype is self, "This better be true." return _w_descriptors[self.typeid] + def cast(self, data): + return rffi.cast(lltype.Ptr(arraytype), data) + + def dump(self, data): + return ', '.join([str(x) for x in self.cast(data)]) + for type in [lltype.Signed, lltype.Float]: def get_type(self, data, index): value = self.getitem(data, index) Modified: pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py Fri Aug 6 10:35:57 2010 @@ -1,17 +1,16 @@ ?from pypy.interpreter.baseobjspace import ObjSpace from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.baseobjspace import W_Root +from pypy.objspace.std.sliceobject import W_SliceObject from pypy.interpreter.error import OperationError from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.gateway import interp2app from pypy.interpreter.gateway import NoneNotWrapped -from pypy.rlib.debug import make_sure_not_resized - from pypy.module import micronumpy from pypy.module.micronumpy.array import BaseNumArray -from pypy.module.micronumpy.array import construct_array, infer_shape +from pypy.module.micronumpy.array import infer_shape from pypy.module.micronumpy.dtype import null_data from pypy.module.micronumpy.array import stride_row, stride_column @@ -26,23 +25,23 @@ return space.int_w(space.index(w_index)) class MicroIter(Wrappable): - _immutable_fields_ = ['array', 'stride'] + _immutable_fields_ = ['array'] def __init__(self, array): self.array = array - self.i = 0 + self.i = array.offsets[0] def descr_iter(self, space): return space.wrap(self) descr_iter.unwrap_spec = ['self', ObjSpace] def descr_next(self, space): - if self.i < self.array.shape[0]: #will change for row/column + if self.i < self.array.shape[0]: if len(self.array.shape) > 1: - ar = MicroArray(self.array.shape[1:], # ok for column major? + ar = MicroArray(self.array.shape[1:], self.array.dtype, parent=self.array, strides=self.array.strides[1:], - offset=self.array.get_offset(self.array.flatten_index([self.i]))) + offsets=self.array.offsets + [self.i]) self.i += 1 return space.wrap(ar) elif len(self.array.shape) == 1: @@ -74,26 +73,32 @@ ) class MicroArray(BaseNumArray): - _immutable_fields_ = ['parent', 'offset'] - def __init__(self, shape, dtype, order='C', strides=None, parent=None, offset=0): + _immutable_fields_ = ['parent', 'shape', 'strides', 'offset', 'slice_starts'] + def __init__(self, shape, dtype, order='C', strides=[], parent=None, offset=0, slice_starts=[], slice_steps=[]): + assert dtype is not None + self.shape = shape self.dtype = dtype self.parent = parent + self.order = order self.offset = offset - self.order = order + self.slice_starts = slice_starts[:] + for i in range(len(shape) - len(slice_starts)): + self.slice_starts.append(0) + + self.slice_steps = slice_steps[:] + for i in range(len(shape) - len(slice_steps)): + self.slice_steps.append(1) - assert self.dtype is not None dtype = dtype.dtype #XXX: ugly size = size_from_shape(shape) - if strides is not None: - self.strides = strides - else: - self.strides = [0] * len(self.shape) - for i in range(len(self.shape)): - self.strides[i] = self.stride(i) # XXX calling self.stride repeatedly is a bit wasteful + self.strides = strides + stridelen = len(self.strides) + for i in range(len(self.shape) - stridelen): + self.strides.append(self.stride(stridelen + i)) # XXX calling self.stride repeatedly is a bit wasteful if size > 0 and parent is None: self.data = dtype.alloc(size) @@ -102,25 +107,25 @@ else: self.data = null_data - def get_offset(self, index): - return self.offset + index - def descr_len(self, space): return space.wrap(self.shape[0]) descr_len.unwrap_spec = ['self', ObjSpace] + def absolute_offset(self, index): + return self.offset + index # XXX: need more descriptive name + def getitem(self, space, index): try: dtype = self.dtype.dtype #XXX: kinda ugly return dtype.w_getitem(space, self.data, - self.get_offset(index)) + self.absolute_offset(index)) except IndexError, e: raise OperationError(space.w_IndexError, space.wrap("index out of bounds")) def setitem(self, space, index, w_value): dtype = self.dtype.dtype #XXX: kinda ugly - dtype.w_setitem(space, self.data, self.get_offset(index), w_value) #maybe hang onto w_dtype separately? + dtype.w_setitem(space, self.data, index, w_value) #maybe hang onto w_dtype separately? def flatten_applevel_index(self, space, w_index): try: @@ -142,7 +147,7 @@ def flatten_index(self, index): offset = 0 for i in range(len(index)): - offset += index[i] * self.strides[i] + offset += (self.slice_starts[i] + self.slice_steps[i] * index[i]) * self.strides[i] return offset def stride(self, i): @@ -163,61 +168,226 @@ raise OperationError(space.w_NotImplementedError, space.wrap("Unknown order: '%s'" % self.order)) - def descr_getitem(self, space, w_index): + def index2slices(self, space, w_index): + dtype = self.dtype.dtype try: - index = self.flatten_applevel_index(space, w_index) + index = space.int_w(space.index(w_index)) + return (self.flatten_index([index]), + self.slice_starts[1:], self.shape[1:], self.slice_steps[1:]) + except OperationError, e: + if e.match(space, space.w_TypeError): pass + else:raise + + try: + indices = space.fixedview(w_index) + + indexlen = len(indices) + slack = len(self.shape) - indexlen + + assert slack >= 0 - try: - index_dimension = space.int_w(space.len(w_index)) - except OperationError, e: - if e.match(space, space.w_TypeError): - index_dimension = 1 - else: raise + slice_starts = self.slice_starts[:] + slice_steps = self.slice_steps[:] + strides = self.strides[:] + shape = self.shape[:] + + for i in range(indexlen): + w_index = indices[i] + try: + index = space.int_w(space.index(w_index)) + slice_starts[i] += index + shape[i] = 1 + continue - if index_dimension == len(self.shape): + except OperationError, e: + if e.match(space, space.w_TypeError): pass + else: raise + + if isinstance(w_index, W_SliceObject): + start, stop, step, length = w_index.indices4(space, self.shape[i]) + slice_starts[i] += start + shape[i] = length + slice_steps[i] *= step + elif space.is_w(w_index, space.w_Ellipsis): + pass # I can't think of anything we need to do + else: + # XXX: no exception handling because we *want* to blow up + print 'type(w_index) = %s, w_index = %s' % (type(w_index), w_index) + index = space.str(w_index) + raise OperationError(space.w_NotImplementedError, + space.wrap("Don't support records yet.")) # XXX: and we may never + + prefix = 0 + while shape[prefix] == 1: prefix += 1 + + index = 0 + for offset in slice_starts[:prefix]: + index += self.strides[offset] + + return (self.flatten_index(slice_starts[:prefix]), + slice_starts[prefix:], shape[prefix:], slice_steps[prefix:]) + + except IndexError, e: + raise OperationError(space.w_IndexError, + space.wrap("invalid index")) + + def descr_getitem(self, space, w_index): + offset, slice_starts, shape, slice_steps = self.index2slices(space, w_index) + + size = size_from_shape(shape) + + if size == 1: + return self.getitem(space, + self.flatten_index(slice_starts)) + else: + ar = MicroArray(shape, + dtype=dtype, + parent=self, + offset=self.offset + offset, # XXX: need to figure out how to name things better + slice_starts=slice_starts, # XXX: begin renaming slice_starts + slice_steps=slice_steps) + return space.wrap(ar) + + def descr_getitem_2(self, space, w_index): # XXX: condemned + dtype = self.dtype.dtype + try: # XXX: this section should be shared with + # the iterator + index = space.int_w(space.index(w_index)) + if len(self.shape) == 1: return self.getitem(space, index) - elif index_dimension < len(self.shape): - assert index_dimension > 0 - array = MicroArray(self.shape[index_dimension:], self.dtype, - parent=self, offset=self.get_offset(index)) - return space.wrap(array) else: - raise OperationError(space.w_IndexError, - space.wrap("invalid index")) - + ar = MicroArray(self.shape[1:], # ok for column major? + self.dtype, + parent=self, + strides=self.strides[1:], + slice_starts=self.slice_starts[1:], + slice_steps=self.slice_steps[1:], + offset=self.absolute_offset(index) # XXX: are we sure? + ) + return space.wrap(ar) except OperationError, e: - if e.match(space, space.w_TypeError): pass # is there any way this can be caught here? - else: raise - - # XXX: here be demons + if e.match(space, space.w_TypeError): pass + else:raise + single_element = True try: - indices = [] - index = space.fixedview(w_index) - if len(index) > len(self.shape): - raise OperationError(space.w_ValueError, - space.wrap("Index has more dimensions (%d) than array (%d)" % (len(index), len(self.shape)))) + indices = space.fixedview(w_index) + + indexlen = len(indices) + slack = len(self.shape) - indexlen - for i in range(len(index)): - indices.append([]) + assert slack >= 0 + + slice_starts = self.slice_starts[:] + slice_steps = self.slice_steps[:] + strides = self.strides[:] + shape = self.shape[:] - for subindex in index: + for i in range(indexlen): + w_index = indices[i] try: - raise OperationError(space.w_NotImplementedError, - space.wrap("Haven't implemented newaxis.")) + index = space.int_w(space.index(w_index)) + slice_starts[i] += index + shape[i] = 1 + continue + except OperationError, e: - pass - + if e.match(space, space.w_TypeError): pass + else: raise + + single_element = False + + if isinstance(w_index, W_SliceObject): + start, stop, step, length = w_index.indices4(space, self.shape[i]) + slice_starts[i] += start + shape[i] = length + slice_steps[i] *= step + elif space.is_w(w_index, space.w_Ellipsis): + pass # I can't think of anything we need to do + else: + # XXX: no exception handling because we *want* to blow up + index = space.str(w_index) + raise OperationError(space.w_NotImplementedError, + space.wrap("Don't support records yet.")) # XXX: and we may never + + if single_element: + offset = self.flatten_index(slice_starts) + return self.getitem(space, offset) #XXX: maybe? + else: + size = size_from_shape(shape) + if size == 1: + offset = self.flatten_index(slice_starts) + return self.getitem(space, offset) + + prefix = 0 + while shape[prefix] == 1: j += prefix + + index = 0 + for offset in slice_starts[:prefix]: + index += self.strides[offset] + + ar = MicroArray(shape[prefix:], # ok for column major? + self.dtype, + parent=self, + strides=self.strides[prefix:], + offset=self.absolute_offset(offset), + slice_starts=slice_starts[prefix:], + slice_steps=slice_steps[prefix:]) + return space.wrap(ar) + + except IndexError, e: + raise OperationError(space.w_IndexError, + space.wrap("invalid index")) except OperationError, e: - if e.match(space, space.w_StopIteration): pass + if e.match(space, space.w_TypeError): raise # XXX: I think we want to blow up else: raise + descr_getitem.unwrap_spec = ['self', ObjSpace, W_Root] + def set_slice_single_value(self, space, shape, offsets, steps, w_value): + index = offsets + if len(shape) > 1: + for i in shape[0]: + self.set_slice(space, shape[1:], index, steps, w_value) + index[len(shape)-1] += 1 # XXX: reason + else: + dtype = self.dtype.dtype + for i in range(0, shape[0]): + dtype.w_setitem(space, data, + self.flatten_index(index), + value) + index[len(index)-1] += steps[0] # XXX: don't do steps in range + + def set_slice(self, space, shape, offsets, steps, w_value): + if self.dtype.w_is_compatible(space, w_value): + self.set_slice_single_value(space, shape, offsets, steps, w_value) + else: + length = space.int_w(space.len(w_value)) - descr_getitem.unwrap_spec = ['self', ObjSpace, W_Root] + if length == 1: + self.set_slice_single_value(space, shape, offsets, steps, w_value) + else: + value_shape = infer_shape(space, w_value) + if value_shape != shape: + raise OperationError(space.w_ValueError, + space.wrap("shape mismatch: objects cannot" + " be broadcast to a single shape")) + raise OperationError(space.w_NotImplementedError, + space.wrap("TODO")) def descr_setitem(self, space, w_index, w_value): - index = self.flatten_applevel_index(space, w_index) - self.setitem(space, index, w_value) + dtype = self.dtype.dtype + + offset, slice_starts, shape, slice_steps = self.index2slices(space, w_index) + + size = size_from_shape(shape) + + if size == 1: + self.setitem(space, + offset + selfflatten_index(slice_starts), # XXX: yeah? + w_value) + else: + self.set_slice(space, shape, slice_starts, slice_steps, w_value) + descr_setitem.unwrap_spec = ['self', ObjSpace, W_Root, W_Root] def descr_repr(self, space): @@ -234,7 +404,8 @@ descr_iter.unwrap_spec = ['self', ObjSpace] def __del__(self): - if self.parent is None and self.data != null_data: + if self.parent is None: + assert self.data != null_data dtype = self.dtype.dtype dtype.free(self.data) Modified: pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py Fri Aug 6 10:35:57 2010 @@ -438,9 +438,9 @@ ar = array(space, w_data, w_dtype=w_int_descr, order='C') #C for C not column for i in range(memlen): array_element = space.unwrap(ar.getitem(space, i)) # ugly, but encapsulates everything - assert array_element == row_major[i] + assert array_element == row_major[i], "Array Data: %r, Array Index: %d (%s != %s)" % (ar.dtype.dtype.dump(ar.data), i, array_element, row_major[i]) ar = array(space, w_data, w_dtype=w_int_descr, order='F') for i in range(memlen): array_element = space.unwrap(ar.getitem(space, i)) # ugly, but encapsulates everything - assert array_element == column_major[i] + assert array_element == column_major[i], "Array Data: %r, Array Index: %d (%s != %s)" % (ar.dtype.dtype.dump(ar.data), i, array_element, row_major[i]) From dan at codespeak.net Fri Aug 6 10:37:06 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Fri, 6 Aug 2010 10:37:06 +0200 (CEST) Subject: [pypy-svn] r76498 - pypy/branch/micronumpy/pypy/module/micronumpy Message-ID: <20100806083706.B4ACB282BE8@codespeak.net> Author: dan Date: Fri Aug 6 10:37:05 2010 New Revision: 76498 Modified: pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py Log: Culling the method that inspired the last push. Modified: pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py Fri Aug 6 10:37:05 2010 @@ -248,101 +248,6 @@ slice_steps=slice_steps) return space.wrap(ar) - def descr_getitem_2(self, space, w_index): # XXX: condemned - dtype = self.dtype.dtype - try: # XXX: this section should be shared with - # the iterator - index = space.int_w(space.index(w_index)) - if len(self.shape) == 1: - return self.getitem(space, index) - else: - ar = MicroArray(self.shape[1:], # ok for column major? - self.dtype, - parent=self, - strides=self.strides[1:], - slice_starts=self.slice_starts[1:], - slice_steps=self.slice_steps[1:], - offset=self.absolute_offset(index) # XXX: are we sure? - ) - return space.wrap(ar) - except OperationError, e: - if e.match(space, space.w_TypeError): pass - else:raise - - single_element = True - try: - indices = space.fixedview(w_index) - - indexlen = len(indices) - slack = len(self.shape) - indexlen - - assert slack >= 0 - - slice_starts = self.slice_starts[:] - slice_steps = self.slice_steps[:] - strides = self.strides[:] - shape = self.shape[:] - - for i in range(indexlen): - w_index = indices[i] - try: - index = space.int_w(space.index(w_index)) - slice_starts[i] += index - shape[i] = 1 - continue - - except OperationError, e: - if e.match(space, space.w_TypeError): pass - else: raise - - single_element = False - - if isinstance(w_index, W_SliceObject): - start, stop, step, length = w_index.indices4(space, self.shape[i]) - slice_starts[i] += start - shape[i] = length - slice_steps[i] *= step - elif space.is_w(w_index, space.w_Ellipsis): - pass # I can't think of anything we need to do - else: - # XXX: no exception handling because we *want* to blow up - index = space.str(w_index) - raise OperationError(space.w_NotImplementedError, - space.wrap("Don't support records yet.")) # XXX: and we may never - - if single_element: - offset = self.flatten_index(slice_starts) - return self.getitem(space, offset) #XXX: maybe? - else: - size = size_from_shape(shape) - if size == 1: - offset = self.flatten_index(slice_starts) - return self.getitem(space, offset) - - prefix = 0 - while shape[prefix] == 1: j += prefix - - index = 0 - for offset in slice_starts[:prefix]: - index += self.strides[offset] - - ar = MicroArray(shape[prefix:], # ok for column major? - self.dtype, - parent=self, - strides=self.strides[prefix:], - offset=self.absolute_offset(offset), - slice_starts=slice_starts[prefix:], - slice_steps=slice_steps[prefix:]) - return space.wrap(ar) - - except IndexError, e: - raise OperationError(space.w_IndexError, - space.wrap("invalid index")) - except OperationError, e: - if e.match(space, space.w_TypeError): raise # XXX: I think we want to blow up - else: raise - descr_getitem.unwrap_spec = ['self', ObjSpace, W_Root] - def set_slice_single_value(self, space, shape, offsets, steps, w_value): index = offsets if len(shape) > 1: From arigo at codespeak.net Fri Aug 6 11:22:13 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 6 Aug 2010 11:22:13 +0200 (CEST) Subject: [pypy-svn] r76499 - pypy/trunk/pypy/rpython Message-ID: <20100806092213.A9670282BFE@codespeak.net> Author: arigo Date: Fri Aug 6 11:22:12 2010 New Revision: 76499 Modified: pypy/trunk/pypy/rpython/rstr.py Log: Fix for r76143. Modified: pypy/trunk/pypy/rpython/rstr.py ============================================================================== --- pypy/trunk/pypy/rpython/rstr.py (original) +++ pypy/trunk/pypy/rpython/rstr.py Fri Aug 6 11:22:12 2010 @@ -288,8 +288,8 @@ if not hop.args_s[1].is_constant(): raise TyperError("encoding must be constant") encoding = hop.args_s[1].const - if encoding == "ascii": - expect = self.lowleveltype # can be a UniChar + if encoding == "ascii" and self.lowleveltype == UniChar: + expect = UniChar # only for unichar.encode('ascii') else: expect = self.repr # must be a regular unicode string v_self = hop.inputarg(expect, 0) From arigo at codespeak.net Fri Aug 6 11:22:34 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 6 Aug 2010 11:22:34 +0200 (CEST) Subject: [pypy-svn] r76500 - pypy/trunk/pypy/rpython/tool Message-ID: <20100806092234.B27B7282BFE@codespeak.net> Author: arigo Date: Fri Aug 6 11:22:33 2010 New Revision: 76500 Modified: pypy/trunk/pypy/rpython/tool/rfficache.py Log: Write a '\n' at the end of the file. Modified: pypy/trunk/pypy/rpython/tool/rfficache.py ============================================================================== --- pypy/trunk/pypy/rpython/tool/rfficache.py (original) +++ pypy/trunk/pypy/rpython/tool/rfficache.py Fri Aug 6 11:22:33 2010 @@ -29,7 +29,7 @@ } ''' % (include_string, add_source, str(question))) c_file = udir.join("gcctest.c") - c_file.write(c_source) + c_file.write(str(c_source) + '\n') eci = ExternalCompilationInfo() return build_executable_cache([c_file], eci) From arigo at codespeak.net Fri Aug 6 11:50:56 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 6 Aug 2010 11:50:56 +0200 (CEST) Subject: [pypy-svn] r76501 - pypy/trunk/pypy/translator/c Message-ID: <20100806095056.5A14A282BFE@codespeak.net> Author: arigo Date: Fri Aug 6 11:50:54 2010 New Revision: 76501 Modified: pypy/trunk/pypy/translator/c/genc.py pypy/trunk/pypy/translator/c/node.py Log: Remove two slots from ContainerNode, as an attempt to reduce memory usage of a full translation. Modified: pypy/trunk/pypy/translator/c/genc.py ============================================================================== --- pypy/trunk/pypy/translator/c/genc.py (original) +++ pypy/trunk/pypy/translator/c/genc.py Fri Aug 6 11:50:54 2010 @@ -1,7 +1,6 @@ import autopath import py import sys, os -from pypy.translator.c.node import PyObjectNode, FuncNode from pypy.translator.c.database import LowLevelDatabase from pypy.translator.c.extfunc import pre_include_code_lines from pypy.translator.llsupport.wrapper import new_wrapper @@ -196,7 +195,7 @@ all = [] for node in self.db.globalcontainers(): - eci = getattr(node, 'compilation_info', None) + eci = node.compilation_info() if eci: all.append(eci) self.merge_eci(*all) @@ -222,7 +221,7 @@ graphs = db.all_graphs() db.gctransformer.prepare_inline_helpers(graphs) for node in db.containerlist: - if isinstance(node, FuncNode): + if hasattr(node, 'funcgens'): for funcgen in node.funcgens: funcgen.patch_graph(copy_graph=False) return db Modified: pypy/trunk/pypy/translator/c/node.py ============================================================================== --- pypy/trunk/pypy/translator/c/node.py (original) +++ pypy/trunk/pypy/translator/c/node.py Fri Aug 6 11:50:54 2010 @@ -466,15 +466,15 @@ class ContainerNode(object): - if USESLOTS: - __slots__ = """db T obj + if USESLOTS: # keep the number of slots down! + __slots__ = """db obj typename implementationtypename - name ptrname compilation_info + name ptrname globalcontainer""".split() + eci_name = '_compilation_info' def __init__(self, db, T, obj): self.db = db - self.T = T self.obj = obj #self.dependencies = {} self.typename = db.gettype(T) #, who_asks=self) @@ -489,16 +489,22 @@ else: self.globalcontainer = False parentnode = db.getcontainernode(parent) - defnode = db.gettypedefnode(parentnode.T) + defnode = db.gettypedefnode(parentnode.getTYPE()) self.name = defnode.access_expr(parentnode.name, parentindex) if self.typename != self.implementationtypename: if db.gettypedefnode(T).extra_union_for_varlength: self.name += '.b' - self.compilation_info = getattr(obj, '_compilation_info', None) self.ptrname = '(&%s)' % self.name + def getTYPE(self): + return typeOf(self.obj) + def is_thread_local(self): - return hasattr(self.T, "_hints") and self.T._hints.get('thread_local') + T = self.getTYPE() + return hasattr(T, "_hints") and T._hints.get('thread_local') + + def compilation_info(self): + return getattr(self.obj, self.eci_name, None) def get_declaration(self): if self.name[-2:] == '.b': @@ -546,27 +552,31 @@ __slots__ = () def basename(self): - return self.T._name + T = self.getTYPE() + return T._name def enum_dependencies(self): - for name in self.T._names: + T = self.getTYPE() + for name in T._names: yield getattr(self.obj, name) def getlength(self): - if self.T._arrayfld is None: + T = self.getTYPE() + if T._arrayfld is None: return 1 else: - array = getattr(self.obj, self.T._arrayfld) + array = getattr(self.obj, T._arrayfld) return len(array.items) def initializationexpr(self, decoration=''): + T = self.getTYPE() is_empty = True yield '{' - defnode = self.db.gettypedefnode(self.T) + defnode = self.db.gettypedefnode(T) data = [] - if needs_gcheader(self.T): + if needs_gcheader(T): gc_init = self.db.gcpolicy.struct_gcheader_initdata(self) data.append(('gcheader', gc_init)) @@ -578,16 +588,16 @@ # '.fieldname = value'. But here we don't know which of the # fields need initialization, so XXX we pick the first one # arbitrarily. - if hasattr(self.T, "_hints") and self.T._hints.get('union'): + if hasattr(T, "_hints") and T._hints.get('union'): data = data[0:1] - if 'get_padding_drop' in self.T._hints: + if 'get_padding_drop' in T._hints: d = {} for name, _ in data: - T = defnode.c_struct_field_type(name) - typename = self.db.gettype(T) + T1 = defnode.c_struct_field_type(name) + typename = self.db.gettype(T1) d[name] = cdecl(typename, '') - padding_drop = self.T._hints['get_padding_drop'](d) + padding_drop = T._hints['get_padding_drop'](d) else: padding_drop = [] @@ -617,9 +627,10 @@ return 'struct _hashT_%s @' % self.name def forward_declaration(self): + T = self.getTYPE() assert self.typename == self.implementationtypename # no array part hash_typename = self.get_hash_typename() - hash_offset = self.db.gctransformer.get_hash_offset(self.T) + hash_offset = self.db.gctransformer.get_hash_offset(T) yield '%s {' % cdecl(hash_typename, '') yield '\tunion {' yield '\t\t%s;' % cdecl(self.implementationtypename, 'head') @@ -671,22 +682,23 @@ return len(self.obj.items) def initializationexpr(self, decoration=''): - defnode = self.db.gettypedefnode(self.T) + T = self.getTYPE() + defnode = self.db.gettypedefnode(T) yield '{' - if needs_gcheader(self.T): + if needs_gcheader(T): gc_init = self.db.gcpolicy.array_gcheader_initdata(self) lines = generic_initializationexpr(self.db, gc_init, 'gcheader', '%sgcheader' % (decoration,)) for line in lines: yield line - if self.T._hints.get('nolength', False): + if T._hints.get('nolength', False): length = '' else: length = '%d, ' % len(self.obj.items) - if self.T.OF is Void or len(self.obj.items) == 0: + if T.OF is Void or len(self.obj.items) == 0: yield '\t%s' % length.rstrip(', ') yield '}' - elif self.T.OF == Char: + elif T.OF == Char: if len(self.obj.items) and self.obj.items[0] is None: s = ''.join([self.obj.getitem(i) for i in range(len(self.obj.items))]) else: @@ -694,7 +706,7 @@ yield '\t%s%s' % (length, c_char_array_constant(s)) yield '}' else: - barebone = barebonearray(self.T) + barebone = barebonearray(T) if not barebone: yield '\t%s{' % length for j in range(len(self.obj.items)): @@ -722,7 +734,8 @@ self.ptrname = self.name def basename(self): - return self.T._name + T = self.getTYPE() + return T._name def enum_dependencies(self): for i in range(self.obj.getlength()): @@ -732,11 +745,12 @@ return 1 # not variable-sized! def initializationexpr(self, decoration=''): + T = self.getTYPE() assert self.typename == self.implementationtypename # not var-sized is_empty = True yield '{' # _names == ['item0', 'item1', ...] - for j, name in enumerate(self.T._names): + for j, name in enumerate(T._names): value = getattr(self.obj, name) lines = generic_initializationexpr(self.db, value, '%s[%d]' % (self.name, j), @@ -777,6 +791,7 @@ class FuncNode(ContainerNode): nodekind = 'func' + eci_name = 'compilation_info' # there not so many node of this kind, slots should not # be necessary @@ -794,7 +809,6 @@ else: self.name = (forcename or db.namespace.uniquename('g_' + self.basename())) - self.compilation_info = getattr(obj, 'compilation_info', None) self.make_funcgens() #self.dependencies = {} self.typename = db.gettype(T) #, who_asks=self) @@ -939,18 +953,20 @@ return [] def initializationexpr(self, decoration=''): - yield 'RPyOpaque_INITEXPR_%s' % (self.T.tag,) + T = self.getTYPE() + yield 'RPyOpaque_INITEXPR_%s' % (T.tag,) def startupcode(self): + T = self.getTYPE() args = [self.ptrname] # XXX how to make this code more generic? - if self.T.tag == 'ThreadLock': + if T.tag == 'ThreadLock': lock = self.obj.externalobj if lock.locked(): args.append('1') else: args.append('0') - yield 'RPyOpaque_SETUP_%s(%s);' % (self.T.tag, ', '.join(args)) + yield 'RPyOpaque_SETUP_%s(%s);' % (T.tag, ', '.join(args)) def opaquenode_factory(db, T, obj): From arigo at codespeak.net Fri Aug 6 11:51:42 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 6 Aug 2010 11:51:42 +0200 (CEST) Subject: [pypy-svn] r76502 - in pypy/trunk/pypy/jit/backend/x86: . test Message-ID: <20100806095142.381D4282BFE@codespeak.net> Author: arigo Date: Fri Aug 6 11:51:40 2010 New Revision: 76502 Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/test/test_assembler.py Log: Use a Struct instead of an Array for debug_counter. Fixes test_ztranslation, which is unhappy about this being the only array in the translated program. Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Fri Aug 6 11:51:40 2010 @@ -151,7 +151,7 @@ # XXX: 32 is pulled out of the air return 32 + len(self.desc_bytes) -DEBUG_COUNTER = rffi.CArray(lltype.Signed) +DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed)) class Assembler386(object): mc = None @@ -182,7 +182,7 @@ self.pending_guard_tokens = None self.setup_failure_recovery() self._debug = False - self.debug_counter_descr = cpu.arraydescrof(DEBUG_COUNTER) + self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') def leave_jitted_hook(self): ptrs = self.fail_boxes_ptr.ar @@ -240,8 +240,8 @@ assert output_log is not None f = open_file_as_stream(output_log, "w") for i in range(len(self.loop_run_counters)): - name, arr = self.loop_run_counters[i] - f.write(name + ":" + str(arr[0]) + "\n") + name, struct = self.loop_run_counters[i] + f.write(name + ":" + str(struct.i) + "\n") f.close() def _build_float_constants(self): @@ -394,9 +394,9 @@ funcname = "" % len(self.loop_run_counters) # invent the counter, so we don't get too confused if self._debug: - arr = lltype.malloc(DEBUG_COUNTER, 1, flavor='raw') - arr[0] = 0 - self.loop_run_counters.append((funcname, arr)) + struct = lltype.malloc(DEBUG_COUNTER, flavor='raw') + struct.i = 0 + self.loop_run_counters.append((funcname, struct)) return funcname def patch_jump_for_descr(self, faildescr, adr_new_target): @@ -426,11 +426,10 @@ self.loop_run_counters[-1][1])) box = BoxInt() box2 = BoxInt() - ops = [ResOperation(rop.GETARRAYITEM_RAW, [c_adr, ConstInt(0)], + ops = [ResOperation(rop.GETFIELD_RAW, [c_adr], box, descr=self.debug_counter_descr), ResOperation(rop.INT_ADD, [box, ConstInt(1)], box2), - ResOperation(rop.SETARRAYITEM_RAW, [c_adr, ConstInt(0), - box2], + ResOperation(rop.SETFIELD_RAW, [c_adr, box2], None, descr=self.debug_counter_descr)] operations = ops + operations # # we need one register free (a bit of a hack, but whatever) Modified: pypy/trunk/pypy/jit/backend/x86/test/test_assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_assembler.py Fri Aug 6 11:51:40 2010 @@ -15,7 +15,7 @@ supports_floats = True NUM_REGS = ACTUAL_CPU.NUM_REGS - def arraydescrof(self, ARR): + def fielddescrof(self, STRUCT, name): return 42 class FakeMC: From arigo at codespeak.net Fri Aug 6 12:00:11 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 6 Aug 2010 12:00:11 +0200 (CEST) Subject: [pypy-svn] r76503 - pypy/trunk/pypy/jit/backend/x86/test Message-ID: <20100806100011.B5B51282BFE@codespeak.net> Author: arigo Date: Fri Aug 6 12:00:10 2010 New Revision: 76503 Modified: pypy/trunk/pypy/jit/backend/x86/test/test_runner.py Log: This fix goes with r76502. Modified: pypy/trunk/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_runner.py Fri Aug 6 12:00:10 2010 @@ -498,9 +498,9 @@ self.cpu.set_future_value_int(0, 0) self.cpu.execute_token(ops.token) # check debugging info - name, arr = self.cpu.assembler.loop_run_counters[0] + name, struct = self.cpu.assembler.loop_run_counters[0] assert name == 'xyz' - assert arr[0] == 10 + assert struct.i == 10 self.cpu.finish_once() lines = py.path.local(self.logfile + ".count").readlines() assert lines[0] == 'xyz:10\n' From arigo at codespeak.net Fri Aug 6 12:55:00 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 6 Aug 2010 12:55:00 +0200 (CEST) Subject: [pypy-svn] r76504 - pypy/trunk/pypy/jit/backend/x86 Message-ID: <20100806105500.EAD5736C230@codespeak.net> Author: arigo Date: Fri Aug 6 12:54:59 2010 New Revision: 76504 Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/regalloc.py pypy/trunk/pypy/jit/backend/x86/regloc.py Log: Cancel r76493. The fix works; test_ztranslation failed for other reasons already before (and is fixed now). Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Fri Aug 6 12:54:59 2010 @@ -818,10 +818,8 @@ for i in range(start, len(arglocs)): loc = arglocs[i] - assert isinstance(loc, RegLoc) or isinstance(loc, ImmedLoc) or isinstance(loc, StackLoc) - # XXX: Should be much simplier to tell whether a location is a float! - if (isinstance(loc, RegLoc) and loc.is_xmm) or (isinstance(loc, StackLoc) and loc.type == FLOAT): + if (isinstance(loc, RegLoc) and loc.is_xmm) or (loc.is_memory_reference() and loc.type == FLOAT): if len(unused_xmm) > 0: xmm_src_locs.append(loc) xmm_dst_locs.append(unused_xmm.pop()) Modified: pypy/trunk/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regalloc.py Fri Aug 6 12:54:59 2010 @@ -53,6 +53,7 @@ def __init__(self): self.cur_array_free = 0 + self.const_id = 0 def _get_new_array(self): n = self.BASE_CONSTANT_SIZE @@ -68,7 +69,8 @@ n = self.cur_array_free - 1 arr[n] = floatval self.cur_array_free = n - return rffi.cast(lltype.Signed, arr) + n * 8 + self.const_id += 1 + return (self.const_id, rffi.cast(lltype.Signed, arr) + n * 8) class X86XMMRegisterManager(RegisterManager): @@ -89,8 +91,8 @@ self.float_constants = assembler._float_constants def convert_to_imm(self, c): - adr = self.float_constants.record_float(c.getfloat()) - return AddressLoc(ImmedLoc(adr), ImmedLoc(0), 0, 0) + const_id, adr = self.float_constants.record_float(c.getfloat()) + return ConstFloatLoc(adr, const_id) def after_call(self, v): # the result is stored in st0, but we don't have this around, Modified: pypy/trunk/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regloc.py Fri Aug 6 12:54:59 2010 @@ -5,6 +5,7 @@ from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.objectmodel import specialize from pypy.rlib.rarithmetic import intmask +from pypy.jit.metainterp.history import FLOAT # # This module adds support for "locations", which can be either in a Const, @@ -145,6 +146,28 @@ def value_m(self): return self.loc_m +class ConstFloatLoc(AssemblerLocation): + # XXX: We have to use this class instead of just AddressLoc because + # AddressLoc is "untyped" and also we to have need some sort of unique + # identifier that we can use in _getregkey (for jump.py) + + _immutable_ = True + + width = 8 + type = FLOAT + + def __init__(self, address, const_id): + self.value = address + self.const_id = const_id + + def _getregkey(self): + # XXX: 1000 is kind of magic: We just don't want to be confused + # with any registers + return 1000 + self.const_id + + def location_code(self): + return 'j' + REGLOCS = [RegLoc(i, is_xmm=False) for i in range(16)] XMMREGLOCS = [RegLoc(i, is_xmm=True) for i in range(16)] eax, ecx, edx, ebx, esp, ebp, esi, edi, r8, r9, r10, r11, r12, r13, r14, r15 = REGLOCS From getxsick at codespeak.net Fri Aug 6 13:34:53 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Fri, 6 Aug 2010 13:34:53 +0200 (CEST) Subject: [pypy-svn] r76505 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100806113453.13283282BFE@codespeak.net> Author: getxsick Date: Fri Aug 6 13:34:52 2010 New Revision: 76505 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: keep CallDescrs in one place Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Fri Aug 6 13:34:52 2010 @@ -83,9 +83,9 @@ if self.res_type == 'i': cls = SignedCallDescr elif self.res_type == 'f': - cls = descr.FloatCallDescr + cls = FloatCallDescr elif self.res_type == 'v': - cls = descr.VoidCallDescr + cls = VoidCallDescr else: raise NotImplementedError('Unknown type of descr: %s' % self.res_type) @@ -129,3 +129,6 @@ _clsname = 'SignedCallDescr' def get_result_size(self, translate_support_code): return symbolic.get_size(lltype.Signed, translate_support_code) + +FloatCallDescr = descr.FloatCallDescr +VoidCallDescr = descr.VoidCallDescr From agaynor at codespeak.net Fri Aug 6 15:05:46 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Fri, 6 Aug 2010 15:05:46 +0200 (CEST) Subject: [pypy-svn] r76506 - in pypy/trunk/lib_pypy: . pypy_test Message-ID: <20100806130546.6A161282BFE@codespeak.net> Author: agaynor Date: Fri Aug 6 15:05:43 2010 New Revision: 76506 Modified: pypy/trunk/lib_pypy/pypy_test/test_coroutine.py pypy/trunk/lib_pypy/stackless.py Log: Fix for failing (and hanging) stackless and coroutine tests. Modified: pypy/trunk/lib_pypy/pypy_test/test_coroutine.py ============================================================================== --- pypy/trunk/lib_pypy/pypy_test/test_coroutine.py (original) +++ pypy/trunk/lib_pypy/pypy_test/test_coroutine.py Fri Aug 6 15:05:43 2010 @@ -2,7 +2,7 @@ from py.test import skip, raises try: - from ..stackless import coroutine + from ..stackless import coroutine, CoroutineExit except ImportError, e: skip('cannot import stackless: %s' % (e,)) Modified: pypy/trunk/lib_pypy/stackless.py ============================================================================== --- pypy/trunk/lib_pypy/stackless.py (original) +++ pypy/trunk/lib_pypy/stackless.py Fri Aug 6 15:05:43 2010 @@ -14,9 +14,11 @@ import traceback import sys try: - from _stackless import coroutine, greenlet + from _stackless import coroutine, greenlet, CoroutineExit, TaskletExit except ImportError: # we are running from CPython - from greenlet import greenlet + from greenlet import greenlet, GreenletExit + TaskletExit = CoroutineExit = GreenletExit + del GreenletExit try: from functools import partial except ImportError: # we are not running python 2.5 From agaynor at codespeak.net Fri Aug 6 15:23:44 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Fri, 6 Aug 2010 15:23:44 +0200 (CEST) Subject: [pypy-svn] r76507 - pypy/trunk/lib_pypy Message-ID: <20100806132344.6B662282BFE@codespeak.net> Author: agaynor Date: Fri Aug 6 15:23:42 2010 New Revision: 76507 Modified: pypy/trunk/lib_pypy/stackless.py Log: Remove some extraneous imports that made test_distributed fail, these are automatically available when running in a stackless enviroment. Modified: pypy/trunk/lib_pypy/stackless.py ============================================================================== --- pypy/trunk/lib_pypy/stackless.py (original) +++ pypy/trunk/lib_pypy/stackless.py Fri Aug 6 15:23:42 2010 @@ -14,7 +14,9 @@ import traceback import sys try: - from _stackless import coroutine, greenlet, CoroutineExit, TaskletExit + # If _stackless can be imported then TaskletExit and CoroutineExit are + # automatically added to the builtins. + from _stackless import coroutine, greenlet except ImportError: # we are running from CPython from greenlet import greenlet, GreenletExit TaskletExit = CoroutineExit = GreenletExit From arigo at codespeak.net Fri Aug 6 15:32:58 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 6 Aug 2010 15:32:58 +0200 (CEST) Subject: [pypy-svn] r76508 - pypy/branch/rsre2/pypy/rlib/rsre_jit Message-ID: <20100806133258.5FB03282BFE@codespeak.net> Author: arigo Date: Fri Aug 6 15:32:56 2010 New Revision: 76508 Removed: pypy/branch/rsre2/pypy/rlib/rsre_jit/ Log: Delete experimental directory. From arigo at codespeak.net Fri Aug 6 15:33:38 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 6 Aug 2010 15:33:38 +0200 (CEST) Subject: [pypy-svn] r76509 - in pypy/branch/rsre2/pypy/rlib/rsre: . test Message-ID: <20100806133338.EB4FC282BFE@codespeak.net> Author: arigo Date: Fri Aug 6 15:33:37 2010 New Revision: 76509 Added: pypy/branch/rsre2/pypy/rlib/rsre/rsre_core.py - copied, changed from r75994, pypy/branch/rsre2/pypy/rlib/rsre/rsre.py pypy/branch/rsre2/pypy/rlib/rsre/rsre_re.py (contents, props changed) pypy/branch/rsre2/pypy/rlib/rsre/test/test_re.py - copied, changed from r76325, pypy/trunk/lib-python/modified-2.5.2/test/test_re.py pypy/branch/rsre2/pypy/rlib/rsre/test/test_zinterp.py (contents, props changed) Removed: pypy/branch/rsre2/pypy/rlib/rsre/rsre.py pypy/branch/rsre2/pypy/rlib/rsre/rsre_gen.py Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py pypy/branch/rsre2/pypy/rlib/rsre/test/test_more.py pypy/branch/rsre2/pypy/rlib/rsre/test/test_search.py pypy/branch/rsre2/pypy/rlib/rsre/test/test_zexternal.py Log: Finish and polish the regexp engine. It supports a subset of the interface of CPython. The idea will be to try to link it with pypy/module/_sre now. Copied: pypy/branch/rsre2/pypy/rlib/rsre/rsre_core.py (from r75994, pypy/branch/rsre2/pypy/rlib/rsre/rsre.py) ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre_core.py Fri Aug 6 15:33:37 2010 @@ -1,5 +1,7 @@ +import sys from pypy.rlib.debug import check_nonneg from pypy.rlib.rsre import rsre_char +from pypy.rlib import jit OPCODE_FAILURE = 0 @@ -42,11 +44,12 @@ match_marks = None match_marks_flat = None - def __init__(self, pattern, string, match_start, flags): - assert match_start >= 0 + def __init__(self, pattern, string, match_start, end, flags): self.pattern = pattern self.string = string - self.end = len(string) + if end > len(string): + end = len(string) + self.end = end self.match_start = match_start self.flags = flags @@ -70,6 +73,10 @@ if self.match_marks_flat is None: self.match_marks_flat = [self.match_start, self.match_end] mark = self.match_marks + if mark is not None: + self.match_lastindex = mark.gid + else: + self.match_lastindex = -1 while mark is not None: index = mark.gid + 2 while index >= len(self.match_marks_flat): @@ -88,12 +95,13 @@ return (-1, -1) return (fmarks[groupnum], fmarks[groupnum+1]) - def group(self, group=0): + def group(self, groupnum=0): # compatibility - frm, to = self.span(group) - if frm < 0 or to < frm: + frm, to = self.span(groupnum) + if 0 <= frm <= to: + return self.string[frm:to] + else: return None - return self.string[frm:to] def at_boundary(self, ptr, word_checker): if self.end == 0: @@ -153,6 +161,7 @@ self.start_ptr = ptr self.start_marks = marks + @jit.unroll_safe # there are only a few branch alternatives def find_first_result(self, ctx): ppos = self.ppos while ctx.pat(ppos): @@ -221,7 +230,15 @@ self.tailppos = tailppos self.cur_ptr = ptr self.cur_marks = marks - self.pending = [] + self.pending = None + self.num_pending = 0 + +class Pending(object): + def __init__(self, ptr, marks, enum, next): + self.ptr = ptr + self.marks = marks + self.enum = enum + self.next = next # chained list class MaxUntilMatchResult(AbstractUntilMatchResult): @@ -242,13 +259,14 @@ while True: if enum is not None: # matched one more 'item'. record it and continue - self.pending.append((ptr, marks, enum)) + self.pending = Pending(ptr, marks, enum, self.pending) + self.num_pending += 1 ptr = ctx.match_end marks = ctx.match_marks break else: # 'item' no longer matches. - if not resume and len(self.pending) >= min: + if not resume and self.num_pending >= min: # try to match 'tail' if we have enough 'item' result = sre_match(ctx, self.tailppos, ptr, marks) if result is not None: @@ -257,12 +275,16 @@ self.cur_marks = marks return self resume = False - if len(self.pending) == 0: + p = self.pending + if p is None: return - ptr, marks, enum = self.pending.pop() - enum = enum.move_to_next_result(ctx) + self.pending = p.next + self.num_pending -= 1 + ptr = p.ptr + marks = p.marks + enum = p.enum.move_to_next_result(ctx) # - if max == 65535 or len(self.pending) < max: + if max == 65535 or self.num_pending < max: # try to match one more 'item' enum = sre_match(ctx, ppos + 3, ptr, marks) else: @@ -284,7 +306,7 @@ marks = self.cur_marks while True: # try to match 'tail' if we have enough 'item' - if not resume and len(self.pending) >= min: + if not resume and self.num_pending >= min: result = sre_match(ctx, self.tailppos, ptr, marks) if result is not None: self.subresult = result @@ -293,7 +315,7 @@ return self resume = False - if max == 65535 or len(self.pending) < max: + if max == 65535 or self.num_pending < max: # try to match one more 'item' enum = sre_match(ctx, ppos + 3, ptr, marks) else: @@ -302,18 +324,25 @@ while enum is None: # 'item' does not match; try to get further results from # the 'pending' list. - if len(self.pending) == 0: + p = self.pending + if p is None: return - ptr, marks, enum = self.pending.pop() - enum = enum.move_to_next_result(ctx) + self.pending = p.next + self.num_pending -= 1 + ptr = p.ptr + marks = p.marks + enum = p.enum.move_to_next_result(ctx) # matched one more 'item'. record it and continue - self.pending.append((ptr, marks, enum)) + self.pending = Pending(ptr, marks, enum, self.pending) + self.num_pending += 1 ptr = ctx.match_end marks = ctx.match_marks # ____________________________________________________________ + at jit.unroll_safe # it's safe to unroll the main 'while' loop: + # 'ppos' is only ever incremented in this function def sre_match(ctx, ppos, ptr, marks): """Returns either None or a MatchResult object. Usually we only need the first result, but there is the case of REPEAT...UNTIL where we @@ -383,35 +412,34 @@ elif op == OPCODE_GROUPREF: # match backreference # - gid = ctx.pat(ppos) * 2 - startptr = find_mark(marks, gid) - if startptr < 0: - return - endptr = find_mark(marks, gid + 1) - if endptr < startptr: # also includes the case "endptr == -1" - return - for i in range(startptr, endptr): - if ptr >= ctx.end or ctx.str(ptr) != ctx.str(i): - return - ptr += 1 + startptr, length = get_group_ref(marks, ctx.pat(ppos)) + if length < 0: + return # group was not previously defined + if not match_repeated(ctx, ptr, startptr, length): + return # no match + ptr += length ppos += 1 elif op == OPCODE_GROUPREF_IGNORE: # match backreference # - gid = ctx.pat(ppos) * 2 - startptr = find_mark(marks, gid) - if startptr < 0: - return - endptr = find_mark(marks, gid + 1) - if endptr < startptr: # also includes the case "endptr == -1" - return - for i in range(startptr, endptr): - if ptr >= ctx.end or ctx.lowstr(ptr) != ctx.lowstr(i): - return - ptr += 1 + startptr, length = get_group_ref(marks, ctx.pat(ppos)) + if length < 0: + return # group was not previously defined + if not match_repeated_ignore(ctx, ptr, startptr, length): + return # no match + ptr += length ppos += 1 + elif op == OPCODE_GROUPREF_EXISTS: + # conditional match depending on the existence of a group + # codeyes codeno ... + _, length = get_group_ref(marks, ctx.pat(ppos)) + if length >= 0: + ppos += 2 # jump to 'codeyes' + else: + ppos += ctx.pat(ppos+1) # jump to 'codeno' + elif op == OPCODE_IN: # match set member (or non_member) # @@ -561,6 +589,31 @@ assert 0, "bad pattern code %d" % op +def get_group_ref(marks, groupnum): + gid = groupnum * 2 + startptr = find_mark(marks, gid) + if startptr < 0: + return 0, -1 + endptr = find_mark(marks, gid + 1) + length = endptr - startptr # < 0 if endptr < startptr (or if endptr=-1) + return startptr, length + +def match_repeated(ctx, ptr, oldptr, length): + if ptr + length > ctx.end: + return False + for i in range(length): + if ctx.str(ptr + i) != ctx.str(oldptr + i): + return False + return True + +def match_repeated_ignore(ctx, ptr, oldptr, length): + if ptr + length > ctx.end: + return False + for i in range(length): + if ctx.lowstr(ptr + i) != ctx.lowstr(oldptr + i): + return False + return True + def find_repetition_end(ctx, ppos, ptr, maxcount): end = ctx.end # adjust end @@ -570,54 +623,60 @@ end = end1 op = ctx.pat(ppos) + if op == OPCODE_ANY: return fre_ANY(ctx, ptr, end) + if op == OPCODE_ANY_ALL: return end + if op == OPCODE_IN: return fre_IN(ctx, ptr, end, ppos) + if op == OPCODE_IN_IGNORE: return fre_IN_IGNORE(ctx, ptr, end, ppos) + if op == OPCODE_LITERAL: return fre_LITERAL(ctx, ptr, end, ppos) + if op == OPCODE_LITERAL_IGNORE: return fre_LITERAL_IGNORE(ctx,ptr,end,ppos) + if op == OPCODE_NOT_LITERAL: return fre_NOT_LITERAL(ctx, ptr, end, ppos) + if op == OPCODE_NOT_LITERAL_IGNORE: return fre_NOT_LITERAL_IGNORE(ctx,ptr, + end,ppos) + raise NotImplementedError("rsre.find_repetition_end[%d]" % op) + +def fre_ANY(ctx, ptr, end): + # repeated dot wildcard. + while ptr < end and not rsre_char.is_linebreak(ctx.str(ptr)): + ptr += 1 + return ptr - if op == OPCODE_ANY: - # repeated dot wildcard. - while ptr < end and not rsre_char.is_linebreak(ctx.str(ptr)): - ptr += 1 - - elif op == OPCODE_ANY_ALL: - # repeated dot wildcare. skip to the end of the target - # string, and backtrack from there - ptr = end - - elif op == OPCODE_IN: - # repeated set - while ptr < end and rsre_char.check_charset(ctx.pattern, ppos+2, - ctx.str(ptr)): - ptr += 1 - - elif op == OPCODE_IN_IGNORE: - # repeated set - while ptr < end and rsre_char.check_charset(ctx.pattern, ppos+2, - ctx.lowstr(ptr)): - ptr += 1 - - elif op == OPCODE_LITERAL: - chr = ctx.pat(ppos+1) - while ptr < end and ctx.str(ptr) == chr: - ptr += 1 - - elif op == OPCODE_LITERAL_IGNORE: - chr = ctx.pat(ppos+1) - while ptr < end and ctx.lowstr(ptr) == chr: - ptr += 1 +def fre_IN(ctx, ptr, end, ppos): + # repeated set + while ptr < end and rsre_char.check_charset(ctx.pattern, ppos+2, + ctx.str(ptr)): + ptr += 1 + return ptr - elif op == OPCODE_NOT_LITERAL: - chr = ctx.pat(ppos+1) - while ptr < end and ctx.str(ptr) != chr: - ptr += 1 +def fre_IN_IGNORE(ctx, ptr, end, ppos): + # repeated set + while ptr < end and rsre_char.check_charset(ctx.pattern, ppos+2, + ctx.lowstr(ptr)): + ptr += 1 + return ptr - elif op == OPCODE_NOT_LITERAL_IGNORE: - chr = ctx.pat(ppos+1) - while ptr < end and ctx.lowstr(ptr) != chr: - ptr += 1 +def fre_LITERAL(ctx, ptr, end, ppos): + chr = ctx.pat(ppos+1) + while ptr < end and ctx.str(ptr) == chr: + ptr += 1 + return ptr - else: - raise NotImplementedError("rsre.find_repetition_end[%d]" % op) +def fre_LITERAL_IGNORE(ctx, ptr, end, ppos): + chr = ctx.pat(ppos+1) + while ptr < end and ctx.lowstr(ptr) == chr: + ptr += 1 + return ptr +def fre_NOT_LITERAL(ctx, ptr, end, ppos): + chr = ctx.pat(ppos+1) + while ptr < end and ctx.str(ptr) != chr: + ptr += 1 return ptr +def fre_NOT_LITERAL_IGNORE(ctx, ptr, end, ppos): + chr = ctx.pat(ppos+1) + while ptr < end and ctx.lowstr(ptr) != chr: + ptr += 1 + return ptr ##### At dispatch @@ -676,14 +735,18 @@ # ____________________________________________________________ -def match(pattern, string, start=0, flags=0): - ctx = MatchContext(pattern, string, start, flags) +def match(pattern, string, start=0, end=sys.maxint, flags=0): + if start < 0: start = 0 + if end < start: return None + ctx = MatchContext(pattern, string, start, end, flags) if sre_match(ctx, 0, start, None) is not None: return ctx return None -def search(pattern, string, start=0, flags=0): - ctx = MatchContext(pattern, string, start, flags) +def search(pattern, string, start=0, end=sys.maxint, flags=0): + if start < 0: start = 0 + if end < start: return None + ctx = MatchContext(pattern, string, start, end, flags) if ctx.pat(0) == OPCODE_INFO: if ctx.pat(2) & rsre_char.SRE_INFO_PREFIX and ctx.pat(5) > 1: return fast_search(ctx) Added: pypy/branch/rsre2/pypy/rlib/rsre/rsre_re.py ============================================================================== --- (empty file) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre_re.py Fri Aug 6 15:33:37 2010 @@ -0,0 +1,171 @@ +import re, sys +from pypy.rlib.rsre import rsre_core, rsre_char +from pypy.rlib.rsre.test.test_match import get_code as _get_code +from pypy.module.unicodedata import unicodedb_3_2_0 +rsre_char.unicodedb = unicodedb_3_2_0 + + +I = IGNORECASE = 2 # ignore case +L = LOCALE = 4 # assume current 8-bit locale +U = UNICODE = 32 # assume unicode locale +M = MULTILINE = 8 # make anchors look for newline +S = DOTALL = 16 # make dot match newline +X = VERBOSE = 64 # ignore whitespace and comments + + +def match(pattern, string, flags=0): + return compile(pattern, flags).match(string) + +def search(pattern, string, flags=0): + return compile(pattern, flags).search(string) + +def findall(pattern, string, flags=0): + return compile(pattern, flags).findall(string) + +def finditer(pattern, string, flags=0): + return compile(pattern, flags).finditer(string) + +def compile(pattern, flags=0): + code, flags, args = _get_code(pattern, flags, allargs=True) + return RSREPattern(pattern, code, flags, *args) + +escape = re.escape +error = re.error + + +class RSREPattern(object): + + def __init__(self, pattern, code, flags, + num_groups, groupindex, indexgroup): + self._code = code + self.pattern = pattern + self.flags = flags + self.groups = num_groups + self.groupindex = groupindex + self._indexgroup = indexgroup + + def match(self, string, pos=0, endpos=sys.maxint): + return self._make_match(rsre_core.match(self._code, string, + pos, endpos, + flags=self.flags)) + + def search(self, string, pos=0, endpos=sys.maxint): + return self._make_match(rsre_core.search(self._code, string, + pos, endpos, + flags=self.flags)) + + def findall(self, string): + matchlist = [] + for match in self.finditer(string): + if self.groups == 0 or self.groups == 1: + item = match.group(self.groups) + else: + item = match.groups("") + matchlist.append(item) + return matchlist + + def finditer(self, string): + matchlist = [] + start = 0 + while True: + match = rsre_core.search(self._code, string, start, + flags=self.flags) + if match is None: + break + end = match.match_end + yield RSREMatch(self, match) + if start == end: + start += 1 + if start > len(string): + break + else: + start = end + + def _make_match(self, res): + if res is None: + return None + return RSREMatch(self, res) + + +class RSREMatch(object): + + def __init__(self, pattern, ctx): + self.re = pattern + self._ctx = ctx + + def span(self, groupnum=0): + if not isinstance(groupnum, (int, long)): + groupnum = self.re.groupindex[groupnum] + return self._ctx.span(groupnum) + + def start(self, groupnum=0): + return self.span(groupnum)[0] + + def end(self, groupnum=0): + return self.span(groupnum)[1] + + def group(self, *groups): + groups = groups or (0,) + result = [] + for group in groups: + frm, to = self.span(group) + if 0 <= frm <= to: + result.append(self._ctx.string[frm:to]) + else: + result.append(None) + if len(result) > 1: + return tuple(result) + else: + return result[0] + + def groups(self, default=None): + fmarks = self._ctx.flatten_marks() + grps = [] + for i in range(1, self.re.groups+1): + grp = self.group(i) + if grp is None: grp = default + grps.append(grp) + return tuple(grps) + + def groupdict(self, default=None): + d = {} + for key, value in self.re.groupindex.iteritems(): + grp = self.group(value) + if grp is None: grp = default + d[key] = grp + return d + + def expand(self, template): + return re._expand(self.re, self, template) + + @property + def regs(self): + fmarks = self._ctx.flatten_marks() + return tuple([(fmarks[i], fmarks[i+1]) + for i in range(0, len(fmarks), 2)]) + + @property + def lastindex(self): + self._ctx.flatten_marks() + if self._ctx.match_lastindex < 0: + return None + return self._ctx.match_lastindex // 2 + 1 + + @property + def lastgroup(self): + lastindex = self.lastindex + if lastindex < 0 or lastindex >= len(self.re._indexgroup): + return None + return self.re._indexgroup[lastindex] + + @property + def string(self): + return self._ctx.string + + @property + def pos(self): + return self._ctx.match_start + + @property + def endpos(self): + return self._ctx.end Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_match.py Fri Aug 6 15:33:37 2010 @@ -1,25 +1,28 @@ import _sre, re, sre_compile -from pypy.rlib.rsre import rsre +from pypy.rlib.rsre import rsre_core -def get_code(regexp): +def get_code(regexp, flags=0, allargs=False): class GotIt(Exception): pass def my_compile(pattern, flags, code, *args): print code - raise GotIt(code) + raise GotIt(code, flags, args) saved = _sre.compile try: _sre.compile = my_compile try: - sre_compile.compile(regexp, 0) + sre_compile.compile(regexp, flags) except GotIt, e: pass else: raise ValueError("did not reach _sre.compile()!") finally: _sre.compile = saved - return e.args[0] + if allargs: + return e.args + else: + return e.args[0] def get_code_and_re(regexp): return get_code(regexp), re.compile(regexp) @@ -34,61 +37,61 @@ def test_or(self): r = get_code(r"a|bc|def") - assert rsre.match(r, "a") - assert rsre.match(r, "bc") - assert rsre.match(r, "def") - assert not rsre.match(r, "ghij") + assert rsre_core.match(r, "a") + assert rsre_core.match(r, "bc") + assert rsre_core.match(r, "def") + assert not rsre_core.match(r, "ghij") def test_any(self): r = get_code(r"ab.cd") - assert rsre.match(r, "abXcdef") - assert not rsre.match(r, "ab\ncdef") - assert not rsre.match(r, "abXcDef") + assert rsre_core.match(r, "abXcdef") + assert not rsre_core.match(r, "ab\ncdef") + assert not rsre_core.match(r, "abXcDef") def test_any_repetition(self): r = get_code(r"ab.*cd") - assert rsre.match(r, "abXXXXcdef") - assert rsre.match(r, "abcdef") - assert not rsre.match(r, "abX\nXcdef") - assert not rsre.match(r, "abXXXXcDef") + assert rsre_core.match(r, "abXXXXcdef") + assert rsre_core.match(r, "abcdef") + assert not rsre_core.match(r, "abX\nXcdef") + assert not rsre_core.match(r, "abXXXXcDef") def test_any_all(self): r = get_code(r"(?s)ab.cd") - assert rsre.match(r, "abXcdef") - assert rsre.match(r, "ab\ncdef") - assert not rsre.match(r, "ab\ncDef") + assert rsre_core.match(r, "abXcdef") + assert rsre_core.match(r, "ab\ncdef") + assert not rsre_core.match(r, "ab\ncDef") def test_any_all_repetition(self): r = get_code(r"(?s)ab.*cd") - assert rsre.match(r, "abXXXXcdef") - assert rsre.match(r, "abcdef") - assert rsre.match(r, "abX\nXcdef") - assert not rsre.match(r, "abX\nXcDef") + assert rsre_core.match(r, "abXXXXcdef") + assert rsre_core.match(r, "abcdef") + assert rsre_core.match(r, "abX\nXcdef") + assert not rsre_core.match(r, "abX\nXcDef") def test_assert(self): r = get_code(r"abc(?=def)(.)") - res = rsre.match(r, "abcdefghi") + res = rsre_core.match(r, "abcdefghi") assert res is not None and res.get_mark(1) == 4 - assert not rsre.match(r, "abcdeFghi") + assert not rsre_core.match(r, "abcdeFghi") def test_assert_not(self): r = get_code(r"abc(?!def)(.)") - res = rsre.match(r, "abcdeFghi") + res = rsre_core.match(r, "abcdeFghi") assert res is not None and res.get_mark(1) == 4 - assert not rsre.match(r, "abcdefghi") + assert not rsre_core.match(r, "abcdefghi") def test_lookbehind(self): r = get_code(r"([a-z]*)(?<=de)") - assert rsre.match(r, "ade") - res = rsre.match(r, "adefg") + assert rsre_core.match(r, "ade") + res = rsre_core.match(r, "adefg") assert res is not None and res.get_mark(1) == 3 - assert not rsre.match(r, "abc") - assert not rsre.match(r, "X") - assert not rsre.match(r, "eX") + assert not rsre_core.match(r, "abc") + assert not rsre_core.match(r, "X") + assert not rsre_core.match(r, "eX") def test_negative_lookbehind(self): def found(s): - res = rsre.match(r, s) + res = rsre_core.match(r, s) assert res is not None return res.get_mark(1) r = get_code(r"([a-z]*)(?\s*(.*?)') - res = rsre.match(r_code1, " abcdef") + res = rsre_core.match(r_code1, " abcdef") assert res is not None #groups = state.create_regs(1) #assert groups[0] == (3, 29) @@ -14,13 +14,13 @@ def test_max_until_0_65535(self): r_code2 = get_code(r'(?:xy)*xy') - #res = rsre.match(r_code2, 'def') + #res = rsre_core.match(r_code2, 'def') #assert res is None - #res = rsre.match(r_code2, 'xydef') + #res = rsre_core.match(r_code2, 'xydef') #assert res is not None - res = rsre.match(r_code2, 'xyxyxydef') + res = rsre_core.match(r_code2, 'xyxyxydef') assert res is not None - res = rsre.match(r_code2, '' + 'xy'*1000 + 'def') + res = rsre_core.match(r_code2, '' + 'xy'*1000 + 'def') assert res is not None def test_max_until_3_5(self): @@ -28,18 +28,18 @@ for i in range(8): s = '' + 'xy'*i + 'defdefdefdefdef' assert (r.match(s) is not None) is (3 <= i-1 <= 5) - res = rsre.match(r_code2, s) + res = rsre_core.match(r_code2, s) assert (res is not None) is (3 <= i-1 <= 5) def test_min_until_0_65535(self): r_code2 = get_code(r'(?:xy)*?xy') - res = rsre.match(r_code2, 'def') + res = rsre_core.match(r_code2, 'def') assert res is None - res = rsre.match(r_code2, 'xydef') + res = rsre_core.match(r_code2, 'xydef') assert res is not None - res = rsre.match(r_code2, 'xyxyxydef') + res = rsre_core.match(r_code2, 'xyxyxydef') assert res is not None - res = rsre.match(r_code2, '' + 'xy'*1000 + 'def') + res = rsre_core.match(r_code2, '' + 'xy'*1000 + 'def') assert res is not None def test_min_until_3_5(self): @@ -47,44 +47,44 @@ for i in range(8): s = '' + 'xy'*i + 'defdefdefdefdef' assert (r.match(s) is not None) is (3 <= i-1 <= 5) - res = rsre.match(r_code2, s) + res = rsre_core.match(r_code2, s) assert (res is not None) is (3 <= i-1 <= 5) def test_min_repeat_one(self): r_code3 = get_code(r'.{3,5}?y') for i in range(8): - res = rsre.match(r_code3, '' + 'x'*i + 'y') + res = rsre_core.match(r_code3, '' + 'x'*i + 'y') assert (res is not None) is (3 <= i <= 5) def test_simple_group(self): r_code4 = get_code(r'(x.)') - res = rsre.match(r_code4, 'xadef') + res = rsre_core.match(r_code4, 'xadef') assert res is not None assert res.get_mark(0) == 5 assert res.get_mark(1) == 7 def test_max_until_groups(self): r_code4 = get_code(r'(x.)*xy') - res = rsre.match(r_code4, 'xaxbxydef') + res = rsre_core.match(r_code4, 'xaxbxydef') assert res is not None assert res.get_mark(0) == 7 assert res.get_mark(1) == 9 def test_group_branch(self): r_code5 = get_code(r'(ab|c)') - res = rsre.match(r_code5, 'abdef') + res = rsre_core.match(r_code5, 'abdef') assert (res.get_mark(0), res.get_mark(1)) == (5, 7) - res = rsre.match(r_code5, 'cdef') + res = rsre_core.match(r_code5, 'cdef') assert (res.get_mark(0), res.get_mark(1)) == (5, 6) - res = rsre.match(r_code5, 'dedef') + res = rsre_core.match(r_code5, 'dedef') assert res is None def test_group_branch_max_until(self): r_code6 = get_code(r'(ab|c)*a') - res = rsre.match(r_code6, 'ccabcccabadef') + res = rsre_core.match(r_code6, 'ccabcccabadef') assert (res.get_mark(0), res.get_mark(1)) == (12, 14) r_code7 = get_code(r'((ab)|(c))*a') - res = rsre.match(r_code7, 'ccabcccabadef') + res = rsre_core.match(r_code7, 'ccabcccabadef') assert (res.get_mark(0), res.get_mark(1)) == (12, 14) assert (res.get_mark(2), res.get_mark(3)) == (12, 14) assert (res.get_mark(4), res.get_mark(5)) == (11, 12) @@ -95,7 +95,7 @@ assert match.span(1) == (12, 13) assert match.span(3) == (12, 13) assert match.span(2) == (8, 9) - res = rsre.match(r_code7, 'bbbabbbb') + res = rsre_core.match(r_code7, 'bbbabbbb') assert (res.get_mark(0), res.get_mark(1)) == (12, 13) assert (res.get_mark(4), res.get_mark(5)) == (12, 13) assert (res.get_mark(2), res.get_mark(3)) == (8, 9) @@ -106,7 +106,7 @@ assert match.span(1) == (6, 7) assert match.span(3) == (6, 7) assert match.span(2) == (5, 6) - res = rsre.match(r_code8, 'ab') + res = rsre_core.match(r_code8, 'ab') assert (res.get_mark(0), res.get_mark(1)) == (6, 7) assert (res.get_mark(4), res.get_mark(5)) == (6, 7) assert (res.get_mark(2), res.get_mark(3)) == (5, 6) @@ -116,7 +116,7 @@ match = r9.match('xyzxc') assert match.span(1) == (3, 4) assert match.span(2) == (-1, -1) - res = rsre.match(r_code9, 'xyzxc') + res = rsre_core.match(r_code9, 'xyzxc') assert (res.get_mark(0), res.get_mark(1)) == (3, 4) assert (res.get_mark(2), res.get_mark(3)) == (-1, -1) @@ -125,6 +125,6 @@ match = r9.match('xycxyzxc') assert match.span(2) == (6, 7) #assert match.span(3) == (1, 2) --- bug of CPython - res = rsre.match(r_code9, 'xycxyzxc') + res = rsre_core.match(r_code9, 'xycxyzxc') assert (res.get_mark(2), res.get_mark(3)) == (6, 7) assert (res.get_mark(4), res.get_mark(5)) == (1, 2) Copied: pypy/branch/rsre2/pypy/rlib/rsre/test/test_re.py (from r76325, pypy/trunk/lib-python/modified-2.5.2/test/test_re.py) ============================================================================== --- pypy/trunk/lib-python/modified-2.5.2/test/test_re.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_re.py Fri Aug 6 15:33:37 2010 @@ -1,89 +1,71 @@ -import sys -sys.path = ['.'] + sys.path +import sys, os +from pypy.rlib.rsre.test.test_match import get_code +from pypy.rlib.rsre import rsre_re as re -from test.test_support import verbose, run_unittest -import re -from re import Scanner -import sys, os #, traceback -#from weakref import proxy - -# Misc tests from Tim Peters' re.doc - -# WARNING: Don't change details in these tests if you don't know -# what you're doing. Some of these tests were carefuly modeled to -# cover most of the code. - -import unittest - -class ReTests(unittest.TestCase): - - def DONOTtest_weakref(self): - s = 'QabbbcR' - x = re.compile('ab+c') - y = proxy(x) - self.assertEqual(x.findall('QabbbcR'), y.findall('QabbbcR')) + +class TestRe: def test_search_star_plus(self): - self.assertEqual(re.search('x*', 'axx').span(0), (0, 0)) - self.assertEqual(re.search('x*', 'axx').span(), (0, 0)) - self.assertEqual(re.search('x+', 'axx').span(0), (1, 3)) - self.assertEqual(re.search('x+', 'axx').span(), (1, 3)) - self.assertEqual(re.search('x', 'aaa'), None) - self.assertEqual(re.match('a*', 'xxx').span(0), (0, 0)) - self.assertEqual(re.match('a*', 'xxx').span(), (0, 0)) - self.assertEqual(re.match('x*', 'xxxa').span(0), (0, 3)) - self.assertEqual(re.match('x*', 'xxxa').span(), (0, 3)) - self.assertEqual(re.match('a+', 'xxx'), None) + assert re.search('x*', 'axx').span(0) == (0, 0) + assert re.search('x*', 'axx').span() == (0, 0) + assert re.search('x+', 'axx').span(0) == (1, 3) + assert re.search('x+', 'axx').span() == (1, 3) + assert re.search('x', 'aaa') == None + assert re.match('a*', 'xxx').span(0) == (0, 0) + assert re.match('a*', 'xxx').span() == (0, 0) + assert re.match('x*', 'xxxa').span(0) == (0, 3) + assert re.match('x*', 'xxxa').span() == (0, 3) + assert re.match('a+', 'xxx') == None def bump_num(self, matchobj): int_value = int(matchobj.group(0)) return str(int_value + 1) - def test_basic_re_sub(self): - self.assertEqual(re.sub("(?i)b+", "x", "bbbb BBBB"), 'x x') - self.assertEqual(re.sub(r'\d+', self.bump_num, '08.2 -2 23x99y'), + def XXXtest_basic_re_sub(self): + assert re.sub("(?i)b+", "x", "bbbb BBBB") == 'x x' + assert re.sub(r'\d+', self.bump_num, '08.2 -2 23x99y') == ( '9.3 -3 24x100y') - self.assertEqual(re.sub(r'\d+', self.bump_num, '08.2 -2 23x99y', 3), + assert re.sub(r'\d+', self.bump_num, '08.2 -2 23x99y', 3) == ( '9.3 -3 23x99y') - self.assertEqual(re.sub('.', lambda m: r"\n", 'x'), '\\n') - self.assertEqual(re.sub('.', r"\n", 'x'), '\n') + assert re.sub('.', lambda m: r"\n", 'x') == '\\n' + assert re.sub('.', r"\n", 'x') == '\n' s = r"\1\1" - self.assertEqual(re.sub('(.)', s, 'x'), 'xx') - self.assertEqual(re.sub('(.)', re.escape(s), 'x'), s) - self.assertEqual(re.sub('(.)', lambda m: s, 'x'), s) - - self.assertEqual(re.sub('(?Px)', '\g\g', 'xx'), 'xxxx') - self.assertEqual(re.sub('(?Px)', '\g\g<1>', 'xx'), 'xxxx') - self.assertEqual(re.sub('(?Px)', '\g\g', 'xx'), 'xxxx') - self.assertEqual(re.sub('(?Px)', '\g<1>\g<1>', 'xx'), 'xxxx') + assert re.sub('(.)', s, 'x') == 'xx' + assert re.sub('(.)', re.escape(s), 'x') == s + assert re.sub('(.)', lambda m: s, 'x') == s + + assert re.sub('(?Px)', '\g\g', 'xx') == 'xxxx' + assert re.sub('(?Px)', '\g\g<1>', 'xx') == 'xxxx' + assert re.sub('(?Px)', '\g\g', 'xx') == 'xxxx' + assert re.sub('(?Px)', '\g<1>\g<1>', 'xx') == 'xxxx' - self.assertEqual(re.sub('a',r'\t\n\v\r\f\a\b\B\Z\a\A\w\W\s\S\d\D','a'), + assert re.sub('a',r'\t\n\v\r\f\a\b\B\Z\a\A\w\W\s\S\d\D','a') == ( '\t\n\v\r\f\a\b\\B\\Z\a\\A\\w\\W\\s\\S\\d\\D') - self.assertEqual(re.sub('a', '\t\n\v\r\f\a', 'a'), '\t\n\v\r\f\a') - self.assertEqual(re.sub('a', '\t\n\v\r\f\a', 'a'), + assert re.sub('a', '\t\n\v\r\f\a', 'a') == '\t\n\v\r\f\a' + assert re.sub('a', '\t\n\v\r\f\a', 'a') == ( (chr(9)+chr(10)+chr(11)+chr(13)+chr(12)+chr(7))) - self.assertEqual(re.sub('^\s*', 'X', 'test'), 'Xtest') + assert re.sub('^\s*', 'X', 'test') == 'Xtest' - def test_bug_449964(self): + def XXXtest_bug_449964(self): # fails for group followed by other escape - self.assertEqual(re.sub(r'(?Px)', '\g<1>\g<1>\\b', 'xx'), + assert re.sub(r'(?Px)', '\g<1>\g<1>\\b', 'xx') == ( 'xx\bxx\b') - def test_bug_449000(self): + def XXXtest_bug_449000(self): # Test for sub() on escaped characters - self.assertEqual(re.sub(r'\r\n', r'\n', 'abc\r\ndef\r\n'), + assert re.sub(r'\r\n', r'\n', 'abc\r\ndef\r\n') == ( 'abc\ndef\n') - self.assertEqual(re.sub('\r\n', r'\n', 'abc\r\ndef\r\n'), + assert re.sub('\r\n', r'\n', 'abc\r\ndef\r\n') == ( 'abc\ndef\n') - self.assertEqual(re.sub(r'\r\n', '\n', 'abc\r\ndef\r\n'), + assert re.sub(r'\r\n', '\n', 'abc\r\ndef\r\n') == ( 'abc\ndef\n') - self.assertEqual(re.sub('\r\n', '\n', 'abc\r\ndef\r\n'), + assert re.sub('\r\n', '\n', 'abc\r\ndef\r\n') == ( 'abc\ndef\n') - def test_bug_1140(self): + def XXXtest_bug_1140(self): # re.sub(x, y, u'') should return u'', not '', and # re.sub(x, y, '') should return '', not u''. # Also: @@ -93,346 +75,345 @@ for x in 'x', u'x': for y in 'y', u'y': z = re.sub(x, y, u'') - self.assertEqual(z, u'') - self.assertEqual(type(z), unicode) + assert z == u'' + assert type(z) == unicode # z = re.sub(x, y, '') - self.assertEqual(z, '') - self.assertEqual(type(z), str) + assert z == '' + assert type(z) == str # z = re.sub(x, y, unicode(x)) - self.assertEqual(z, y) - self.assertEqual(type(z), unicode) + assert z == y + assert type(z) == unicode # z = re.sub(x, y, str(x)) - self.assertEqual(z, y) - self.assertEqual(type(z), type(y)) + assert z == y + assert type(z) == type(y) - def test_sub_template_numeric_escape(self): + def XXXtest_sub_template_numeric_escape(self): # bug 776311 and friends - self.assertEqual(re.sub('x', r'\0', 'x'), '\0') - self.assertEqual(re.sub('x', r'\000', 'x'), '\000') - self.assertEqual(re.sub('x', r'\001', 'x'), '\001') - self.assertEqual(re.sub('x', r'\008', 'x'), '\0' + '8') - self.assertEqual(re.sub('x', r'\009', 'x'), '\0' + '9') - self.assertEqual(re.sub('x', r'\111', 'x'), '\111') - self.assertEqual(re.sub('x', r'\117', 'x'), '\117') - - self.assertEqual(re.sub('x', r'\1111', 'x'), '\1111') - self.assertEqual(re.sub('x', r'\1111', 'x'), '\111' + '1') - - self.assertEqual(re.sub('x', r'\00', 'x'), '\x00') - self.assertEqual(re.sub('x', r'\07', 'x'), '\x07') - self.assertEqual(re.sub('x', r'\08', 'x'), '\0' + '8') - self.assertEqual(re.sub('x', r'\09', 'x'), '\0' + '9') - self.assertEqual(re.sub('x', r'\0a', 'x'), '\0' + 'a') - - self.assertEqual(re.sub('x', r'\400', 'x'), '\0') - self.assertEqual(re.sub('x', r'\777', 'x'), '\377') - - self.assertRaises(re.error, re.sub, 'x', r'\1', 'x') - self.assertRaises(re.error, re.sub, 'x', r'\8', 'x') - self.assertRaises(re.error, re.sub, 'x', r'\9', 'x') - self.assertRaises(re.error, re.sub, 'x', r'\11', 'x') - self.assertRaises(re.error, re.sub, 'x', r'\18', 'x') - self.assertRaises(re.error, re.sub, 'x', r'\1a', 'x') - self.assertRaises(re.error, re.sub, 'x', r'\90', 'x') - self.assertRaises(re.error, re.sub, 'x', r'\99', 'x') - self.assertRaises(re.error, re.sub, 'x', r'\118', 'x') # r'\11' + '8' - self.assertRaises(re.error, re.sub, 'x', r'\11a', 'x') - self.assertRaises(re.error, re.sub, 'x', r'\181', 'x') # r'\18' + '1' - self.assertRaises(re.error, re.sub, 'x', r'\800', 'x') # r'\80' + '0' + assert re.sub('x', r'\0', 'x') == '\0' + assert re.sub('x', r'\000', 'x') == '\000' + assert re.sub('x', r'\001', 'x') == '\001' + assert re.sub('x', r'\008', 'x') == '\0' + '8' + assert re.sub('x', r'\009', 'x') == '\0' + '9' + assert re.sub('x', r'\111', 'x') == '\111' + assert re.sub('x', r'\117', 'x') == '\117' + + assert re.sub('x', r'\1111', 'x') == '\1111' + assert re.sub('x', r'\1111', 'x') == '\111' + '1' + + assert re.sub('x', r'\00', 'x') == '\x00' + assert re.sub('x', r'\07', 'x') == '\x07' + assert re.sub('x', r'\08', 'x') == '\0' + '8' + assert re.sub('x', r'\09', 'x') == '\0' + '9' + assert re.sub('x', r'\0a', 'x') == '\0' + 'a' + + assert re.sub('x', r'\400', 'x') == '\0' + assert re.sub('x', r'\777', 'x') == '\377' + + raises(re.error, re.sub, 'x', r'\1', 'x') + raises(re.error, re.sub, 'x', r'\8', 'x') + raises(re.error, re.sub, 'x', r'\9', 'x') + raises(re.error, re.sub, 'x', r'\11', 'x') + raises(re.error, re.sub, 'x', r'\18', 'x') + raises(re.error, re.sub, 'x', r'\1a', 'x') + raises(re.error, re.sub, 'x', r'\90', 'x') + raises(re.error, re.sub, 'x', r'\99', 'x') + raises(re.error, re.sub, 'x', r'\118', 'x') # r'\11' + '8' + raises(re.error, re.sub, 'x', r'\11a', 'x') + raises(re.error, re.sub, 'x', r'\181', 'x') # r'\18' + '1' + raises(re.error, re.sub, 'x', r'\800', 'x') # r'\80' + '0' # in python2.3 (etc), these loop endlessly in sre_parser.py - self.assertEqual(re.sub('(((((((((((x)))))))))))', r'\11', 'x'), 'x') - self.assertEqual(re.sub('((((((((((y))))))))))(.)', r'\118', 'xyz'), + assert re.sub('(((((((((((x)))))))))))', r'\11', 'x') == 'x' + assert re.sub('((((((((((y))))))))))(.)', r'\118', 'xyz') == ( 'xz8') - self.assertEqual(re.sub('((((((((((y))))))))))(.)', r'\11a', 'xyz'), + assert re.sub('((((((((((y))))))))))(.)', r'\11a', 'xyz') == ( 'xza') - def test_qualified_re_sub(self): - self.assertEqual(re.sub('a', 'b', 'aaaaa'), 'bbbbb') - self.assertEqual(re.sub('a', 'b', 'aaaaa', 1), 'baaaa') + def XXXtest_qualified_re_sub(self): + assert re.sub('a', 'b', 'aaaaa') == 'bbbbb' + assert re.sub('a', 'b', 'aaaaa', 1) == 'baaaa' - def test_bug_114660(self): - self.assertEqual(re.sub(r'(\S)\s+(\S)', r'\1 \2', 'hello there'), + def XXXtest_bug_114660(self): + assert re.sub(r'(\S)\s+(\S)', r'\1 \2', 'hello there') == ( 'hello there') - def test_bug_462270(self): + def XXXtest_bug_462270(self): # Test for empty sub() behaviour, see SF bug #462270 - self.assertEqual(re.sub('x*', '-', 'abxd'), '-a-b-d-') - self.assertEqual(re.sub('x+', '-', 'abxd'), 'ab-d') + assert re.sub('x*', '-', 'abxd') == '-a-b-d-' + assert re.sub('x+', '-', 'abxd') == 'ab-d' - def test_symbolic_refs(self): - self.assertRaises(re.error, re.sub, '(?Px)', '\gx)', '\g<', 'xx') - self.assertRaises(re.error, re.sub, '(?Px)', '\g', 'xx') - self.assertRaises(re.error, re.sub, '(?Px)', '\g', 'xx') - self.assertRaises(re.error, re.sub, '(?Px)', '\g<1a1>', 'xx') - self.assertRaises(IndexError, re.sub, '(?Px)', '\g', 'xx') - self.assertRaises(re.error, re.sub, '(?Px)|(?Py)', '\g', 'xx') - self.assertRaises(re.error, re.sub, '(?Px)|(?Py)', '\\2', 'xx') - self.assertRaises(re.error, re.sub, '(?Px)', '\g<-1>', 'xx') - - def test_re_subn(self): - self.assertEqual(re.subn("(?i)b+", "x", "bbbb BBBB"), ('x x', 2)) - self.assertEqual(re.subn("b+", "x", "bbbb BBBB"), ('x BBBB', 1)) - self.assertEqual(re.subn("b+", "x", "xyz"), ('xyz', 0)) - self.assertEqual(re.subn("b*", "x", "xyz"), ('xxxyxzx', 4)) - self.assertEqual(re.subn("b*", "x", "xyz", 2), ('xxxyz', 2)) - - def test_re_split(self): - self.assertEqual(re.split(":", ":a:b::c"), ['', 'a', 'b', '', 'c']) - self.assertEqual(re.split(":*", ":a:b::c"), ['', 'a', 'b', 'c']) - self.assertEqual(re.split("(:*)", ":a:b::c"), + def XXXtest_symbolic_refs(self): + raises(re.error, re.sub, '(?Px)', '\gx)', '\g<', 'xx') + raises(re.error, re.sub, '(?Px)', '\g', 'xx') + raises(re.error, re.sub, '(?Px)', '\g', 'xx') + raises(re.error, re.sub, '(?Px)', '\g<1a1>', 'xx') + raises(IndexError, re.sub, '(?Px)', '\g', 'xx') + raises(re.error, re.sub, '(?Px)|(?Py)', '\g', 'xx') + raises(re.error, re.sub, '(?Px)|(?Py)', '\\2', 'xx') + raises(re.error, re.sub, '(?Px)', '\g<-1>', 'xx') + + def XXXtest_re_subn(self): + assert re.subn("(?i)b+", "x", "bbbb BBBB") == ('x x', 2) + assert re.subn("b+", "x", "bbbb BBBB") == ('x BBBB', 1) + assert re.subn("b+", "x", "xyz") == ('xyz', 0) + assert re.subn("b*", "x", "xyz") == ('xxxyxzx', 4) + assert re.subn("b*", "x", "xyz", 2) == ('xxxyz', 2) + + def XXXtest_re_split(self): + assert re.split(":", ":a:b::c") == ['', 'a', 'b', '', 'c'] + assert re.split(":*", ":a:b::c") == ['', 'a', 'b', 'c'] + assert re.split("(:*)", ":a:b::c") == ( ['', ':', 'a', ':', 'b', '::', 'c']) - self.assertEqual(re.split("(?::*)", ":a:b::c"), ['', 'a', 'b', 'c']) - self.assertEqual(re.split("(:)*", ":a:b::c"), + assert re.split("(?::*)", ":a:b::c") == ['', 'a', 'b', 'c'] + assert re.split("(:)*", ":a:b::c") == ( ['', ':', 'a', ':', 'b', ':', 'c']) - self.assertEqual(re.split("([b:]+)", ":a:b::c"), + assert re.split("([b:]+)", ":a:b::c") == ( ['', ':', 'a', ':b::', 'c']) - self.assertEqual(re.split("(b)|(:+)", ":a:b::c"), + assert re.split("(b)|(:+)", ":a:b::c") == ( ['', None, ':', 'a', None, ':', '', 'b', None, '', None, '::', 'c']) - self.assertEqual(re.split("(?:b)|(?::+)", ":a:b::c"), + assert re.split("(?:b)|(?::+)", ":a:b::c") == ( ['', 'a', '', '', 'c']) - def test_qualified_re_split(self): - self.assertEqual(re.split(":", ":a:b::c", 2), ['', 'a', 'b::c']) - self.assertEqual(re.split(':', 'a:b:c:d', 2), ['a', 'b', 'c:d']) - self.assertEqual(re.split("(:)", ":a:b::c", 2), + def XXXtest_qualified_re_split(self): + assert re.split(":", ":a:b::c", 2) == ['', 'a', 'b::c'] + assert re.split(':', 'a:b:c:d', 2) == ['a', 'b', 'c:d'] + assert re.split("(:)", ":a:b::c", 2) == ( ['', ':', 'a', ':', 'b::c']) - self.assertEqual(re.split("(:*)", ":a:b::c", 2), + assert re.split("(:*)", ":a:b::c", 2) == ( ['', ':', 'a', ':', 'b::c']) def test_re_findall(self): - self.assertEqual(re.findall(":+", "abc"), []) - self.assertEqual(re.findall(":+", "a:b::c:::d"), [":", "::", ":::"]) - self.assertEqual(re.findall("(:+)", "a:b::c:::d"), [":", "::", ":::"]) - self.assertEqual(re.findall("(:)(:*)", "a:b::c:::d"), [(":", ""), + assert re.findall(":+", "abc") == [] + assert re.findall(":+", "a:b::c:::d") == [":", "::", ":::"] + assert re.findall("(:+)", "a:b::c:::d") == [":", "::", ":::"] + assert re.findall("(:)(:*)", "a:b::c:::d") == [(":", ""), (":", ":"), - (":", "::")]) + (":", "::")] def test_bug_117612(self): - self.assertEqual(re.findall(r"(a|(b))", "aba"), + assert re.findall(r"(a|(b))", "aba") == ( [("a", ""),("b", "b"),("a", "")]) def test_re_match(self): - self.assertEqual(re.match('a', 'a').groups(), ()) - self.assertEqual(re.match('(a)', 'a').groups(), ('a',)) - self.assertEqual(re.match(r'(a)', 'a').group(0), 'a') - self.assertEqual(re.match(r'(a)', 'a').group(1), 'a') - self.assertEqual(re.match(r'(a)', 'a').group(1, 1), ('a', 'a')) + assert re.match('a', 'a').groups() == () + assert re.match('(a)', 'a').groups() == ('a',) + assert re.match(r'(a)', 'a').group(0) == 'a' + assert re.match(r'(a)', 'a').group(1) == 'a' + assert re.match(r'(a)', 'a').group(1, 1) == ('a', 'a') pat = re.compile('((a)|(b))(c)?') - self.assertEqual(pat.match('a').groups(), ('a', 'a', None, None)) - self.assertEqual(pat.match('b').groups(), ('b', None, 'b', None)) - self.assertEqual(pat.match('ac').groups(), ('a', 'a', None, 'c')) - self.assertEqual(pat.match('bc').groups(), ('b', None, 'b', 'c')) - self.assertEqual(pat.match('bc').groups(""), ('b', "", 'b', 'c')) + assert pat.match('a').groups() == ('a', 'a', None, None) + assert pat.match('b').groups() == ('b', None, 'b', None) + assert pat.match('ac').groups() == ('a', 'a', None, 'c') + assert pat.match('bc').groups() == ('b', None, 'b', 'c') + assert pat.match('bc').groups("") == ('b', "", 'b', 'c') # A single group m = re.match('(a)', 'a') - self.assertEqual(m.group(0), 'a') - self.assertEqual(m.group(0), 'a') - self.assertEqual(m.group(1), 'a') - self.assertEqual(m.group(1, 1), ('a', 'a')) + assert m.group(0) == 'a' + assert m.group(0) == 'a' + assert m.group(1) == 'a' + assert m.group(1, 1) == ('a', 'a') pat = re.compile('(?:(?Pa)|(?Pb))(?Pc)?') - self.assertEqual(pat.match('a').group(1, 2, 3), ('a', None, None)) - self.assertEqual(pat.match('b').group('a1', 'b2', 'c3'), + assert pat.match('a').group(1, 2, 3) == ('a', None, None) + assert pat.match('b').group('a1', 'b2', 'c3') == ( (None, 'b', None)) - self.assertEqual(pat.match('ac').group(1, 'b2', 3), ('a', None, 'c')) + assert pat.match('ac').group(1, 'b2', 3) == ('a', None, 'c') def test_re_groupref_exists(self): - self.assertEqual(re.match('^(\()?([^()]+)(?(1)\))$', '(a)').groups(), + assert re.match('^(\()?([^()]+)(?(1)\))$', '(a)').groups() == ( ('(', 'a')) - self.assertEqual(re.match('^(\()?([^()]+)(?(1)\))$', 'a').groups(), + assert re.match('^(\()?([^()]+)(?(1)\))$', 'a').groups() == ( (None, 'a')) - self.assertEqual(re.match('^(\()?([^()]+)(?(1)\))$', 'a)'), None) - self.assertEqual(re.match('^(\()?([^()]+)(?(1)\))$', '(a'), None) - self.assertEqual(re.match('^(?:(a)|c)((?(1)b|d))$', 'ab').groups(), + assert re.match('^(\()?([^()]+)(?(1)\))$', 'a)') == None + assert re.match('^(\()?([^()]+)(?(1)\))$', '(a') == None + assert re.match('^(?:(a)|c)((?(1)b|d))$', 'ab').groups() == ( ('a', 'b')) - self.assertEqual(re.match('^(?:(a)|c)((?(1)b|d))$', 'cd').groups(), + assert re.match('^(?:(a)|c)((?(1)b|d))$', 'cd').groups() == ( (None, 'd')) - self.assertEqual(re.match('^(?:(a)|c)((?(1)|d))$', 'cd').groups(), + assert re.match('^(?:(a)|c)((?(1)|d))$', 'cd').groups() == ( (None, 'd')) - self.assertEqual(re.match('^(?:(a)|c)((?(1)|d))$', 'a').groups(), + assert re.match('^(?:(a)|c)((?(1)|d))$', 'a').groups() == ( ('a', '')) # Tests for bug #1177831: exercise groups other than the first group p = re.compile('(?Pa)(?Pb)?((?(g2)c|d))') - self.assertEqual(p.match('abc').groups(), + assert p.match('abc').groups() == ( ('a', 'b', 'c')) - self.assertEqual(p.match('ad').groups(), + assert p.match('ad').groups() == ( ('a', None, 'd')) - self.assertEqual(p.match('abd'), None) - self.assertEqual(p.match('ac'), None) + assert p.match('abd') == None + assert p.match('ac') == None def test_re_groupref(self): - self.assertEqual(re.match(r'^(\|)?([^()]+)\1$', '|a|').groups(), + assert re.match(r'^(\|)?([^()]+)\1$', '|a|').groups() == ( ('|', 'a')) - self.assertEqual(re.match(r'^(\|)?([^()]+)\1?$', 'a').groups(), + assert re.match(r'^(\|)?([^()]+)\1?$', 'a').groups() == ( (None, 'a')) - self.assertEqual(re.match(r'^(\|)?([^()]+)\1$', 'a|'), None) - self.assertEqual(re.match(r'^(\|)?([^()]+)\1$', '|a'), None) - self.assertEqual(re.match(r'^(?:(a)|c)(\1)$', 'aa').groups(), + assert re.match(r'^(\|)?([^()]+)\1$', 'a|') == None + assert re.match(r'^(\|)?([^()]+)\1$', '|a') == None + assert re.match(r'^(?:(a)|c)(\1)$', 'aa').groups() == ( ('a', 'a')) - self.assertEqual(re.match(r'^(?:(a)|c)(\1)?$', 'c').groups(), + assert re.match(r'^(?:(a)|c)(\1)?$', 'c').groups() == ( (None, None)) def test_groupdict(self): - self.assertEqual(re.match('(?Pfirst) (?Psecond)', - 'first second').groupdict(), + assert re.match('(?Pfirst) (?Psecond)', + 'first second').groupdict() == ( {'first':'first', 'second':'second'}) def test_expand(self): - self.assertEqual(re.match("(?Pfirst) (?Psecond)", + assert (re.match("(?Pfirst) (?Psecond)", "first second") - .expand(r"\2 \1 \g \g"), + .expand(r"\2 \1 \g \g")) == ( "second first second first") def test_repeat_minmax(self): - self.assertEqual(re.match("^(\w){1}$", "abc"), None) - self.assertEqual(re.match("^(\w){1}?$", "abc"), None) - self.assertEqual(re.match("^(\w){1,2}$", "abc"), None) - self.assertEqual(re.match("^(\w){1,2}?$", "abc"), None) - - self.assertEqual(re.match("^(\w){3}$", "abc").group(1), "c") - self.assertEqual(re.match("^(\w){1,3}$", "abc").group(1), "c") - self.assertEqual(re.match("^(\w){1,4}$", "abc").group(1), "c") - self.assertEqual(re.match("^(\w){3,4}?$", "abc").group(1), "c") - self.assertEqual(re.match("^(\w){3}?$", "abc").group(1), "c") - self.assertEqual(re.match("^(\w){1,3}?$", "abc").group(1), "c") - self.assertEqual(re.match("^(\w){1,4}?$", "abc").group(1), "c") - self.assertEqual(re.match("^(\w){3,4}?$", "abc").group(1), "c") - - self.assertEqual(re.match("^x{1}$", "xxx"), None) - self.assertEqual(re.match("^x{1}?$", "xxx"), None) - self.assertEqual(re.match("^x{1,2}$", "xxx"), None) - self.assertEqual(re.match("^x{1,2}?$", "xxx"), None) - - self.assertNotEqual(re.match("^x{3}$", "xxx"), None) - self.assertNotEqual(re.match("^x{1,3}$", "xxx"), None) - self.assertNotEqual(re.match("^x{1,4}$", "xxx"), None) - self.assertNotEqual(re.match("^x{3,4}?$", "xxx"), None) - self.assertNotEqual(re.match("^x{3}?$", "xxx"), None) - self.assertNotEqual(re.match("^x{1,3}?$", "xxx"), None) - self.assertNotEqual(re.match("^x{1,4}?$", "xxx"), None) - self.assertNotEqual(re.match("^x{3,4}?$", "xxx"), None) + assert re.match("^(\w){1}$", "abc") == None + assert re.match("^(\w){1}?$", "abc") == None + assert re.match("^(\w){1,2}$", "abc") == None + assert re.match("^(\w){1,2}?$", "abc") == None + + assert re.match("^(\w){3}$", "abc").group(1) == "c" + assert re.match("^(\w){1,3}$", "abc").group(1) == "c" + assert re.match("^(\w){1,4}$", "abc").group(1) == "c" + assert re.match("^(\w){3,4}?$", "abc").group(1) == "c" + assert re.match("^(\w){3}?$", "abc").group(1) == "c" + assert re.match("^(\w){1,3}?$", "abc").group(1) == "c" + assert re.match("^(\w){1,4}?$", "abc").group(1) == "c" + assert re.match("^(\w){3,4}?$", "abc").group(1) == "c" + + assert re.match("^x{1}$", "xxx") == None + assert re.match("^x{1}?$", "xxx") == None + assert re.match("^x{1,2}$", "xxx") == None + assert re.match("^x{1,2}?$", "xxx") == None + + assert re.match("^x{3}$", "xxx") != None + assert re.match("^x{1,3}$", "xxx") != None + assert re.match("^x{1,4}$", "xxx") != None + assert re.match("^x{3,4}?$", "xxx") != None + assert re.match("^x{3}?$", "xxx") != None + assert re.match("^x{1,3}?$", "xxx") != None + assert re.match("^x{1,4}?$", "xxx") != None + assert re.match("^x{3,4}?$", "xxx") != None - self.assertEqual(re.match("^x{}$", "xxx"), None) - self.assertNotEqual(re.match("^x{}$", "x{}"), None) + assert re.match("^x{}$", "xxx") == None + assert re.match("^x{}$", "x{}") != None def test_getattr(self): - self.assertEqual(re.match("(a)", "a").pos, 0) - self.assertEqual(re.match("(a)", "a").endpos, 1) - self.assertEqual(re.match("(a)", "a").string, "a") - self.assertEqual(re.match("(a)", "a").regs, ((0, 1), (0, 1))) - self.assertNotEqual(re.match("(a)", "a").re, None) + assert re.match("(a)", "a").pos == 0 + assert re.match("(a)", "a").endpos == 1 + assert re.match("(a)", "a").string == "a" + assert re.match("(a)", "a").regs == ((0, 1), (0, 1)) + assert re.match("(a)", "a").re != None def test_special_escapes(self): - self.assertEqual(re.search(r"\b(b.)\b", - "abcd abc bcd bx").group(1), "bx") - self.assertEqual(re.search(r"\B(b.)\B", - "abc bcd bc abxd").group(1), "bx") - self.assertEqual(re.search(r"\b(b.)\b", - "abcd abc bcd bx", re.LOCALE).group(1), "bx") - self.assertEqual(re.search(r"\B(b.)\B", - "abc bcd bc abxd", re.LOCALE).group(1), "bx") - self.assertEqual(re.search(r"\b(b.)\b", - "abcd abc bcd bx", re.UNICODE).group(1), "bx") - self.assertEqual(re.search(r"\B(b.)\B", - "abc bcd bc abxd", re.UNICODE).group(1), "bx") - self.assertEqual(re.search(r"^abc$", "\nabc\n", re.M).group(0), "abc") - self.assertEqual(re.search(r"^\Aabc\Z$", "abc", re.M).group(0), "abc") - self.assertEqual(re.search(r"^\Aabc\Z$", "\nabc\n", re.M), None) - self.assertEqual(re.search(r"\b(b.)\b", - u"abcd abc bcd bx").group(1), "bx") - self.assertEqual(re.search(r"\B(b.)\B", - u"abc bcd bc abxd").group(1), "bx") - self.assertEqual(re.search(r"^abc$", u"\nabc\n", re.M).group(0), "abc") - self.assertEqual(re.search(r"^\Aabc\Z$", u"abc", re.M).group(0), "abc") - self.assertEqual(re.search(r"^\Aabc\Z$", u"\nabc\n", re.M), None) - self.assertEqual(re.search(r"\d\D\w\W\s\S", - "1aa! a").group(0), "1aa! a") - self.assertEqual(re.search(r"\d\D\w\W\s\S", - "1aa! a", re.LOCALE).group(0), "1aa! a") - self.assertEqual(re.search(r"\d\D\w\W\s\S", - "1aa! a", re.UNICODE).group(0), "1aa! a") + assert re.search(r"\b(b.)\b", + "abcd abc bcd bx").group(1) == "bx" + assert re.search(r"\B(b.)\B", + "abc bcd bc abxd").group(1) == "bx" + assert re.search(r"\b(b.)\b", + "abcd abc bcd bx", re.LOCALE).group(1) == "bx" + assert re.search(r"\B(b.)\B", + "abc bcd bc abxd", re.LOCALE).group(1) == "bx" + assert re.search(r"\b(b.)\b", + "abcd abc bcd bx", re.UNICODE).group(1) == "bx" + assert re.search(r"\B(b.)\B", + "abc bcd bc abxd", re.UNICODE).group(1) == "bx" + assert re.search(r"^abc$", "\nabc\n", re.M).group(0) == "abc" + assert re.search(r"^\Aabc\Z$", "abc", re.M).group(0) == "abc" + assert re.search(r"^\Aabc\Z$", "\nabc\n", re.M) == None + assert re.search(r"\b(b.)\b", + u"abcd abc bcd bx").group(1) == "bx" + assert re.search(r"\B(b.)\B", + u"abc bcd bc abxd").group(1) == "bx" + assert re.search(r"^abc$", u"\nabc\n", re.M).group(0) == "abc" + assert re.search(r"^\Aabc\Z$", u"abc", re.M).group(0) == "abc" + assert re.search(r"^\Aabc\Z$", u"\nabc\n", re.M) == None + assert re.search(r"\d\D\w\W\s\S", + "1aa! a").group(0) == "1aa! a" + assert re.search(r"\d\D\w\W\s\S", + "1aa! a", re.LOCALE).group(0) == "1aa! a" + assert re.search(r"\d\D\w\W\s\S", + "1aa! a", re.UNICODE).group(0) == "1aa! a" def test_ignore_case(self): - self.assertEqual(re.match("abc", "ABC", re.I).group(0), "ABC") - self.assertEqual(re.match("abc", u"ABC", re.I).group(0), "ABC") + assert re.match("abc", "ABC", re.I).group(0) == "ABC" + assert re.match("abc", u"ABC", re.I).group(0) == "ABC" def test_bigcharset(self): - self.assertEqual(re.match(u"([\u2222\u2223])", - u"\u2222").group(1), u"\u2222") - self.assertEqual(re.match(u"([\u2222\u2223])", - u"\u2222", re.UNICODE).group(1), u"\u2222") + assert re.match(u"([\u2222\u2223])", + u"\u2222").group(1) == u"\u2222" + assert re.match(u"([\u2222\u2223])", + u"\u2222", re.UNICODE).group(1) == u"\u2222" def test_anyall(self): - self.assertEqual(re.match("a.b", "a\nb", re.DOTALL).group(0), + assert re.match("a.b", "a\nb", re.DOTALL).group(0) == ( "a\nb") - self.assertEqual(re.match("a.*b", "a\n\nb", re.DOTALL).group(0), + assert re.match("a.*b", "a\n\nb", re.DOTALL).group(0) == ( "a\n\nb") def test_non_consuming(self): - self.assertEqual(re.match("(a(?=\s[^a]))", "a b").group(1), "a") - self.assertEqual(re.match("(a(?=\s[^a]*))", "a b").group(1), "a") - self.assertEqual(re.match("(a(?=\s[abc]))", "a b").group(1), "a") - self.assertEqual(re.match("(a(?=\s[abc]*))", "a bc").group(1), "a") - self.assertEqual(re.match(r"(a)(?=\s\1)", "a a").group(1), "a") - self.assertEqual(re.match(r"(a)(?=\s\1*)", "a aa").group(1), "a") - self.assertEqual(re.match(r"(a)(?=\s(abc|a))", "a a").group(1), "a") - - self.assertEqual(re.match(r"(a(?!\s[^a]))", "a a").group(1), "a") - self.assertEqual(re.match(r"(a(?!\s[abc]))", "a d").group(1), "a") - self.assertEqual(re.match(r"(a)(?!\s\1)", "a b").group(1), "a") - self.assertEqual(re.match(r"(a)(?!\s(abc|a))", "a b").group(1), "a") + assert re.match("(a(?=\s[^a]))", "a b").group(1) == "a" + assert re.match("(a(?=\s[^a]*))", "a b").group(1) == "a" + assert re.match("(a(?=\s[abc]))", "a b").group(1) == "a" + assert re.match("(a(?=\s[abc]*))", "a bc").group(1) == "a" + assert re.match(r"(a)(?=\s\1)", "a a").group(1) == "a" + assert re.match(r"(a)(?=\s\1*)", "a aa").group(1) == "a" + assert re.match(r"(a)(?=\s(abc|a))", "a a").group(1) == "a" + + assert re.match(r"(a(?!\s[^a]))", "a a").group(1) == "a" + assert re.match(r"(a(?!\s[abc]))", "a d").group(1) == "a" + assert re.match(r"(a)(?!\s\1)", "a b").group(1) == "a" + assert re.match(r"(a)(?!\s(abc|a))", "a b").group(1) == "a" def test_ignore_case(self): - self.assertEqual(re.match(r"(a\s[^a])", "a b", re.I).group(1), "a b") - self.assertEqual(re.match(r"(a\s[^a]*)", "a bb", re.I).group(1), "a bb") - self.assertEqual(re.match(r"(a\s[abc])", "a b", re.I).group(1), "a b") - self.assertEqual(re.match(r"(a\s[abc]*)", "a bb", re.I).group(1), "a bb") - self.assertEqual(re.match(r"((a)\s\2)", "a a", re.I).group(1), "a a") - self.assertEqual(re.match(r"((a)\s\2*)", "a aa", re.I).group(1), "a aa") - self.assertEqual(re.match(r"((a)\s(abc|a))", "a a", re.I).group(1), "a a") - self.assertEqual(re.match(r"((a)\s(abc|a)*)", "a aa", re.I).group(1), "a aa") + assert re.match(r"(a\s[^a])", "a b", re.I).group(1) == "a b" + assert re.match(r"(a\s[^a]*)", "a bb", re.I).group(1) == "a bb" + assert re.match(r"(a\s[abc])", "a b", re.I).group(1) == "a b" + assert re.match(r"(a\s[abc]*)", "a bb", re.I).group(1) == "a bb" + assert re.match(r"((a)\s\2)", "a a", re.I).group(1) == "a a" + assert re.match(r"((a)\s\2*)", "a aa", re.I).group(1) == "a aa" + assert re.match(r"((a)\s(abc|a))", "a a", re.I).group(1) == "a a" + assert re.match(r"((a)\s(abc|a)*)", "a aa", re.I).group(1) == "a aa" def test_category(self): - self.assertEqual(re.match(r"(\s)", " ").group(1), " ") + assert re.match(r"(\s)", " ").group(1) == " " def test_getlower(self): import _sre - self.assertEqual(_sre.getlower(ord('A'), 0), ord('a')) - self.assertEqual(_sre.getlower(ord('A'), re.LOCALE), ord('a')) - self.assertEqual(_sre.getlower(ord('A'), re.UNICODE), ord('a')) + assert _sre.getlower(ord('A'), 0) == ord('a') + assert _sre.getlower(ord('A'), re.LOCALE) == ord('a') + assert _sre.getlower(ord('A'), re.UNICODE) == ord('a') - self.assertEqual(re.match("abc", "ABC", re.I).group(0), "ABC") - self.assertEqual(re.match("abc", u"ABC", re.I).group(0), "ABC") + assert re.match("abc", "ABC", re.I).group(0) == "ABC" + assert re.match("abc", u"ABC", re.I).group(0) == "ABC" def test_not_literal(self): - self.assertEqual(re.search("\s([^a])", " b").group(1), "b") - self.assertEqual(re.search("\s([^a]*)", " bb").group(1), "bb") + assert re.search("\s([^a])", " b").group(1) == "b" + assert re.search("\s([^a]*)", " bb").group(1) == "bb" def test_search_coverage(self): - self.assertEqual(re.search("\s(b)", " b").group(1), "b") - self.assertEqual(re.search("a\s", "a ").group(0), "a ") + assert re.search("\s(b)", " b").group(1) == "b" + assert re.search("a\s", "a ").group(0) == "a " def test_re_escape(self): p="" for i in range(0, 256): p = p + chr(i) - self.assertEqual(re.match(re.escape(chr(i)), chr(i)) is not None, - True) - self.assertEqual(re.match(re.escape(chr(i)), chr(i)).span(), (0,1)) + assert re.match(re.escape(chr(i)), chr(i)) is not None + assert re.match(re.escape(chr(i)), chr(i)).span() == (0,1) pat=re.compile(re.escape(p)) - self.assertEqual(pat.match(p) is not None, True) - self.assertEqual(pat.match(p).span(), (0,256)) + assert pat.match(p) is not None + assert pat.match(p).span() == (0,256) def test_pickling(self): import pickle @@ -457,85 +438,81 @@ # not seem to preserve that in all cases (observed on an UCS-4 build # of 2.4.1). #self.assertEqual(oldpat, newpat) - self.assertEqual(oldpat.__dict__, newpat.__dict__) + assert oldpat.__dict__ == newpat.__dict__ def test_constants(self): - self.assertEqual(re.I, re.IGNORECASE) - self.assertEqual(re.L, re.LOCALE) - self.assertEqual(re.M, re.MULTILINE) - self.assertEqual(re.S, re.DOTALL) - self.assertEqual(re.X, re.VERBOSE) + assert re.I == re.IGNORECASE + assert re.L == re.LOCALE + assert re.M == re.MULTILINE + assert re.S == re.DOTALL + assert re.X == re.VERBOSE def test_flags(self): for flag in [re.I, re.M, re.X, re.S, re.L]: - self.assertNotEqual(re.compile('^pattern$', flag), None) + assert re.compile('^pattern$', flag) != None def test_sre_character_literals(self): for i in [0, 8, 16, 32, 64, 127, 128, 255]: - self.assertNotEqual(re.match(r"\%03o" % i, chr(i)), None) - self.assertNotEqual(re.match(r"\%03o0" % i, chr(i)+"0"), None) - self.assertNotEqual(re.match(r"\%03o8" % i, chr(i)+"8"), None) - self.assertNotEqual(re.match(r"\x%02x" % i, chr(i)), None) - self.assertNotEqual(re.match(r"\x%02x0" % i, chr(i)+"0"), None) - self.assertNotEqual(re.match(r"\x%02xz" % i, chr(i)+"z"), None) - self.assertRaises(re.error, re.match, "\911", "") + assert re.match(r"\%03o" % i, chr(i)) != None + assert re.match(r"\%03o0" % i, chr(i)+"0") != None + assert re.match(r"\%03o8" % i, chr(i)+"8") != None + assert re.match(r"\x%02x" % i, chr(i)) != None + assert re.match(r"\x%02x0" % i, chr(i)+"0") != None + assert re.match(r"\x%02xz" % i, chr(i)+"z") != None + raises(re.error, re.match, "\911", "") def test_sre_character_class_literals(self): for i in [0, 8, 16, 32, 64, 127, 128, 255]: - self.assertNotEqual(re.match(r"[\%03o]" % i, chr(i)), None) - self.assertNotEqual(re.match(r"[\%03o0]" % i, chr(i)), None) - self.assertNotEqual(re.match(r"[\%03o8]" % i, chr(i)), None) - self.assertNotEqual(re.match(r"[\x%02x]" % i, chr(i)), None) - self.assertNotEqual(re.match(r"[\x%02x0]" % i, chr(i)), None) - self.assertNotEqual(re.match(r"[\x%02xz]" % i, chr(i)), None) - self.assertRaises(re.error, re.match, "[\911]", "") + assert re.match(r"[\%03o]" % i, chr(i)) != None + assert re.match(r"[\%03o0]" % i, chr(i)) != None + assert re.match(r"[\%03o8]" % i, chr(i)) != None + assert re.match(r"[\x%02x]" % i, chr(i)) != None + assert re.match(r"[\x%02x0]" % i, chr(i)) != None + assert re.match(r"[\x%02xz]" % i, chr(i)) != None + raises(re.error, re.match, "[\911]", "") def test_bug_113254(self): - self.assertEqual(re.match(r'(a)|(b)', 'b').start(1), -1) - self.assertEqual(re.match(r'(a)|(b)', 'b').end(1), -1) - self.assertEqual(re.match(r'(a)|(b)', 'b').span(1), (-1, -1)) + assert re.match(r'(a)|(b)', 'b').start(1) == -1 + assert re.match(r'(a)|(b)', 'b').end(1) == -1 + assert re.match(r'(a)|(b)', 'b').span(1) == (-1, -1) def test_bug_527371(self): # bug described in patches 527371/672491 - self.assertEqual(re.match(r'(a)?a','a').lastindex, None) - self.assertEqual(re.match(r'(a)(b)?b','ab').lastindex, 1) - self.assertEqual(re.match(r'(?Pa)(?Pb)?b','ab').lastgroup, 'a') - self.assertEqual(re.match("(?Pa(b))", "ab").lastgroup, 'a') - self.assertEqual(re.match("((a))", "a").lastindex, 1) + assert re.match(r'(a)?a','a').lastindex == None + assert re.match(r'(a)(b)?b','ab').lastindex == 1 + assert re.match(r'(?Pa)(?Pb)?b','ab').lastgroup == 'a' + assert re.match("(?Pa(b))", "ab").lastgroup == 'a' + assert re.match("((a))", "a").lastindex == 1 def test_bug_545855(self): # bug 545855 -- This pattern failed to cause a compile error as it # should, instead provoking a TypeError. - self.assertRaises(re.error, re.compile, 'foo[a-') + raises(re.error, re.compile, 'foo[a-') - def DONOTtest_bug_418626(self): - # XXX disabled for PyPy, too time-consuming. But our implementation is - # in fact non-recursive as well. + def test_bug_418626(self): # bugs 418626 at al. -- Testing Greg Chapman's addition of op code # SRE_OP_MIN_REPEAT_ONE for eliminating recursion on simple uses of # pattern '*?' on a long string. - self.assertEqual(re.match('.*?c', 10000*'ab'+'cd').end(0), 20001) - self.assertEqual(re.match('.*?cd', 5000*'ab'+'c'+5000*'ab'+'cde').end(0), + assert re.match('.*?c', 10000*'ab'+'cd').end(0) == 20001 + assert re.match('.*?cd', 5000*'ab'+'c'+5000*'ab'+'cde').end(0) == ( 20003) - self.assertEqual(re.match('.*?cd', 20000*'abc'+'de').end(0), 60001) + assert re.match('.*?cd', 20000*'abc'+'de').end(0) == 60001 # non-simple '*?' still used to hit the recursion limit, before the # non-recursive scheme was implemented. - self.assertEqual(re.search('(a|b)*?c', 10000*'ab'+'cd').end(0), 20001) + assert re.search('(a|b)*?c', 10000*'ab'+'cd').end(0) == 20001 def test_bug_612074(self): pat=u"["+re.escape(u"\u2039")+u"]" - self.assertEqual(re.compile(pat) and 1, 1) + assert re.compile(pat) and 1 == 1 - def DONOTtest_stack_overflow(self): - # XXX disabled for PyPy, too time-consuming. But our implementation is - # in fact non-recursive as well. + def test_stack_overflow(self): # nasty cases that used to overflow the straightforward recursive # implementation of repeated groups. - self.assertEqual(re.match('(x)*', 50000*'x').group(1), 'x') - self.assertEqual(re.match('(x)*y', 50000*'x'+'y').group(1), 'x') - self.assertEqual(re.match('(x)*?y', 50000*'x'+'y').group(1), 'x') + assert re.match('(x)*', 50000*'x').group(1) == 'x' + assert re.match('(x)*y', 50000*'x'+'y').group(1) == 'x' + assert re.match('(x)*?y', 50000*'x'+'y').group(1) == 'x' - def test_scanner(self): + def XXXtest_scanner(self): def s_ident(scanner, token): return token def s_operator(scanner, token): return "op%s" % token def s_float(scanner, token): return float(token) @@ -549,9 +526,9 @@ (r"\s+", None), ]) - self.assertNotEqual(scanner.scanner.scanner("").pattern, None) + assert scanner.scanner.scanner("").pattern != None - self.assertEqual(scanner.scan("sum = 3*foo + 312.50 + bar"), + assert scanner.scan("sum = 3*foo + 312.50 + bar") == ( (['sum', 'op=', 3, 'op*', 'foo', 'op+', 312.5, 'op+', 'bar'], '')) @@ -559,35 +536,35 @@ # bug 448951 (similar to 429357, but with single char match) # (Also test greedy matches.) for op in '','?','*': - self.assertEqual(re.match(r'((.%s):)?z'%op, 'z').groups(), + assert re.match(r'((.%s):)?z'%op, 'z').groups() == ( (None, None)) - self.assertEqual(re.match(r'((.%s):)?z'%op, 'a:z').groups(), + assert re.match(r'((.%s):)?z'%op, 'a:z').groups() == ( ('a:', 'a')) def test_bug_725106(self): # capturing groups in alternatives in repeats - self.assertEqual(re.match('^((a)|b)*', 'abc').groups(), + assert re.match('^((a)|b)*', 'abc').groups() == ( ('b', 'a')) - self.assertEqual(re.match('^(([ab])|c)*', 'abc').groups(), + assert re.match('^(([ab])|c)*', 'abc').groups() == ( ('c', 'b')) - self.assertEqual(re.match('^((d)|[ab])*', 'abc').groups(), + assert re.match('^((d)|[ab])*', 'abc').groups() == ( ('b', None)) - self.assertEqual(re.match('^((a)c|[ab])*', 'abc').groups(), + assert re.match('^((a)c|[ab])*', 'abc').groups() == ( ('b', None)) - self.assertEqual(re.match('^((a)|b)*?c', 'abc').groups(), + assert re.match('^((a)|b)*?c', 'abc').groups() == ( ('b', 'a')) - self.assertEqual(re.match('^(([ab])|c)*?d', 'abcd').groups(), + assert re.match('^(([ab])|c)*?d', 'abcd').groups() == ( ('c', 'b')) - self.assertEqual(re.match('^((d)|[ab])*?c', 'abc').groups(), + assert re.match('^((d)|[ab])*?c', 'abc').groups() == ( ('b', None)) - self.assertEqual(re.match('^((a)c|[ab])*?c', 'abc').groups(), + assert re.match('^((a)c|[ab])*?c', 'abc').groups() == ( ('b', None)) def test_bug_725149(self): # mark_stack_base restoring before restoring marks - self.assertEqual(re.match('(a)(?:(?=(b)*)c)*', 'abb').groups(), + assert re.match('(a)(?:(?=(b)*)c)*', 'abb').groups() == ( ('a', None)) - self.assertEqual(re.match('(a)((?!(b)*))*', 'abb').groups(), + assert re.match('(a)((?!(b)*))*', 'abb').groups() == ( ('a', None, None)) def test_bug_764548(self): @@ -598,11 +575,11 @@ return # no problem if we have no unicode class my_unicode(unicode): pass pat = re.compile(my_unicode("abc")) - self.assertEqual(pat.match("xyz"), None) + assert pat.match("xyz") == None def test_finditer(self): iter = re.finditer(r":+", "a:b::c:::d") - self.assertEqual([item.group(0) for item in iter], + assert [item.group(0) for item in iter] == ( [":", "::", ":::"]) def test_bug_926075(self): @@ -610,40 +587,41 @@ unicode except NameError: return # no problem if we have no unicode - self.assert_(re.compile('bug_926075') is not + assert (re.compile('bug_926075') is not re.compile(eval("u'bug_926075'"))) - def test_bug_931848(self): + def XXXtest_bug_931848(self): try: unicode except NameError: pass pattern = eval('u"[\u002E\u3002\uFF0E\uFF61]"') - self.assertEqual(re.compile(pattern).split("a.b.c"), + assert re.compile(pattern).split("a.b.c") == ( ['a','b','c']) def test_bug_581080(self): iter = re.finditer(r"\s", "a b") - self.assertEqual(iter.next().span(), (1,2)) - self.assertRaises(StopIteration, iter.next) + assert iter.next().span() == (1,2) + raises(StopIteration, iter.next) - scanner = re.compile(r"\s").scanner("a b") - self.assertEqual(scanner.search().span(), (1, 2)) - self.assertEqual(scanner.search(), None) + if 0: # XXX + scanner = re.compile(r"\s").scanner("a b") + assert scanner.search().span() == (1, 2) + assert scanner.search() == None def test_bug_817234(self): iter = re.finditer(r".*", "asdf") - self.assertEqual(iter.next().span(), (0, 4)) - self.assertEqual(iter.next().span(), (4, 4)) - self.assertRaises(StopIteration, iter.next) + assert iter.next().span() == (0, 4) + assert iter.next().span() == (4, 4) + raises(StopIteration, iter.next) def test_empty_array(self): # SF buf 1647541 import array for typecode in 'cbBuhHiIlLfd': a = array.array(typecode) - self.assertEqual(re.compile("bla").match(a), None) - self.assertEqual(re.compile("").match(a).groups(), ()) + assert re.compile("bla").match(a) == None + assert re.compile("").match(a).groups() == () def test_inline_flags(self): # Bug #1700 @@ -652,159 +630,24 @@ p = re.compile(upper_char, re.I | re.U) q = p.match(lower_char) - self.assertNotEqual(q, None) + assert q != None p = re.compile(lower_char, re.I | re.U) q = p.match(upper_char) - self.assertNotEqual(q, None) + assert q != None p = re.compile('(?i)' + upper_char, re.U) q = p.match(lower_char) - self.assertNotEqual(q, None) + assert q != None p = re.compile('(?i)' + lower_char, re.U) q = p.match(upper_char) - self.assertNotEqual(q, None) + assert q != None p = re.compile('(?iu)' + upper_char) q = p.match(lower_char) - self.assertNotEqual(q, None) + assert q != None p = re.compile('(?iu)' + lower_char) q = p.match(upper_char) - self.assertNotEqual(q, None) - - -def run_re_tests(): - from test.re_tests import benchmarks, tests, SUCCEED, FAIL, SYNTAX_ERROR - if verbose: - print 'Running re_tests test suite' - else: - # To save time, only run the first and last 10 tests - #tests = tests[:10] + tests[-10:] - pass - - for t in tests: - sys.stdout.flush() - pattern = s = outcome = repl = expected = None - if len(t) == 5: - pattern, s, outcome, repl, expected = t - elif len(t) == 3: - pattern, s, outcome = t - else: - raise ValueError, ('Test tuples should have 3 or 5 fields', t) - - try: - obj = re.compile(pattern) - except re.error: - if outcome == SYNTAX_ERROR: pass # Expected a syntax error - else: - print '=== Syntax error:', t - except KeyboardInterrupt: raise KeyboardInterrupt - except: - print '*** Unexpected error ***', t - # Traceback disabled in PyPy for speed reasons - #if verbose: - # traceback.print_exc(file=sys.stdout) - else: - try: - result = obj.search(s) - except re.error, msg: - print '=== Unexpected exception', t, repr(msg) - if outcome == SYNTAX_ERROR: - # This should have been a syntax error; forget it. - pass - elif outcome == FAIL: - if result is None: pass # No match, as expected - else: print '=== Succeeded incorrectly', t - elif outcome == SUCCEED: - if result is not None: - # Matched, as expected, so now we compute the - # result string and compare it to our expected result. - start, end = result.span(0) - vardict={'found': result.group(0), - 'groups': result.group(), - 'flags': result.re.flags} - for i in range(1, 100): - try: - gi = result.group(i) - # Special hack because else the string concat fails: - if gi is None: - gi = "None" - except IndexError: - gi = "Error" - vardict['g%d' % i] = gi - for i in result.re.groupindex.keys(): - try: - gi = result.group(i) - if gi is None: - gi = "None" - except IndexError: - gi = "Error" - vardict[i] = gi - repl = eval(repl, vardict) - if repl != expected: - print '=== grouping error', t, - print repr(repl) + ' should be ' + repr(expected) - else: - print '=== Failed incorrectly', t - - # Try the match on a unicode string, and check that it - # still succeeds. - try: - result = obj.search(unicode(s, "latin-1")) - if result is None: - print '=== Fails on unicode match', t - except NameError: - continue # 1.5.2 - except TypeError: - continue # unicode test case - - # Try the match on a unicode pattern, and check that it - # still succeeds. - obj=re.compile(unicode(pattern, "latin-1")) - result = obj.search(s) - if result is None: - print '=== Fails on unicode pattern match', t - - # Try the match with the search area limited to the extent - # of the match and see if it still succeeds. \B will - # break (because it won't match at the end or start of a - # string), so we'll ignore patterns that feature it. - - if pattern[:2] != '\\B' and pattern[-2:] != '\\B' \ - and result is not None: - obj = re.compile(pattern) - result = obj.search(s, result.start(0), result.end(0) + 1) - if result is None: - print '=== Failed on range-limited match', t - - # Try the match with IGNORECASE enabled, and check that it - # still succeeds. - obj = re.compile(pattern, re.IGNORECASE) - result = obj.search(s) - if result is None: - print '=== Fails on case-insensitive match', t - - # Try the match with LOCALE enabled, and check that it - # still succeeds. - obj = re.compile(pattern, re.LOCALE) - result = obj.search(s) - if result is None: - print '=== Fails on locale-sensitive match', t - - # Try the match with UNICODE locale enabled, and check - # that it still succeeds. - obj = re.compile(pattern, re.UNICODE) - result = obj.search(s) - if result is None: - print '=== Fails on unicode-sensitive match', t - -def test_main(): - run_unittest(ReTests) - # XXX Disabled re_tests for PyPy because they take approximately forever - # to run ... - #run_re_tests() - -if __name__ == "__main__": - test_main() + assert q != None Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_search.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/test_search.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_search.py Fri Aug 6 15:33:37 2010 @@ -1,4 +1,4 @@ -from pypy.rlib.rsre import rsre +from pypy.rlib.rsre import rsre_core from pypy.rlib.rsre.test.test_match import get_code @@ -6,20 +6,20 @@ def test_code1(self): r_code1 = get_code(r'[abc][def][ghi]') - res = rsre.search(r_code1, "fooahedixxx") + res = rsre_core.search(r_code1, "fooahedixxx") assert res is None - res = rsre.search(r_code1, "fooahcdixxx") + res = rsre_core.search(r_code1, "fooahcdixxx") assert res is not None assert res.span() == (5, 8) def test_code2(self): r_code2 = get_code(r'\s*(.*?)') - res = rsre.search(r_code2, "foo bar abcdef") + res = rsre_core.search(r_code2, "foo bar abcdef") assert res is not None assert res.span() == (8, 34) def test_pure_literal(self): r_code3 = get_code(r'foobar') - res = rsre.search(r_code3, "foo bar foobar baz") + res = rsre_core.search(r_code3, "foo bar foobar baz") assert res is not None assert res.span() == (8, 14) Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_zexternal.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/test_zexternal.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_zexternal.py Fri Aug 6 15:33:37 2010 @@ -1,6 +1,6 @@ import re from pypy.rlib.rsre.test.test_match import get_code -from pypy.rlib.rsre import rsre +from pypy.rlib.rsre import rsre_core def test_external_match(): @@ -31,11 +31,11 @@ raise Exception("this should have been a syntax error") # if use_search: - result = rsre.search(obj, s) + result = rsre_core.search(obj, s) else: # Emulate a poor man's search() with repeated match()s for i in range(len(s)+1): - result = rsre.match(obj, s, start=i) + result = rsre_core.match(obj, s, start=i) if result: break # Added: pypy/branch/rsre2/pypy/rlib/rsre/test/test_zinterp.py ============================================================================== --- (empty file) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_zinterp.py Fri Aug 6 15:33:37 2010 @@ -0,0 +1,15 @@ +# minimal test: just checks that (parts of) rsre can be translated + +from pypy.rpython.test.test_llinterp import gengraph +from pypy.rlib.rsre import rsre_core + +def main(n): + assert n >= 0 + pattern = [n] * n + string = chr(n) * n + rsre_core.search(pattern, string) + return 0 + + +def test_gengraph(): + t, typer, graph = gengraph(main, [int]) From arigo at codespeak.net Fri Aug 6 15:36:06 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 6 Aug 2010 15:36:06 +0200 (CEST) Subject: [pypy-svn] r76510 - pypy/branch/kill-caninline/pypy/jit/metainterp Message-ID: <20100806133606.8341E282BFE@codespeak.net> Author: arigo Date: Fri Aug 6 15:36:05 2010 New Revision: 76510 Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/pyjitpl.py Log: Kill unnecessary assert. Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/pyjitpl.py Fri Aug 6 15:36:05 2010 @@ -151,7 +151,6 @@ def make_result_of_lastop(self, resultbox): got_type = resultbox.type if not we_are_translated(): - assert resultbox is not None typeof = {'i': history.INT, 'r': history.REF, 'f': history.FLOAT} From getxsick at codespeak.net Fri Aug 6 16:22:20 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Fri, 6 Aug 2010 16:22:20 +0200 (CEST) Subject: [pypy-svn] r76511 - pypy/branch/fast-ctypes/pypy/jit/backend/llsupport Message-ID: <20100806142220.7AB16282BEB@codespeak.net> Author: getxsick Date: Fri Aug 6 16:22:18 2010 New Revision: 76511 Modified: pypy/branch/fast-ctypes/pypy/jit/backend/llsupport/descr.py Log: add SignedCallDescr Modified: pypy/branch/fast-ctypes/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/llsupport/descr.py Fri Aug 6 16:22:18 2010 @@ -308,6 +308,11 @@ def get_result_size(self, translate_support_code): return symbolic.get_size_of_ptr(translate_support_code) +class SignedCallDescr(BaseIntCallDescr): + _clsname = 'SignedCallDescr' + def get_result_size(self, translate_support_code): + return symbolic.get_size(lltype.Signed, translate_support_code) + class GcPtrCallDescr(BaseCallDescr): _clsname = 'GcPtrCallDescr' _return_type = history.REF From getxsick at codespeak.net Fri Aug 6 16:23:29 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Fri, 6 Aug 2010 16:23:29 +0200 (CEST) Subject: [pypy-svn] r76512 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100806142329.26DC1282BEB@codespeak.net> Author: getxsick Date: Fri Aug 6 16:23:27 2010 New Revision: 76512 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: update descr handling after changes in r76511 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Fri Aug 6 16:23:27 2010 @@ -81,11 +81,11 @@ def get_calldescr(self): if self.res_type == 'i': - cls = SignedCallDescr + cls = descr.SignedCallDescr elif self.res_type == 'f': - cls = FloatCallDescr + cls = descr.FloatCallDescr elif self.res_type == 'v': - cls = VoidCallDescr + cls = descr.VoidCallDescr else: raise NotImplementedError('Unknown type of descr: %s' % self.res_type) @@ -121,14 +121,3 @@ def push_float(self, value): self.cpu.set_future_value_float(self.esp, value) self.esp += 1 - -# ____________________________________________________________ -# CallDescrs - -class SignedCallDescr(descr.BaseIntCallDescr): - _clsname = 'SignedCallDescr' - def get_result_size(self, translate_support_code): - return symbolic.get_size(lltype.Signed, translate_support_code) - -FloatCallDescr = descr.FloatCallDescr -VoidCallDescr = descr.VoidCallDescr From arigo at codespeak.net Fri Aug 6 18:14:55 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 6 Aug 2010 18:14:55 +0200 (CEST) Subject: [pypy-svn] r76516 - in pypy/branch/rsre2/pypy: annotation rlib/rsre rlib/rsre/test Message-ID: <20100806161455.0B1D4282BEB@codespeak.net> Author: arigo Date: Fri Aug 6 18:14:53 2010 New Revision: 76516 Modified: pypy/branch/rsre2/pypy/annotation/specialize.py pypy/branch/rsre2/pypy/rlib/rsre/rsre_core.py pypy/branch/rsre2/pypy/rlib/rsre/rsre_re.py pypy/branch/rsre2/pypy/rlib/rsre/test/test_zinterp.py Log: Unicode support, with appropriate specializations in order to translate different versions of some of the functions in rsre_core depending on whether they operate on strings or unicodes. Modified: pypy/branch/rsre2/pypy/annotation/specialize.py ============================================================================== --- pypy/branch/rsre2/pypy/annotation/specialize.py (original) +++ pypy/branch/rsre2/pypy/annotation/specialize.py Fri Aug 6 18:14:53 2010 @@ -354,6 +354,12 @@ def specialize_argtype(funcdesc, args_s, *argindices): key = tuple([args_s[i].knowntype for i in argindices]) + for cls in key: + try: + assert '_must_specialize_' not in cls.classdesc.pyobj.__dict__, ( + "%s has the tag _must_specialize_" % (cls,)) + except AttributeError: + pass return maybe_star_args(funcdesc, key, args_s) def specialize_arglistitemtype(funcdesc, args_s, i): Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre_core.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre_core.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre_core.py Fri Aug 6 18:14:53 2010 @@ -2,6 +2,7 @@ from pypy.rlib.debug import check_nonneg from pypy.rlib.rsre import rsre_char from pypy.rlib import jit +from pypy.tool.sourcetools import func_with_new_name OPCODE_FAILURE = 0 @@ -37,20 +38,50 @@ #OPCODE_SUBPATTERN = 30 OPCODE_MIN_REPEAT_ONE = 31 +# ____________________________________________________________ + +def specializectx(func): + """A decorator that specializes 'func' for each concrete subclass + XyzMatchContext. It must then be called as func(ctx,...) where + ctx is known to be of a specific subclass. Use this for the case + of very common and small methods; for large methods where an + indirect call is ok, use @specializectxmethod. + """ + func._annspecialcase_ = 'specialize:argtype(0)' + return func + +def specializectxmethod(func): + """A decorator that puts 'func' as a method on each concrete + subclass XyzMatchContext. This is an annotation trick to allow a + different version of the function to be called (as a method call) + depending on whether it operates on strings or unicodes. It is ok + to do ctx.func(...) even if ctx is a general AbstractMatchContext; + it becomes an indirect call in the C version. + """ + name = func.__name__ + setattr(StrMatchContext, name, + func_with_new_name(func, name + '_str')) + setattr(UnicodeMatchContext, name, + func_with_new_name(func, name + '_unicode')) + return NotImplemented # the original decorated function is not available -class MatchContext(object): +# ____________________________________________________________ + +class AbstractMatchContext(object): + """Abstract base class""" + _must_specialize_ = True match_start = 0 match_end = 0 match_marks = None match_marks_flat = None - def __init__(self, pattern, string, match_start, end, flags): + def __init__(self, pattern, match_start, end, flags): + # here, 'end' must be at most len(string) self.pattern = pattern - self.string = string - if end > len(string): - end = len(string) - self.end = end + if match_start < 0: + match_start = 0 self.match_start = match_start + self.end = end self.flags = flags def pat(self, index): @@ -58,9 +89,14 @@ return self.pattern[index] def str(self, index): - check_nonneg(index) - return ord(self.string[index]) + """NOT_RPYTHON: Must be overridden in a concrete subclass. + The line below is used to generate a translation-time crash + if there is a call to str() that is indirect. All calls must + be direct for performance reasons; you need to specialize the + caller with @specializectx or @specializectxmethod.""" + raise NotImplementedError + @specializectx def lowstr(self, index): c = self.str(index) return rsre_char.getlower(c, self.flags) @@ -88,7 +124,7 @@ return self.match_marks_flat def span(self, groupnum=0): - # compatibility + "NOT_RPYTHON" # compatibility fmarks = self.flatten_marks() groupnum *= 2 if groupnum >= len(fmarks): @@ -96,31 +132,40 @@ return (fmarks[groupnum], fmarks[groupnum+1]) def group(self, groupnum=0): - # compatibility + "NOT_RPYTHON" # compatibility frm, to = self.span(groupnum) if 0 <= frm <= to: - return self.string[frm:to] + return self._string[frm:to] else: return None - def at_boundary(self, ptr, word_checker): - if self.end == 0: - return False - prevptr = ptr - 1 - that = prevptr >= 0 and word_checker(self.str(prevptr)) - this = ptr < self.end and word_checker(self.str(ptr)) - return this != that - at_boundary._annspecialcase_ = 'specialize:arg(2)' +class StrMatchContext(AbstractMatchContext): + """Concrete subclass for matching in a plain string.""" - def at_non_boundary(self, ptr, word_checker): - if self.end == 0: - return False - prevptr = ptr - 1 - that = prevptr >= 0 and word_checker(self.str(prevptr)) - this = ptr < self.end and word_checker(self.str(ptr)) - return this == that - at_non_boundary._annspecialcase_ = 'specialize:arg(2)' + def __init__(self, pattern, string, match_start, end, flags): + if end > len(string): + end = len(string) + AbstractMatchContext.__init__(self, pattern, match_start, end, flags) + self._string = string + def str(self, index): + check_nonneg(index) + return ord(self._string[index]) + +class UnicodeMatchContext(AbstractMatchContext): + """Concrete subclass for matching in a unicode string.""" + + def __init__(self, pattern, unicodestr, match_start, end, flags): + if end > len(unicodestr): + end = len(unicodestr) + AbstractMatchContext.__init__(self, pattern, match_start, end, flags) + self._unicodestr = unicodestr + + def str(self, index): + check_nonneg(index) + return ord(self._unicodestr[index]) + +# ____________________________________________________________ class Mark(object): _immutable_ = True @@ -137,6 +182,7 @@ mark = mark.prev return -1 +# ____________________________________________________________ class MatchResult(object): subresult = None @@ -165,7 +211,7 @@ def find_first_result(self, ctx): ppos = self.ppos while ctx.pat(ppos): - result = sre_match(ctx, ppos + 1, self.start_ptr, self.start_marks) + result = ctx.sre_match(ppos + 1, self.start_ptr, self.start_marks) ppos += ctx.pat(ppos) if result is not None: self.subresult = result @@ -184,7 +230,7 @@ def find_first_result(self, ctx): ptr = self.start_ptr while ptr >= self.minptr: - result = sre_match(ctx, self.nextppos, ptr, self.start_marks) + result = ctx.sre_match(self.nextppos, ptr, self.start_marks) ptr -= 1 if result is not None: self.subresult = result @@ -205,19 +251,19 @@ def find_first_result(self, ctx): ptr = self.start_ptr while ptr <= self.maxptr: - result = sre_match(ctx, self.nextppos, ptr, self.start_marks) + result = ctx.sre_match(self.nextppos, ptr, self.start_marks) if result is not None: self.subresult = result self.start_ptr = ptr return self - ptr1 = find_repetition_end(ctx, self.ppos3, ptr, 1) + ptr1 = ctx.find_repetition_end(self.ppos3, ptr, 1) if ptr1 == ptr: break ptr = ptr1 def find_next_result(self, ctx): ptr = self.start_ptr - ptr1 = find_repetition_end(ctx, self.ppos3, ptr, 1) + ptr1 = ctx.find_repetition_end(self.ppos3, ptr, 1) if ptr1 == ptr: return self.start_ptr = ptr1 @@ -243,7 +289,7 @@ class MaxUntilMatchResult(AbstractUntilMatchResult): def find_first_result(self, ctx): - enum = sre_match(ctx, self.ppos + 3, self.cur_ptr, self.cur_marks) + enum = ctx.sre_match(self.ppos + 3, self.cur_ptr, self.cur_marks) return self.search_next(ctx, enum, resume=False) def find_next_result(self, ctx): @@ -268,7 +314,7 @@ # 'item' no longer matches. if not resume and self.num_pending >= min: # try to match 'tail' if we have enough 'item' - result = sre_match(ctx, self.tailppos, ptr, marks) + result = ctx.sre_match(self.tailppos, ptr, marks) if result is not None: self.subresult = result self.cur_ptr = ptr @@ -286,7 +332,7 @@ # if max == 65535 or self.num_pending < max: # try to match one more 'item' - enum = sre_match(ctx, ppos + 3, ptr, marks) + enum = ctx.sre_match(ppos + 3, ptr, marks) else: enum = None # 'max' reached, no more matches @@ -307,7 +353,7 @@ while True: # try to match 'tail' if we have enough 'item' if not resume and self.num_pending >= min: - result = sre_match(ctx, self.tailppos, ptr, marks) + result = ctx.sre_match(self.tailppos, ptr, marks) if result is not None: self.subresult = result self.cur_ptr = ptr @@ -317,7 +363,7 @@ if max == 65535 or self.num_pending < max: # try to match one more 'item' - enum = sre_match(ctx, ppos + 3, ptr, marks) + enum = ctx.sre_match(ppos + 3, ptr, marks) else: enum = None # 'max' reached, no more matches @@ -341,6 +387,7 @@ # ____________________________________________________________ + at specializectxmethod @jit.unroll_safe # it's safe to unroll the main 'while' loop: # 'ppos' is only ever incremented in this function def sre_match(ctx, ppos, ptr, marks): @@ -380,7 +427,7 @@ # assert subpattern # <0=skip> <1=back> ptr1 = ptr - ctx.pat(ppos+1) - if ptr1 < 0 or sre_match(ctx, ppos + 2, ptr1, marks) is None: + if ptr1 < 0 or ctx.sre_match(ppos + 2, ptr1, marks) is None: return marks = ctx.match_marks ppos += ctx.pat(ppos) @@ -389,7 +436,7 @@ # assert not subpattern # <0=skip> <1=back> ptr1 = ptr - ctx.pat(ppos+1) - if ptr1 >= 0 and sre_match(ctx, ppos + 2, ptr1, marks) is not None: + if ptr1 >= 0 and ctx.sre_match(ppos + 2, ptr1, marks) is not None: return ppos += ctx.pat(ppos) @@ -548,7 +595,7 @@ minptr = start + ctx.pat(ppos+1) if minptr > ctx.end: return # cannot match - ptr = find_repetition_end(ctx, ppos+3, start, ctx.pat(ppos+2)) + ptr = ctx.find_repetition_end(ppos+3, start, ctx.pat(ppos+2)) # when we arrive here, ptr points to the tail of the target # string. check if the rest of the pattern matches, # and backtrack if not. @@ -570,7 +617,7 @@ if minptr > ctx.end: return # cannot match # count using pattern min as the maximum - ptr = find_repetition_end(ctx, ppos+3, ptr, min) + ptr = ctx.find_repetition_end(ppos+3, ptr, min) if ptr < minptr: return # did not match minimum number of times @@ -598,6 +645,7 @@ length = endptr - startptr # < 0 if endptr < startptr (or if endptr=-1) return startptr, length + at specializectx def match_repeated(ctx, ptr, oldptr, length): if ptr + length > ctx.end: return False @@ -606,6 +654,7 @@ return False return True + at specializectx def match_repeated_ignore(ctx, ptr, oldptr, length): if ptr + length > ctx.end: return False @@ -614,6 +663,7 @@ return False return True + at specializectxmethod def find_repetition_end(ctx, ppos, ptr, maxcount): end = ctx.end # adjust end @@ -634,12 +684,14 @@ end,ppos) raise NotImplementedError("rsre.find_repetition_end[%d]" % op) + at specializectx def fre_ANY(ctx, ptr, end): # repeated dot wildcard. while ptr < end and not rsre_char.is_linebreak(ctx.str(ptr)): ptr += 1 return ptr + at specializectx def fre_IN(ctx, ptr, end, ppos): # repeated set while ptr < end and rsre_char.check_charset(ctx.pattern, ppos+2, @@ -647,6 +699,7 @@ ptr += 1 return ptr + at specializectx def fre_IN_IGNORE(ctx, ptr, end, ppos): # repeated set while ptr < end and rsre_char.check_charset(ctx.pattern, ppos+2, @@ -654,24 +707,28 @@ ptr += 1 return ptr + at specializectx def fre_LITERAL(ctx, ptr, end, ppos): chr = ctx.pat(ppos+1) while ptr < end and ctx.str(ptr) == chr: ptr += 1 return ptr + at specializectx def fre_LITERAL_IGNORE(ctx, ptr, end, ppos): chr = ctx.pat(ppos+1) while ptr < end and ctx.lowstr(ptr) == chr: ptr += 1 return ptr + at specializectx def fre_NOT_LITERAL(ctx, ptr, end, ppos): chr = ctx.pat(ppos+1) while ptr < end and ctx.str(ptr) != chr: ptr += 1 return ptr + at specializectx def fre_NOT_LITERAL_IGNORE(ctx, ptr, end, ppos): chr = ctx.pat(ppos+1) while ptr < end and ctx.lowstr(ptr) != chr: @@ -693,6 +750,7 @@ AT_UNI_BOUNDARY = 10 AT_UNI_NON_BOUNDARY = 11 + at specializectx def sre_at(ctx, atcode, ptr): if (atcode == AT_BEGINNING or atcode == AT_BEGINNING_STRING): @@ -703,10 +761,10 @@ return prevptr < 0 or rsre_char.is_linebreak(ctx.str(prevptr)) elif atcode == AT_BOUNDARY: - return ctx.at_boundary(ptr, rsre_char.is_word) + return at_boundary(ctx, ptr, rsre_char.is_word) elif atcode == AT_NON_BOUNDARY: - return ctx.at_non_boundary(ptr, rsre_char.is_word) + return at_non_boundary(ctx, ptr, rsre_char.is_word) elif atcode == AT_END: remaining_chars = ctx.end - ptr @@ -720,47 +778,79 @@ return ptr == ctx.end elif atcode == AT_LOC_BOUNDARY: - return ctx.at_boundary(ptr, rsre_char.is_loc_word) + return at_boundary(ctx, ptr, rsre_char.is_loc_word) elif atcode == AT_LOC_NON_BOUNDARY: - return ctx.at_non_boundary(ptr, rsre_char.is_loc_word) + return at_non_boundary(ctx, ptr, rsre_char.is_loc_word) elif atcode == AT_UNI_BOUNDARY: - return ctx.at_boundary(ptr, rsre_char.is_uni_word) + return at_boundary(ctx, ptr, rsre_char.is_uni_word) elif atcode == AT_UNI_NON_BOUNDARY: - return ctx.at_non_boundary(ptr, rsre_char.is_uni_word) + return at_non_boundary(ctx, ptr, rsre_char.is_uni_word) return False + at specializectx +def at_boundary(ctx, ptr, word_checker): + if ctx.end == 0: + return False + prevptr = ptr - 1 + that = prevptr >= 0 and word_checker(ctx.str(prevptr)) + this = ptr < ctx.end and word_checker(ctx.str(ptr)) + return this != that + + at specializectx +def at_non_boundary(ctx, ptr, word_checker): + if ctx.end == 0: + return False + prevptr = ptr - 1 + that = prevptr >= 0 and word_checker(ctx.str(prevptr)) + this = ptr < ctx.end and word_checker(ctx.str(ptr)) + return this == that + # ____________________________________________________________ def match(pattern, string, start=0, end=sys.maxint, flags=0): - if start < 0: start = 0 - if end < start: return None - ctx = MatchContext(pattern, string, start, end, flags) - if sre_match(ctx, 0, start, None) is not None: + ctx = StrMatchContext(pattern, string, start, end, flags) + if match_context(ctx) is not None: return ctx return None def search(pattern, string, start=0, end=sys.maxint, flags=0): - if start < 0: start = 0 - if end < start: return None - ctx = MatchContext(pattern, string, start, end, flags) + ctx = StrMatchContext(pattern, string, start, end, flags) + if search_context(ctx) is not None: + return ctx + return None + + at specializectx +def match_context(ctx): + if ctx.end < ctx.match_start: + return None + if ctx.sre_match(0, ctx.match_start, None) is not None: + return ctx + return None + + at specializectx +def search_context(ctx): + if ctx.end < ctx.match_start: + return None if ctx.pat(0) == OPCODE_INFO: if ctx.pat(2) & rsre_char.SRE_INFO_PREFIX and ctx.pat(5) > 1: return fast_search(ctx) return regular_search(ctx) + at specializectx def regular_search(ctx): start = ctx.match_start while start <= ctx.end: - if sre_match(ctx, 0, start, None) is not None: + if ctx.sre_match(0, start, None) is not None: ctx.match_start = start return ctx start += 1 return None + at specializectx def fast_search(ctx): # skips forward in a string as fast as possible using information from # an optimization info block @@ -800,7 +890,7 @@ ctx.match_marks = None return ctx ppos = pattern_offset + 2 * prefix_skip - if sre_match(ctx, ppos, ptr, None) is not None: + if ctx.sre_match(ppos, ptr, None) is not None: ctx.match_start = start return ctx i = ctx.pat(overlap_offset + i) Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre_re.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre_re.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre_re.py Fri Aug 6 18:14:53 2010 @@ -110,7 +110,7 @@ for group in groups: frm, to = self.span(group) if 0 <= frm <= to: - result.append(self._ctx.string[frm:to]) + result.append(self._ctx._string[frm:to]) else: result.append(None) if len(result) > 1: @@ -160,7 +160,7 @@ @property def string(self): - return self._ctx.string + return self._ctx._string @property def pos(self): Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_zinterp.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/test_zinterp.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_zinterp.py Fri Aug 6 18:14:53 2010 @@ -8,6 +8,12 @@ pattern = [n] * n string = chr(n) * n rsre_core.search(pattern, string) + # + unicodestr = unichr(n) * n + ctx = rsre_core.UnicodeMatchContext(pattern, unicodestr, + 0, len(unicodestr), 0) + rsre_core.search_context(ctx) + # return 0 From arigo at codespeak.net Fri Aug 6 18:17:34 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 6 Aug 2010 18:17:34 +0200 (CEST) Subject: [pypy-svn] r76517 - in pypy/branch/rsre2/pypy/rlib/rsre: . test Message-ID: <20100806161734.6A242282BEB@codespeak.net> Author: arigo Date: Fri Aug 6 18:17:33 2010 New Revision: 76517 Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre_core.py pypy/branch/rsre2/pypy/rlib/rsre/test/targetrsre.py Log: Fixes. Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre_core.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre_core.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre_core.py Fri Aug 6 18:17:33 2010 @@ -124,7 +124,7 @@ return self.match_marks_flat def span(self, groupnum=0): - "NOT_RPYTHON" # compatibility + # compatibility fmarks = self.flatten_marks() groupnum *= 2 if groupnum >= len(fmarks): Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/targetrsre.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/targetrsre.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/targetrsre.py Fri Aug 6 18:17:33 2010 @@ -1,6 +1,6 @@ #!/usr/bin/env python from pypy.rlib.rarithmetic import intmask -from pypy.rlib.rsre import rsre +from pypy.rlib.rsre import rsre_core import os, time @@ -26,7 +26,7 @@ data = read(filename) p = 0 while True: - res = rsre.search(r_code1, data, p) + res = rsre_core.search(r_code1, data, p) if res is None: break matchstart, matchstop = res.span(1) From arigo at codespeak.net Fri Aug 6 18:27:43 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 6 Aug 2010 18:27:43 +0200 (CEST) Subject: [pypy-svn] r76518 - pypy/branch/rsre2/pypy/rlib/rsre Message-ID: <20100806162743.6F55E282BEB@codespeak.net> Author: arigo Date: Fri Aug 6 18:27:41 2010 New Revision: 76518 Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre_core.py Log: Simplifies the code, specialize a bit more. Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre_core.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre_core.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre_core.py Fri Aug 6 18:27:41 2010 @@ -43,28 +43,12 @@ def specializectx(func): """A decorator that specializes 'func' for each concrete subclass XyzMatchContext. It must then be called as func(ctx,...) where - ctx is known to be of a specific subclass. Use this for the case - of very common and small methods; for large methods where an - indirect call is ok, use @specializectxmethod. + ctx is known to be of a specific subclass. """ - func._annspecialcase_ = 'specialize:argtype(0)' + i = list(func.func_code.co_varnames).index('ctx') + func._annspecialcase_ = 'specialize:argtype(%d)' % i return func -def specializectxmethod(func): - """A decorator that puts 'func' as a method on each concrete - subclass XyzMatchContext. This is an annotation trick to allow a - different version of the function to be called (as a method call) - depending on whether it operates on strings or unicodes. It is ok - to do ctx.func(...) even if ctx is a general AbstractMatchContext; - it becomes an indirect call in the C version. - """ - name = func.__name__ - setattr(StrMatchContext, name, - func_with_new_name(func, name + '_str')) - setattr(UnicodeMatchContext, name, - func_with_new_name(func, name + '_unicode')) - return NotImplemented # the original decorated function is not available - # ____________________________________________________________ class AbstractMatchContext(object): @@ -93,13 +77,13 @@ The line below is used to generate a translation-time crash if there is a call to str() that is indirect. All calls must be direct for performance reasons; you need to specialize the - caller with @specializectx or @specializectxmethod.""" + caller with @specializectx.""" raise NotImplementedError @specializectx - def lowstr(self, index): - c = self.str(index) - return rsre_char.getlower(c, self.flags) + def lowstr(ctx, index): + c = ctx.str(index) + return rsre_char.getlower(c, ctx.flags) def get_mark(self, gid): return find_mark(self.match_marks, gid) @@ -187,6 +171,7 @@ class MatchResult(object): subresult = None + @specializectx def move_to_next_result(self, ctx): result = self.subresult if result is None: @@ -207,11 +192,12 @@ self.start_ptr = ptr self.start_marks = marks + @specializectx @jit.unroll_safe # there are only a few branch alternatives def find_first_result(self, ctx): ppos = self.ppos while ctx.pat(ppos): - result = ctx.sre_match(ppos + 1, self.start_ptr, self.start_marks) + result = sre_match(ctx, ppos + 1, self.start_ptr, self.start_marks) ppos += ctx.pat(ppos) if result is not None: self.subresult = result @@ -227,10 +213,11 @@ self.start_ptr = ptr self.start_marks = marks + @specializectx def find_first_result(self, ctx): ptr = self.start_ptr while ptr >= self.minptr: - result = ctx.sre_match(self.nextppos, ptr, self.start_marks) + result = sre_match(ctx, self.nextppos, ptr, self.start_marks) ptr -= 1 if result is not None: self.subresult = result @@ -248,22 +235,24 @@ self.start_ptr = ptr self.start_marks = marks + @specializectx def find_first_result(self, ctx): ptr = self.start_ptr while ptr <= self.maxptr: - result = ctx.sre_match(self.nextppos, ptr, self.start_marks) + result = sre_match(ctx, self.nextppos, ptr, self.start_marks) if result is not None: self.subresult = result self.start_ptr = ptr return self - ptr1 = ctx.find_repetition_end(self.ppos3, ptr, 1) + ptr1 = find_repetition_end(ctx, self.ppos3, ptr, 1) if ptr1 == ptr: break ptr = ptr1 + @specializectx def find_next_result(self, ctx): ptr = self.start_ptr - ptr1 = ctx.find_repetition_end(self.ppos3, ptr, 1) + ptr1 = find_repetition_end(ctx, self.ppos3, ptr, 1) if ptr1 == ptr: return self.start_ptr = ptr1 @@ -288,13 +277,16 @@ class MaxUntilMatchResult(AbstractUntilMatchResult): + @specializectx def find_first_result(self, ctx): - enum = ctx.sre_match(self.ppos + 3, self.cur_ptr, self.cur_marks) + enum = sre_match(ctx, self.ppos + 3, self.cur_ptr, self.cur_marks) return self.search_next(ctx, enum, resume=False) + @specializectx def find_next_result(self, ctx): return self.search_next(ctx, None, resume=True) + @specializectx def search_next(self, ctx, enum, resume): ppos = self.ppos min = ctx.pat(ppos+1) @@ -314,7 +306,7 @@ # 'item' no longer matches. if not resume and self.num_pending >= min: # try to match 'tail' if we have enough 'item' - result = ctx.sre_match(self.tailppos, ptr, marks) + result = sre_match(ctx, self.tailppos, ptr, marks) if result is not None: self.subresult = result self.cur_ptr = ptr @@ -332,18 +324,21 @@ # if max == 65535 or self.num_pending < max: # try to match one more 'item' - enum = ctx.sre_match(ppos + 3, ptr, marks) + enum = sre_match(ctx, ppos + 3, ptr, marks) else: enum = None # 'max' reached, no more matches class MinUntilMatchResult(AbstractUntilMatchResult): + @specializectx def find_first_result(self, ctx): return self.search_next(ctx, resume=False) + @specializectx def find_next_result(self, ctx): return self.search_next(ctx, resume=True) + @specializectx def search_next(self, ctx, resume): ppos = self.ppos min = ctx.pat(ppos+1) @@ -353,7 +348,7 @@ while True: # try to match 'tail' if we have enough 'item' if not resume and self.num_pending >= min: - result = ctx.sre_match(self.tailppos, ptr, marks) + result = sre_match(ctx, self.tailppos, ptr, marks) if result is not None: self.subresult = result self.cur_ptr = ptr @@ -363,7 +358,7 @@ if max == 65535 or self.num_pending < max: # try to match one more 'item' - enum = ctx.sre_match(ppos + 3, ptr, marks) + enum = sre_match(ctx, ppos + 3, ptr, marks) else: enum = None # 'max' reached, no more matches @@ -387,7 +382,7 @@ # ____________________________________________________________ - at specializectxmethod + at specializectx @jit.unroll_safe # it's safe to unroll the main 'while' loop: # 'ppos' is only ever incremented in this function def sre_match(ctx, ppos, ptr, marks): @@ -427,7 +422,7 @@ # assert subpattern # <0=skip> <1=back> ptr1 = ptr - ctx.pat(ppos+1) - if ptr1 < 0 or ctx.sre_match(ppos + 2, ptr1, marks) is None: + if ptr1 < 0 or sre_match(ctx, ppos + 2, ptr1, marks) is None: return marks = ctx.match_marks ppos += ctx.pat(ppos) @@ -436,7 +431,7 @@ # assert not subpattern # <0=skip> <1=back> ptr1 = ptr - ctx.pat(ppos+1) - if ptr1 >= 0 and ctx.sre_match(ppos + 2, ptr1, marks) is not None: + if ptr1 >= 0 and sre_match(ctx, ppos + 2, ptr1, marks) is not None: return ppos += ctx.pat(ppos) @@ -595,7 +590,7 @@ minptr = start + ctx.pat(ppos+1) if minptr > ctx.end: return # cannot match - ptr = ctx.find_repetition_end(ppos+3, start, ctx.pat(ppos+2)) + ptr = find_repetition_end(ctx, ppos+3, start, ctx.pat(ppos+2)) # when we arrive here, ptr points to the tail of the target # string. check if the rest of the pattern matches, # and backtrack if not. @@ -617,7 +612,7 @@ if minptr > ctx.end: return # cannot match # count using pattern min as the maximum - ptr = ctx.find_repetition_end(ppos+3, ptr, min) + ptr = find_repetition_end(ctx, ppos+3, ptr, min) if ptr < minptr: return # did not match minimum number of times @@ -663,7 +658,7 @@ return False return True - at specializectxmethod + at specializectx def find_repetition_end(ctx, ppos, ptr, maxcount): end = ctx.end # adjust end @@ -827,7 +822,7 @@ def match_context(ctx): if ctx.end < ctx.match_start: return None - if ctx.sre_match(0, ctx.match_start, None) is not None: + if sre_match(ctx, 0, ctx.match_start, None) is not None: return ctx return None @@ -844,7 +839,7 @@ def regular_search(ctx): start = ctx.match_start while start <= ctx.end: - if ctx.sre_match(0, start, None) is not None: + if sre_match(ctx, 0, start, None) is not None: ctx.match_start = start return ctx start += 1 @@ -890,7 +885,7 @@ ctx.match_marks = None return ctx ppos = pattern_offset + 2 * prefix_skip - if ctx.sre_match(ppos, ptr, None) is not None: + if sre_match(ctx, ppos, ptr, None) is not None: ctx.match_start = start return ctx i = ctx.pat(overlap_offset + i) From arigo at codespeak.net Fri Aug 6 18:33:02 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 6 Aug 2010 18:33:02 +0200 (CEST) Subject: [pypy-svn] r76519 - pypy/branch/rsre2/pypy/rlib/rsre Message-ID: <20100806163302.9B57D282BEB@codespeak.net> Author: arigo Date: Fri Aug 6 18:33:01 2010 New Revision: 76519 Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre_core.py Log: Reintroduce specialization on the function. Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre_core.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre_core.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre_core.py Fri Aug 6 18:33:01 2010 @@ -756,10 +756,10 @@ return prevptr < 0 or rsre_char.is_linebreak(ctx.str(prevptr)) elif atcode == AT_BOUNDARY: - return at_boundary(ctx, ptr, rsre_char.is_word) + return at_boundary(ctx, ptr) elif atcode == AT_NON_BOUNDARY: - return at_non_boundary(ctx, ptr, rsre_char.is_word) + return at_non_boundary(ctx, ptr) elif atcode == AT_END: remaining_chars = ctx.end - ptr @@ -773,36 +773,41 @@ return ptr == ctx.end elif atcode == AT_LOC_BOUNDARY: - return at_boundary(ctx, ptr, rsre_char.is_loc_word) + return at_loc_boundary(ctx, ptr) elif atcode == AT_LOC_NON_BOUNDARY: - return at_non_boundary(ctx, ptr, rsre_char.is_loc_word) + return at_loc_non_boundary(ctx, ptr) elif atcode == AT_UNI_BOUNDARY: - return at_boundary(ctx, ptr, rsre_char.is_uni_word) + return at_uni_boundary(ctx, ptr) elif atcode == AT_UNI_NON_BOUNDARY: - return at_non_boundary(ctx, ptr, rsre_char.is_uni_word) + return at_uni_non_boundary(ctx, ptr) return False - at specializectx -def at_boundary(ctx, ptr, word_checker): - if ctx.end == 0: - return False - prevptr = ptr - 1 - that = prevptr >= 0 and word_checker(ctx.str(prevptr)) - this = ptr < ctx.end and word_checker(ctx.str(ptr)) - return this != that - - at specializectx -def at_non_boundary(ctx, ptr, word_checker): - if ctx.end == 0: - return False - prevptr = ptr - 1 - that = prevptr >= 0 and word_checker(ctx.str(prevptr)) - this = ptr < ctx.end and word_checker(ctx.str(ptr)) - return this == that +def _make_boundary(word_checker): + @specializectx + def at_boundary(ctx, ptr): + if ctx.end == 0: + return False + prevptr = ptr - 1 + that = prevptr >= 0 and word_checker(ctx.str(prevptr)) + this = ptr < ctx.end and word_checker(ctx.str(ptr)) + return this != that + @specializectx + def at_non_boundary(ctx, ptr): + if ctx.end == 0: + return False + prevptr = ptr - 1 + that = prevptr >= 0 and word_checker(ctx.str(prevptr)) + this = ptr < ctx.end and word_checker(ctx.str(ptr)) + return this == that + return at_boundary, at_non_boundary + +at_boundary, at_non_boundary = _make_boundary(rsre_char.is_word) +at_loc_boundary, at_loc_non_boundary = _make_boundary(rsre_char.is_loc_word) +at_uni_boundary, at_uni_non_boundary = _make_boundary(rsre_char.is_uni_word) # ____________________________________________________________ From getxsick at codespeak.net Sat Aug 7 17:16:32 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Sat, 7 Aug 2010 17:16:32 +0200 (CEST) Subject: [pypy-svn] r76520 - in pypy/branch/fast-ctypes/pypy: jit/backend/x86 jit/codewriter jit/metainterp rlib Message-ID: <20100807151632.4D789282B90@codespeak.net> Author: getxsick Date: Sat Aug 7 17:16:30 2010 New Revision: 76520 Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/regalloc.py pypy/branch/fast-ctypes/pypy/jit/codewriter/heaptracker.py pypy/branch/fast-ctypes/pypy/jit/metainterp/resoperation.py pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: some hacks to push translation process (thanks to antocuni) Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/regalloc.py Sat Aug 7 17:16:30 2010 @@ -637,6 +637,7 @@ def consider_call_assembler(self, op, guard_op): portal_calldescr = self.assembler.cpu.portal_calldescr + assert portal_calldescr is not None size = portal_calldescr.get_result_size(self.translate_support_code) # descr = op.descr Modified: pypy/branch/fast-ctypes/pypy/jit/codewriter/heaptracker.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/codewriter/heaptracker.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/codewriter/heaptracker.py Sat Aug 7 17:16:30 2010 @@ -85,6 +85,7 @@ # This is necessary because the 'vtables' are just pointers to # static data, so they can't be used as keys in prebuilt dicts. d = cpu._vtable_to_descr_dict + assert d is not None if d is None: d = cpu._vtable_to_descr_dict = {} for descr in cpu._all_size_descrs_with_vtable: Modified: pypy/branch/fast-ctypes/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/metainterp/resoperation.py Sat Aug 7 17:16:30 2010 @@ -5,7 +5,7 @@ """The central ResOperation class, representing one operation.""" # for 'guard_*' - fail_args = None + fail_args = [] # debug name = "" Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Sat Aug 7 17:16:30 2010 @@ -8,6 +8,9 @@ from pypy.jit.metainterp.typesystem import deref GLOBAL_CPU = get_cpu() +GLOBAL_CPU.portal_calldescr = None +GLOBAL_CPU._vtable_to_descr_dict = None +GLOBAL_CPU.setup() class CDLL(object): def __init__(self, name, load=True): From getxsick at codespeak.net Sat Aug 7 18:08:32 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Sat, 7 Aug 2010 18:08:32 +0200 (CEST) Subject: [pypy-svn] r76521 - pypy/branch/fast-ctypes/pypy/module/jitffi Message-ID: <20100807160832.81CA0282B90@codespeak.net> Author: getxsick Date: Sat Aug 7 18:08:31 2010 New Revision: 76521 Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Log: use interp2app for wrap_* Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Sat Aug 7 18:08:31 2010 @@ -71,13 +71,13 @@ if res_type == 'i': self.rget = rjitffi._Get(cpu, lib, func, args_type, - res_type, self.wrap_int, cache=True) + res_type, self.wrap_int_w, cache=True) elif res_type == 'f': self.rget = rjitffi._Get(cpu, lib, func, args_type, - res_type, self.wrap_float, cache=True) + res_type, self.wrap_float_w, cache=True) elif res_type == 'v': self.rget = rjitffi._Get(cpu, lib, func, args_type, - res_type, self.wrap_void, cache=True) + res_type, self.wrap_void_w, cache=True) else: raise OperationError( space.w_ValueError, @@ -119,9 +119,9 @@ i += 1 return self.rget.call() - wrap_int = lambda self, value: self.space.wrap(value) - wrap_float = lambda self, value: self.space.wrap(value) - wrap_void = lambda self, value: self.space.wrap(value) + wrap_int_w = lambda self, value: self.space.wrap(value) + wrap_float_w = lambda self, value: self.space.wrap(value) + wrap_void_w = lambda self, w_value: value #def W_Get___new__(space, w_type, cpu, lib, func, args_type, res_type): # try: @@ -132,5 +132,8 @@ W_Get.typedef = TypeDef( 'Get', #__new__ = interp2app(W_Get___new__, unwrap_spec=[ObjSpace, W_Root, W_Root, W_Root, str, W_Root, str]), - call = interp2app(W_Get.call_w, unwrap_spec=['self', ObjSpace, W_Root]) + call = interp2app(W_Get.call_w, unwrap_spec=['self', ObjSpace, W_Root]), + wrap_int = interp2app(W_Get.wrap_int_w, unwrap_spec=['self', int]), + wrap_float = interp2app(W_Get.wrap_float_w, unwrap_spec=['self', float]), + wrap_void = interp2app(W_Get.wrap_void_w, unwrap_spec=['self', W_Root]) ) From getxsick at codespeak.net Sat Aug 7 19:01:52 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Sat, 7 Aug 2010 19:01:52 +0200 (CEST) Subject: [pypy-svn] r76522 - in pypy/branch/fast-ctypes/pypy/jit: backend/x86 codewriter metainterp Message-ID: <20100807170152.14D6A282B90@codespeak.net> Author: getxsick Date: Sat Aug 7 19:01:50 2010 New Revision: 76522 Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/regalloc.py pypy/branch/fast-ctypes/pypy/jit/codewriter/heaptracker.py pypy/branch/fast-ctypes/pypy/jit/metainterp/resoperation.py Log: add XXX comments to hacks added in r76520 Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/regalloc.py Sat Aug 7 19:01:50 2010 @@ -637,7 +637,7 @@ def consider_call_assembler(self, op, guard_op): portal_calldescr = self.assembler.cpu.portal_calldescr - assert portal_calldescr is not None + assert portal_calldescr is not None # XXX is it correct? size = portal_calldescr.get_result_size(self.translate_support_code) # descr = op.descr Modified: pypy/branch/fast-ctypes/pypy/jit/codewriter/heaptracker.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/codewriter/heaptracker.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/codewriter/heaptracker.py Sat Aug 7 19:01:50 2010 @@ -85,7 +85,7 @@ # This is necessary because the 'vtables' are just pointers to # static data, so they can't be used as keys in prebuilt dicts. d = cpu._vtable_to_descr_dict - assert d is not None + assert d is not None # XXX this is wrong if d is None: d = cpu._vtable_to_descr_dict = {} for descr in cpu._all_size_descrs_with_vtable: Modified: pypy/branch/fast-ctypes/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/metainterp/resoperation.py Sat Aug 7 19:01:50 2010 @@ -5,7 +5,7 @@ """The central ResOperation class, representing one operation.""" # for 'guard_*' - fail_args = [] + fail_args = [] # XXX is it correct? # debug name = "" From getxsick at codespeak.net Sat Aug 7 20:35:53 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Sat, 7 Aug 2010 20:35:53 +0200 (CEST) Subject: [pypy-svn] r76523 - pypy/branch/fast-ctypes/pypy/rlib/test Message-ID: <20100807183553.402C7282B90@codespeak.net> Author: getxsick Date: Sat Aug 7 20:35:51 2010 New Revision: 76523 Modified: pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Log: add ztranslation test Modified: pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Sat Aug 7 20:35:51 2010 @@ -2,6 +2,8 @@ from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.translator.platform import platform from pypy.rpython.lltypesystem import rffi, lltype +from pypy.rpython.test.test_llinterp import interpret +from pypy.jit.backend.llsupport import descr import py @@ -186,3 +188,48 @@ py.test.raises(ValueError, lib.get, 'fvoid', ['i','xxxfoo888baryyy']) py.test.raises(ValueError, lib.get, 'fvoid', ['xxxfoo888baryyy'],'i') py.test.raises(ValueError, lib.get, 'fvoid', [], 'xxxfoo888baryyy') + +class TestTranslation(object): + @staticmethod + def preprare_c_example(): + from pypy.tool.udir import udir + c_file = udir.ensure("test_rjitffi", dir=True).join("xlib.c") + c_file.write(py.code.Source(''' + int add_int(int a, int b) + { + return a+b; + } + float add_float(float a, float b) + { + return a+b; + } + void ret_void(int a) + { + } + ''' + )) + + symbols = ['add_int', 'add_float', 'ret_void'] + eci = ExternalCompilationInfo(export_symbols=symbols) + + return str(platform.compile([c_file], eci, 'x', standalone=False)) + + def setup_class(cls): + cls.lib_name = cls.preprare_c_example() + + def test_get_calldescr(self): + lib = rjitffi.CDLL(self.lib_name) + def ret_int(): + func = lib.get('add_int', ['i', 'i'], 'i') + assert isinstance(func.get_calldescr(), descr.SignedCallDescr) + interpret(ret_int, []) + + def ret_float(): + func = lib.get('add_float', ['f', 'f'], 'f') + assert isinstance(func.get_calldescr(), descr.FloatCallDescr) + interpret(ret_float, []) + + def ret_void(): + func = lib.get('ret_void', ['i'], 'v') + assert isinstance(func.get_calldescr(), descr.VoidCallDescr) + interpret(ret_void, []) From getxsick at codespeak.net Sat Aug 7 20:36:16 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Sat, 7 Aug 2010 20:36:16 +0200 (CEST) Subject: [pypy-svn] r76524 - pypy/branch/fast-ctypes/pypy/rlib/test Message-ID: <20100807183616.0751F282B90@codespeak.net> Author: getxsick Date: Sat Aug 7 20:36:14 2010 New Revision: 76524 Modified: pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Log: typo Modified: pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Sat Aug 7 20:36:14 2010 @@ -11,7 +11,7 @@ @staticmethod def preprare_c_example(): from pypy.tool.udir import udir - c_file = udir.ensure("test_jitffi", dir=True).join("xlib.c") + c_file = udir.ensure("test_rjitffi", dir=True).join("xlib.c") c_file.write(py.code.Source(''' int add_integers(int a, int b) { From agaynor at codespeak.net Sun Aug 8 17:13:28 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sun, 8 Aug 2010 17:13:28 +0200 (CEST) Subject: [pypy-svn] r76526 - pypy/trunk/pypy/jit/metainterp/test Message-ID: <20100808151328.1E859282B90@codespeak.net> Author: agaynor Date: Sun Aug 8 17:13:26 2010 New Revision: 76526 Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Log: Remove a skip from a passing test. Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Sun Aug 8 17:13:26 2010 @@ -411,7 +411,6 @@ self.optimize_loop(ops, 'Not', expected) def test_int_is_true_1(self): - py.test.skip("XXX implement me") ops = """ [i0] i1 = int_is_true(i0) From hakanardo at codespeak.net Mon Aug 9 09:49:26 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Mon, 9 Aug 2010 09:49:26 +0200 (CEST) Subject: [pypy-svn] r76540 - in pypy/branch/interplevel-array/pypy: jit/tl module/array module/array/benchmark Message-ID: <20100809074926.B836C282B9C@codespeak.net> Author: hakanardo Date: Mon Aug 9 09:49:25 2010 New Revision: 76540 Modified: pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py pypy/branch/interplevel-array/pypy/module/array/benchmark/sumtst.py pypy/branch/interplevel-array/pypy/module/array/interp_array.py Log: Somewhat nicer? Modified: pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py (original) +++ pypy/branch/interplevel-array/pypy/jit/tl/pypyjit_demo.py Mon Aug 9 09:49:25 2010 @@ -42,7 +42,7 @@ def f(img): i=0 sa=0 - while i<4: + while i < img.__len__(): sa+=img[i] i+=1 return sa Modified: pypy/branch/interplevel-array/pypy/module/array/benchmark/sumtst.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/benchmark/sumtst.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/benchmark/sumtst.py Mon Aug 9 09:49:25 2010 @@ -1,5 +1,5 @@ #!/usr/bin/python -from array import array, simple_array +from array import array #img=array('d',(0,)*640*480); def f(img): @@ -10,12 +10,9 @@ i+=1 return l -if True: - img=array('d', '\x00'*640*480) - #img=array('d', [0]*640*480) - #img=array('d', (0,))*(640*480) -else: - img=simple_array(640*480) +img=array('d', (0,)) * (640*480) +#img=array('d', [0]*640*480) +#img=array('d', (0,))*(640*480) for l in range(500): f(img) #print f(img) Modified: pypy/branch/interplevel-array/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/array/interp_array.py (original) +++ pypy/branch/interplevel-array/pypy/module/array/interp_array.py Mon Aug 9 09:49:25 2010 @@ -13,7 +13,7 @@ from pypy.objspace.std.register_all import register_all from pypy.objspace.std.model import W_Object from pypy.interpreter.argument import Arguments, Signature - +from pypy.module._file.interp_file import W_File def w_array(space, w_cls, typecode, w_initializer=None, w_args=None): if len(w_args.arguments_w) > 0: @@ -502,7 +502,7 @@ return self.space.wrap(s) def array_fromfile__Array_ANY_ANY(space, self, w_f, w_n): - if space.type(w_f).name != 'file': + if not isinstance(w_f, W_File): msg = "arg1 must be open file" raise OperationError(space.w_TypeError, space.wrap(msg)) n = space.int_w(w_n) @@ -522,7 +522,7 @@ array_fromstring__Array_ANY(space, self, w_item) def array_tofile__Array_ANY(space, self, w_f): - if space.type(w_f).name != 'file': + if not isinstance(w_f, W_File): msg = "arg1 must be open file" raise OperationError(space.w_TypeError, space.wrap(msg)) w_s = array_tostring__Array(space, self) From arigo at codespeak.net Mon Aug 9 11:32:58 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 9 Aug 2010 11:32:58 +0200 (CEST) Subject: [pypy-svn] r76541 - in pypy/branch/rsre2/pypy: jit/metainterp/test rpython rpython/test Message-ID: <20100809093258.A36E1282B9C@codespeak.net> Author: arigo Date: Mon Aug 9 11:32:56 2010 New Revision: 76541 Modified: pypy/branch/rsre2/pypy/jit/metainterp/test/test_immutable.py pypy/branch/rsre2/pypy/rpython/rclass.py pypy/branch/rsre2/pypy/rpython/test/test_rclass.py Log: Fix _immutable_fields_ to account for fields that only show up on a subclass, depending on translation details. Modified: pypy/branch/rsre2/pypy/jit/metainterp/test/test_immutable.py ============================================================================== --- pypy/branch/rsre2/pypy/jit/metainterp/test/test_immutable.py (original) +++ pypy/branch/rsre2/pypy/jit/metainterp/test/test_immutable.py Mon Aug 9 11:32:56 2010 @@ -17,6 +17,38 @@ assert res == 28 self.check_operations_history(getfield_gc=0, getfield_gc_pure=1, int_add=1) + def test_fields_subclass(self): + class X(object): + _immutable_fields_ = ["x"] + + def __init__(self, x): + self.x = x + + class Y(X): + _immutable_fields_ = ["y"] + + def __init__(self, x, y): + X.__init__(self, x) + self.y = y + + def f(x, y): + X(x) # force the field 'x' to be on class 'X' + z = Y(x, y) + return z.x + z.y + 5 + res = self.interp_operations(f, [23, 11]) + assert res == 39 + self.check_operations_history(getfield_gc=0, getfield_gc_pure=2, + int_add=2) + + def f(x, y): + # this time, the field 'x' only shows up on subclass 'Y' + z = Y(x, y) + return z.x + z.y + 5 + res = self.interp_operations(f, [23, 11]) + assert res == 39 + self.check_operations_history(getfield_gc=0, getfield_gc_pure=2, + int_add=2) + def test_array(self): class X(object): _immutable_fields_ = ["y[*]"] Modified: pypy/branch/rsre2/pypy/rpython/rclass.py ============================================================================== --- pypy/branch/rsre2/pypy/rpython/rclass.py (original) +++ pypy/branch/rsre2/pypy/rpython/rclass.py Mon Aug 9 11:32:56 2010 @@ -156,9 +156,14 @@ if '_immutable_' in self.classdef.classdesc.classdict: hints = hints.copy() hints['immutable'] = True - if '_immutable_fields_' in self.classdef.classdesc.classdict: + if self.classdef.classdesc.lookup('_immutable_fields_') is not None: hints = hints.copy() - self.immutable_field_list = self.classdef.classdesc.classdict['_immutable_fields_'].value + immutable_fields = self.classdef.classdesc.classdict.get( + '_immutable_fields_') + if immutable_fields is not None: + self.immutable_field_list = immutable_fields.value + else: + self.immutable_field_list = [] accessor = FieldListAccessor() hints['immutable_fields'] = accessor return hints @@ -181,7 +186,13 @@ hints = self.object_type._hints if "immutable_fields" in hints: accessor = hints["immutable_fields"] - self._parse_field_list(self.immutable_field_list, accessor) + immutable_fields = {} + rbase = self + while rbase.classdef is not None: + immutable_fields.update( + dict.fromkeys(rbase.immutable_field_list)) + rbase = rbase.rbase + self._parse_field_list(immutable_fields, accessor) def _parse_field_list(self, fields, accessor): with_suffix = {} @@ -191,7 +202,10 @@ suffix = '[*]' else: suffix = '' - mangled_name, r = self._get_field(name) + try: + mangled_name, r = self._get_field(name) + except KeyError: + continue with_suffix[mangled_name] = suffix accessor.initialize(self.object_type, with_suffix) return with_suffix Modified: pypy/branch/rsre2/pypy/rpython/test/test_rclass.py ============================================================================== --- pypy/branch/rsre2/pypy/rpython/test/test_rclass.py (original) +++ pypy/branch/rsre2/pypy/rpython/test/test_rclass.py Mon Aug 9 11:32:56 2010 @@ -738,6 +738,45 @@ assert accessor.fields == {"inst_x" : "", "inst_y" : "[*]"} or \ accessor.fields == {"ox" : "", "oy" : "[*]"} # for ootype + def test_immutable_fields_subclass_1(self): + from pypy.jit.metainterp.typesystem import deref + class A(object): + _immutable_fields_ = ["x"] + def __init__(self, x): + self.x = x + class B(A): + def __init__(self, x, y): + A.__init__(self, x) + self.y = y + + def f(): + return B(3, 5) + t, typer, graph = self.gengraph(f, []) + B_TYPE = deref(graph.getreturnvar().concretetype) + accessor = B_TYPE._hints["immutable_fields"] + assert accessor.fields == {"inst_x" : ""} or \ + accessor.fields == {"ox" : ""} # for ootype + + def test_immutable_fields_subclass_2(self): + from pypy.jit.metainterp.typesystem import deref + class A(object): + _immutable_fields_ = ["x"] + def __init__(self, x): + self.x = x + class B(A): + _immutable_fields_ = ["y"] + def __init__(self, x, y): + A.__init__(self, x) + self.y = y + + def f(): + return B(3, 5) + t, typer, graph = self.gengraph(f, []) + B_TYPE = deref(graph.getreturnvar().concretetype) + accessor = B_TYPE._hints["immutable_fields"] + assert accessor.fields == {"inst_x" : "", "inst_y" : ""} or \ + accessor.fields == {"ox" : "", "oy" : ""} # for ootype + def test_immutable_inheritance(self): class I(object): def __init__(self, v): From arigo at codespeak.net Mon Aug 9 13:20:15 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 9 Aug 2010 13:20:15 +0200 (CEST) Subject: [pypy-svn] r76542 - pypy/trunk/pypy/rlib Message-ID: <20100809112015.4435A282B9C@codespeak.net> Author: arigo Date: Mon Aug 9 13:20:12 2010 New Revision: 76542 Modified: pypy/trunk/pypy/rlib/rmmap.py Log: Reported by mvt: some systems (some versions of OS/X?) complain if they are passed a non-zero address as the first argument to mmap(). Modified: pypy/trunk/pypy/rlib/rmmap.py ============================================================================== --- pypy/trunk/pypy/rlib/rmmap.py (original) +++ pypy/trunk/pypy/rlib/rmmap.py Mon Aug 9 13:20:12 2010 @@ -646,8 +646,14 @@ hintp = rffi.cast(PTR, hint.pos) res = c_mmap_safe(hintp, map_size, prot, flags, -1, 0) if res == rffi.cast(PTR, -1): - raise MemoryError - hint.pos += map_size + # some systems (some versions of OS/X?) complain if they + # are passed a non-zero address. Try again. + hintp = rffi.cast(PTR, 0) + res = c_mmap_safe(hintp, map_size, prot, flags, -1, 0) + if res == rffi.cast(PTR, -1): + raise MemoryError + else: + hint.pos += map_size return res alloc._annenforceargs_ = (int,) From jcreigh at codespeak.net Mon Aug 9 16:09:39 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Mon, 9 Aug 2010 16:09:39 +0200 (CEST) Subject: [pypy-svn] r76545 - pypy/build/bot2/pypybuildbot Message-ID: <20100809140939.53B90282B9C@codespeak.net> Author: jcreigh Date: Mon Aug 9 16:09:36 2010 New Revision: 76545 Modified: pypy/build/bot2/pypybuildbot/master.py Log: add buildbot target for 64-bit JIT Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Mon Aug 9 16:09:36 2010 @@ -72,8 +72,7 @@ lib_python=True, app_tests=True) -jit_translation_args = ['-Ojit', '--gc=hybrid', - '--gcrootfinder=asmgcc'] +jit_translation_args = ['-Ojit'] pypyJITTranslatedTestFactory = pypybuilds.Translated( translationArgs=jit_translation_args, @@ -121,6 +120,7 @@ STACKLESSAPPLVLFREEBSD64 = 'pypy-c-stackless-app-level-freebsd-7-x86-64' JITLINUX32 = "pypy-c-jit-linux-x86-32" +JITLINUX64 = "pypy-c-jit-linux-x86-64" OJITLINUX32 = "pypy-c-Ojit-no-jit-linux-x86-32" JITMACOSX32 = "pypy-c-jit-macosx-x86-32" JITWIN32 = "pypy-c-jit-win-x86-32" @@ -210,6 +210,12 @@ 'factory' : pypyJITTranslatedTestFactory, 'category' : 'jit', }, + {'name': JITLINUX64, + 'slavenames': ['tannit64'], + 'builddir': JITLINUX64, + 'factory': pypyJITTranslatedTestFactory, + 'category': 'jit', + }, {"name" : JITMACOSX32, "slavenames": ["minime"], 'builddir' : JITMACOSX32, From jcreigh at codespeak.net Mon Aug 9 16:32:12 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Mon, 9 Aug 2010 16:32:12 +0200 (CEST) Subject: [pypy-svn] r76546 - pypy/trunk/pypy/jit/backend/x86 Message-ID: <20100809143212.EBADE282B9C@codespeak.net> Author: jcreigh Date: Mon Aug 9 16:32:11 2010 New Revision: 76546 Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/regloc.py Log: hack to make 64-bit JIT translate (should really fix issue r76491 tried to fix) Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Mon Aug 9 16:32:11 2010 @@ -20,7 +20,7 @@ r12, r13, r14, r15, X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG, - RegLoc, StackLoc, + RegLoc, StackLoc, ConstFloatLoc, ImmedLoc, AddressLoc, imm) from pypy.rlib.objectmodel import we_are_translated, specialize @@ -818,8 +818,13 @@ for i in range(start, len(arglocs)): loc = arglocs[i] - # XXX: Should be much simplier to tell whether a location is a float! - if (isinstance(loc, RegLoc) and loc.is_xmm) or (loc.is_memory_reference() and loc.type == FLOAT): + # XXX: Should be much simplier to tell whether a location is a + # float! It's so ugly because we have to "guard" the access to + # .type with isinstance, since not all AssemblerLocation classes + # are "typed" + if ((isinstance(loc, RegLoc) and loc.is_xmm) or + (isinstance(loc, StackLoc) and loc.type == FLOAT) or + (isinstance(loc, ConstFloatLoc))): if len(unused_xmm) > 0: xmm_src_locs.append(loc) xmm_dst_locs.append(unused_xmm.pop()) Modified: pypy/trunk/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regloc.py Mon Aug 9 16:32:11 2010 @@ -154,7 +154,6 @@ _immutable_ = True width = 8 - type = FLOAT def __init__(self, address, const_id): self.value = address From getxsick at codespeak.net Mon Aug 9 18:26:47 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Mon, 9 Aug 2010 18:26:47 +0200 (CEST) Subject: [pypy-svn] r76547 - pypy/branch/fast-ctypes/pypy/rlib/test Message-ID: <20100809162647.D76B3282B9C@codespeak.net> Author: getxsick Date: Mon Aug 9 18:26:46 2010 New Revision: 76547 Modified: pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Log: use different names for testing C libs to avoid conflicts Modified: pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Mon Aug 9 18:26:46 2010 @@ -11,7 +11,7 @@ @staticmethod def preprare_c_example(): from pypy.tool.udir import udir - c_file = udir.ensure("test_rjitffi", dir=True).join("xlib.c") + c_file = udir.ensure("test_rjitffi", dir=True).join("xlib1.c") c_file.write(py.code.Source(''' int add_integers(int a, int b) { @@ -59,7 +59,7 @@ 'return_float', 'max3', 'fvoid', 'return_void'] eci = ExternalCompilationInfo(export_symbols=symbols) - return str(platform.compile([c_file], eci, 'x', standalone=False)) + return str(platform.compile([c_file], eci, 'x1', standalone=False)) def setup_class(cls): cls.lib_name = cls.preprare_c_example() @@ -193,7 +193,7 @@ @staticmethod def preprare_c_example(): from pypy.tool.udir import udir - c_file = udir.ensure("test_rjitffi", dir=True).join("xlib.c") + c_file = udir.ensure("test_rjitffi", dir=True).join("xlib2.c") c_file.write(py.code.Source(''' int add_int(int a, int b) { @@ -212,7 +212,7 @@ symbols = ['add_int', 'add_float', 'ret_void'] eci = ExternalCompilationInfo(export_symbols=symbols) - return str(platform.compile([c_file], eci, 'x', standalone=False)) + return str(platform.compile([c_file], eci, 'x2', standalone=False)) def setup_class(cls): cls.lib_name = cls.preprare_c_example() From getxsick at codespeak.net Mon Aug 9 18:28:28 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Mon, 9 Aug 2010 18:28:28 +0200 (CEST) Subject: [pypy-svn] r76548 - in pypy/branch/fast-ctypes/pypy/rlib: . test Message-ID: <20100809162828.A1420282B9C@codespeak.net> Author: getxsick Date: Mon Aug 9 18:28:27 2010 New Revision: 76548 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Log: add support for C pointers as func arguments Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Mon Aug 9 18:28:27 2010 @@ -61,6 +61,8 @@ bargs.append(BoxInt()) elif arg == 'f': bargs.append(BoxFloat()) + elif arg == 'p': + bargs.append(BoxInt()) else: raise ValueError(arg) @@ -120,6 +122,7 @@ self.cpu.set_future_value_int(self.esp, value) self.esp += 1 push_funcaddr = push_int + push_ref = push_int def push_float(self, value): self.cpu.set_future_value_float(self.esp, value) Modified: pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Mon Aug 9 18:28:27 2010 @@ -52,11 +52,16 @@ int c; c = a + b; } + int return_ptrvalue(int *a) + { + return *a; + } ''' )) symbols = ['add_integers', 'add_floats', 'add_intfloat', - 'return_float', 'max3', 'fvoid', 'return_void'] + 'return_float', 'max3', 'fvoid', 'return_void', + 'return_ptrvalue'] eci = ExternalCompilationInfo(export_symbols=symbols) return str(platform.compile([c_file], eci, 'x1', standalone=False)) @@ -167,6 +172,16 @@ func.push_float(1.3) assert func.call() == 1 + def test_ptrargs(self): + lib = rjitffi.CDLL(self.lib_name) + + func = lib.get('return_ptrvalue', ['p'], 'i', self.push_result) + intp = lltype.malloc(rffi.INTP.TO, 1, flavor='raw') + intp[0] = 5 + func.push_ref(rffi.cast(lltype.Signed, intp)) + lltype.free(intp, flavor='raw') + assert func.call() == 5 + def test_nocache(self): lib = rjitffi.CDLL(self.lib_name) From jcreigh at codespeak.net Mon Aug 9 19:07:40 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Mon, 9 Aug 2010 19:07:40 +0200 (CEST) Subject: [pypy-svn] r76549 - in pypy/branch/asmgcc-64/pypy: jit/backend/llsupport jit/backend/x86 jit/backend/x86/test rpython/memory/gctransform Message-ID: <20100809170740.3824E282BD4@codespeak.net> Author: jcreigh Date: Mon Aug 9 19:07:38 2010 New Revision: 76549 Modified: pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/gc.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/regalloc.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/regloc.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_gc_integration.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_zrpy_gc.py pypy/branch/asmgcc-64/pypy/rpython/memory/gctransform/asmgcroot.py Log: some work on trying to make asmgcc-64 play nice with the JIT Modified: pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/gc.py Mon Aug 9 19:07:38 2010 @@ -251,13 +251,25 @@ if oldgcmap: lltype.free(oldgcmap, flavor='raw') - def get_basic_shape(self): - return [chr(self.LOC_EBP_PLUS | 4), # return addr: at 4(%ebp) - chr(self.LOC_EBP_MINUS | 4), # saved %ebx: at -4(%ebp) - chr(self.LOC_EBP_MINUS | 8), # saved %esi: at -8(%ebp) - chr(self.LOC_EBP_MINUS | 12), # saved %edi: at -12(%ebp) - chr(self.LOC_EBP_PLUS | 0), # saved %ebp: at (%ebp) - chr(0)] + def get_basic_shape(self, is_64_bit=False): + # XXX: Should this code even really know about stack frame layout of + # the JIT? + if is_64_bit: + return [chr(self.LOC_EBP_PLUS | 8), + chr(self.LOC_EBP_MINUS | 8), + chr(self.LOC_EBP_MINUS | 16), + chr(self.LOC_EBP_MINUS | 24), + chr(self.LOC_EBP_MINUS | 32), + chr(self.LOC_EBP_MINUS | 40), + chr(self.LOC_EBP_PLUS | 0), + chr(0)] + else: + return [chr(self.LOC_EBP_PLUS | 4), # return addr: at 4(%ebp) + chr(self.LOC_EBP_MINUS | 4), # saved %ebx: at -4(%ebp) + chr(self.LOC_EBP_MINUS | 8), # saved %esi: at -8(%ebp) + chr(self.LOC_EBP_MINUS | 12), # saved %edi: at -12(%ebp) + chr(self.LOC_EBP_PLUS | 0), # saved %ebp: at (%ebp) + chr(0)] def _encode_num(self, shape, number): assert number >= 0 @@ -276,6 +288,9 @@ num = self.LOC_EBP_MINUS | (-offset) self._encode_num(shape, num) + def add_callee_save_reg(self, shape, reg_index): + shape.append(chr(self.LOC_REG | (reg_index << 2))) + def add_ebx(self, shape): shape.append(chr(self.LOC_REG | 4)) Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/regalloc.py Mon Aug 9 19:07:38 2010 @@ -26,6 +26,12 @@ no_lower_byte_regs = [esi, edi] save_around_call_regs = [eax, edx, ecx] + REGLOC_TO_GCROOTMAP_REG_INDEX = { + ebx: 1, + esi: 2, + edi: 3, + } + def call_result_location(self, v): return eax @@ -47,6 +53,13 @@ no_lower_byte_regs = [] save_around_call_regs = [eax, ecx, edx, esi, edi, r8, r9, r10] + REGLOC_TO_GCROOTMAP_REG_INDEX = { + ebx: 1, + r12: 2, + r13: 3, + r14: 4, + r15: 5, + } class FloatConstants(object): BASE_CONSTANT_SIZE = 1000 @@ -695,21 +708,14 @@ gc_ll_descr = self.assembler.cpu.gc_ll_descr tmp0 = TempBox() self.rm.force_allocate_reg(op.result, selected_reg=eax) - self.rm.force_allocate_reg(tmp0, selected_reg=edx) - # XXX about the next 10 lines: why not just say - # force_allocate_reg(tmp1, selected_reg=ecx)????? - for v, reg in self.rm.reg_bindings.items(): - if reg is ecx: - to_sync = v - break - else: - to_sync = None - if to_sync is not None: - self.rm._sync_var(to_sync) - del self.rm.reg_bindings[to_sync] - self.rm.free_regs.append(ecx) - # we need to do it here, so edx is not in reg_bindings - self.rm.possibly_free_var(tmp0) + for reg in self.rm.save_around_call_regs: + if reg is not eax: + # FIXME: Is this the right way to spill a register? And do we + # need to do this for all non-callee-save regs? + tmp_box = TempBox() + self.rm.force_allocate_reg(tmp_box, selected_reg=reg) + self.rm.possibly_free_var(tmp_box) + self.assembler.malloc_cond_fixedsize( gc_ll_descr.get_nursery_free_addr(), gc_ll_descr.get_nursery_top_addr(), @@ -961,7 +967,7 @@ pass def get_mark_gc_roots(self, gcrootmap): - shape = gcrootmap.get_basic_shape() + shape = gcrootmap.get_basic_shape(IS_X86_64) for v, val in self.fm.frame_bindings.items(): if (isinstance(v, BoxPtr) and self.rm.stays_alive(v)): assert isinstance(val, StackLoc) @@ -970,15 +976,8 @@ if reg is eax: continue # ok to ignore this one if (isinstance(v, BoxPtr) and self.rm.stays_alive(v)): - if reg is ebx: - gcrootmap.add_ebx(shape) - elif reg is esi: - gcrootmap.add_esi(shape) - elif reg is edi: - gcrootmap.add_edi(shape) - else: - print "[get_mark_gc_roots] bogus register", reg - assert False + assert reg in self.rm.REGLOC_TO_GCROOTMAP_REG_INDEX + gcrootmap.add_callee_save_reg(shape, self.rm.REGLOC_TO_GCROOTMAP_REG_INDEX[reg]) return gcrootmap.compress_callshape(shape) def consider_force_token(self, op): Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/regloc.py Mon Aug 9 19:07:38 2010 @@ -1,7 +1,7 @@ from pypy.jit.metainterp.history import AbstractValue, ConstInt from pypy.jit.backend.x86 import rx86 from pypy.rlib.unroll import unrolling_iterable -from pypy.jit.backend.x86.arch import WORD +from pypy.jit.backend.x86.arch import WORD, IS_X86_32, IS_X86_64 from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.objectmodel import specialize Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_gc_integration.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_gc_integration.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_gc_integration.py Mon Aug 9 19:07:38 2010 @@ -26,16 +26,13 @@ CPU = getcpuclass() class MockGcRootMap(object): - def get_basic_shape(self): + def get_basic_shape(self, is_64_bit): return ['shape'] def add_ebp_offset(self, shape, offset): shape.append(offset) - def add_ebx(self, shape): - shape.append('ebx') - def add_esi(self, shape): - shape.append('esi') - def add_edi(self, shape): - shape.append('edi') + def add_callee_save_reg(self, shape, reg_index): + index_to_name = { 1: 'ebx', 2: 'esi', 3: 'edi' } + shape.append(index_to_name[reg_index]) def compress_callshape(self, shape): assert shape[0] == 'shape' return ['compressed'] + shape[1:] Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_zrpy_gc.py Mon Aug 9 19:07:38 2010 @@ -128,10 +128,6 @@ class TestCompileHybrid(object): def setup_class(cls): - if IS_X86_64: - # No hybrid GC on 64-bit for the time being - py.test.skip() - funcs = [] name_to_func = {} for fullname in dir(cls): Modified: pypy/branch/asmgcc-64/pypy/rpython/memory/gctransform/asmgcroot.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rpython/memory/gctransform/asmgcroot.py (original) +++ pypy/branch/asmgcc-64/pypy/rpython/memory/gctransform/asmgcroot.py Mon Aug 9 19:07:38 2010 @@ -18,6 +18,7 @@ # The .s file produced by GCC is then parsed by trackgcroot.py. # +IS_64_BITS = sys.maxint > 2147483647 class AsmGcRootFrameworkGCTransformer(FrameworkGCTransformer): _asmgcc_save_restore_arguments = None @@ -465,16 +466,15 @@ # - frame address (actually the addr of the retaddr of the current function; # that's the last word of the frame in memory) # -""" -CALLEE_SAVED_REGS = 4 # there are 4 callee-saved registers -INDEX_OF_EBP = 3 -FRAME_PTR = CALLEE_SAVED_REGS # the frame is at index 4 in the array -""" - -# FIXME: hard-coded for 64-bit -CALLEE_SAVED_REGS = 6 -INDEX_OF_EBP = 5 -FRAME_PTR = CALLEE_SAVED_REGS + +if IS_64_BITS: + CALLEE_SAVED_REGS = 6 + INDEX_OF_EBP = 5 + FRAME_PTR = CALLEE_SAVED_REGS +else: + CALLEE_SAVED_REGS = 4 # there are 4 callee-saved registers + INDEX_OF_EBP = 3 + FRAME_PTR = CALLEE_SAVED_REGS # the frame is at index 4 in the array ASM_CALLBACK_PTR = lltype.Ptr(lltype.FuncType([], lltype.Void)) From jcreigh at codespeak.net Mon Aug 9 19:26:21 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Mon, 9 Aug 2010 19:26:21 +0200 (CEST) Subject: [pypy-svn] r76550 - pypy/branch/asmgcc-64/pypy/config Message-ID: <20100809172621.D4808282BD4@codespeak.net> Author: jcreigh Date: Mon Aug 9 19:26:20 2010 New Revision: 76550 Modified: pypy/branch/asmgcc-64/pypy/config/translationoption.py Log: Don't force boehm when translating 64-bit JIT Modified: pypy/branch/asmgcc-64/pypy/config/translationoption.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/config/translationoption.py (original) +++ pypy/branch/asmgcc-64/pypy/config/translationoption.py Mon Aug 9 19:26:20 2010 @@ -342,10 +342,6 @@ 'jit': 'hybrid extraopts jit', } -# For now, 64-bit JIT requires boehm -if IS_64_BITS: - OPT_TABLE['jit'] = OPT_TABLE['jit'].replace('hybrid', 'boehm') - def set_opt_level(config, level): """Apply optimization suggestions on the 'config'. The optimizations depend on the selected level and possibly on the backend. From getxsick at codespeak.net Mon Aug 9 19:27:01 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Mon, 9 Aug 2010 19:27:01 +0200 (CEST) Subject: [pypy-svn] r76551 - pypy/branch/fast-ctypes/pypy/rlib/test Message-ID: <20100809172701.37381282BD4@codespeak.net> Author: getxsick Date: Mon Aug 9 19:26:59 2010 New Revision: 76551 Modified: pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Log: more tests for C pointers Modified: pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Mon Aug 9 19:26:59 2010 @@ -52,16 +52,34 @@ int c; c = a + b; } - int return_ptrvalue(int *a) + int return_ptrvalue(int a, int *b) { - return *a; + return a+(*b); + } + int sum_intarray(int *a) + { + int i; + int sum = 0; + for(i=0; i<5; i++) + { + sum += *a+i; + } + return sum; + } + void a2b(char *txt) + { + int i; + for(i=0; txt[i] != '\0'; i++) + { + if (txt[i] == 'a') txt[i] = 'b'; + } } ''' )) symbols = ['add_integers', 'add_floats', 'add_intfloat', 'return_float', 'max3', 'fvoid', 'return_void', - 'return_ptrvalue'] + 'return_ptrvalue', 'sum_intarray', 'a2b'] eci = ExternalCompilationInfo(export_symbols=symbols) return str(platform.compile([c_file], eci, 'x1', standalone=False)) @@ -175,12 +193,34 @@ def test_ptrargs(self): lib = rjitffi.CDLL(self.lib_name) - func = lib.get('return_ptrvalue', ['p'], 'i', self.push_result) - intp = lltype.malloc(rffi.INTP.TO, 1, flavor='raw') - intp[0] = 5 - func.push_ref(rffi.cast(lltype.Signed, intp)) - lltype.free(intp, flavor='raw') - assert func.call() == 5 + func = lib.get('return_ptrvalue', ['i', 'p'], 'i', self.push_result) + func.push_int(20) + try: + intp = lltype.malloc(rffi.INTP.TO, 1, flavor='raw') + intp[0] = 10 + func.push_ref(rffi.cast(lltype.Signed, intp)) + assert func.call() == 30 + finally: + lltype.free(intp, flavor='raw') + + func = lib.get('sum_intarray', ['p'], 'i', self.push_result) + intp = lltype.malloc(rffi.INTP.TO, 5, flavor='raw') + try: + for i in xrange(5): + intp[i] = i + func.push_ref(rffi.cast(lltype.Signed, intp)) + assert func.call() == 10 + finally: + lltype.free(intp, flavor='raw') + + func = lib.get('a2b', ['p'], push_result=self.push_result) + charp = rffi.str2charp('xaxaxa') + try: + func.push_ref(rffi.cast(lltype.Signed, charp)) + func.call() + assert rffi.charp2str(charp) == 'xbxbxb' + finally: + rffi.free_charp(charp) def test_nocache(self): lib = rjitffi.CDLL(self.lib_name) From dan at codespeak.net Tue Aug 10 09:07:24 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Tue, 10 Aug 2010 09:07:24 +0200 (CEST) Subject: [pypy-svn] r76552 - in pypy/branch/micronumpy/pypy/module/micronumpy: . test Message-ID: <20100810070724.2E08B282B9C@codespeak.net> Author: dan Date: Tue Aug 10 09:07:22 2010 New Revision: 76552 Modified: pypy/branch/micronumpy/pypy/module/micronumpy/array.py pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py Log: Almost passing everything again, slicing almost done. Modified: pypy/branch/micronumpy/pypy/module/micronumpy/array.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/array.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/array.py Tue Aug 10 09:07:22 2010 @@ -20,6 +20,59 @@ i -= 1 return stride +def size_from_shape(shape): + size = 1 + for dimension in shape: + size *= dimension + return size + +def normalize_slice_starts(starts, shape): + result = starts[:] + for i in range(len(result)): + if result[i] < 0: + result[i] += shape[i] + elif result[i] >= shape[i]: + raise IndexError("invalid index") + return result + +def squeeze_slice(current_shape, starts, shape, step): + current_shape = current_shape[:] + i = 0 + stop = len(shape) + while i < stop: + if shape[i] == 1: + if i == 0: + offset += starts[i] # FIXME: eh? + offset *= current_shape[i] + else: + step[i-1] += starts[i] # FIXME: eh? + step[i-1] *= current_shape[i] + + del current_shape[i] + del starts[i] # XXX: I think this needs to be incorporated... + del shape[i] + del step[i] + else: + i += 1 + return offset + +def shape_prefix(shape): + prefix = 0 + try: + while shape[prefix] == 1: prefix += 1 + except IndexError, e: + prefix = len(shape) # XXX - 1? + + return prefix + +def shape_suffix(shape): + suffix = len(shape) + try: + while shape[suffix - 1] == 1: suffix -= 1 + except IndexError, e: + suffix = 0 + return suffix + def validate_index(array, space, w_i): index_dimensionality = space.int_w(space.len(w_i)) array_dimensionality = len(array.shape) Modified: pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py Tue Aug 10 09:07:22 2010 @@ -9,17 +9,12 @@ from pypy.interpreter.gateway import NoneNotWrapped from pypy.module import micronumpy -from pypy.module.micronumpy.array import BaseNumArray -from pypy.module.micronumpy.array import infer_shape from pypy.module.micronumpy.dtype import null_data -from pypy.module.micronumpy.array import stride_row, stride_column - -def size_from_shape(shape): - size = 1 - for dimension in shape: - size *= dimension - return size +from pypy.module.micronumpy.array import BaseNumArray +from pypy.module.micronumpy.array import infer_shape +from pypy.module.micronumpy.array import stride_row, stride_column, size_from_shape +from pypy.module.micronumpy.array import normalize_slice_starts def index_w(space, w_index): return space.int_w(space.index(w_index)) @@ -28,24 +23,27 @@ _immutable_fields_ = ['array'] def __init__(self, array): self.array = array - self.i = array.offsets[0] - + self.i = array.slice_starts[0] + self.step = array.slice_steps[0] + self.stop = self.i + array.shape[0] + self.ndim = len(self.array.shape) def descr_iter(self, space): return space.wrap(self) descr_iter.unwrap_spec = ['self', ObjSpace] def descr_next(self, space): - if self.i < self.array.shape[0]: - if len(self.array.shape) > 1: + if self.i < self.stop: + if self.ndim > 1: ar = MicroArray(self.array.shape[1:], self.array.dtype, parent=self.array, strides=self.array.strides[1:], - offsets=self.array.offsets + [self.i]) + slice_starts=self.array.offsets + [self.i], + slice_steps=self.array.slice_steps[1:]) self.i += 1 return space.wrap(ar) - elif len(self.array.shape) == 1: - next = self.array.getitem(space, self.array.flatten_index([self.i])) + elif self.ndim == 1: + next = self.array.getitem(space, self.array.offset + self.array.flatten_index([self.i])) self.i += 1 return next else: @@ -111,14 +109,12 @@ return space.wrap(self.shape[0]) descr_len.unwrap_spec = ['self', ObjSpace] - def absolute_offset(self, index): - return self.offset + index # XXX: need more descriptive name - - def getitem(self, space, index): + def getitem(self, space, offset): + """Helper function. + Grabs a value at an offset into the data.""" try: dtype = self.dtype.dtype #XXX: kinda ugly - return dtype.w_getitem(space, self.data, - self.absolute_offset(index)) + return dtype.w_getitem(space, self.data, offset) except IndexError, e: raise OperationError(space.w_IndexError, space.wrap("index out of bounds")) @@ -127,24 +123,9 @@ dtype = self.dtype.dtype #XXX: kinda ugly dtype.w_setitem(space, self.data, index, w_value) #maybe hang onto w_dtype separately? - def flatten_applevel_index(self, space, w_index): - try: - index = space.int_w(w_index) - return index - except OperationError, e: - if not e.match(space, space.w_TypeError): - raise - - index = space.fixedview(w_index) - index = [index_w(space, w_x) for w_x in index] - - # Normalize indices - for i in range(len(index)): - if index[i] < 0: - index[i] = self.shape[i] + index[i] - return self.flatten_index(index) - def flatten_index(self, index): + """Computes offset into subarray from all information. + Gives offset into subarray, not into data.""" offset = 0 for i in range(len(index)): offset += (self.slice_starts[i] + self.slice_steps[i] * index[i]) * self.strides[i] @@ -172,8 +153,9 @@ dtype = self.dtype.dtype try: index = space.int_w(space.index(w_index)) - return (self.flatten_index([index]), - self.slice_starts[1:], self.shape[1:], self.slice_steps[1:]) + slice_starts = self.slice_starts[:] + slice_starts[0] += index + return slice_starts, self.shape[1:], self.slice_steps[1:] except OperationError, e: if e.match(space, space.w_TypeError): pass else:raise @@ -217,28 +199,25 @@ raise OperationError(space.w_NotImplementedError, space.wrap("Don't support records yet.")) # XXX: and we may never - prefix = 0 - while shape[prefix] == 1: prefix += 1 + try: + slice_starts = normalize_slice_starts(slice_starts, self.shape) # XXX: could make in-place + except IndexError, e: + #print "starts=%s, shape=%s" % (slice_starts, shape) + raise OperationError(space.w_IndexError, + space.wrap("invalid index")) - index = 0 - for offset in slice_starts[:prefix]: - index += self.strides[offset] + return slice_starts, shape, slice_steps - return (self.flatten_index(slice_starts[:prefix]), - slice_starts[prefix:], shape[prefix:], slice_steps[prefix:]) - - except IndexError, e: - raise OperationError(space.w_IndexError, - space.wrap("invalid index")) + finally: pass def descr_getitem(self, space, w_index): - offset, slice_starts, shape, slice_steps = self.index2slices(space, w_index) + slice_starts, shape, slice_steps = self.index2slices(space, w_index) size = size_from_shape(shape) if size == 1: return self.getitem(space, - self.flatten_index(slice_starts)) + self.offset + self.flatten_index(slice_starts)) else: ar = MicroArray(shape, dtype=dtype, @@ -247,19 +226,20 @@ slice_starts=slice_starts, # XXX: begin renaming slice_starts slice_steps=slice_steps) return space.wrap(ar) + descr_getitem.unwrap_spec = ['self', ObjSpace, W_Root] - def set_slice_single_value(self, space, shape, offsets, steps, w_value): + def set_slice_single_value(self, space, offset, shape, offsets, steps, w_value): index = offsets if len(shape) > 1: for i in shape[0]: - self.set_slice(space, shape[1:], index, steps, w_value) - index[len(shape)-1] += 1 # XXX: reason + self.set_slice_single_value(space, offset, shape[1:], index, steps[1:], w_value) + index[len(index) - len(shape)] += steps[0] # XXX: reason else: dtype = self.dtype.dtype for i in range(0, shape[0]): - dtype.w_setitem(space, data, - self.flatten_index(index), - value) + self.set_slice_single_value(self, space, + offset, shape, index, steps, + w_value) index[len(index)-1] += steps[0] # XXX: don't do steps in range def set_slice(self, space, shape, offsets, steps, w_value): @@ -271,27 +251,41 @@ if length == 1: self.set_slice_single_value(space, shape, offsets, steps, w_value) else: - value_shape = infer_shape(space, w_value) - if value_shape != shape: - raise OperationError(space.w_ValueError, - space.wrap("shape mismatch: objects cannot" - " be broadcast to a single shape")) raise OperationError(space.w_NotImplementedError, space.wrap("TODO")) def descr_setitem(self, space, w_index, w_value): dtype = self.dtype.dtype - offset, slice_starts, shape, slice_steps = self.index2slices(space, w_index) + slice_starts, shape, slice_steps = self.index2slices(space, w_index) size = size_from_shape(shape) if size == 1: self.setitem(space, - offset + selfflatten_index(slice_starts), # XXX: yeah? + self.offset + self.flatten_index(slice_starts), w_value) else: - self.set_slice(space, shape, slice_starts, slice_steps, w_value) + try: + if space.int_w(space.len(w_value)) == 1: + w_value = space.getitem(w_value, space.wrap(0)) + value_shape = infer_shape(space, w_value) + value_size = size_from_shape(value_shape) + + except OperationError, e: + if e.match(space, space.w_TypeError): + value_size = 1 + else: raise + + if value_size == 1: + pass # TODO: set one value many times + + if value_shape != shape: + raise OperationError(space.w_ValueError, + space.wrap("shape mismatch: objects cannot" + " be broadcast to a single shape")) + else: + self.set_slice(space, offset, shape, slice_starts, slice_steps, w_value) descr_setitem.unwrap_spec = ['self', ObjSpace, W_Root, W_Root] Modified: pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py Tue Aug 10 09:07:22 2010 @@ -400,8 +400,41 @@ assert typecode == infer_from_iterable(space, w_xs).typecode class TestMicroArray(object): + def test_index2strides(self, space): + from pypy.module.micronumpy.microarray import MicroArray + from pypy.module.micronumpy.dtype import int_descr + + w_int_descr = int_descr.wrappable_dtype() + + ar = MicroArray(shape=[2, 3, 4], + dtype=w_int_descr) + + w_index = space.wrap([0, 0, 1]) + start, shape, step = ar.index2slices(space, w_index) + + assert start == [0, 0, 1] + assert shape == [1, 1, 1] + assert step == [1, 1, 1] + + ar = MicroArray(shape=[5, 4, 3, 2], + dtype=w_int_descr) + + w_index = space.wrap([1, Ellipsis, Ellipsis, Ellipsis]) + start, shape, step = ar.index2slices(space, w_index) + + assert start == [1, 0, 0, 0] + assert shape == [1, 4, 3, 2] + assert step == [1, 1, 1, 1] + + w_index = space.wrap([Ellipsis, 2, Ellipsis, Ellipsis]) + start, shape, step = ar.index2slices(space, w_index) + + assert start == [0, 2, 0, 0] + assert shape == [5, 1, 3, 2] + assert step == [1, 1, 1, 1] + def test_strides(self, space): - from pypy.module.micronumpy.microarray import stride_row, stride_column + from pypy.module.micronumpy.array import stride_row, stride_column shape = (2, 3) assert stride_row(shape, 0) == 3 From dan at codespeak.net Tue Aug 10 10:06:30 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Tue, 10 Aug 2010 10:06:30 +0200 (CEST) Subject: [pypy-svn] r76553 - in pypy/branch/micronumpy/pypy/module/micronumpy: . test Message-ID: <20100810080630.930B1282B9C@codespeak.net> Author: dan Date: Tue Aug 10 10:06:29 2010 New Revision: 76553 Modified: pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py Log: Passing everything again, woo! Modified: pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py Tue Aug 10 10:06:29 2010 @@ -20,13 +20,14 @@ return space.int_w(space.index(w_index)) class MicroIter(Wrappable): - _immutable_fields_ = ['array'] + _immutable_fields_ = ['array', 'step', 'stop', 'ndim'] def __init__(self, array): self.array = array self.i = array.slice_starts[0] self.step = array.slice_steps[0] self.stop = self.i + array.shape[0] self.ndim = len(self.array.shape) + def descr_iter(self, space): return space.wrap(self) descr_iter.unwrap_spec = ['self', ObjSpace] @@ -53,18 +54,6 @@ raise OperationError(space.w_StopIteration, space.wrap("")) descr_next.unwrap_spec = ['self', ObjSpace] - def __str__(self): - from pypy.rlib.rStringIO import RStringIO as StringIO - out = StringIO() - out.write('iterator(i=') - out.write(str(self.i)) - out.write(', array=') - out.write(repr(self.array)) - out.write(')') - result = out.getvalue() - out.close() - return result - MicroIter.typedef = TypeDef('iterator', __iter__ = interp2app(MicroIter.descr_iter), next = interp2app(MicroIter.descr_next), @@ -93,7 +82,7 @@ size = size_from_shape(shape) - self.strides = strides + self.strides = strides[:] stridelen = len(self.strides) for i in range(len(self.shape) - stridelen): self.strides.append(self.stride(stridelen + i)) # XXX calling self.stride repeatedly is a bit wasteful @@ -202,7 +191,6 @@ try: slice_starts = normalize_slice_starts(slice_starts, self.shape) # XXX: could make in-place except IndexError, e: - #print "starts=%s, shape=%s" % (slice_starts, shape) raise OperationError(space.w_IndexError, space.wrap("invalid index")) Modified: pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py Tue Aug 10 10:06:29 2010 @@ -402,9 +402,7 @@ class TestMicroArray(object): def test_index2strides(self, space): from pypy.module.micronumpy.microarray import MicroArray - from pypy.module.micronumpy.dtype import int_descr - - w_int_descr = int_descr.wrappable_dtype() + from pypy.module.micronumpy.dtype import w_int_descr ar = MicroArray(shape=[2, 3, 4], dtype=w_int_descr) @@ -436,22 +434,45 @@ def test_strides(self, space): from pypy.module.micronumpy.array import stride_row, stride_column - shape = (2, 3) - assert stride_row(shape, 0) == 3 - assert stride_row(shape, 1) == 1 + def strides(shape, f): + result = [0] * len(shape) + for i in range(len(shape)): + result[i] = f(shape, i) + return result - assert stride_column(shape, 0) == 1 - assert stride_column(shape, 1) == 2 + row_strides = lambda shape: strides(shape, stride_row) + column_strides = lambda shape: strides(shape, stride_column) + + shape = (2, 3) + assert row_strides(shape) == [3, 1] + assert column_strides(shape) == [1, 2] shape = (5, 4, 3) - assert stride_row(shape, 0) == 12 - assert stride_row(shape, 1) == 3 - assert stride_row(shape, 2) == 1 - - assert stride_column(shape, 0) == 1 - assert stride_column(shape, 1) == 5 - assert stride_column(shape, 2) == 20 - # TODO: more, N-dimensional too + assert row_strides(shape) == [12, 3, 1] + assert column_strides(shape) == [1, 5, 20] + + shape = (7, 5, 3, 2) + assert row_strides(shape) == [30, 6, 2, 1] + assert column_strides(shape) == [1, 7, 35, 105] + + def test_flatten_index(self, space): + from pypy.module.micronumpy.microarray import MicroArray + from pypy.module.micronumpy.dtype import w_int_descr + + ar = MicroArray(shape=[7, 5, 3, 2], + dtype=w_int_descr) + + offset = ar.flatten_index([0, 0, 0, 0]) + assert offset == 0 + + offset = ar.flatten_index([0, 0, 0, 1]) + assert offset == 1 + + offset = ar.flatten_index([0, 0, 1, 1]) + assert offset == 3 + + offset = ar.flatten_index([0, 2, 0, 1]) + assert offset == 13 def test_memory_layout(self, space): from pypy.module.micronumpy.microarray import MicroArray From getxsick at codespeak.net Tue Aug 10 14:19:48 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Tue, 10 Aug 2010 14:19:48 +0200 (CEST) Subject: [pypy-svn] r76554 - in pypy/branch/fast-ctypes/pypy/rlib: . test Message-ID: <20100810121948.548EF282B9C@codespeak.net> Author: getxsick Date: Tue Aug 10 14:19:46 2010 New Revision: 76554 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Log: add support for C functions returning pointers Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Tue Aug 10 14:19:46 2010 @@ -70,6 +70,8 @@ bres = BoxInt() elif self.res_type == 'f': bres = BoxFloat() + elif self.res_type == 'p': + bres = BoxInt() elif self.res_type == 'v': bres = NULLBOX else: @@ -89,6 +91,8 @@ cls = descr.SignedCallDescr elif self.res_type == 'f': cls = descr.FloatCallDescr + elif self.res_type == 'p': + cls = descr.SignedCallDescr elif self.res_type == 'v': cls = descr.VoidCallDescr else: @@ -106,6 +110,8 @@ r = self.push_result(self.cpu.get_latest_value_int(0)) elif self.res_type == 'f': r = self.push_result(self.cpu.get_latest_value_float(0)) + elif self.res_type == 'p': + r = self.push_result(self.cpu.get_latest_value_int(0)) elif self.res_type == 'v': r = None else: Modified: pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Tue Aug 10 14:19:46 2010 @@ -74,12 +74,18 @@ if (txt[i] == 'a') txt[i] = 'b'; } } + int *return_intptr(int a) + { + int *x = malloc(sizeof(int)); + *x = a; + return x; + } ''' )) symbols = ['add_integers', 'add_floats', 'add_intfloat', 'return_float', 'max3', 'fvoid', 'return_void', - 'return_ptrvalue', 'sum_intarray', 'a2b'] + 'return_ptrvalue', 'sum_intarray', 'a2b', 'return_intptr'] eci = ExternalCompilationInfo(export_symbols=symbols) return str(platform.compile([c_file], eci, 'x1', standalone=False)) @@ -222,6 +228,15 @@ finally: rffi.free_charp(charp) + def test_get_ptr(self): + lib = rjitffi.CDLL(self.lib_name) + + func = lib.get('return_intptr', ['i'], 'p', self.push_result) + func.push_int(22) + addr = func.call() + ret = rffi.cast(rffi.INTP, addr) + assert ret[0] == 22 + def test_nocache(self): lib = rjitffi.CDLL(self.lib_name) From fijal at codespeak.net Tue Aug 10 14:41:49 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 10 Aug 2010 14:41:49 +0200 (CEST) Subject: [pypy-svn] r76555 - pypy/trunk/pypy/module/_demo Message-ID: <20100810124149.CA3A7282B9C@codespeak.net> Author: fijal Date: Tue Aug 10 14:41:48 2010 New Revision: 76555 Modified: pypy/trunk/pypy/module/_demo/demo.py Log: An (incomplete) approach to fix translation of demo - it needs tests badly Modified: pypy/trunk/pypy/module/_demo/demo.py ============================================================================== --- pypy/trunk/pypy/module/_demo/demo.py (original) +++ pypy/trunk/pypy/module/_demo/demo.py Tue Aug 10 14:41:48 2010 @@ -4,11 +4,14 @@ from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.tool import rffi_platform +from pypy.translator.tool.cbuild import ExternalCompilationInfo import sys, math time_t = rffi_platform.getsimpletype('time_t', '#include ', rffi.LONG) -time = rffi.llexternal('time', [rffi.VOIDP], time_t, includes=['time.h']) +eci = ExternalCompilationInfo(includes=['time.h']) +time = rffi.llexternal('time', [int], time_t, + compilation_info=eci) def get(space, name): w_module = space.getbuiltinmodule('_demo') @@ -20,10 +23,10 @@ w_DemoError = get(space, 'DemoError') msg = "repetition count must be > 0" raise OperationError(w_DemoError, space.wrap(msg)) - starttime = time(None) + starttime = time(0) for i in range(repetitions): space.call_function(w_callable) - endtime = time(None) + endtime = time(0) return space.wrap(endtime - starttime) measuretime.unwrap_spec = [ObjSpace, int, W_Root] @@ -62,11 +65,16 @@ self.x = space.int_w(w_value) def mytype_new(space, w_subtype, x): + if x == 3: + return space.wrap(MySubType(space, x)) return space.wrap(W_MyType(space, x)) mytype_new.unwrap_spec = [ObjSpace, W_Root, int] getset_x = GetSetProperty(W_MyType.fget_x, W_MyType.fset_x, cls=W_MyType) +class MySubType(W_MyType): + pass + W_MyType.typedef = TypeDef('MyType', __new__ = interp2app(mytype_new), x = getset_x, From fijal at codespeak.net Tue Aug 10 14:58:08 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 10 Aug 2010 14:58:08 +0200 (CEST) Subject: [pypy-svn] r76557 - in pypy/branch/interplevel-array: . lib-python lib_pypy lib_pypy/pypy_test pypy/annotation pypy/interpreter pypy/jit/backend/x86 pypy/jit/backend/x86/test pypy/jit/backend/x86/tool pypy/jit/metainterp/test pypy/module/_stackless pypy/module/array/benchmark pypy/module/test_lib_pypy/ctypes_tests pypy/objspace/std pypy/rlib pypy/rpython pypy/rpython/tool pypy/translator/c Message-ID: <20100810125808.0BD7B282B9C@codespeak.net> Author: fijal Date: Tue Aug 10 14:58:06 2010 New Revision: 76557 Removed: pypy/branch/interplevel-array/lib_pypy/greenlet.py Modified: pypy/branch/interplevel-array/ (props changed) pypy/branch/interplevel-array/lib-python/ (props changed) pypy/branch/interplevel-array/lib_pypy/ (props changed) pypy/branch/interplevel-array/lib_pypy/pypy_test/test_coroutine.py pypy/branch/interplevel-array/lib_pypy/stackless.py pypy/branch/interplevel-array/pypy/annotation/annrpython.py pypy/branch/interplevel-array/pypy/interpreter/pyopcode.py pypy/branch/interplevel-array/pypy/jit/backend/x86/assembler.py pypy/branch/interplevel-array/pypy/jit/backend/x86/regalloc.py pypy/branch/interplevel-array/pypy/jit/backend/x86/regloc.py pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_assembler.py pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_runner.py pypy/branch/interplevel-array/pypy/jit/backend/x86/tool/viewcode.py pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/interplevel-array/pypy/module/_stackless/interp_coroutine.py pypy/branch/interplevel-array/pypy/module/array/benchmark/Makefile (props changed) pypy/branch/interplevel-array/pypy/module/array/benchmark/intimg.c (props changed) pypy/branch/interplevel-array/pypy/module/array/benchmark/intimgtst.c (props changed) pypy/branch/interplevel-array/pypy/module/array/benchmark/intimgtst.py (props changed) pypy/branch/interplevel-array/pypy/module/array/benchmark/loop.c (props changed) pypy/branch/interplevel-array/pypy/module/array/benchmark/sum.c (props changed) pypy/branch/interplevel-array/pypy/module/array/benchmark/sumtst.c (props changed) pypy/branch/interplevel-array/pypy/module/array/benchmark/sumtst.py (props changed) pypy/branch/interplevel-array/pypy/module/test_lib_pypy/ctypes_tests/ (props changed) pypy/branch/interplevel-array/pypy/objspace/std/callmethod.py pypy/branch/interplevel-array/pypy/rlib/rmmap.py pypy/branch/interplevel-array/pypy/rpython/rstr.py pypy/branch/interplevel-array/pypy/rpython/tool/rfficache.py pypy/branch/interplevel-array/pypy/translator/c/genc.py pypy/branch/interplevel-array/pypy/translator/c/node.py Log: Merge from trunk Modified: pypy/branch/interplevel-array/lib_pypy/pypy_test/test_coroutine.py ============================================================================== --- pypy/branch/interplevel-array/lib_pypy/pypy_test/test_coroutine.py (original) +++ pypy/branch/interplevel-array/lib_pypy/pypy_test/test_coroutine.py Tue Aug 10 14:58:06 2010 @@ -2,7 +2,7 @@ from py.test import skip, raises try: - from ..stackless import coroutine + from ..stackless import coroutine, CoroutineExit except ImportError, e: skip('cannot import stackless: %s' % (e,)) Modified: pypy/branch/interplevel-array/lib_pypy/stackless.py ============================================================================== --- pypy/branch/interplevel-array/lib_pypy/stackless.py (original) +++ pypy/branch/interplevel-array/lib_pypy/stackless.py Tue Aug 10 14:58:06 2010 @@ -14,9 +14,13 @@ import traceback import sys try: + # If _stackless can be imported then TaskletExit and CoroutineExit are + # automatically added to the builtins. from _stackless import coroutine, greenlet except ImportError: # we are running from CPython - from greenlet import greenlet + from greenlet import greenlet, GreenletExit + TaskletExit = CoroutineExit = GreenletExit + del GreenletExit try: from functools import partial except ImportError: # we are not running python 2.5 Modified: pypy/branch/interplevel-array/pypy/annotation/annrpython.py ============================================================================== --- pypy/branch/interplevel-array/pypy/annotation/annrpython.py (original) +++ pypy/branch/interplevel-array/pypy/annotation/annrpython.py Tue Aug 10 14:58:06 2010 @@ -495,6 +495,12 @@ self.blocked_blocks[block] = graph def bindinputargs(self, graph, block, inputcells, called_from_graph=None): + if 'getitem__Array' in graph.name: + if len(inputcells) > 2 and isinstance(inputcells[1], annmodel.SomeInstance) and 'Base' in str(inputcells[1].classdef): + import pdb + pdb.set_trace() + print graph + print inputcells # Create the initial bindings for the input args of a block. assert len(block.inputargs) == len(inputcells) where = (graph, block, None) Modified: pypy/branch/interplevel-array/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/interplevel-array/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/interplevel-array/pypy/interpreter/pyopcode.py Tue Aug 10 14:58:06 2010 @@ -988,23 +988,9 @@ self.dropvalues(nargs) self.pushvalue(w_result) - def LOOKUP_METHOD(self, nameindex, next_instr): - # overridden by faster version in the standard object space. - space = self.space - w_obj = self.popvalue() - w_name = self.getname_w(nameindex) - w_value = space.getattr(w_obj, w_name) - self.pushvalue(w_value) - - def CALL_METHOD(self, nargs, next_instr): - # overridden by faster version in the standard object space. - # 'nargs' is the argument count excluding the implicit 'self' - w_callable = self.peekvalue(nargs) - try: - w_result = self.space.call_valuestack(w_callable, nargs, self) - finally: - self.dropvalues(nargs + 1) - self.pushvalue(w_result) + # overridden by faster version in the standard object space. + LOOKUP_METHOD = LOAD_ATTR + CALL_METHOD = CALL_FUNCTION def MISSING_OPCODE(self, oparg, next_instr): ofs = self.last_instr Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/assembler.py Tue Aug 10 14:58:06 2010 @@ -20,7 +20,7 @@ r12, r13, r14, r15, X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG, - RegLoc, StackLoc, + RegLoc, StackLoc, ConstFloatLoc, ImmedLoc, AddressLoc, imm) from pypy.rlib.objectmodel import we_are_translated, specialize @@ -151,7 +151,7 @@ # XXX: 32 is pulled out of the air return 32 + len(self.desc_bytes) -DEBUG_COUNTER = rffi.CArray(lltype.Signed) +DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed)) class Assembler386(object): mc = None @@ -182,7 +182,7 @@ self.pending_guard_tokens = None self.setup_failure_recovery() self._debug = False - self.debug_counter_descr = cpu.arraydescrof(DEBUG_COUNTER) + self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') def leave_jitted_hook(self): ptrs = self.fail_boxes_ptr.ar @@ -240,8 +240,8 @@ assert output_log is not None f = open_file_as_stream(output_log, "w") for i in range(len(self.loop_run_counters)): - name, arr = self.loop_run_counters[i] - f.write(name + ":" + str(arr[0]) + "\n") + name, struct = self.loop_run_counters[i] + f.write(name + ":" + str(struct.i) + "\n") f.close() def _build_float_constants(self): @@ -394,9 +394,9 @@ funcname = "" % len(self.loop_run_counters) # invent the counter, so we don't get too confused if self._debug: - arr = lltype.malloc(DEBUG_COUNTER, 1, flavor='raw') - arr[0] = 0 - self.loop_run_counters.append((funcname, arr)) + struct = lltype.malloc(DEBUG_COUNTER, flavor='raw') + struct.i = 0 + self.loop_run_counters.append((funcname, struct)) return funcname def patch_jump_for_descr(self, faildescr, adr_new_target): @@ -426,11 +426,10 @@ self.loop_run_counters[-1][1])) box = BoxInt() box2 = BoxInt() - ops = [ResOperation(rop.GETARRAYITEM_RAW, [c_adr, ConstInt(0)], + ops = [ResOperation(rop.GETFIELD_RAW, [c_adr], box, descr=self.debug_counter_descr), ResOperation(rop.INT_ADD, [box, ConstInt(1)], box2), - ResOperation(rop.SETARRAYITEM_RAW, [c_adr, ConstInt(0), - box2], + ResOperation(rop.SETFIELD_RAW, [c_adr, box2], None, descr=self.debug_counter_descr)] operations = ops + operations # # we need one register free (a bit of a hack, but whatever) @@ -819,10 +818,13 @@ for i in range(start, len(arglocs)): loc = arglocs[i] - assert isinstance(loc, RegLoc) or isinstance(loc, ImmedLoc) or isinstance(loc, StackLoc) - - # XXX: Should be much simplier to tell whether a location is a float! - if (isinstance(loc, RegLoc) and loc.is_xmm) or (isinstance(loc, StackLoc) and loc.type == FLOAT): + # XXX: Should be much simplier to tell whether a location is a + # float! It's so ugly because we have to "guard" the access to + # .type with isinstance, since not all AssemblerLocation classes + # are "typed" + if ((isinstance(loc, RegLoc) and loc.is_xmm) or + (isinstance(loc, StackLoc) and loc.type == FLOAT) or + (isinstance(loc, ConstFloatLoc))): if len(unused_xmm) > 0: xmm_src_locs.append(loc) xmm_dst_locs.append(unused_xmm.pop()) Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/regalloc.py Tue Aug 10 14:58:06 2010 @@ -53,6 +53,7 @@ def __init__(self): self.cur_array_free = 0 + self.const_id = 0 def _get_new_array(self): n = self.BASE_CONSTANT_SIZE @@ -68,7 +69,8 @@ n = self.cur_array_free - 1 arr[n] = floatval self.cur_array_free = n - return rffi.cast(lltype.Signed, arr) + n * 8 + self.const_id += 1 + return (self.const_id, rffi.cast(lltype.Signed, arr) + n * 8) class X86XMMRegisterManager(RegisterManager): @@ -89,8 +91,8 @@ self.float_constants = assembler._float_constants def convert_to_imm(self, c): - adr = self.float_constants.record_float(c.getfloat()) - return AddressLoc(ImmedLoc(adr), ImmedLoc(0), 0, 0) + const_id, adr = self.float_constants.record_float(c.getfloat()) + return ConstFloatLoc(adr, const_id) def after_call(self, v): # the result is stored in st0, but we don't have this around, Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/regloc.py Tue Aug 10 14:58:06 2010 @@ -5,6 +5,7 @@ from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.objectmodel import specialize from pypy.rlib.rarithmetic import intmask +from pypy.jit.metainterp.history import FLOAT # # This module adds support for "locations", which can be either in a Const, @@ -145,6 +146,27 @@ def value_m(self): return self.loc_m +class ConstFloatLoc(AssemblerLocation): + # XXX: We have to use this class instead of just AddressLoc because + # AddressLoc is "untyped" and also we to have need some sort of unique + # identifier that we can use in _getregkey (for jump.py) + + _immutable_ = True + + width = 8 + + def __init__(self, address, const_id): + self.value = address + self.const_id = const_id + + def _getregkey(self): + # XXX: 1000 is kind of magic: We just don't want to be confused + # with any registers + return 1000 + self.const_id + + def location_code(self): + return 'j' + REGLOCS = [RegLoc(i, is_xmm=False) for i in range(16)] XMMREGLOCS = [RegLoc(i, is_xmm=True) for i in range(16)] eax, ecx, edx, ebx, esp, ebp, esi, edi, r8, r9, r10, r11, r12, r13, r14, r15 = REGLOCS Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_assembler.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_assembler.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_assembler.py Tue Aug 10 14:58:06 2010 @@ -15,6 +15,9 @@ supports_floats = True NUM_REGS = ACTUAL_CPU.NUM_REGS + def fielddescrof(self, STRUCT, name): + return 42 + class FakeMC: def __init__(self, base_address=0): self.content = [] Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/test/test_runner.py Tue Aug 10 14:58:06 2010 @@ -1,8 +1,9 @@ import py from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rstr, rclass +from pypy.rpython.annlowlevel import llhelper from pypy.jit.metainterp.history import ResOperation, LoopToken -from pypy.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstPtr, - Box, BoxFloat, BasicFailDescr) +from pypy.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstFloat, + ConstPtr, Box, BoxFloat, BasicFailDescr) from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.backend.x86.arch import WORD from pypy.jit.backend.llsupport import symbolic @@ -376,6 +377,19 @@ res = self.cpu.get_latest_value_int(0) assert res == 20 + def test_call_with_const_floats(self): + def func(f1, f2): + return f1 + f2 + + FUNC = self.FuncType([lltype.Float, lltype.Float], lltype.Float) + FPTR = self.Ptr(FUNC) + calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + func_ptr = llhelper(FPTR, func) + funcbox = self.get_funcbox(self.cpu, func_ptr) + res = self.execute_operation(rop.CALL, [funcbox, ConstFloat(1.5), ConstFloat(2.5)], 'float', descr=calldescr) + assert res.value == 4.0 + + class TestX86OverflowMC(TestX86): def setup_method(self, meth): @@ -484,9 +498,9 @@ self.cpu.set_future_value_int(0, 0) self.cpu.execute_token(ops.token) # check debugging info - name, arr = self.cpu.assembler.loop_run_counters[0] + name, struct = self.cpu.assembler.loop_run_counters[0] assert name == 'xyz' - assert arr[0] == 10 + assert struct.i == 10 self.cpu.finish_once() lines = py.path.local(self.logfile + ".count").readlines() assert lines[0] == 'xyz:10\n' Modified: pypy/branch/interplevel-array/pypy/jit/backend/x86/tool/viewcode.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/backend/x86/tool/viewcode.py (original) +++ pypy/branch/interplevel-array/pypy/jit/backend/x86/tool/viewcode.py Tue Aug 10 14:58:06 2010 @@ -35,6 +35,7 @@ objdump_backend_option = { 'x86': 'i386', 'x86_64': 'x86-64', + 'i386': 'i386', } objdump = ('objdump -M intel,%(backend)s -b binary -m i386 ' '--adjust-vma=%(origin)d -D %(file)s') Modified: pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/interplevel-array/pypy/jit/metainterp/test/test_optimizeopt.py Tue Aug 10 14:58:06 2010 @@ -479,7 +479,6 @@ self.optimize_loop(ops, 'Not', expected) def test_int_is_true_1(self): - py.test.skip("XXX implement me") ops = """ [i0] i1 = int_is_true(i0) Modified: pypy/branch/interplevel-array/pypy/module/_stackless/interp_coroutine.py ============================================================================== --- pypy/branch/interplevel-array/pypy/module/_stackless/interp_coroutine.py (original) +++ pypy/branch/interplevel-array/pypy/module/_stackless/interp_coroutine.py Tue Aug 10 14:58:06 2010 @@ -265,10 +265,14 @@ instr += 1 oparg = ord(code[instr]) | ord(code[instr + 1]) << 8 nargs = oparg & 0xff + nkwds = (oparg >> 8) & 0xff if space.config.objspace.opcodes.CALL_METHOD and opcode == map['CALL_METHOD']: - chain = resume_state_create(chain, 'CALL_METHOD', frame, - nargs) - elif opcode == map['CALL_FUNCTION'] and (oparg >> 8) & 0xff == 0: + if nkwds == 0: # only positional arguments + chain = resume_state_create(chain, 'CALL_METHOD', frame, + nargs) + else: # includes keyword arguments + chain = resume_state_create(chain, 'CALL_METHOD_KW', frame) + elif opcode == map['CALL_FUNCTION'] and nkwds == 0: # Only positional arguments # case1: ("CALL_FUNCTION", f, nargs, returns=w_result) chain = resume_state_create(chain, 'CALL_FUNCTION', frame, Modified: pypy/branch/interplevel-array/pypy/objspace/std/callmethod.py ============================================================================== --- pypy/branch/interplevel-array/pypy/objspace/std/callmethod.py (original) +++ pypy/branch/interplevel-array/pypy/objspace/std/callmethod.py Tue Aug 10 14:58:06 2010 @@ -62,12 +62,13 @@ n_args = oparg & 0xff n_kwargs = (oparg >> 8) & 0xff w_self = f.peekvalue(n_args + (2 * n_kwargs)) - w_callable = f.peekvalue(n_args + (2 * n_kwargs) + 1) n = n_args + (w_self is not None) if not n_kwargs: + w_callable = f.peekvalue(n_args + (2 * n_kwargs) + 1) try: w_result = f.space.call_valuestack(w_callable, n, f) + rstack.resume_point("CALL_METHOD", f, n_args, returns=w_result) finally: f.dropvalues(n_args + 2) else: @@ -83,14 +84,13 @@ keywords[n_kwargs] = key keywords_w[n_kwargs] = w_value - arguments = f.popvalues(n) + arguments = f.popvalues(n) # includes w_self if it is not None args = f.argument_factory(arguments, keywords, keywords_w, None, None) - - try: - w_result = f.space.call_args(w_callable, args) - finally: - f.dropvalues(1 + (w_self is None)) - rstack.resume_point("CALL_METHOD", f, returns=w_result) + if w_self is None: + f.popvalue() # removes w_self, which is None + w_callable = f.popvalue() + w_result = f.space.call_args(w_callable, args) + rstack.resume_point("CALL_METHOD_KW", f, returns=w_result) f.pushvalue(w_result) Modified: pypy/branch/interplevel-array/pypy/rlib/rmmap.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rlib/rmmap.py (original) +++ pypy/branch/interplevel-array/pypy/rlib/rmmap.py Tue Aug 10 14:58:06 2010 @@ -646,8 +646,14 @@ hintp = rffi.cast(PTR, hint.pos) res = c_mmap_safe(hintp, map_size, prot, flags, -1, 0) if res == rffi.cast(PTR, -1): - raise MemoryError - hint.pos += map_size + # some systems (some versions of OS/X?) complain if they + # are passed a non-zero address. Try again. + hintp = rffi.cast(PTR, 0) + res = c_mmap_safe(hintp, map_size, prot, flags, -1, 0) + if res == rffi.cast(PTR, -1): + raise MemoryError + else: + hint.pos += map_size return res alloc._annenforceargs_ = (int,) Modified: pypy/branch/interplevel-array/pypy/rpython/rstr.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rpython/rstr.py (original) +++ pypy/branch/interplevel-array/pypy/rpython/rstr.py Tue Aug 10 14:58:06 2010 @@ -288,8 +288,8 @@ if not hop.args_s[1].is_constant(): raise TyperError("encoding must be constant") encoding = hop.args_s[1].const - if encoding == "ascii": - expect = self.lowleveltype # can be a UniChar + if encoding == "ascii" and self.lowleveltype == UniChar: + expect = UniChar # only for unichar.encode('ascii') else: expect = self.repr # must be a regular unicode string v_self = hop.inputarg(expect, 0) Modified: pypy/branch/interplevel-array/pypy/rpython/tool/rfficache.py ============================================================================== --- pypy/branch/interplevel-array/pypy/rpython/tool/rfficache.py (original) +++ pypy/branch/interplevel-array/pypy/rpython/tool/rfficache.py Tue Aug 10 14:58:06 2010 @@ -29,7 +29,7 @@ } ''' % (include_string, add_source, str(question))) c_file = udir.join("gcctest.c") - c_file.write(c_source) + c_file.write(str(c_source) + '\n') eci = ExternalCompilationInfo() return build_executable_cache([c_file], eci) Modified: pypy/branch/interplevel-array/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/interplevel-array/pypy/translator/c/genc.py (original) +++ pypy/branch/interplevel-array/pypy/translator/c/genc.py Tue Aug 10 14:58:06 2010 @@ -1,7 +1,6 @@ import autopath import py import sys, os -from pypy.translator.c.node import PyObjectNode, FuncNode from pypy.translator.c.database import LowLevelDatabase from pypy.translator.c.extfunc import pre_include_code_lines from pypy.translator.llsupport.wrapper import new_wrapper @@ -196,7 +195,7 @@ all = [] for node in self.db.globalcontainers(): - eci = getattr(node, 'compilation_info', None) + eci = node.compilation_info() if eci: all.append(eci) self.merge_eci(*all) @@ -222,7 +221,7 @@ graphs = db.all_graphs() db.gctransformer.prepare_inline_helpers(graphs) for node in db.containerlist: - if isinstance(node, FuncNode): + if hasattr(node, 'funcgens'): for funcgen in node.funcgens: funcgen.patch_graph(copy_graph=False) return db Modified: pypy/branch/interplevel-array/pypy/translator/c/node.py ============================================================================== --- pypy/branch/interplevel-array/pypy/translator/c/node.py (original) +++ pypy/branch/interplevel-array/pypy/translator/c/node.py Tue Aug 10 14:58:06 2010 @@ -466,15 +466,15 @@ class ContainerNode(object): - if USESLOTS: - __slots__ = """db T obj + if USESLOTS: # keep the number of slots down! + __slots__ = """db obj typename implementationtypename - name ptrname compilation_info + name ptrname globalcontainer""".split() + eci_name = '_compilation_info' def __init__(self, db, T, obj): self.db = db - self.T = T self.obj = obj #self.dependencies = {} self.typename = db.gettype(T) #, who_asks=self) @@ -489,16 +489,22 @@ else: self.globalcontainer = False parentnode = db.getcontainernode(parent) - defnode = db.gettypedefnode(parentnode.T) + defnode = db.gettypedefnode(parentnode.getTYPE()) self.name = defnode.access_expr(parentnode.name, parentindex) if self.typename != self.implementationtypename: if db.gettypedefnode(T).extra_union_for_varlength: self.name += '.b' - self.compilation_info = getattr(obj, '_compilation_info', None) self.ptrname = '(&%s)' % self.name + def getTYPE(self): + return typeOf(self.obj) + def is_thread_local(self): - return hasattr(self.T, "_hints") and self.T._hints.get('thread_local') + T = self.getTYPE() + return hasattr(T, "_hints") and T._hints.get('thread_local') + + def compilation_info(self): + return getattr(self.obj, self.eci_name, None) def get_declaration(self): if self.name[-2:] == '.b': @@ -546,27 +552,31 @@ __slots__ = () def basename(self): - return self.T._name + T = self.getTYPE() + return T._name def enum_dependencies(self): - for name in self.T._names: + T = self.getTYPE() + for name in T._names: yield getattr(self.obj, name) def getlength(self): - if self.T._arrayfld is None: + T = self.getTYPE() + if T._arrayfld is None: return 1 else: - array = getattr(self.obj, self.T._arrayfld) + array = getattr(self.obj, T._arrayfld) return len(array.items) def initializationexpr(self, decoration=''): + T = self.getTYPE() is_empty = True yield '{' - defnode = self.db.gettypedefnode(self.T) + defnode = self.db.gettypedefnode(T) data = [] - if needs_gcheader(self.T): + if needs_gcheader(T): gc_init = self.db.gcpolicy.struct_gcheader_initdata(self) data.append(('gcheader', gc_init)) @@ -578,16 +588,16 @@ # '.fieldname = value'. But here we don't know which of the # fields need initialization, so XXX we pick the first one # arbitrarily. - if hasattr(self.T, "_hints") and self.T._hints.get('union'): + if hasattr(T, "_hints") and T._hints.get('union'): data = data[0:1] - if 'get_padding_drop' in self.T._hints: + if 'get_padding_drop' in T._hints: d = {} for name, _ in data: - T = defnode.c_struct_field_type(name) - typename = self.db.gettype(T) + T1 = defnode.c_struct_field_type(name) + typename = self.db.gettype(T1) d[name] = cdecl(typename, '') - padding_drop = self.T._hints['get_padding_drop'](d) + padding_drop = T._hints['get_padding_drop'](d) else: padding_drop = [] @@ -617,9 +627,10 @@ return 'struct _hashT_%s @' % self.name def forward_declaration(self): + T = self.getTYPE() assert self.typename == self.implementationtypename # no array part hash_typename = self.get_hash_typename() - hash_offset = self.db.gctransformer.get_hash_offset(self.T) + hash_offset = self.db.gctransformer.get_hash_offset(T) yield '%s {' % cdecl(hash_typename, '') yield '\tunion {' yield '\t\t%s;' % cdecl(self.implementationtypename, 'head') @@ -671,22 +682,23 @@ return len(self.obj.items) def initializationexpr(self, decoration=''): - defnode = self.db.gettypedefnode(self.T) + T = self.getTYPE() + defnode = self.db.gettypedefnode(T) yield '{' - if needs_gcheader(self.T): + if needs_gcheader(T): gc_init = self.db.gcpolicy.array_gcheader_initdata(self) lines = generic_initializationexpr(self.db, gc_init, 'gcheader', '%sgcheader' % (decoration,)) for line in lines: yield line - if self.T._hints.get('nolength', False): + if T._hints.get('nolength', False): length = '' else: length = '%d, ' % len(self.obj.items) - if self.T.OF is Void or len(self.obj.items) == 0: + if T.OF is Void or len(self.obj.items) == 0: yield '\t%s' % length.rstrip(', ') yield '}' - elif self.T.OF == Char: + elif T.OF == Char: if len(self.obj.items) and self.obj.items[0] is None: s = ''.join([self.obj.getitem(i) for i in range(len(self.obj.items))]) else: @@ -694,7 +706,7 @@ yield '\t%s%s' % (length, c_char_array_constant(s)) yield '}' else: - barebone = barebonearray(self.T) + barebone = barebonearray(T) if not barebone: yield '\t%s{' % length for j in range(len(self.obj.items)): @@ -722,7 +734,8 @@ self.ptrname = self.name def basename(self): - return self.T._name + T = self.getTYPE() + return T._name def enum_dependencies(self): for i in range(self.obj.getlength()): @@ -732,11 +745,12 @@ return 1 # not variable-sized! def initializationexpr(self, decoration=''): + T = self.getTYPE() assert self.typename == self.implementationtypename # not var-sized is_empty = True yield '{' # _names == ['item0', 'item1', ...] - for j, name in enumerate(self.T._names): + for j, name in enumerate(T._names): value = getattr(self.obj, name) lines = generic_initializationexpr(self.db, value, '%s[%d]' % (self.name, j), @@ -777,6 +791,7 @@ class FuncNode(ContainerNode): nodekind = 'func' + eci_name = 'compilation_info' # there not so many node of this kind, slots should not # be necessary @@ -794,7 +809,6 @@ else: self.name = (forcename or db.namespace.uniquename('g_' + self.basename())) - self.compilation_info = getattr(obj, 'compilation_info', None) self.make_funcgens() #self.dependencies = {} self.typename = db.gettype(T) #, who_asks=self) @@ -939,18 +953,20 @@ return [] def initializationexpr(self, decoration=''): - yield 'RPyOpaque_INITEXPR_%s' % (self.T.tag,) + T = self.getTYPE() + yield 'RPyOpaque_INITEXPR_%s' % (T.tag,) def startupcode(self): + T = self.getTYPE() args = [self.ptrname] # XXX how to make this code more generic? - if self.T.tag == 'ThreadLock': + if T.tag == 'ThreadLock': lock = self.obj.externalobj if lock.locked(): args.append('1') else: args.append('0') - yield 'RPyOpaque_SETUP_%s(%s);' % (self.T.tag, ', '.join(args)) + yield 'RPyOpaque_SETUP_%s(%s);' % (T.tag, ', '.join(args)) def opaquenode_factory(db, T, obj): From fijal at codespeak.net Tue Aug 10 15:13:02 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 10 Aug 2010 15:13:02 +0200 (CEST) Subject: [pypy-svn] r76558 - in pypy/trunk: . lib_pypy pypy/annotation pypy/jit/backend/test pypy/jit/backend/x86 pypy/jit/metainterp pypy/jit/metainterp/test pypy/jit/tl pypy/module/array pypy/module/array/benchmark pypy/module/array/test pypy/module/pypyjit pypy/module/pypyjit/test pypy/module/test_lib_pypy pypy/objspace/std pypy/translator/platform Message-ID: <20100810131302.E7B80282B9C@codespeak.net> Author: fijal Date: Tue Aug 10 15:13:00 2010 New Revision: 76558 Added: pypy/trunk/lib_pypy/apparray.py - copied unchanged from r76557, pypy/branch/interplevel-array/lib_pypy/apparray.py pypy/trunk/pypy/module/array/ - copied from r76557, pypy/branch/interplevel-array/pypy/module/array/ Removed: pypy/trunk/lib_pypy/array.py pypy/trunk/pypy/module/test_lib_pypy/test_array.py Modified: pypy/trunk/ (props changed) pypy/trunk/pypy/annotation/annrpython.py pypy/trunk/pypy/jit/backend/test/runner_test.py pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/resoperation.py pypy/trunk/pypy/jit/metainterp/test/test_executor.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py pypy/trunk/pypy/jit/tl/pypyjit.py pypy/trunk/pypy/jit/tl/pypyjit_demo.py pypy/trunk/pypy/module/array/benchmark/Makefile (props changed) pypy/trunk/pypy/module/array/benchmark/intimg.c (props changed) pypy/trunk/pypy/module/array/benchmark/intimgtst.c (props changed) pypy/trunk/pypy/module/array/benchmark/intimgtst.py (props changed) pypy/trunk/pypy/module/array/benchmark/loop.c (props changed) pypy/trunk/pypy/module/array/benchmark/sum.c (props changed) pypy/trunk/pypy/module/array/benchmark/sumtst.c (props changed) pypy/trunk/pypy/module/array/benchmark/sumtst.py (props changed) pypy/trunk/pypy/module/array/test/test_array_old.py (props changed) pypy/trunk/pypy/module/pypyjit/policy.py pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py pypy/trunk/pypy/objspace/std/itertype.py pypy/trunk/pypy/objspace/std/model.py pypy/trunk/pypy/translator/platform/posix.py Log: (hakanardo, fijal reviewing) merge interplevel-array branch. This branch contains: * an RPython level implementation of array module * some simple JIT optimization techniques * changes in backend enough to make array work on top of the JIT Modified: pypy/trunk/pypy/annotation/annrpython.py ============================================================================== --- pypy/trunk/pypy/annotation/annrpython.py (original) +++ pypy/trunk/pypy/annotation/annrpython.py Tue Aug 10 15:13:00 2010 @@ -495,6 +495,12 @@ self.blocked_blocks[block] = graph def bindinputargs(self, graph, block, inputcells, called_from_graph=None): + if 'getitem__Array' in graph.name: + if len(inputcells) > 2 and isinstance(inputcells[1], annmodel.SomeInstance) and 'Base' in str(inputcells[1].classdef): + import pdb + pdb.set_trace() + print graph + print inputcells # Create the initial bindings for the input args of a block. assert len(block.inputargs) == len(inputcells) where = (graph, block, None) Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/runner_test.py (original) +++ pypy/trunk/pypy/jit/backend/test/runner_test.py Tue Aug 10 15:13:00 2010 @@ -657,6 +657,21 @@ assert r.value == 1 def test_array_basic(self): + a_box, A = self.alloc_array_of(rffi.SHORT, 342) + arraydescr = self.cpu.arraydescrof(A) + assert not arraydescr.is_array_of_pointers() + # + r = self.execute_operation(rop.ARRAYLEN_GC, [a_box], + 'int', descr=arraydescr) + assert r.value == 342 + r = self.execute_operation(rop.SETARRAYITEM_GC, [a_box, BoxInt(310), + BoxInt(744)], + 'void', descr=arraydescr) + assert r is None + r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(310)], + 'int', descr=arraydescr) + assert r.value == 744 + a_box, A = self.alloc_array_of(lltype.Signed, 342) arraydescr = self.cpu.arraydescrof(A) assert not arraydescr.is_array_of_pointers() Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Tue Aug 10 15:13:00 2010 @@ -1066,11 +1066,14 @@ if scale.value == 0: self.mc.MOVZX8(resloc, addr_add(base_loc, ofs_loc, ofs.value, scale.value)) + elif scale.value == 1: + self.mc.MOVZX16(resloc, addr_add(base_loc, ofs_loc, ofs.value, + scale.value)) elif (1 << scale.value) == WORD: self.mc.MOV(resloc, addr_add(base_loc, ofs_loc, ofs.value, scale.value)) else: - print "[asmgen]setarrayitem unsupported size: %d" % scale.value + print "[asmgen]getarrayitem unsupported size: %d" % scale.value raise NotImplementedError() genop_getarrayitem_gc_pure = genop_getarrayitem_gc @@ -1103,6 +1106,8 @@ else: if (1 << scale_loc.value) == WORD: self.mc.MOV(dest_addr, value_loc) + elif scale_loc.value == 1: + self.mc.MOV16(dest_addr, value_loc) elif scale_loc.value == 0: self.mc.MOV8(dest_addr, value_loc.lowest8bits()) else: Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Tue Aug 10 15:13:00 2010 @@ -1,7 +1,7 @@ from pypy.jit.metainterp.history import Box, BoxInt, LoopToken, BoxFloat,\ ConstFloat from pypy.jit.metainterp.history import Const, ConstInt, ConstPtr, ConstObj, REF -from pypy.jit.metainterp.resoperation import rop, ResOperation +from pypy.jit.metainterp.resoperation import rop, ResOperation, opboolinvers, opboolreflex from pypy.jit.metainterp import jitprof from pypy.jit.metainterp.executor import execute_nonspec from pypy.jit.metainterp.specnode import SpecNode, NotSpecNode, ConstantSpecNode @@ -612,12 +612,59 @@ assert oldop.opnum == op.opnum self.make_equal_to(op.result, self.getvalue(oldop.result)) return + elif self.find_rewriteable_bool(op, args): + return else: self.pure_operations[args] = op # otherwise, the operation remains self.emit_operation(op) + + def try_boolinvers(self, op, targs): + oldop = self.pure_operations.get(targs, None) + if oldop is not None and oldop.descr is op.descr: + value = self.getvalue(oldop.result) + if value.is_constant(): + if value.box is CONST_1: + self.make_constant(op.result, CONST_0) + return True + elif value.box is CONST_0: + self.make_constant(op.result, CONST_1) + return True + return False + + + def find_rewriteable_bool(self, op, args): + try: + oldopnum = opboolinvers[op.opnum] + targs = [args[0], args[1], ConstInt(oldopnum)] + if self.try_boolinvers(op, targs): + return True + except KeyError: + pass + + try: + oldopnum = opboolreflex[op.opnum] + targs = [args[1], args[0], ConstInt(oldopnum)] + oldop = self.pure_operations.get(targs, None) + if oldop is not None and oldop.descr is op.descr: + self.make_equal_to(op.result, self.getvalue(oldop.result)) + return True + except KeyError: + pass + + try: + oldopnum = opboolinvers[opboolreflex[op.opnum]] + targs = [args[1], args[0], ConstInt(oldopnum)] + if self.try_boolinvers(op, targs): + return True + except KeyError: + pass + + return False + + def optimize_JUMP(self, op): orgop = self.loop.operations[-1] exitargs = [] Modified: pypy/trunk/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resoperation.py (original) +++ pypy/trunk/pypy/jit/metainterp/resoperation.py Tue Aug 10 15:13:00 2010 @@ -274,3 +274,51 @@ setup(__name__ == '__main__') # print out the table when run directly del _oplist + +opboolinvers = { + rop.INT_EQ: rop.INT_NE, + rop.INT_NE: rop.INT_EQ, + rop.INT_LT: rop.INT_GE, + rop.INT_GE: rop.INT_LT, + rop.INT_GT: rop.INT_LE, + rop.INT_LE: rop.INT_GT, + + rop.UINT_LT: rop.UINT_GE, + rop.UINT_GE: rop.UINT_LT, + rop.UINT_GT: rop.UINT_LE, + rop.UINT_LE: rop.UINT_GT, + + rop.FLOAT_EQ: rop.FLOAT_NE, + rop.FLOAT_NE: rop.FLOAT_EQ, + rop.FLOAT_LT: rop.FLOAT_GE, + rop.FLOAT_GE: rop.FLOAT_LT, + rop.FLOAT_GT: rop.FLOAT_LE, + rop.FLOAT_LE: rop.FLOAT_GT, + + rop.PTR_EQ: rop.PTR_NE, + rop.PTR_NE: rop.PTR_EQ, + } + +opboolreflex = { + rop.INT_EQ: rop.INT_EQ, + rop.INT_NE: rop.INT_NE, + rop.INT_LT: rop.INT_GT, + rop.INT_GE: rop.INT_LE, + rop.INT_GT: rop.INT_LT, + rop.INT_LE: rop.INT_GE, + + rop.UINT_LT: rop.UINT_GT, + rop.UINT_GE: rop.UINT_LE, + rop.UINT_GT: rop.UINT_LT, + rop.UINT_LE: rop.UINT_GE, + + rop.FLOAT_EQ: rop.FLOAT_EQ, + rop.FLOAT_NE: rop.FLOAT_NE, + rop.FLOAT_LT: rop.FLOAT_GT, + rop.FLOAT_GE: rop.FLOAT_LE, + rop.FLOAT_GT: rop.FLOAT_LT, + rop.FLOAT_LE: rop.FLOAT_GE, + + rop.PTR_EQ: rop.PTR_EQ, + rop.PTR_NE: rop.PTR_NE, + } Modified: pypy/trunk/pypy/jit/metainterp/test/test_executor.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_executor.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_executor.py Tue Aug 10 15:13:00 2010 @@ -4,14 +4,14 @@ from pypy.rpython.lltypesystem import lltype, llmemory from pypy.jit.metainterp.executor import execute from pypy.jit.metainterp.executor import execute_varargs, execute_nonspec -from pypy.jit.metainterp.resoperation import rop +from pypy.jit.metainterp.resoperation import rop, opboolinvers, opboolreflex, opname from pypy.jit.metainterp.history import BoxInt, ConstInt from pypy.jit.metainterp.history import BoxPtr, ConstPtr from pypy.jit.metainterp.history import BoxFloat, ConstFloat from pypy.jit.metainterp.history import AbstractDescr, Box from pypy.jit.metainterp import history from pypy.jit.backend.model import AbstractCPU - +from pypy.rpython.lltypesystem import llmemory, rffi class FakeDescr(AbstractDescr): pass @@ -312,3 +312,40 @@ assert box.getint() == retvalue else: assert 0, "rettype is %r" % (rettype,) + +def make_args_for_op(op, a, b): + n=opname[op] + if n[0:3] == 'INT' or n[0:4] == 'UINT': + arg1 = ConstInt(a) + arg2 = ConstInt(b) + elif n[0:5] == 'FLOAT': + arg1 = ConstFloat(float(a)) + arg2 = ConstFloat(float(b)) + elif n[0:3] == 'PTR': + arg1 = ConstPtr(rffi.cast(llmemory.GCREF, a)) + arg2 = ConstPtr(rffi.cast(llmemory.GCREF, b)) + else: + raise NotImplementedError( + "Don't know how to make args for " + n) + return arg1, arg2 + + +def test_opboolinvers(): + cpu = FakeCPU() + for op1, op2 in opboolinvers.items(): + for a in (1,2,3): + for b in (1,2,3): + arg1, arg2 = make_args_for_op(op1, a, b) + box1 = execute(cpu, None, op1, None, arg1, arg2) + box2 = execute(cpu, None, op2, None, arg1, arg2) + assert box1.value == (not box2.value) + +def test_opboolreflex(): + cpu = FakeCPU() + for op1, op2 in opboolreflex.items(): + for a in (1,2,3): + for b in (1,2,3): + arg1, arg2 = make_args_for_op(op1, a, b) + box1 = execute(cpu, None, op1, None, arg1, arg2) + box2 = execute(cpu, None, op2, None, arg2, arg1) + assert box1.value == box2.value Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Tue Aug 10 15:13:00 2010 @@ -364,6 +364,74 @@ """ self.optimize_loop(ops, 'Not', expected) + def test_constant_boolrewrite_lt(self): + ops = """ + [i0] + i1 = int_lt(i0, 0) + guard_true(i1) [] + i2 = int_ge(i0, 0) + guard_false(i2) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_lt(i0, 0) + guard_true(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_constant_boolrewrite_gt(self): + ops = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + i2 = int_le(i0, 0) + guard_false(i2) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_constant_boolrewrite_reflex(self): + ops = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + i2 = int_lt(0, i0) + guard_true(i2) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_constant_boolrewrite_reflex_invers(self): + ops = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + i2 = int_ge(0, i0) + guard_false(i2) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + def test_remove_consecutive_guard_value_constfold(self): ops = """ [] @@ -805,16 +873,10 @@ guard_nonnull(p0) [] i7 = ptr_ne(p0, p1) guard_true(i7) [] - i8 = ptr_eq(p0, p1) - guard_false(i8) [] i9 = ptr_ne(p0, p2) guard_true(i9) [] - i10 = ptr_eq(p0, p2) - guard_false(i10) [] i11 = ptr_ne(p2, p1) guard_true(i11) [] - i12 = ptr_eq(p2, p1) - guard_false(i12) [] jump(p0, p1, p2) """ self.optimize_loop(ops, 'Not, Not, Not', expected2) Modified: pypy/trunk/pypy/jit/tl/pypyjit.py ============================================================================== --- pypy/trunk/pypy/jit/tl/pypyjit.py (original) +++ pypy/trunk/pypy/jit/tl/pypyjit.py Tue Aug 10 15:13:00 2010 @@ -37,6 +37,7 @@ set_opt_level(config, level='jit') config.objspace.allworkingmodules = False config.objspace.usemodules.pypyjit = True +config.objspace.usemodules.array = True config.objspace.usemodules._weakref = False config.objspace.usemodules._sre = False set_pypy_opt_level(config, level='jit') Modified: pypy/trunk/pypy/jit/tl/pypyjit_demo.py ============================================================================== --- pypy/trunk/pypy/jit/tl/pypyjit_demo.py (original) +++ pypy/trunk/pypy/jit/tl/pypyjit_demo.py Tue Aug 10 15:13:00 2010 @@ -1,38 +1,64 @@ -base = object +## base = object -class Number(base): - __slots__ = ('val', ) - def __init__(self, val=0): - self.val = val - - def __add__(self, other): - if not isinstance(other, int): - other = other.val - return Number(val=self.val + other) +## class Number(base): +## __slots__ = ('val', ) +## def __init__(self, val=0): +## self.val = val + +## def __add__(self, other): +## if not isinstance(other, int): +## other = other.val +## return Number(val=self.val + other) - def __cmp__(self, other): - val = self.val - if not isinstance(other, int): - other = other.val - return cmp(val, other) - - def __nonzero__(self): - return bool(self.val) - -def g(x, inc=2): - return x + inc - -def f(n, x, inc): - while x < n: - x = g(x, inc=1) - return x - -import time -#t1 = time.time() -#f(10000000, Number(), 1) -#t2 = time.time() -#print t2 - t1 -t1 = time.time() -f(10000000, 0, 1) -t2 = time.time() -print t2 - t1 +## def __cmp__(self, other): +## val = self.val +## if not isinstance(other, int): +## other = other.val +## return cmp(val, other) + +## def __nonzero__(self): +## return bool(self.val) + +## def g(x, inc=2): +## return x + inc + +## def f(n, x, inc): +## while x < n: +## x = g(x, inc=1) +## return x + +## import time +## #t1 = time.time() +## #f(10000000, Number(), 1) +## #t2 = time.time() +## #print t2 - t1 +## t1 = time.time() +## f(10000000, 0, 1) +## t2 = time.time() +## print t2 - t1 + +try: + from array import array + def f(img): + i=0 + sa=0 + while i < img.__len__(): + sa+=img[i] + i+=1 + return sa + + img=array('h',(1,2,3,4)) + print f(img) +except Exception, e: + print "Exception: ", type(e) + print e + +## def f(): +## a=7 +## i=0 +## while i<4: +## if i<0: break +## if i<0: break +## i+=1 + +## f() Modified: pypy/trunk/pypy/module/pypyjit/policy.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/policy.py (original) +++ pypy/trunk/pypy/module/pypyjit/policy.py Tue Aug 10 15:13:00 2010 @@ -11,7 +11,7 @@ if '.' in modname: modname, _ = modname.split('.', 1) if modname in ['pypyjit', 'signal', 'micronumpy', 'math', 'exceptions', - 'imp', 'sys']: + 'imp', 'sys', 'array']: return True return False Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Tue Aug 10 15:13:00 2010 @@ -615,6 +615,237 @@ return total ''', 170, ([], 4999450000L)) + def test_boolrewrite_invers(self): + for a, b, res, ops in (('2000', '2000', 20001000, 51), + ( '500', '500', 15001500, 81), + ( '300', '600', 16001700, 83), + ( 'a', 'b', 16001700, 89), + ( 'a', 'a', 13001700, 85)): + + self.run_source(''' + def main(): + sa = 0 + a = 300 + b = 600 + for i in range(1000): + if i < %s: sa += 1 + else: sa += 2 + if i >= %s: sa += 10000 + else: sa += 20000 + return sa + '''%(a, b), ops, ([], res)) + + def test_boolrewrite_reflex(self): + for a, b, res, ops in (('2000', '2000', 10001000, 51), + ( '500', '500', 15001500, 81), + ( '300', '600', 14001700, 83), + ( 'a', 'b', 14001700, 89), + ( 'a', 'a', 17001700, 85)): + + self.run_source(''' + def main(): + sa = 0 + a = 300 + b = 600 + for i in range(1000): + if i < %s: sa += 1 + else: sa += 2 + if %s > i: sa += 10000 + else: sa += 20000 + return sa + '''%(a, b), ops, ([], res)) + + + def test_boolrewrite_correct_invers(self): + def opval(i, op, a): + if eval('%d %s %d' % (i, op, a)): return 1 + return 2 + + ops = ('<', '>', '<=', '>=', '==', '!=') + for op1 in ops: + for op2 in ops: + for a,b in ((500, 500), (300, 600)): + res = 0 + res += opval(a-1, op1, a) * (a) + res += opval( a, op1, a) + res += opval(a+1, op1, a) * (1000 - a - 1) + res += opval(b-1, op2, b) * 10000 * (b) + res += opval( b, op2, b) * 10000 + res += opval(b+1, op2, b) * 10000 * (1000 - b - 1) + + self.run_source(''' + def main(): + sa = 0 + for i in range(1000): + if i %s %d: sa += 1 + else: sa += 2 + if i %s %d: sa += 10000 + else: sa += 20000 + return sa + '''%(op1, a, op2, b), 83, ([], res)) + + self.run_source(''' + def main(): + sa = 0 + i = 0.0 + while i < 250.0: + if i %s %f: sa += 1 + else: sa += 2 + if i %s %f: sa += 10000 + else: sa += 20000 + i += 0.25 + return sa + '''%(op1, float(a)/4.0, op2, float(b)/4.0), 109, ([], res)) + + + def test_boolrewrite_correct_reflex(self): + def opval(i, op, a): + if eval('%d %s %d' % (i, op, a)): return 1 + return 2 + + ops = ('<', '>', '<=', '>=', '==', '!=') + for op1 in ops: + for op2 in ops: + for a,b in ((500, 500), (300, 600)): + res = 0 + res += opval(a-1, op1, a) * (a) + res += opval( a, op1, a) + res += opval(a+1, op1, a) * (1000 - a - 1) + res += opval(b, op2, b-1) * 10000 * (b) + res += opval(b, op2, b) * 10000 + res += opval(b, op2, b+1) * 10000 * (1000 - b - 1) + + self.run_source(''' + def main(): + sa = 0 + for i in range(1000): + if i %s %d: sa += 1 + else: sa += 2 + if %d %s i: sa += 10000 + else: sa += 20000 + return sa + '''%(op1, a, b, op2), 83, ([], res)) + + self.run_source(''' + def main(): + sa = 0 + i = 0.0 + while i < 250.0: + if i %s %f: sa += 1 + else: sa += 2 + if %f %s i: sa += 10000 + else: sa += 20000 + i += 0.25 + return sa + '''%(op1, float(a)/4.0, float(b)/4.0, op2), 109, ([], res)) + + def test_boolrewrite_ptr(self): + compares = ('a == b', 'b == a', 'a != b', 'b != a', 'a == c', 'c != b') + for e1 in compares: + for e2 in compares: + a, b, c = 1, 2, 3 + if eval(e1): res = 752 * 1 + else: res = 752 * 2 + if eval(e2): res += 752 * 10000 + else: res += 752 * 20000 + a = b + if eval(e1): res += 248 * 1 + else: res += 248 * 2 + if eval(e2): res += 248 * 10000 + else: res += 248 * 20000 + + + if 'c' in e1 or 'c' in e2: + n = 337 + else: + n = 215 + + self.run_source(''' + class tst: + pass + def main(): + a = tst() + b = tst() + c = tst() + sa = 0 + for i in range(1000): + if %s: sa += 1 + else: sa += 2 + if %s: sa += 10000 + else: sa += 20000 + if i > 750: a = b + return sa + '''%(e1, e2), n, ([], res)) + + def test_array_sum(self): + for tc, maxops in zip('bhilBHILfd', (38,) * 6 + (40, 40, 41, 38)): + res = 19352859 + if tc in 'IL': + res = long(res) + elif tc in 'fd': + res = float(res) + + self.run_source(''' + from array import array + + def main(): + img = array("%s", range(127) * 5) * 484 + l, i = 0, 0 + while i < 640 * 480: + l += img[i] + i += 1 + return l + ''' % tc, maxops, ([], res)) + + def test_array_sum_char(self): + self.run_source(''' + from array import array + + def main(): + img = array("c", "Hello") * 130 * 480 + l, i = 0, 0 + while i < 640 * 480: + l += ord(img[i]) + i += 1 + return l + ''', 60, ([], 30720000)) + + def test_array_sum_unicode(self): + self.run_source(''' + from array import array + + def main(): + img = array("u", u"Hello") * 130 * 480 + l, i = 0, 0 + while i < 640 * 480: + if img[i] == u"l": + l += 1 + i += 1 + return l + ''', 65, ([], 122880)) + + def test_array_intimg(self): + for tc, maxops in zip('ilILd', (67, 67, 69, 69, 61)): + res = 73574560 + if tc in 'IL': + res = long(res) + elif tc in 'fd': + res = float(res) + + self.run_source(''' + from array import array + + def main(tc): + img = array(tc, range(3)) * (350 * 480) + intimg = array(tc, (0,)) * (640 * 480) + l, i = 0, 640 + while i < 640 * 480: + l = l + img[i] + intimg[i] = (intimg[i-640] + l) + i += 1 + return intimg[i - 1] + ''', maxops, ([tc], res)) + class AppTestJIT(PyPyCJITTests): def setup_class(cls): if not option.runappdirect: @@ -637,6 +868,7 @@ cls.counter = 0 cls.pypy_c = option.pypy_c + def has_info(pypy_c, option): g = os.popen('"%s" --info' % pypy_c, 'r') lines = g.readlines() Modified: pypy/trunk/pypy/objspace/std/itertype.py ============================================================================== --- pypy/trunk/pypy/objspace/std/itertype.py (original) +++ pypy/trunk/pypy/objspace/std/itertype.py Tue Aug 10 15:13:00 2010 @@ -1,5 +1,6 @@ from pypy.interpreter import gateway from pypy.objspace.std.stdtypedef import StdTypeDef +from pypy.interpreter.error import OperationError # ____________________________________________________________ @@ -8,6 +9,11 @@ XXX to do: remove this __reduce__ method and do a registration with copy_reg, instead. """ + + # cpython does not support pickling iterators + msg = 'Pickling for iterators dissabled as cpython does not support it' + raise OperationError(space.w_TypeError, space.wrap(msg)) + from pypy.objspace.std.iterobject import W_AbstractSeqIterObject assert isinstance(w_self, W_AbstractSeqIterObject) from pypy.interpreter.mixedmodule import MixedModule Modified: pypy/trunk/pypy/objspace/std/model.py ============================================================================== --- pypy/trunk/pypy/objspace/std/model.py (original) +++ pypy/trunk/pypy/objspace/std/model.py Tue Aug 10 15:13:00 2010 @@ -88,6 +88,7 @@ import pypy.objspace.std.default # register a few catch-all multimethods import pypy.objspace.std.marshal_impl # install marshal multimethods + import pypy.module.array # the set of implementation types self.typeorder = { @@ -140,6 +141,8 @@ # check if we missed implementations for implcls in _registered_implementations: + if hasattr(implcls, 'register'): + implcls.register(self.typeorder) assert (implcls in self.typeorder or implcls in self.imported_but_not_registered), ( "please add %r in StdTypeModel.typeorder" % (implcls,)) Modified: pypy/trunk/pypy/translator/platform/posix.py ============================================================================== --- pypy/trunk/pypy/translator/platform/posix.py (original) +++ pypy/trunk/pypy/translator/platform/posix.py Tue Aug 10 15:13:00 2010 @@ -14,7 +14,10 @@ def __init__(self, cc=None): if cc is None: - cc = 'gcc' + try: + cc = os.environ['CC'] + except KeyError: + cc = 'gcc' self.cc = cc def _libs(self, libraries): From fijal at codespeak.net Tue Aug 10 15:14:01 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 10 Aug 2010 15:14:01 +0200 (CEST) Subject: [pypy-svn] r76559 - pypy/branch/interplevel-array Message-ID: <20100810131401.B8743282B9C@codespeak.net> Author: fijal Date: Tue Aug 10 15:14:00 2010 New Revision: 76559 Removed: pypy/branch/interplevel-array/ Log: Remove merged branch From fijal at codespeak.net Tue Aug 10 15:18:38 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 10 Aug 2010 15:18:38 +0200 (CEST) Subject: [pypy-svn] r76560 - pypy/trunk/lib_pypy Message-ID: <20100810131838.AC5BA282B9C@codespeak.net> Author: fijal Date: Tue Aug 10 15:18:37 2010 New Revision: 76560 Added: pypy/trunk/lib_pypy/array.py - copied unchanged from r76558, pypy/trunk/lib_pypy/apparray.py Removed: pypy/trunk/lib_pypy/apparray.py Log: Move back apparray to array - it won't hurt since we use interp-level whenever possible From getxsick at codespeak.net Tue Aug 10 15:19:59 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Tue, 10 Aug 2010 15:19:59 +0200 (CEST) Subject: [pypy-svn] r76561 - pypy/branch/fast-ctypes/pypy/module/jitffi Message-ID: <20100810131959.9A890282B9C@codespeak.net> Author: getxsick Date: Tue Aug 10 15:19:58 2010 New Revision: 76561 Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Log: fix translation Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Tue Aug 10 15:19:58 2010 @@ -119,9 +119,14 @@ i += 1 return self.rget.call() - wrap_int_w = lambda self, value: self.space.wrap(value) - wrap_float_w = lambda self, value: self.space.wrap(value) - wrap_void_w = lambda self, w_value: value + def wrap_int_w(self, value): + return self.space.wrap(value) + + def wrap_float_w(self, value): + return self.space.wrap(value) + + def wrap_void_w(self, w_value): + return w_value #def W_Get___new__(space, w_type, cpu, lib, func, args_type, res_type): # try: From fijal at codespeak.net Tue Aug 10 15:20:14 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 10 Aug 2010 15:20:14 +0200 (CEST) Subject: [pypy-svn] r76562 - in pypy/trunk/pypy: config doc/config Message-ID: <20100810132014.4EA37282B9C@codespeak.net> Author: fijal Date: Tue Aug 10 15:20:12 2010 New Revision: 76562 Added: pypy/trunk/pypy/doc/config/objspace.usemodules.array.txt Modified: pypy/trunk/pypy/config/pypyoption.py Log: Add a doc file, put array into allworkingmodules Modified: pypy/trunk/pypy/config/pypyoption.py ============================================================================== --- pypy/trunk/pypy/config/pypyoption.py (original) +++ pypy/trunk/pypy/config/pypyoption.py Tue Aug 10 15:20:12 2010 @@ -30,7 +30,7 @@ "rctime" , "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios", "zlib", "struct", "md5", "sha", "bz2", "_minimal_curses", "cStringIO", - "thread", "itertools", "pyexpat", "_ssl", "cpyext"] + "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array"] )) working_oo_modules = default_modules.copy() Added: pypy/trunk/pypy/doc/config/objspace.usemodules.array.txt ============================================================================== --- (empty file) +++ pypy/trunk/pypy/doc/config/objspace.usemodules.array.txt Tue Aug 10 15:20:12 2010 @@ -0,0 +1 @@ +Use interpreter-level version of array module (on by default). From fijal at codespeak.net Tue Aug 10 15:21:38 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 10 Aug 2010 15:21:38 +0200 (CEST) Subject: [pypy-svn] r76563 - pypy/trunk/pypy/module/array Message-ID: <20100810132138.198BA282B9C@codespeak.net> Author: fijal Date: Tue Aug 10 15:21:36 2010 New Revision: 76563 Modified: pypy/trunk/pypy/module/array/__init__.py Log: strike debugging print Modified: pypy/trunk/pypy/module/array/__init__.py ============================================================================== --- pypy/trunk/pypy/module/array/__init__.py (original) +++ pypy/trunk/pypy/module/array/__init__.py Tue Aug 10 15:21:36 2010 @@ -3,7 +3,6 @@ from pypy.objspace.std.model import registerimplementation for mytype in types.values(): - print mytype.w_class registerimplementation(mytype.w_class) From agaynor at codespeak.net Tue Aug 10 15:34:06 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Tue, 10 Aug 2010 15:34:06 +0200 (CEST) Subject: [pypy-svn] r76564 - pypy/trunk/pypy/annotation Message-ID: <20100810133406.10347282B9C@codespeak.net> Author: agaynor Date: Tue Aug 10 15:34:05 2010 New Revision: 76564 Modified: pypy/trunk/pypy/annotation/annrpython.py Log: Remove debugging stuff from interp level array. Modified: pypy/trunk/pypy/annotation/annrpython.py ============================================================================== --- pypy/trunk/pypy/annotation/annrpython.py (original) +++ pypy/trunk/pypy/annotation/annrpython.py Tue Aug 10 15:34:05 2010 @@ -495,12 +495,6 @@ self.blocked_blocks[block] = graph def bindinputargs(self, graph, block, inputcells, called_from_graph=None): - if 'getitem__Array' in graph.name: - if len(inputcells) > 2 and isinstance(inputcells[1], annmodel.SomeInstance) and 'Base' in str(inputcells[1].classdef): - import pdb - pdb.set_trace() - print graph - print inputcells # Create the initial bindings for the input args of a block. assert len(block.inputargs) == len(inputcells) where = (graph, block, None) From fijal at codespeak.net Tue Aug 10 15:54:16 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 10 Aug 2010 15:54:16 +0200 (CEST) Subject: [pypy-svn] r76565 - pypy/trunk/pypy/doc Message-ID: <20100810135416.0050C282B9C@codespeak.net> Author: fijal Date: Tue Aug 10 15:54:15 2010 New Revision: 76565 Modified: pypy/trunk/pypy/doc/faq.txt Log: fix faq (thanks timonator!) Modified: pypy/trunk/pypy/doc/faq.txt ============================================================================== --- pypy/trunk/pypy/doc/faq.txt (original) +++ pypy/trunk/pypy/doc/faq.txt Tue Aug 10 15:54:15 2010 @@ -47,8 +47,8 @@ There is also an experimental support for CPython extension modules, so they'll run without change (from current observation, rather with little -change) on trunk. It has not been released yet, although it should be a major -point of the next pypy release. +change) on trunk. It has been a part of 1.3 release, but support is still +in alpha phase. .. _`extension modules`: cpython_differences.html#extension-modules .. _`cpython_differences`: cpython_differences.html @@ -373,7 +373,7 @@ -------------------------------------------- No. PyPy always runs your code in its own interpreter, which is a -full and compliant Python 2.4 interpreter. RPython_ is only the +full and compliant Python 2.5 interpreter. RPython_ is only the language in which parts of PyPy itself are written and extension modules for it. The answer to whether something needs to be written as an extension module, apart from the "gluing to external libraries" reason, will From arigo at codespeak.net Tue Aug 10 16:11:06 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 10 Aug 2010 16:11:06 +0200 (CEST) Subject: [pypy-svn] r76566 - pypy/branch/memrecord Message-ID: <20100810141106.78D5A282B9C@codespeak.net> Author: arigo Date: Tue Aug 10 16:11:05 2010 New Revision: 76566 Added: pypy/branch/memrecord/ - copied from r76541, pypy/trunk/ Log: A branch in which to hack in a custom recording of all memory writes of GC pointers. From jcreigh at codespeak.net Tue Aug 10 16:36:55 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Tue, 10 Aug 2010 16:36:55 +0200 (CEST) Subject: [pypy-svn] r76567 - in pypy/branch/asmgcc-64: . lib_pypy lib_pypy/pypy_test pypy/config pypy/doc pypy/doc/config pypy/interpreter pypy/jit/backend/llsupport/test pypy/jit/backend/test pypy/jit/backend/x86 pypy/jit/backend/x86/test pypy/jit/backend/x86/tool pypy/jit/metainterp pypy/jit/metainterp/test pypy/jit/tl pypy/module/_demo pypy/module/_stackless pypy/module/array pypy/module/pypyjit pypy/module/pypyjit/test pypy/module/test_lib_pypy pypy/objspace/std pypy/rlib pypy/rpython pypy/rpython/tool pypy/translator/c pypy/translator/platform Message-ID: <20100810143655.278A1282B9C@codespeak.net> Author: jcreigh Date: Tue Aug 10 16:36:50 2010 New Revision: 76567 Added: pypy/branch/asmgcc-64/lib_pypy/array.py - copied unchanged from r76565, pypy/trunk/lib_pypy/array.py pypy/branch/asmgcc-64/pypy/doc/config/objspace.usemodules.array.txt - copied unchanged from r76565, pypy/trunk/pypy/doc/config/objspace.usemodules.array.txt pypy/branch/asmgcc-64/pypy/module/array/ - copied from r76565, pypy/trunk/pypy/module/array/ Removed: pypy/branch/asmgcc-64/lib_pypy/greenlet.py pypy/branch/asmgcc-64/pypy/module/test_lib_pypy/test_array.py Modified: pypy/branch/asmgcc-64/ (props changed) pypy/branch/asmgcc-64/lib_pypy/pypy_test/test_coroutine.py pypy/branch/asmgcc-64/lib_pypy/stackless.py pypy/branch/asmgcc-64/pypy/config/pypyoption.py pypy/branch/asmgcc-64/pypy/doc/faq.txt pypy/branch/asmgcc-64/pypy/interpreter/pyopcode.py pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/test/test_regalloc.py pypy/branch/asmgcc-64/pypy/jit/backend/test/runner_test.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/assembler.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/regalloc.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/regloc.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/rx86.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_assembler.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_regloc.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_runner.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_rx86.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/tool/viewcode.py pypy/branch/asmgcc-64/pypy/jit/metainterp/optimizeopt.py pypy/branch/asmgcc-64/pypy/jit/metainterp/resoperation.py pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_executor.py pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/asmgcc-64/pypy/jit/tl/pypyjit.py pypy/branch/asmgcc-64/pypy/jit/tl/pypyjit_demo.py pypy/branch/asmgcc-64/pypy/module/_demo/demo.py pypy/branch/asmgcc-64/pypy/module/_stackless/interp_coroutine.py pypy/branch/asmgcc-64/pypy/module/pypyjit/policy.py pypy/branch/asmgcc-64/pypy/module/pypyjit/test/test_pypy_c.py pypy/branch/asmgcc-64/pypy/objspace/std/callmethod.py pypy/branch/asmgcc-64/pypy/objspace/std/itertype.py pypy/branch/asmgcc-64/pypy/objspace/std/model.py pypy/branch/asmgcc-64/pypy/rlib/rmmap.py pypy/branch/asmgcc-64/pypy/rpython/rstr.py pypy/branch/asmgcc-64/pypy/rpython/tool/rfficache.py pypy/branch/asmgcc-64/pypy/translator/c/genc.py pypy/branch/asmgcc-64/pypy/translator/c/node.py pypy/branch/asmgcc-64/pypy/translator/platform/posix.py Log: merged changes from trunk through r76565 Modified: pypy/branch/asmgcc-64/lib_pypy/pypy_test/test_coroutine.py ============================================================================== --- pypy/branch/asmgcc-64/lib_pypy/pypy_test/test_coroutine.py (original) +++ pypy/branch/asmgcc-64/lib_pypy/pypy_test/test_coroutine.py Tue Aug 10 16:36:50 2010 @@ -2,7 +2,7 @@ from py.test import skip, raises try: - from ..stackless import coroutine + from ..stackless import coroutine, CoroutineExit except ImportError, e: skip('cannot import stackless: %s' % (e,)) Modified: pypy/branch/asmgcc-64/lib_pypy/stackless.py ============================================================================== --- pypy/branch/asmgcc-64/lib_pypy/stackless.py (original) +++ pypy/branch/asmgcc-64/lib_pypy/stackless.py Tue Aug 10 16:36:50 2010 @@ -14,9 +14,13 @@ import traceback import sys try: + # If _stackless can be imported then TaskletExit and CoroutineExit are + # automatically added to the builtins. from _stackless import coroutine, greenlet except ImportError: # we are running from CPython - from greenlet import greenlet + from greenlet import greenlet, GreenletExit + TaskletExit = CoroutineExit = GreenletExit + del GreenletExit try: from functools import partial except ImportError: # we are not running python 2.5 Modified: pypy/branch/asmgcc-64/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/config/pypyoption.py (original) +++ pypy/branch/asmgcc-64/pypy/config/pypyoption.py Tue Aug 10 16:36:50 2010 @@ -30,7 +30,7 @@ "rctime" , "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios", "zlib", "struct", "md5", "sha", "bz2", "_minimal_curses", "cStringIO", - "thread", "itertools", "pyexpat", "_ssl", "cpyext"] + "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array"] )) working_oo_modules = default_modules.copy() Modified: pypy/branch/asmgcc-64/pypy/doc/faq.txt ============================================================================== --- pypy/branch/asmgcc-64/pypy/doc/faq.txt (original) +++ pypy/branch/asmgcc-64/pypy/doc/faq.txt Tue Aug 10 16:36:50 2010 @@ -47,8 +47,8 @@ There is also an experimental support for CPython extension modules, so they'll run without change (from current observation, rather with little -change) on trunk. It has not been released yet, although it should be a major -point of the next pypy release. +change) on trunk. It has been a part of 1.3 release, but support is still +in alpha phase. .. _`extension modules`: cpython_differences.html#extension-modules .. _`cpython_differences`: cpython_differences.html @@ -373,7 +373,7 @@ -------------------------------------------- No. PyPy always runs your code in its own interpreter, which is a -full and compliant Python 2.4 interpreter. RPython_ is only the +full and compliant Python 2.5 interpreter. RPython_ is only the language in which parts of PyPy itself are written and extension modules for it. The answer to whether something needs to be written as an extension module, apart from the "gluing to external libraries" reason, will Modified: pypy/branch/asmgcc-64/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/asmgcc-64/pypy/interpreter/pyopcode.py Tue Aug 10 16:36:50 2010 @@ -988,23 +988,9 @@ self.dropvalues(nargs) self.pushvalue(w_result) - def LOOKUP_METHOD(self, nameindex, next_instr): - # overridden by faster version in the standard object space. - space = self.space - w_obj = self.popvalue() - w_name = self.getname_w(nameindex) - w_value = space.getattr(w_obj, w_name) - self.pushvalue(w_value) - - def CALL_METHOD(self, nargs, next_instr): - # overridden by faster version in the standard object space. - # 'nargs' is the argument count excluding the implicit 'self' - w_callable = self.peekvalue(nargs) - try: - w_result = self.space.call_valuestack(w_callable, nargs, self) - finally: - self.dropvalues(nargs + 1) - self.pushvalue(w_result) + # overridden by faster version in the standard object space. + LOOKUP_METHOD = LOAD_ATTR + CALL_METHOD = CALL_FUNCTION def MISSING_OPCODE(self, oparg, next_instr): ofs = self.last_instr Modified: pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/test/test_regalloc.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/test/test_regalloc.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/test/test_regalloc.py Tue Aug 10 16:36:50 2010 @@ -1,5 +1,5 @@ -from pypy.jit.metainterp.history import BoxInt, ConstInt, BoxFloat +from pypy.jit.metainterp.history import BoxInt, ConstInt, BoxFloat, INT, FLOAT from pypy.jit.backend.llsupport.regalloc import FrameManager from pypy.jit.backend.llsupport.regalloc import RegisterManager as BaseRegMan @@ -26,9 +26,20 @@ def convert_to_imm(self, v): return v +class FakeFramePos(object): + def __init__(self, pos, box_type): + self.pos = pos + self.box_type = box_type + + def frame_size(self): + if self.box_type == FLOAT: + return 2 + else: + return 1 + class TFrameManager(FrameManager): - def frame_pos(self, i, size): - return i + def frame_pos(self, i, box_type): + return FakeFramePos(i, box_type) class MockAsm(object): def __init__(self): @@ -146,8 +157,8 @@ rm.next_instruction() # allocate a stack position b0, b1, b2, b3, b4 = boxes - sp = fm.loc(b0, 1) - assert sp == 0 + sp = fm.loc(b0) + assert sp.pos == 0 loc = rm.make_sure_var_in_reg(b0) assert isinstance(loc, FakeReg) rm._check_invariants() @@ -207,13 +218,13 @@ asm = MockAsm() rm = RegisterManager(longevity, frame_manager=fm, assembler=asm) rm.next_instruction() - fm.loc(b0, 1) + fm.loc(b0) rm.force_result_in_reg(b1, b0) rm._check_invariants() loc = rm.loc(b1) assert isinstance(loc, FakeReg) loc = rm.loc(b0) - assert isinstance(loc, int) + assert isinstance(loc, FakeFramePos) assert len(asm.moves) == 1 def test_return_constant(self): @@ -304,7 +315,7 @@ def test_different_frame_width(self): class XRegisterManager(RegisterManager): - reg_width = 2 + pass fm = TFrameManager() b0 = BoxInt() Modified: pypy/branch/asmgcc-64/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/test/runner_test.py Tue Aug 10 16:36:50 2010 @@ -657,6 +657,21 @@ assert r.value == 1 def test_array_basic(self): + a_box, A = self.alloc_array_of(rffi.SHORT, 342) + arraydescr = self.cpu.arraydescrof(A) + assert not arraydescr.is_array_of_pointers() + # + r = self.execute_operation(rop.ARRAYLEN_GC, [a_box], + 'int', descr=arraydescr) + assert r.value == 342 + r = self.execute_operation(rop.SETARRAYITEM_GC, [a_box, BoxInt(310), + BoxInt(744)], + 'void', descr=arraydescr) + assert r is None + r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(310)], + 'int', descr=arraydescr) + assert r.value == 744 + a_box, A = self.alloc_array_of(lltype.Signed, 342) arraydescr = self.cpu.arraydescrof(A) assert not arraydescr.is_array_of_pointers() Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/assembler.py Tue Aug 10 16:36:50 2010 @@ -20,17 +20,18 @@ r12, r13, r14, r15, X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG, - RegLoc, StackLoc, + RegLoc, StackLoc, ConstFloatLoc, ImmedLoc, AddressLoc, imm) from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.jit.backend.x86 import rx86, regloc, codebuf -from pypy.jit.metainterp.resoperation import rop +from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.backend.x86.support import values_array from pypy.rlib.debug import debug_print from pypy.rlib import rgc from pypy.jit.backend.x86.jump import remap_frame_layout from pypy.rlib.streamio import open_file_as_stream +from pypy.jit.metainterp.history import ConstInt, BoxInt # darwin requires the stack to be 16 bytes aligned on calls. Same for gcc 4.5.0, # better safe than sorry @@ -150,6 +151,8 @@ # XXX: 32 is pulled out of the air return 32 + len(self.desc_bytes) +DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed)) + class Assembler386(object): mc = None mc_size = MachineCodeBlockWrapper.MC_DEFAULT_SIZE @@ -170,8 +173,7 @@ self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) self.fail_boxes_float = values_array(lltype.Float, failargs_limit) self.fail_ebp = 0 - self.loop_run_counter = values_array(lltype.Signed, 10000) - self.loop_names = [] + self.loop_run_counters = [] # if we have 10000 loops, we have some other problems I guess self.float_const_neg_addr = 0 self.float_const_abs_addr = 0 @@ -179,8 +181,8 @@ self.malloc_fixedsize_slowpath2 = 0 self.pending_guard_tokens = None self.setup_failure_recovery() - self._loop_counter = 0 self._debug = False + self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') def leave_jitted_hook(self): ptrs = self.fail_boxes_ptr.ar @@ -237,9 +239,9 @@ output_log = self._output_loop_log assert output_log is not None f = open_file_as_stream(output_log, "w") - for i in range(self._loop_counter): - f.write(self.loop_names[i] + ":" + - str(self.loop_run_counter.getitem(i)) + "\n") + for i in range(len(self.loop_run_counters)): + name, struct = self.loop_run_counters[i] + f.write(name + ":" + str(struct.i) + "\n") f.close() def _build_float_constants(self): @@ -304,6 +306,7 @@ regalloc = RegAlloc(self, self.cpu.translate_support_code) + operations = self._inject_debugging_code(operations) arglocs = regalloc.prepare_loop(inputargs, operations, looptoken) looptoken._x86_arglocs = arglocs @@ -343,6 +346,7 @@ assert ([loc.assembler() for loc in arglocs] == [loc.assembler() for loc in faildescr._x86_debug_faillocs]) regalloc = RegAlloc(self, self.cpu.translate_support_code) + operations = self._inject_debugging_code(operations) fail_depths = faildescr._x86_current_depths regalloc.prepare_bridge(fail_depths, inputargs, arglocs, operations) @@ -387,11 +391,12 @@ funcname = op.args[0]._get_str() break else: - funcname = "" % self._loop_counter + funcname = "" % len(self.loop_run_counters) # invent the counter, so we don't get too confused if self._debug: - self.loop_names.append(funcname) - self._loop_counter += 1 + struct = lltype.malloc(DEBUG_COUNTER, flavor='raw') + struct.i = 0 + self.loop_run_counters.append((funcname, struct)) return funcname def patch_jump_for_descr(self, faildescr, adr_new_target): @@ -414,17 +419,30 @@ mc.valgrind_invalidated() mc.done() - def _assemble(self, regalloc, operations): - self._regalloc = regalloc + def _inject_debugging_code(self, operations): if self._debug: # before doing anything, let's increase a counter - # we need one register free (a bit of a hack, but whatever) - self.mc.PUSH(eax) - adr = self.loop_run_counter.get_addr_for_num(self._loop_counter - 1) - self.mc.MOV(eax, heap(adr)) - self.mc.ADD(eax, imm(1)) - self.mc.MOV(heap(adr), eax) - self.mc.POP(eax) + c_adr = ConstInt(rffi.cast(lltype.Signed, + self.loop_run_counters[-1][1])) + box = BoxInt() + box2 = BoxInt() + ops = [ResOperation(rop.GETFIELD_RAW, [c_adr], + box, descr=self.debug_counter_descr), + ResOperation(rop.INT_ADD, [box, ConstInt(1)], box2), + ResOperation(rop.SETFIELD_RAW, [c_adr, box2], + None, descr=self.debug_counter_descr)] + operations = ops + operations + # # we need one register free (a bit of a hack, but whatever) + # self.mc.PUSH(eax) + # adr = rffi.cast(lltype.Signed, self.loop_run_counters[-1][1]) + # self.mc.MOV(eax, heap(adr)) + # self.mc.ADD(eax, imm(1)) + # self.mc.MOV(heap(adr), eax) + # self.mc.POP(eax) + return operations + + def _assemble(self, regalloc, operations): + self._regalloc = regalloc regalloc.walk_operations(operations) self.mc.done() if we_are_translated() or self.cpu.dont_keepalive_stuff: @@ -800,10 +818,13 @@ for i in range(start, len(arglocs)): loc = arglocs[i] - assert isinstance(loc, RegLoc) or isinstance(loc, ImmedLoc) or isinstance(loc, StackLoc) - - # XXX: Should be much simplier to tell whether a location is a float! - if (isinstance(loc, RegLoc) and loc.is_xmm) or (isinstance(loc, StackLoc) and loc.type == FLOAT): + # XXX: Should be much simplier to tell whether a location is a + # float! It's so ugly because we have to "guard" the access to + # .type with isinstance, since not all AssemblerLocation classes + # are "typed" + if ((isinstance(loc, RegLoc) and loc.is_xmm) or + (isinstance(loc, StackLoc) and loc.type == FLOAT) or + (isinstance(loc, ConstFloatLoc))): if len(unused_xmm) > 0: xmm_src_locs.append(loc) xmm_dst_locs.append(unused_xmm.pop()) @@ -1045,11 +1066,14 @@ if scale.value == 0: self.mc.MOVZX8(resloc, addr_add(base_loc, ofs_loc, ofs.value, scale.value)) + elif scale.value == 1: + self.mc.MOVZX16(resloc, addr_add(base_loc, ofs_loc, ofs.value, + scale.value)) elif (1 << scale.value) == WORD: self.mc.MOV(resloc, addr_add(base_loc, ofs_loc, ofs.value, scale.value)) else: - print "[asmgen]setarrayitem unsupported size: %d" % scale.value + print "[asmgen]getarrayitem unsupported size: %d" % scale.value raise NotImplementedError() genop_getarrayitem_gc_pure = genop_getarrayitem_gc @@ -1082,6 +1106,8 @@ else: if (1 << scale_loc.value) == WORD: self.mc.MOV(dest_addr, value_loc) + elif scale_loc.value == 1: + self.mc.MOV16(dest_addr, value_loc) elif scale_loc.value == 0: self.mc.MOV8(dest_addr, value_loc.lowest8bits()) else: Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/regalloc.py Tue Aug 10 16:36:50 2010 @@ -66,6 +66,7 @@ def __init__(self): self.cur_array_free = 0 + self.const_id = 0 def _get_new_array(self): n = self.BASE_CONSTANT_SIZE @@ -81,7 +82,8 @@ n = self.cur_array_free - 1 arr[n] = floatval self.cur_array_free = n - return rffi.cast(lltype.Signed, arr) + n * 8 + self.const_id += 1 + return (self.const_id, rffi.cast(lltype.Signed, arr) + n * 8) class X86XMMRegisterManager(RegisterManager): @@ -102,8 +104,8 @@ self.float_constants = assembler._float_constants def convert_to_imm(self, c): - adr = self.float_constants.record_float(c.getfloat()) - return AddressLoc(ImmedLoc(adr), ImmedLoc(0), 0, 0) + const_id, adr = self.float_constants.record_float(c.getfloat()) + return ConstFloatLoc(adr, const_id) def after_call(self, v): # the result is stored in st0, but we don't have this around, Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/regloc.py Tue Aug 10 16:36:50 2010 @@ -4,6 +4,8 @@ from pypy.jit.backend.x86.arch import WORD, IS_X86_32, IS_X86_64 from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.objectmodel import specialize +from pypy.rlib.rarithmetic import intmask +from pypy.jit.metainterp.history import FLOAT # # This module adds support for "locations", which can be either in a Const, @@ -144,6 +146,27 @@ def value_m(self): return self.loc_m +class ConstFloatLoc(AssemblerLocation): + # XXX: We have to use this class instead of just AddressLoc because + # AddressLoc is "untyped" and also we to have need some sort of unique + # identifier that we can use in _getregkey (for jump.py) + + _immutable_ = True + + width = 8 + + def __init__(self, address, const_id): + self.value = address + self.const_id = const_id + + def _getregkey(self): + # XXX: 1000 is kind of magic: We just don't want to be confused + # with any registers + return 1000 + self.const_id + + def location_code(self): + return 'j' + REGLOCS = [RegLoc(i, is_xmm=False) for i in range(16)] XMMREGLOCS = [RegLoc(i, is_xmm=True) for i in range(16)] eax, ecx, edx, ebx, esp, ebp, esi, edi, r8, r9, r10, r11, r12, r13, r14, r15 = REGLOCS @@ -252,7 +275,7 @@ if code == possible_code: val = getattr(loc, "value_" + possible_code)() if possible_code == 'i': - offset = val - (self.tell() + 5) + offset = intmask(val - (self.tell() + 5)) if rx86.fits_in_32bits(offset): _rx86_getattr(self, name + "_l")(val) else: Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/rx86.py Tue Aug 10 16:36:50 2010 @@ -2,6 +2,7 @@ from pypy.rlib.objectmodel import ComputedIntSymbolic, we_are_translated from pypy.rlib.objectmodel import specialize from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib.rarithmetic import intmask from pypy.rpython.lltypesystem import rffi BYTE_REG_FLAG = 0x20 @@ -138,7 +139,7 @@ def encode_relative(mc, target, _, orbyte): assert orbyte == 0 - offset = target - (mc.tell() + 4) + offset = intmask(target - (mc.tell() + 4)) mc.writeimm32(offset) return 0 Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_assembler.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_assembler.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_assembler.py Tue Aug 10 16:36:50 2010 @@ -15,6 +15,9 @@ supports_floats = True NUM_REGS = ACTUAL_CPU.NUM_REGS + def fielddescrof(self, STRUCT, name): + return 42 + class FakeMC: def __init__(self, base_address=0): self.content = [] Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_regloc.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_regloc.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_regloc.py Tue Aug 10 16:36:50 2010 @@ -1,7 +1,9 @@ +import struct from pypy.jit.backend.x86.regloc import * from pypy.jit.backend.x86.test.test_rx86 import CodeBuilder32, CodeBuilder64, assert_encodes_as from pypy.jit.backend.x86.assembler import heap -from pypy.jit.backend.x86.arch import IS_X86_64 +from pypy.jit.backend.x86.arch import IS_X86_64, IS_X86_32 +from pypy.rlib.rarithmetic import intmask import py.test class LocationCodeBuilder32(CodeBuilder32, LocationCodeBuilder): @@ -21,6 +23,27 @@ assert_encodes_as(cb32, "CMP16", (ecx, ebx), '\x66\x39\xD9') assert_encodes_as(cb32, "CMP16", (ecx, ImmedLoc(12345)), '\x66\x81\xF9\x39\x30') +def test_jmp_wraparound(): + if not IS_X86_32: + py.test.skip() + + pos_addr = intmask(0x7FFFFF00) + neg_addr = intmask(0x800000BB) + + # JMP to "negative" address from "positive" address + s = cb32() + s.base_address = pos_addr + s.JMP(ImmedLoc(neg_addr)) + expected_ofs = neg_addr - (pos_addr+5) + assert s.getvalue() == '\xE9' + struct.pack("', rffi.LONG) -time = rffi.llexternal('time', [rffi.VOIDP], time_t, includes=['time.h']) +eci = ExternalCompilationInfo(includes=['time.h']) +time = rffi.llexternal('time', [int], time_t, + compilation_info=eci) def get(space, name): w_module = space.getbuiltinmodule('_demo') @@ -20,10 +23,10 @@ w_DemoError = get(space, 'DemoError') msg = "repetition count must be > 0" raise OperationError(w_DemoError, space.wrap(msg)) - starttime = time(None) + starttime = time(0) for i in range(repetitions): space.call_function(w_callable) - endtime = time(None) + endtime = time(0) return space.wrap(endtime - starttime) measuretime.unwrap_spec = [ObjSpace, int, W_Root] @@ -62,11 +65,16 @@ self.x = space.int_w(w_value) def mytype_new(space, w_subtype, x): + if x == 3: + return space.wrap(MySubType(space, x)) return space.wrap(W_MyType(space, x)) mytype_new.unwrap_spec = [ObjSpace, W_Root, int] getset_x = GetSetProperty(W_MyType.fget_x, W_MyType.fset_x, cls=W_MyType) +class MySubType(W_MyType): + pass + W_MyType.typedef = TypeDef('MyType', __new__ = interp2app(mytype_new), x = getset_x, Modified: pypy/branch/asmgcc-64/pypy/module/_stackless/interp_coroutine.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/_stackless/interp_coroutine.py (original) +++ pypy/branch/asmgcc-64/pypy/module/_stackless/interp_coroutine.py Tue Aug 10 16:36:50 2010 @@ -265,10 +265,14 @@ instr += 1 oparg = ord(code[instr]) | ord(code[instr + 1]) << 8 nargs = oparg & 0xff + nkwds = (oparg >> 8) & 0xff if space.config.objspace.opcodes.CALL_METHOD and opcode == map['CALL_METHOD']: - chain = resume_state_create(chain, 'CALL_METHOD', frame, - nargs) - elif opcode == map['CALL_FUNCTION'] and (oparg >> 8) & 0xff == 0: + if nkwds == 0: # only positional arguments + chain = resume_state_create(chain, 'CALL_METHOD', frame, + nargs) + else: # includes keyword arguments + chain = resume_state_create(chain, 'CALL_METHOD_KW', frame) + elif opcode == map['CALL_FUNCTION'] and nkwds == 0: # Only positional arguments # case1: ("CALL_FUNCTION", f, nargs, returns=w_result) chain = resume_state_create(chain, 'CALL_FUNCTION', frame, Modified: pypy/branch/asmgcc-64/pypy/module/pypyjit/policy.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/pypyjit/policy.py (original) +++ pypy/branch/asmgcc-64/pypy/module/pypyjit/policy.py Tue Aug 10 16:36:50 2010 @@ -11,7 +11,7 @@ if '.' in modname: modname, _ = modname.split('.', 1) if modname in ['pypyjit', 'signal', 'micronumpy', 'math', 'exceptions', - 'imp', 'sys']: + 'imp', 'sys', 'array']: return True return False Modified: pypy/branch/asmgcc-64/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/asmgcc-64/pypy/module/pypyjit/test/test_pypy_c.py Tue Aug 10 16:36:50 2010 @@ -615,6 +615,237 @@ return total ''', 170, ([], 4999450000L)) + def test_boolrewrite_invers(self): + for a, b, res, ops in (('2000', '2000', 20001000, 51), + ( '500', '500', 15001500, 81), + ( '300', '600', 16001700, 83), + ( 'a', 'b', 16001700, 89), + ( 'a', 'a', 13001700, 85)): + + self.run_source(''' + def main(): + sa = 0 + a = 300 + b = 600 + for i in range(1000): + if i < %s: sa += 1 + else: sa += 2 + if i >= %s: sa += 10000 + else: sa += 20000 + return sa + '''%(a, b), ops, ([], res)) + + def test_boolrewrite_reflex(self): + for a, b, res, ops in (('2000', '2000', 10001000, 51), + ( '500', '500', 15001500, 81), + ( '300', '600', 14001700, 83), + ( 'a', 'b', 14001700, 89), + ( 'a', 'a', 17001700, 85)): + + self.run_source(''' + def main(): + sa = 0 + a = 300 + b = 600 + for i in range(1000): + if i < %s: sa += 1 + else: sa += 2 + if %s > i: sa += 10000 + else: sa += 20000 + return sa + '''%(a, b), ops, ([], res)) + + + def test_boolrewrite_correct_invers(self): + def opval(i, op, a): + if eval('%d %s %d' % (i, op, a)): return 1 + return 2 + + ops = ('<', '>', '<=', '>=', '==', '!=') + for op1 in ops: + for op2 in ops: + for a,b in ((500, 500), (300, 600)): + res = 0 + res += opval(a-1, op1, a) * (a) + res += opval( a, op1, a) + res += opval(a+1, op1, a) * (1000 - a - 1) + res += opval(b-1, op2, b) * 10000 * (b) + res += opval( b, op2, b) * 10000 + res += opval(b+1, op2, b) * 10000 * (1000 - b - 1) + + self.run_source(''' + def main(): + sa = 0 + for i in range(1000): + if i %s %d: sa += 1 + else: sa += 2 + if i %s %d: sa += 10000 + else: sa += 20000 + return sa + '''%(op1, a, op2, b), 83, ([], res)) + + self.run_source(''' + def main(): + sa = 0 + i = 0.0 + while i < 250.0: + if i %s %f: sa += 1 + else: sa += 2 + if i %s %f: sa += 10000 + else: sa += 20000 + i += 0.25 + return sa + '''%(op1, float(a)/4.0, op2, float(b)/4.0), 109, ([], res)) + + + def test_boolrewrite_correct_reflex(self): + def opval(i, op, a): + if eval('%d %s %d' % (i, op, a)): return 1 + return 2 + + ops = ('<', '>', '<=', '>=', '==', '!=') + for op1 in ops: + for op2 in ops: + for a,b in ((500, 500), (300, 600)): + res = 0 + res += opval(a-1, op1, a) * (a) + res += opval( a, op1, a) + res += opval(a+1, op1, a) * (1000 - a - 1) + res += opval(b, op2, b-1) * 10000 * (b) + res += opval(b, op2, b) * 10000 + res += opval(b, op2, b+1) * 10000 * (1000 - b - 1) + + self.run_source(''' + def main(): + sa = 0 + for i in range(1000): + if i %s %d: sa += 1 + else: sa += 2 + if %d %s i: sa += 10000 + else: sa += 20000 + return sa + '''%(op1, a, b, op2), 83, ([], res)) + + self.run_source(''' + def main(): + sa = 0 + i = 0.0 + while i < 250.0: + if i %s %f: sa += 1 + else: sa += 2 + if %f %s i: sa += 10000 + else: sa += 20000 + i += 0.25 + return sa + '''%(op1, float(a)/4.0, float(b)/4.0, op2), 109, ([], res)) + + def test_boolrewrite_ptr(self): + compares = ('a == b', 'b == a', 'a != b', 'b != a', 'a == c', 'c != b') + for e1 in compares: + for e2 in compares: + a, b, c = 1, 2, 3 + if eval(e1): res = 752 * 1 + else: res = 752 * 2 + if eval(e2): res += 752 * 10000 + else: res += 752 * 20000 + a = b + if eval(e1): res += 248 * 1 + else: res += 248 * 2 + if eval(e2): res += 248 * 10000 + else: res += 248 * 20000 + + + if 'c' in e1 or 'c' in e2: + n = 337 + else: + n = 215 + + self.run_source(''' + class tst: + pass + def main(): + a = tst() + b = tst() + c = tst() + sa = 0 + for i in range(1000): + if %s: sa += 1 + else: sa += 2 + if %s: sa += 10000 + else: sa += 20000 + if i > 750: a = b + return sa + '''%(e1, e2), n, ([], res)) + + def test_array_sum(self): + for tc, maxops in zip('bhilBHILfd', (38,) * 6 + (40, 40, 41, 38)): + res = 19352859 + if tc in 'IL': + res = long(res) + elif tc in 'fd': + res = float(res) + + self.run_source(''' + from array import array + + def main(): + img = array("%s", range(127) * 5) * 484 + l, i = 0, 0 + while i < 640 * 480: + l += img[i] + i += 1 + return l + ''' % tc, maxops, ([], res)) + + def test_array_sum_char(self): + self.run_source(''' + from array import array + + def main(): + img = array("c", "Hello") * 130 * 480 + l, i = 0, 0 + while i < 640 * 480: + l += ord(img[i]) + i += 1 + return l + ''', 60, ([], 30720000)) + + def test_array_sum_unicode(self): + self.run_source(''' + from array import array + + def main(): + img = array("u", u"Hello") * 130 * 480 + l, i = 0, 0 + while i < 640 * 480: + if img[i] == u"l": + l += 1 + i += 1 + return l + ''', 65, ([], 122880)) + + def test_array_intimg(self): + for tc, maxops in zip('ilILd', (67, 67, 69, 69, 61)): + res = 73574560 + if tc in 'IL': + res = long(res) + elif tc in 'fd': + res = float(res) + + self.run_source(''' + from array import array + + def main(tc): + img = array(tc, range(3)) * (350 * 480) + intimg = array(tc, (0,)) * (640 * 480) + l, i = 0, 640 + while i < 640 * 480: + l = l + img[i] + intimg[i] = (intimg[i-640] + l) + i += 1 + return intimg[i - 1] + ''', maxops, ([tc], res)) + class AppTestJIT(PyPyCJITTests): def setup_class(cls): if not option.runappdirect: @@ -637,6 +868,7 @@ cls.counter = 0 cls.pypy_c = option.pypy_c + def has_info(pypy_c, option): g = os.popen('"%s" --info' % pypy_c, 'r') lines = g.readlines() Modified: pypy/branch/asmgcc-64/pypy/objspace/std/callmethod.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/objspace/std/callmethod.py (original) +++ pypy/branch/asmgcc-64/pypy/objspace/std/callmethod.py Tue Aug 10 16:36:50 2010 @@ -62,13 +62,13 @@ n_args = oparg & 0xff n_kwargs = (oparg >> 8) & 0xff w_self = f.peekvalue(n_args + (2 * n_kwargs)) - w_callable = f.peekvalue(n_args + (2 * n_kwargs) + 1) n = n_args + (w_self is not None) if not n_kwargs: + w_callable = f.peekvalue(n_args + (2 * n_kwargs) + 1) try: w_result = f.space.call_valuestack(w_callable, n, f) - rstack.resume_point("CALL_METHOD_no_kwargs", f, n_args, returns=w_result) + rstack.resume_point("CALL_METHOD", f, n_args, returns=w_result) finally: f.dropvalues(n_args + 2) else: @@ -84,14 +84,13 @@ keywords[n_kwargs] = key keywords_w[n_kwargs] = w_value - arguments = f.popvalues(n) + arguments = f.popvalues(n) # includes w_self if it is not None args = f.argument_factory(arguments, keywords, keywords_w, None, None) - - try: - w_result = f.space.call_args(w_callable, args) - rstack.resume_point("CALL_METHOD", f, w_self, returns=w_result) - finally: - f.dropvalues(1 + (w_self is None)) + if w_self is None: + f.popvalue() # removes w_self, which is None + w_callable = f.popvalue() + w_result = f.space.call_args(w_callable, args) + rstack.resume_point("CALL_METHOD_KW", f, returns=w_result) f.pushvalue(w_result) Modified: pypy/branch/asmgcc-64/pypy/objspace/std/itertype.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/objspace/std/itertype.py (original) +++ pypy/branch/asmgcc-64/pypy/objspace/std/itertype.py Tue Aug 10 16:36:50 2010 @@ -1,5 +1,6 @@ from pypy.interpreter import gateway from pypy.objspace.std.stdtypedef import StdTypeDef +from pypy.interpreter.error import OperationError # ____________________________________________________________ @@ -8,6 +9,11 @@ XXX to do: remove this __reduce__ method and do a registration with copy_reg, instead. """ + + # cpython does not support pickling iterators + msg = 'Pickling for iterators dissabled as cpython does not support it' + raise OperationError(space.w_TypeError, space.wrap(msg)) + from pypy.objspace.std.iterobject import W_AbstractSeqIterObject assert isinstance(w_self, W_AbstractSeqIterObject) from pypy.interpreter.mixedmodule import MixedModule Modified: pypy/branch/asmgcc-64/pypy/objspace/std/model.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/objspace/std/model.py (original) +++ pypy/branch/asmgcc-64/pypy/objspace/std/model.py Tue Aug 10 16:36:50 2010 @@ -88,6 +88,7 @@ import pypy.objspace.std.default # register a few catch-all multimethods import pypy.objspace.std.marshal_impl # install marshal multimethods + import pypy.module.array # the set of implementation types self.typeorder = { @@ -140,6 +141,8 @@ # check if we missed implementations for implcls in _registered_implementations: + if hasattr(implcls, 'register'): + implcls.register(self.typeorder) assert (implcls in self.typeorder or implcls in self.imported_but_not_registered), ( "please add %r in StdTypeModel.typeorder" % (implcls,)) Modified: pypy/branch/asmgcc-64/pypy/rlib/rmmap.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rlib/rmmap.py (original) +++ pypy/branch/asmgcc-64/pypy/rlib/rmmap.py Tue Aug 10 16:36:50 2010 @@ -646,8 +646,14 @@ hintp = rffi.cast(PTR, hint.pos) res = c_mmap_safe(hintp, map_size, prot, flags, -1, 0) if res == rffi.cast(PTR, -1): - raise MemoryError - hint.pos += map_size + # some systems (some versions of OS/X?) complain if they + # are passed a non-zero address. Try again. + hintp = rffi.cast(PTR, 0) + res = c_mmap_safe(hintp, map_size, prot, flags, -1, 0) + if res == rffi.cast(PTR, -1): + raise MemoryError + else: + hint.pos += map_size return res alloc._annenforceargs_ = (int,) Modified: pypy/branch/asmgcc-64/pypy/rpython/rstr.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rpython/rstr.py (original) +++ pypy/branch/asmgcc-64/pypy/rpython/rstr.py Tue Aug 10 16:36:50 2010 @@ -288,8 +288,8 @@ if not hop.args_s[1].is_constant(): raise TyperError("encoding must be constant") encoding = hop.args_s[1].const - if encoding == "ascii": - expect = self.lowleveltype # can be a UniChar + if encoding == "ascii" and self.lowleveltype == UniChar: + expect = UniChar # only for unichar.encode('ascii') else: expect = self.repr # must be a regular unicode string v_self = hop.inputarg(expect, 0) Modified: pypy/branch/asmgcc-64/pypy/rpython/tool/rfficache.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rpython/tool/rfficache.py (original) +++ pypy/branch/asmgcc-64/pypy/rpython/tool/rfficache.py Tue Aug 10 16:36:50 2010 @@ -29,7 +29,7 @@ } ''' % (include_string, add_source, str(question))) c_file = udir.join("gcctest.c") - c_file.write(c_source) + c_file.write(str(c_source) + '\n') eci = ExternalCompilationInfo() return build_executable_cache([c_file], eci) Modified: pypy/branch/asmgcc-64/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/c/genc.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/c/genc.py Tue Aug 10 16:36:50 2010 @@ -1,7 +1,6 @@ import autopath import py import sys, os -from pypy.translator.c.node import PyObjectNode, FuncNode from pypy.translator.c.database import LowLevelDatabase from pypy.translator.c.extfunc import pre_include_code_lines from pypy.translator.llsupport.wrapper import new_wrapper @@ -196,7 +195,7 @@ all = [] for node in self.db.globalcontainers(): - eci = getattr(node, 'compilation_info', None) + eci = node.compilation_info() if eci: all.append(eci) self.merge_eci(*all) @@ -222,7 +221,7 @@ graphs = db.all_graphs() db.gctransformer.prepare_inline_helpers(graphs) for node in db.containerlist: - if isinstance(node, FuncNode): + if hasattr(node, 'funcgens'): for funcgen in node.funcgens: funcgen.patch_graph(copy_graph=False) return db Modified: pypy/branch/asmgcc-64/pypy/translator/c/node.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/c/node.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/c/node.py Tue Aug 10 16:36:50 2010 @@ -466,15 +466,15 @@ class ContainerNode(object): - if USESLOTS: - __slots__ = """db T obj + if USESLOTS: # keep the number of slots down! + __slots__ = """db obj typename implementationtypename - name ptrname compilation_info + name ptrname globalcontainer""".split() + eci_name = '_compilation_info' def __init__(self, db, T, obj): self.db = db - self.T = T self.obj = obj #self.dependencies = {} self.typename = db.gettype(T) #, who_asks=self) @@ -489,16 +489,22 @@ else: self.globalcontainer = False parentnode = db.getcontainernode(parent) - defnode = db.gettypedefnode(parentnode.T) + defnode = db.gettypedefnode(parentnode.getTYPE()) self.name = defnode.access_expr(parentnode.name, parentindex) if self.typename != self.implementationtypename: if db.gettypedefnode(T).extra_union_for_varlength: self.name += '.b' - self.compilation_info = getattr(obj, '_compilation_info', None) self.ptrname = '(&%s)' % self.name + def getTYPE(self): + return typeOf(self.obj) + def is_thread_local(self): - return hasattr(self.T, "_hints") and self.T._hints.get('thread_local') + T = self.getTYPE() + return hasattr(T, "_hints") and T._hints.get('thread_local') + + def compilation_info(self): + return getattr(self.obj, self.eci_name, None) def get_declaration(self): if self.name[-2:] == '.b': @@ -546,27 +552,31 @@ __slots__ = () def basename(self): - return self.T._name + T = self.getTYPE() + return T._name def enum_dependencies(self): - for name in self.T._names: + T = self.getTYPE() + for name in T._names: yield getattr(self.obj, name) def getlength(self): - if self.T._arrayfld is None: + T = self.getTYPE() + if T._arrayfld is None: return 1 else: - array = getattr(self.obj, self.T._arrayfld) + array = getattr(self.obj, T._arrayfld) return len(array.items) def initializationexpr(self, decoration=''): + T = self.getTYPE() is_empty = True yield '{' - defnode = self.db.gettypedefnode(self.T) + defnode = self.db.gettypedefnode(T) data = [] - if needs_gcheader(self.T): + if needs_gcheader(T): gc_init = self.db.gcpolicy.struct_gcheader_initdata(self) data.append(('gcheader', gc_init)) @@ -578,16 +588,16 @@ # '.fieldname = value'. But here we don't know which of the # fields need initialization, so XXX we pick the first one # arbitrarily. - if hasattr(self.T, "_hints") and self.T._hints.get('union'): + if hasattr(T, "_hints") and T._hints.get('union'): data = data[0:1] - if 'get_padding_drop' in self.T._hints: + if 'get_padding_drop' in T._hints: d = {} for name, _ in data: - T = defnode.c_struct_field_type(name) - typename = self.db.gettype(T) + T1 = defnode.c_struct_field_type(name) + typename = self.db.gettype(T1) d[name] = cdecl(typename, '') - padding_drop = self.T._hints['get_padding_drop'](d) + padding_drop = T._hints['get_padding_drop'](d) else: padding_drop = [] @@ -617,9 +627,10 @@ return 'struct _hashT_%s @' % self.name def forward_declaration(self): + T = self.getTYPE() assert self.typename == self.implementationtypename # no array part hash_typename = self.get_hash_typename() - hash_offset = self.db.gctransformer.get_hash_offset(self.T) + hash_offset = self.db.gctransformer.get_hash_offset(T) yield '%s {' % cdecl(hash_typename, '') yield '\tunion {' yield '\t\t%s;' % cdecl(self.implementationtypename, 'head') @@ -671,22 +682,23 @@ return len(self.obj.items) def initializationexpr(self, decoration=''): - defnode = self.db.gettypedefnode(self.T) + T = self.getTYPE() + defnode = self.db.gettypedefnode(T) yield '{' - if needs_gcheader(self.T): + if needs_gcheader(T): gc_init = self.db.gcpolicy.array_gcheader_initdata(self) lines = generic_initializationexpr(self.db, gc_init, 'gcheader', '%sgcheader' % (decoration,)) for line in lines: yield line - if self.T._hints.get('nolength', False): + if T._hints.get('nolength', False): length = '' else: length = '%d, ' % len(self.obj.items) - if self.T.OF is Void or len(self.obj.items) == 0: + if T.OF is Void or len(self.obj.items) == 0: yield '\t%s' % length.rstrip(', ') yield '}' - elif self.T.OF == Char: + elif T.OF == Char: if len(self.obj.items) and self.obj.items[0] is None: s = ''.join([self.obj.getitem(i) for i in range(len(self.obj.items))]) else: @@ -694,7 +706,7 @@ yield '\t%s%s' % (length, c_char_array_constant(s)) yield '}' else: - barebone = barebonearray(self.T) + barebone = barebonearray(T) if not barebone: yield '\t%s{' % length for j in range(len(self.obj.items)): @@ -722,7 +734,8 @@ self.ptrname = self.name def basename(self): - return self.T._name + T = self.getTYPE() + return T._name def enum_dependencies(self): for i in range(self.obj.getlength()): @@ -732,11 +745,12 @@ return 1 # not variable-sized! def initializationexpr(self, decoration=''): + T = self.getTYPE() assert self.typename == self.implementationtypename # not var-sized is_empty = True yield '{' # _names == ['item0', 'item1', ...] - for j, name in enumerate(self.T._names): + for j, name in enumerate(T._names): value = getattr(self.obj, name) lines = generic_initializationexpr(self.db, value, '%s[%d]' % (self.name, j), @@ -777,6 +791,7 @@ class FuncNode(ContainerNode): nodekind = 'func' + eci_name = 'compilation_info' # there not so many node of this kind, slots should not # be necessary @@ -794,7 +809,6 @@ else: self.name = (forcename or db.namespace.uniquename('g_' + self.basename())) - self.compilation_info = getattr(obj, 'compilation_info', None) self.make_funcgens() #self.dependencies = {} self.typename = db.gettype(T) #, who_asks=self) @@ -939,18 +953,20 @@ return [] def initializationexpr(self, decoration=''): - yield 'RPyOpaque_INITEXPR_%s' % (self.T.tag,) + T = self.getTYPE() + yield 'RPyOpaque_INITEXPR_%s' % (T.tag,) def startupcode(self): + T = self.getTYPE() args = [self.ptrname] # XXX how to make this code more generic? - if self.T.tag == 'ThreadLock': + if T.tag == 'ThreadLock': lock = self.obj.externalobj if lock.locked(): args.append('1') else: args.append('0') - yield 'RPyOpaque_SETUP_%s(%s);' % (self.T.tag, ', '.join(args)) + yield 'RPyOpaque_SETUP_%s(%s);' % (T.tag, ', '.join(args)) def opaquenode_factory(db, T, obj): Modified: pypy/branch/asmgcc-64/pypy/translator/platform/posix.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/platform/posix.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/platform/posix.py Tue Aug 10 16:36:50 2010 @@ -14,7 +14,10 @@ def __init__(self, cc=None): if cc is None: - cc = 'gcc' + try: + cc = os.environ['CC'] + except KeyError: + cc = 'gcc' self.cc = cc def _libs(self, libraries): From arigo at codespeak.net Tue Aug 10 16:39:01 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 10 Aug 2010 16:39:01 +0200 (CEST) Subject: [pypy-svn] r76568 - in pypy/branch/memrecord/pypy/translator/c: . src Message-ID: <20100810143901.EB91E282B9C@codespeak.net> Author: arigo Date: Tue Aug 10 16:38:59 2010 New Revision: 76568 Added: pypy/branch/memrecord/pypy/translator/c/src/debug_memrecord.h (contents, props changed) Modified: pypy/branch/memrecord/pypy/translator/c/funcgen.py pypy/branch/memrecord/pypy/translator/c/src/g_include.h pypy/branch/memrecord/pypy/translator/c/src/main.h Log: First attempt, in progress. Modified: pypy/branch/memrecord/pypy/translator/c/funcgen.py ============================================================================== --- pypy/branch/memrecord/pypy/translator/c/funcgen.py (original) +++ pypy/branch/memrecord/pypy/translator/c/funcgen.py Tue Aug 10 16:38:59 2010 @@ -20,6 +20,7 @@ LOCALVAR = 'l_%s' KEEP_INLINED_GRAPHS = False +DMR_COUNT = 0 class FunctionCodeGenerator(object): """ @@ -480,6 +481,12 @@ T = self.lltypemap(op.args[-1]) if T is Void: result = '/* %s */' % result + elif (isinstance(T, Ptr) and T.TO._gckind == 'gc' and + self.lltypemap(op.args[0]).TO._gckind == 'gc'): + global DMR_COUNT + DMR_COUNT += 1 + result = 'RPyWrite(%d, %s, %s);' % ( + DMR_COUNT, targetexpr, newvalue) return result def OP_GETFIELD(self, op, ampersand=''): Added: pypy/branch/memrecord/pypy/translator/c/src/debug_memrecord.h ============================================================================== --- (empty file) +++ pypy/branch/memrecord/pypy/translator/c/src/debug_memrecord.h Tue Aug 10 16:38:59 2010 @@ -0,0 +1,39 @@ +#include + + +extern FILE* debug_memrecord_f; + +void debug_memrecord_startup(void); + +struct debug_memrecord_s { + long a; + void *b, *c, *d; +}; + +#define DEBUG_MEMREC(a1, b1, c1, d1) do { \ + struct debug_memrecord_s _dmr_s; \ + _dmr_s.a = a1; \ + _dmr_s.b = b1; \ + _dmr_s.c = c1; \ + _dmr_s.d = d1; \ + fwrite_unlocked(&_dmr_s, sizeof(struct debug_memrecord_s), 1, \ + debug_memrecord_f); \ +} while(0) + +#define RPyWrite(posid, targetexpr, newvalue) \ + DEBUG_MEMREC(posid, &(targetexpr), targetexpr, newvalue); \ + targetexpr = newvalue + +/************************************************************/ + +#ifndef PYPY_NOT_MAIN_FILE + +FILE* debug_memrecord_f; + +void debug_memrecord_startup(void) +{ + debug_memrecord_f = fopen("debug_memrecord", "wb"); + assert(debug_memrecord_f); +} + +#endif Modified: pypy/branch/memrecord/pypy/translator/c/src/g_include.h ============================================================================== --- pypy/branch/memrecord/pypy/translator/c/src/g_include.h (original) +++ pypy/branch/memrecord/pypy/translator/c/src/g_include.h Tue Aug 10 16:38:59 2010 @@ -61,6 +61,7 @@ #ifdef PYPY_STANDALONE # include "src/allocator.h" +# include "src/debug_memrecord.h" # include "src/main.h" #endif Modified: pypy/branch/memrecord/pypy/translator/c/src/main.h ============================================================================== --- pypy/branch/memrecord/pypy/translator/c/src/main.h (original) +++ pypy/branch/memrecord/pypy/translator/c/src/main.h Tue Aug 10 16:38:59 2010 @@ -41,6 +41,8 @@ pypy_Windows_startup(); #endif + debug_memrecord_startup(); + errmsg = RPython_StartupCode(); if (errmsg) goto error; From arigo at codespeak.net Tue Aug 10 16:48:40 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 10 Aug 2010 16:48:40 +0200 (CEST) Subject: [pypy-svn] r76569 - in pypy/branch/memrecord/pypy: rpython/lltypesystem translator/c/src Message-ID: <20100810144840.A9B73282B9C@codespeak.net> Author: arigo Date: Tue Aug 10 16:48:39 2010 New Revision: 76569 Modified: pypy/branch/memrecord/pypy/rpython/lltypesystem/llarena.py pypy/branch/memrecord/pypy/translator/c/src/debug_memrecord.h Log: Also record the memclears done during GC. Modified: pypy/branch/memrecord/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/branch/memrecord/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/branch/memrecord/pypy/rpython/lltypesystem/llarena.py Tue Aug 10 16:48:39 2010 @@ -447,6 +447,9 @@ clear_large_memory_chunk = llmemory.raw_memclear +clear_large_memory_chunk = llmemory.raw_memclear # XXXXXX + + def llimpl_arena_malloc(nbytes, zero): addr = llmemory.raw_malloc(nbytes) if zero and bool(addr): Modified: pypy/branch/memrecord/pypy/translator/c/src/debug_memrecord.h ============================================================================== --- pypy/branch/memrecord/pypy/translator/c/src/debug_memrecord.h (original) +++ pypy/branch/memrecord/pypy/translator/c/src/debug_memrecord.h Tue Aug 10 16:48:39 2010 @@ -24,6 +24,11 @@ DEBUG_MEMREC(posid, &(targetexpr), targetexpr, newvalue); \ targetexpr = newvalue +#undef OP_RAW_MEMCLEAR +#define OP_RAW_MEMCLEAR(p, size, r) \ + memset((void*)p, 0, size); \ + DEBUG_MEMREC(0, (void*)p, (void*)size, (void*)0) + /************************************************************/ #ifndef PYPY_NOT_MAIN_FILE From jcreigh at codespeak.net Tue Aug 10 17:09:10 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Tue, 10 Aug 2010 17:09:10 +0200 (CEST) Subject: [pypy-svn] r76570 - pypy/branch/asmgcc-64/pypy/translator/c/gcc Message-ID: <20100810150910.19162282B9C@codespeak.net> Author: jcreigh Date: Tue Aug 10 17:09:09 2010 New Revision: 76570 Modified: pypy/branch/asmgcc-64/pypy/translator/c/gcc/trackgcroot.py Log: comments Modified: pypy/branch/asmgcc-64/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/c/gcc/trackgcroot.py Tue Aug 10 17:09:09 2010 @@ -919,14 +919,27 @@ return self._visit_pop(target) def visit_jmp(self, line): - # Check for a very special case that occurs in varags functions. - # See test/elf64/track_varargs_function.s for an example of the - # pattern we are looking for + # On 64-bit, %al is used when calling varargs functions to specify an + # upper-bound on the number of xmm registers used in the call. gcc + # uses %al to compute an indirect jump that looks like: + # + # jmp *[some register] + # movaps %xmm7, [stack location] + # movaps %xmm6, [stack location] + # movaps %xmm5, [stack location] + # movaps %xmm4, [stack location] + # movaps %xmm3, [stack location] + # movaps %xmm2, [stack location] + # movaps %xmm1, [stack location] + # movaps %xmm0, [stack location] + # + # The jmp is always to somewhere in the block of "movaps" + # instructions, according to how many xmm registers need to be saved + # to the stack. The point of all this is that we can safely ignore + # jmp instructions of that form. if (self.currentlineno + 8) < len(self.lines) and self.r_unaryinsn_star.match(line): matches = [self.r_save_xmm_register.match(self.lines[self.currentlineno + 1 + i]) for i in range(8)] if all(m and int(m.group(1)) == (7 - i) for i, m in enumerate(matches)): - # We assume that the JMP is to somewhere in the block of - # movaps instructions, and ignore it. return [] return FunctionGcRootTracker.visit_jmp(self, line) @@ -998,7 +1011,6 @@ EBP = '%rbp' EAX = '%rax' CALLEE_SAVE_REGISTERS = ['%rbx', '%r12', '%r13', '%r14', '%r15', '%rbp'] - # XXX: Is this okay? REG2LOC = dict((_reg, LOC_REG | ((_i+1)<<2)) for _i, _reg in enumerate(CALLEE_SAVE_REGISTERS)) OPERAND = r'(?:[-\w$%+.:@"]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)])' From arigo at codespeak.net Tue Aug 10 18:08:31 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 10 Aug 2010 18:08:31 +0200 (CEST) Subject: [pypy-svn] r76571 - in pypy/branch/memrecord: . pypy/jit/backend/llsupport pypy/jit/backend/x86 pypy/translator/c pypy/translator/c/src Message-ID: <20100810160831.3CBCD282B9C@codespeak.net> Author: arigo Date: Tue Aug 10 18:08:28 2010 New Revision: 76571 Added: pypy/branch/memrecord/dump.py (contents, props changed) Modified: pypy/branch/memrecord/pypy/jit/backend/llsupport/gc.py pypy/branch/memrecord/pypy/jit/backend/llsupport/llmodel.py pypy/branch/memrecord/pypy/jit/backend/x86/assembler.py pypy/branch/memrecord/pypy/translator/c/funcgen.py pypy/branch/memrecord/pypy/translator/c/src/debug_memrecord.h Log: In-progress. Added: pypy/branch/memrecord/dump.py ============================================================================== --- (empty file) +++ pypy/branch/memrecord/dump.py Tue Aug 10 18:08:28 2010 @@ -0,0 +1,10 @@ +#! /usr/bin/env python +import struct + +f = open('debug_memrecord', 'rb') +while 1: + data = f.read(20) + if len(data) < 20: + break + print ' %8x %8x %8x %8x %8x' % struct.unpack("iiiii", data) +f.close() Modified: pypy/branch/memrecord/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/branch/memrecord/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/branch/memrecord/pypy/jit/backend/llsupport/gc.py Tue Aug 10 18:08:28 2010 @@ -623,3 +623,7 @@ raise NotImplementedError("GC transformer %r not supported by " "the JIT backend" % (name,)) return cls(gcdescr, translator, rtyper) + + +debug_memrec = rffi.llexternal('DEBUG_MEMREC', [lltype.Signed]*5, + lltype.Void, _nowrapper=True) Modified: pypy/branch/memrecord/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/memrecord/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/memrecord/pypy/jit/backend/llsupport/llmodel.py Tue Aug 10 18:08:28 2010 @@ -18,6 +18,7 @@ from pypy.jit.backend.llsupport.descr import BaseIntCallDescr, GcPtrCallDescr from pypy.jit.backend.llsupport.descr import FloatCallDescr, VoidCallDescr from pypy.rpython.annlowlevel import cast_instance_to_base_ptr +from pypy.jit.backend.llsupport.gc import debug_memrec class AbstractLLCPU(AbstractCPU): @@ -302,8 +303,15 @@ self.gc_ll_descr.do_write_barrier(gcref, newvalue) # --- start of GC unsafe code (no GC operation!) --- items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + items = rffi.ptradd(items, itemindex * 4) # XXX items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items) - items[itemindex] = self.cast_gcref_to_int(newvalue) + oldvalue = items[0] + items[0] = self.cast_gcref_to_int(newvalue) + debug_memrec(6, + rffi.cast(rffi.CArrayPtr(lltype.Signed), gcref)[0], + rffi.cast(lltype.Signed, items), + oldvalue, + items[0]) # --- end of GC unsafe code --- @specialize.argtype(2) @@ -399,7 +407,13 @@ # --- start of GC unsafe code (no GC operation!) --- fieldptr = rffi.ptradd(rffi.cast(rffi.CCHARP, struct), ofs) fieldptr = rffi.cast(rffi.CArrayPtr(lltype.Signed), fieldptr) + oldvalue = fieldptr[0] fieldptr[0] = self.cast_gcref_to_int(newvalue) + debug_memrec(5, + rffi.cast(rffi.CArrayPtr(lltype.Signed), struct)[0], + rffi.cast(lltype.Signed, fieldptr), + oldvalue, + fieldptr[0]) # --- end of GC unsafe code --- @specialize.argtype(1) Modified: pypy/branch/memrecord/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/memrecord/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/memrecord/pypy/jit/backend/x86/assembler.py Tue Aug 10 18:08:28 2010 @@ -221,6 +221,7 @@ self._build_float_constants() if hasattr(gc_ll_descr, 'get_malloc_fixedsize_slowpath_addr'): self._build_malloc_fixedsize_slowpath() + self._build_debug_memrecord() s = os.environ.get('PYPYLOG') if s: if s.find(':') != -1: @@ -263,6 +264,11 @@ self.float_const_neg_addr = float_constants self.float_const_abs_addr = float_constants + 16 + def _build_debug_memrecord(self): + fptr = llhelper(lltype.Ptr(FOUR_INT_ARGS), + debug_memrecord_setfield_gc) + self.debug_memrecord_setfield_gc_code = rffi.cast(lltype.Signed, fptr) + def _build_malloc_fixedsize_slowpath(self): # ---------- first helper for the slow path of malloc ---------- self.malloc_fixedsize_slowpath1 = self.mc.tell() @@ -1079,6 +1085,24 @@ if isinstance(value_loc, RegLoc) and value_loc.is_xmm: self.mc.MOVSD(dest_addr, value_loc) elif size == WORD: + # + if op.args[0].type == REF and op.args[1].type == REF: + self.mc.PUSH(eax) + self.mc.PUSH(ecx) + self.mc.PUSH(edx) + self.mc.PUSH(value_loc) + self.mc.PUSH(ofs_loc) + self.mc.PUSH(base_loc) + self.mc.PUSH(imm(self.mc.tell())) + self.mc.CALL(imm(self.debug_memrecord_setfield_gc_code)) + self.mc.POP(eax) + self.mc.POP(eax) + self.mc.POP(eax) + self.mc.POP(eax) + self.mc.POP(edx) + self.mc.POP(ecx) + self.mc.POP(eax) + # self.mc.MOV(dest_addr, value_loc) elif size == 2: self.mc.MOV16(dest_addr, value_loc) @@ -1856,3 +1880,13 @@ def heap(addr): return AddressLoc(ImmedLoc(addr), ImmedLoc(0), 0, 0) + + +from pypy.jit.backend.llsupport.gc import debug_memrec +FOUR_INT_ARGS = lltype.FuncType([lltype.Signed]*4, lltype.Void) +def debug_memrecord_setfield_gc(asmaddr, base, ofs, value): + debug_memrec(asmaddr, + rffi.cast(rffi.CArrayPtr(lltype.Signed), base)[0], + base + ofs, + rffi.cast(rffi.CArrayPtr(lltype.Signed), base + ofs)[0], + value) Modified: pypy/branch/memrecord/pypy/translator/c/funcgen.py ============================================================================== --- pypy/branch/memrecord/pypy/translator/c/funcgen.py (original) +++ pypy/branch/memrecord/pypy/translator/c/funcgen.py Tue Aug 10 18:08:28 2010 @@ -20,7 +20,7 @@ LOCALVAR = 'l_%s' KEEP_INLINED_GRAPHS = False -DMR_COUNT = 0 +DMR_COUNT = 16 class FunctionCodeGenerator(object): """ @@ -485,8 +485,8 @@ self.lltypemap(op.args[0]).TO._gckind == 'gc'): global DMR_COUNT DMR_COUNT += 1 - result = 'RPyWrite(%d, %s, %s);' % ( - DMR_COUNT, targetexpr, newvalue) + result = 'RPyWrite(0x%x, %s, %s, %s);' % ( + DMR_COUNT, self.expr(op.args[0]), targetexpr, newvalue) return result def OP_GETFIELD(self, op, ampersand=''): Modified: pypy/branch/memrecord/pypy/translator/c/src/debug_memrecord.h ============================================================================== --- pypy/branch/memrecord/pypy/translator/c/src/debug_memrecord.h (original) +++ pypy/branch/memrecord/pypy/translator/c/src/debug_memrecord.h Tue Aug 10 18:08:28 2010 @@ -7,27 +7,29 @@ struct debug_memrecord_s { long a; - void *b, *c, *d; + void *b, *c, *d, *e; }; -#define DEBUG_MEMREC(a1, b1, c1, d1) do { \ +#define DEBUG_MEMREC(a1, b1, c1, d1, e1) do { \ struct debug_memrecord_s _dmr_s; \ _dmr_s.a = a1; \ _dmr_s.b = b1; \ _dmr_s.c = c1; \ _dmr_s.d = d1; \ + _dmr_s.e = e1; \ fwrite_unlocked(&_dmr_s, sizeof(struct debug_memrecord_s), 1, \ debug_memrecord_f); \ } while(0) -#define RPyWrite(posid, targetexpr, newvalue) \ - DEBUG_MEMREC(posid, &(targetexpr), targetexpr, newvalue); \ +#define RPyWrite(posid, container, targetexpr, newvalue) \ + DEBUG_MEMREC(posid, *(void**)(container), &(targetexpr), \ + targetexpr, newvalue); \ targetexpr = newvalue #undef OP_RAW_MEMCLEAR -#define OP_RAW_MEMCLEAR(p, size, r) \ - memset((void*)p, 0, size); \ - DEBUG_MEMREC(0, (void*)p, (void*)size, (void*)0) +#define OP_RAW_MEMCLEAR(p, size, r) \ + memset((void*)p, 0, size); \ + DEBUG_MEMREC(-1, (void*)p, (void*)size, (void*)0, (void*)0) /************************************************************/ From arigo at codespeak.net Tue Aug 10 18:34:15 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 10 Aug 2010 18:34:15 +0200 (CEST) Subject: [pypy-svn] r76572 - in pypy/branch/memrecord: . pypy/jit/backend/x86 pypy/translator/c/src Message-ID: <20100810163415.387EB282B9C@codespeak.net> Author: arigo Date: Tue Aug 10 18:34:13 2010 New Revision: 76572 Modified: pypy/branch/memrecord/dump.py pypy/branch/memrecord/pypy/jit/backend/x86/assembler.py pypy/branch/memrecord/pypy/translator/c/src/debug_memrecord.h Log: Bug fixes. Modified: pypy/branch/memrecord/dump.py ============================================================================== --- pypy/branch/memrecord/dump.py (original) +++ pypy/branch/memrecord/dump.py Tue Aug 10 18:34:13 2010 @@ -6,5 +6,5 @@ data = f.read(20) if len(data) < 20: break - print ' %8x %8x %8x %8x %8x' % struct.unpack("iiiii", data) + print ' %8x %8x %8x %8x %8x' % struct.unpack("IIIII", data) f.close() Modified: pypy/branch/memrecord/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/memrecord/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/memrecord/pypy/jit/backend/x86/assembler.py Tue Aug 10 18:34:13 2010 @@ -30,7 +30,7 @@ from pypy.rlib.debug import debug_print from pypy.rlib import rgc from pypy.jit.backend.x86.jump import remap_frame_layout -from pypy.rlib.streamio import open_file_as_stream +#from pypy.rlib.streamio import open_file_as_stream from pypy.jit.metainterp.history import ConstInt, BoxInt # darwin requires the stack to be 16 bytes aligned on calls. Same for gcc 4.5.0, @@ -236,14 +236,15 @@ self.pending_guard_tokens = [] def finish_once(self): - if self._debug: - output_log = self._output_loop_log - assert output_log is not None - f = open_file_as_stream(output_log, "w") - for i in range(len(self.loop_run_counters)): - name, struct = self.loop_run_counters[i] - f.write(name + ":" + str(struct.i) + "\n") - f.close() + pass + #if self._debug: + # output_log = self._output_loop_log + # assert output_log is not None + # f = open_file_as_stream(output_log, "w") + # for i in range(len(self.loop_run_counters)): + # name, struct = self.loop_run_counters[i] + # f.write(name + ":" + str(struct.i) + "\n") + # f.close() def _build_float_constants(self): # 44 bytes: 32 bytes for the data, and up to 12 bytes for alignment Modified: pypy/branch/memrecord/pypy/translator/c/src/debug_memrecord.h ============================================================================== --- pypy/branch/memrecord/pypy/translator/c/src/debug_memrecord.h (original) +++ pypy/branch/memrecord/pypy/translator/c/src/debug_memrecord.h Tue Aug 10 18:34:13 2010 @@ -6,14 +6,13 @@ void debug_memrecord_startup(void); struct debug_memrecord_s { - long a; - void *b, *c, *d, *e; + void *a, *b, *c, *d, *e; }; #define DEBUG_MEMREC(a1, b1, c1, d1, e1) do { \ struct debug_memrecord_s _dmr_s; \ - _dmr_s.a = a1; \ - _dmr_s.b = b1; \ + _dmr_s.a = (void*)(a1); \ + _dmr_s.b = (void*)(b1); \ _dmr_s.c = c1; \ _dmr_s.d = d1; \ _dmr_s.e = e1; \ @@ -22,14 +21,16 @@ } while(0) #define RPyWrite(posid, container, targetexpr, newvalue) \ - DEBUG_MEMREC(posid, *(void**)(container), &(targetexpr), \ + DEBUG_MEMREC(posid, \ + ((struct pypy_header0 *)(container))->h_tid, \ + &(targetexpr), \ targetexpr, newvalue); \ targetexpr = newvalue #undef OP_RAW_MEMCLEAR #define OP_RAW_MEMCLEAR(p, size, r) \ memset((void*)p, 0, size); \ - DEBUG_MEMREC(-1, (void*)p, (void*)size, (void*)0, (void*)0) + DEBUG_MEMREC(-1, (void*)p, (void*)size, ((char*)p)+size, (void*)0) /************************************************************/ From arigo at codespeak.net Tue Aug 10 18:58:24 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 10 Aug 2010 18:58:24 +0200 (CEST) Subject: [pypy-svn] r76573 - in pypy/branch/memrecord: . pypy/translator/c/src Message-ID: <20100810165824.0EEED282B9C@codespeak.net> Author: arigo Date: Tue Aug 10 18:58:22 2010 New Revision: 76573 Modified: pypy/branch/memrecord/dump.py pypy/branch/memrecord/pypy/translator/c/src/debug_memrecord.h Log: gzip the file. Reduces it roughly 5x. Modified: pypy/branch/memrecord/dump.py ============================================================================== --- pypy/branch/memrecord/dump.py (original) +++ pypy/branch/memrecord/dump.py Tue Aug 10 18:58:22 2010 @@ -1,7 +1,7 @@ #! /usr/bin/env python -import struct +import struct, os -f = open('debug_memrecord', 'rb') +f = os.popen('zcat debug_memrecord.gz', 'r') while 1: data = f.read(20) if len(data) < 20: Modified: pypy/branch/memrecord/pypy/translator/c/src/debug_memrecord.h ============================================================================== --- pypy/branch/memrecord/pypy/translator/c/src/debug_memrecord.h (original) +++ pypy/branch/memrecord/pypy/translator/c/src/debug_memrecord.h Tue Aug 10 18:58:22 2010 @@ -40,7 +40,7 @@ void debug_memrecord_startup(void) { - debug_memrecord_f = fopen("debug_memrecord", "wb"); + debug_memrecord_f = popen("gzip -c -3 > debug_memrecord.gz", "w"); assert(debug_memrecord_f); } From getxsick at codespeak.net Tue Aug 10 19:20:30 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Tue, 10 Aug 2010 19:20:30 +0200 (CEST) Subject: [pypy-svn] r76574 - in pypy/branch/fast-ctypes/pypy: module/jitffi rlib rlib/test Message-ID: <20100810172030.0DCDA282B9C@codespeak.net> Author: getxsick Date: Tue Aug 10 19:20:28 2010 New Revision: 76574 Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Log: fix translation (i don't like this one) Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Tue Aug 10 19:20:28 2010 @@ -69,20 +69,9 @@ def __init__(self, space, cpu, lib, func, args_type, res_type='v'): self.space = space - if res_type == 'i': - self.rget = rjitffi._Get(cpu, lib, func, args_type, - res_type, self.wrap_int_w, cache=True) - elif res_type == 'f': - self.rget = rjitffi._Get(cpu, lib, func, args_type, - res_type, self.wrap_float_w, cache=True) - elif res_type == 'v': - self.rget = rjitffi._Get(cpu, lib, func, args_type, - res_type, self.wrap_void_w, cache=True) - else: - raise OperationError( - space.w_ValueError, - space.wrap('Unsupported type of result: %s' - % res_type)) + wrap_func = (self.wrap_int_w, self.wrap_float_w, self.wrap_void_w) + self.rget = rjitffi._Get(cpu, lib, func, args_type, res_type, + wrap_func, cache=True) # grab from the cache if possible arg_classes = ''.join(args_type) Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Tue Aug 10 19:20:28 2010 @@ -107,13 +107,13 @@ self.cpu.execute_token(self.looptoken) if self.res_type == 'i': - r = self.push_result(self.cpu.get_latest_value_int(0)) + r = self.push_result[0](self.cpu.get_latest_value_int(0)) elif self.res_type == 'f': - r = self.push_result(self.cpu.get_latest_value_float(0)) + r = self.push_result[1](self.cpu.get_latest_value_float(0)) elif self.res_type == 'p': - r = self.push_result(self.cpu.get_latest_value_int(0)) + r = self.push_result[0](self.cpu.get_latest_value_int(0)) elif self.res_type == 'v': - r = None + r = self.push_result[2](None) else: raise ValueError(self.res_type) Modified: pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Tue Aug 10 19:20:28 2010 @@ -92,9 +92,7 @@ def setup_class(cls): cls.lib_name = cls.preprare_c_example() - - def push_result(self, value): # mock function - return value + cls.push_result = [ lambda x: x for i in xrange(3) ] # mock function def fromcache(self, f, args_type, res_type): if not hasattr(self, 'cache'): From getxsick at codespeak.net Tue Aug 10 20:07:29 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Tue, 10 Aug 2010 20:07:29 +0200 (CEST) Subject: [pypy-svn] r76575 - in pypy/branch/fast-ctypes: . lib_pypy lib_pypy/pypy_test pypy/config pypy/doc pypy/doc/config pypy/interpreter pypy/interpreter/astcompiler pypy/interpreter/test pypy/jit/backend pypy/jit/backend/llsupport pypy/jit/backend/llsupport/test pypy/jit/backend/test pypy/jit/backend/x86 pypy/jit/backend/x86/test pypy/jit/backend/x86/tool pypy/jit/metainterp pypy/jit/metainterp/test pypy/jit/tl pypy/module/_demo pypy/module/_stackless pypy/module/array pypy/module/pypyjit pypy/module/pypyjit/test pypy/module/signal pypy/module/test_lib_pypy pypy/objspace/std pypy/objspace/std/test pypy/rlib pypy/rlib/test pypy/rpython pypy/rpython/lltypesystem pypy/rpython/tool pypy/translator/c pypy/translator/platform Message-ID: <20100810180729.58425282BAD@codespeak.net> Author: getxsick Date: Tue Aug 10 20:07:15 2010 New Revision: 76575 Added: pypy/branch/fast-ctypes/lib_pypy/array.py - copied unchanged from r76574, pypy/trunk/lib_pypy/array.py pypy/branch/fast-ctypes/pypy/doc/config/objspace.usemodules.array.txt - copied unchanged from r76574, pypy/trunk/pypy/doc/config/objspace.usemodules.array.txt pypy/branch/fast-ctypes/pypy/jit/backend/x86/arch.py - copied unchanged from r76574, pypy/trunk/pypy/jit/backend/x86/arch.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/regloc.py - copied unchanged from r76574, pypy/trunk/pypy/jit/backend/x86/regloc.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/rx86.py - copied unchanged from r76574, pypy/trunk/pypy/jit/backend/x86/rx86.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_regloc.py - copied unchanged from r76574, pypy/trunk/pypy/jit/backend/x86/test/test_regloc.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_rx86.py - copied unchanged from r76574, pypy/trunk/pypy/jit/backend/x86/test/test_rx86.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py - copied unchanged from r76574, pypy/trunk/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py - copied unchanged from r76574, pypy/trunk/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/tool/instruction_encoding.sh - copied unchanged from r76574, pypy/trunk/pypy/jit/backend/x86/tool/instruction_encoding.sh pypy/branch/fast-ctypes/pypy/module/array/ - copied from r76574, pypy/trunk/pypy/module/array/ Removed: pypy/branch/fast-ctypes/lib_pypy/greenlet.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/ri386.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/ri386setup.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_ri386.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py pypy/branch/fast-ctypes/pypy/module/test_lib_pypy/test_array.py Modified: pypy/branch/fast-ctypes/ (props changed) pypy/branch/fast-ctypes/lib_pypy/pypy_test/test_coroutine.py pypy/branch/fast-ctypes/lib_pypy/stackless.py pypy/branch/fast-ctypes/pypy/config/pypyoption.py pypy/branch/fast-ctypes/pypy/config/translationoption.py pypy/branch/fast-ctypes/pypy/doc/faq.txt pypy/branch/fast-ctypes/pypy/interpreter/astcompiler/assemble.py pypy/branch/fast-ctypes/pypy/interpreter/astcompiler/codegen.py pypy/branch/fast-ctypes/pypy/interpreter/pyopcode.py pypy/branch/fast-ctypes/pypy/interpreter/test/test_compiler.py pypy/branch/fast-ctypes/pypy/jit/backend/detect_cpu.py pypy/branch/fast-ctypes/pypy/jit/backend/llsupport/regalloc.py pypy/branch/fast-ctypes/pypy/jit/backend/llsupport/test/test_regalloc.py pypy/branch/fast-ctypes/pypy/jit/backend/test/runner_test.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/assembler.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/codebuf.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/jump.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/regalloc.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/runner.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/conftest.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_assembler.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_basic.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_gc_integration.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_jump.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_recompilation.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_regalloc.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_regalloc2.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_runner.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_symbolic_x86.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_zll_random.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_zrpy_gc.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_ztranslation.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/tool/viewcode.py pypy/branch/fast-ctypes/pypy/jit/metainterp/optimizeopt.py pypy/branch/fast-ctypes/pypy/jit/metainterp/resoperation.py pypy/branch/fast-ctypes/pypy/jit/metainterp/test/test_executor.py pypy/branch/fast-ctypes/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/fast-ctypes/pypy/jit/tl/pypyjit.py pypy/branch/fast-ctypes/pypy/jit/tl/pypyjit_demo.py pypy/branch/fast-ctypes/pypy/module/_demo/demo.py pypy/branch/fast-ctypes/pypy/module/_stackless/interp_coroutine.py pypy/branch/fast-ctypes/pypy/module/pypyjit/policy.py pypy/branch/fast-ctypes/pypy/module/pypyjit/test/test_pypy_c.py pypy/branch/fast-ctypes/pypy/module/signal/interp_signal.py pypy/branch/fast-ctypes/pypy/objspace/std/callmethod.py pypy/branch/fast-ctypes/pypy/objspace/std/itertype.py pypy/branch/fast-ctypes/pypy/objspace/std/model.py pypy/branch/fast-ctypes/pypy/objspace/std/test/test_callmethod.py pypy/branch/fast-ctypes/pypy/rlib/objectmodel.py pypy/branch/fast-ctypes/pypy/rlib/rmmap.py pypy/branch/fast-ctypes/pypy/rlib/test/test_objectmodel.py pypy/branch/fast-ctypes/pypy/rpython/lltypesystem/rstr.py pypy/branch/fast-ctypes/pypy/rpython/rstr.py pypy/branch/fast-ctypes/pypy/rpython/tool/rfficache.py pypy/branch/fast-ctypes/pypy/translator/c/genc.py pypy/branch/fast-ctypes/pypy/translator/c/node.py pypy/branch/fast-ctypes/pypy/translator/platform/posix.py Log: merge from trunk Modified: pypy/branch/fast-ctypes/lib_pypy/pypy_test/test_coroutine.py ============================================================================== --- pypy/branch/fast-ctypes/lib_pypy/pypy_test/test_coroutine.py (original) +++ pypy/branch/fast-ctypes/lib_pypy/pypy_test/test_coroutine.py Tue Aug 10 20:07:15 2010 @@ -2,7 +2,7 @@ from py.test import skip, raises try: - from ..stackless import coroutine + from ..stackless import coroutine, CoroutineExit except ImportError, e: skip('cannot import stackless: %s' % (e,)) Modified: pypy/branch/fast-ctypes/lib_pypy/stackless.py ============================================================================== --- pypy/branch/fast-ctypes/lib_pypy/stackless.py (original) +++ pypy/branch/fast-ctypes/lib_pypy/stackless.py Tue Aug 10 20:07:15 2010 @@ -14,9 +14,13 @@ import traceback import sys try: + # If _stackless can be imported then TaskletExit and CoroutineExit are + # automatically added to the builtins. from _stackless import coroutine, greenlet except ImportError: # we are running from CPython - from greenlet import greenlet + from greenlet import greenlet, GreenletExit + TaskletExit = CoroutineExit = GreenletExit + del GreenletExit try: from functools import partial except ImportError: # we are not running python 2.5 Modified: pypy/branch/fast-ctypes/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/config/pypyoption.py (original) +++ pypy/branch/fast-ctypes/pypy/config/pypyoption.py Tue Aug 10 20:07:15 2010 @@ -30,7 +30,7 @@ "rctime" , "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios", "zlib", "struct", "md5", "sha", "bz2", "_minimal_curses", "cStringIO", - "thread", "itertools", "pyexpat", "_ssl", "cpyext"] + "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array"] )) working_oo_modules = default_modules.copy() Modified: pypy/branch/fast-ctypes/pypy/config/translationoption.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/config/translationoption.py (original) +++ pypy/branch/fast-ctypes/pypy/config/translationoption.py Tue Aug 10 20:07:15 2010 @@ -342,6 +342,10 @@ 'jit': 'hybrid extraopts jit', } +# For now, 64-bit JIT requires boehm +if IS_64_BITS: + OPT_TABLE['jit'] = OPT_TABLE['jit'].replace('hybrid', 'boehm') + def set_opt_level(config, level): """Apply optimization suggestions on the 'config'. The optimizations depend on the selected level and possibly on the backend. Modified: pypy/branch/fast-ctypes/pypy/doc/faq.txt ============================================================================== --- pypy/branch/fast-ctypes/pypy/doc/faq.txt (original) +++ pypy/branch/fast-ctypes/pypy/doc/faq.txt Tue Aug 10 20:07:15 2010 @@ -47,8 +47,8 @@ There is also an experimental support for CPython extension modules, so they'll run without change (from current observation, rather with little -change) on trunk. It has not been released yet, although it should be a major -point of the next pypy release. +change) on trunk. It has been a part of 1.3 release, but support is still +in alpha phase. .. _`extension modules`: cpython_differences.html#extension-modules .. _`cpython_differences`: cpython_differences.html @@ -373,7 +373,7 @@ -------------------------------------------- No. PyPy always runs your code in its own interpreter, which is a -full and compliant Python 2.4 interpreter. RPython_ is only the +full and compliant Python 2.5 interpreter. RPython_ is only the language in which parts of PyPy itself are written and extension modules for it. The answer to whether something needs to be written as an extension module, apart from the "gluing to external libraries" reason, will Modified: pypy/branch/fast-ctypes/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/fast-ctypes/pypy/interpreter/astcompiler/assemble.py Tue Aug 10 20:07:15 2010 @@ -566,22 +566,22 @@ return (oparg % 256) + 2 * (oparg / 256) def _compute_CALL_FUNCTION(arg): - return _num_args(arg) + return -_num_args(arg) def _compute_CALL_FUNCTION_VAR(arg): - return _num_args(arg) - 1 + return -_num_args(arg) - 1 def _compute_CALL_FUNCTION_KW(arg): - return _num_args(arg) - 1 + return -_num_args(arg) - 1 def _compute_CALL_FUNCTION_VAR_KW(arg): - return _num_args(arg) - 2 + return -_num_args(arg) - 2 def _compute_CALL_LIKELY_BUILTIN(arg): return -(arg & 0xFF) + 1 def _compute_CALL_METHOD(arg): - return -arg - 1 + return -_num_args(arg) - 1 _stack_effect_computers = {} Modified: pypy/branch/fast-ctypes/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/fast-ctypes/pypy/interpreter/astcompiler/codegen.py Tue Aug 10 20:07:15 2010 @@ -960,9 +960,12 @@ elif call_type == 3: op = ops.CALL_FUNCTION_VAR_KW self.emit_op_arg(op, arg) + + def _call_has_no_star_args(self, call): + return not call.starargs and not call.kwargs def _call_has_simple_args(self, call): - return not call.starargs and not call.kwargs and not call.keywords + return self._call_has_no_star_args(call) and not call.keywords def _optimize_builtin_call(self, call): if not self.space.config.objspace.opcodes.CALL_LIKELY_BUILTIN or \ @@ -988,7 +991,7 @@ def _optimize_method_call(self, call): if not self.space.config.objspace.opcodes.CALL_METHOD or \ - not self._call_has_simple_args(call) or \ + not self._call_has_no_star_args(call) or \ not isinstance(call.func, ast.Attribute): return False attr_lookup = call.func @@ -1000,7 +1003,12 @@ arg_count = len(call.args) else: arg_count = 0 - self.emit_op_arg(ops.CALL_METHOD, arg_count) + if call.keywords: + self.visit_sequence(call.keywords) + kwarg_count = len(call.keywords) + else: + kwarg_count = 0 + self.emit_op_arg(ops.CALL_METHOD, (kwarg_count << 8) | arg_count) return True def _listcomp_generator(self, list_name, gens, gen_index, elt): Modified: pypy/branch/fast-ctypes/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/fast-ctypes/pypy/interpreter/pyopcode.py Tue Aug 10 20:07:15 2010 @@ -988,23 +988,9 @@ self.dropvalues(nargs) self.pushvalue(w_result) - def LOOKUP_METHOD(self, nameindex, next_instr): - # overridden by faster version in the standard object space. - space = self.space - w_obj = self.popvalue() - w_name = self.getname_w(nameindex) - w_value = space.getattr(w_obj, w_name) - self.pushvalue(w_value) - - def CALL_METHOD(self, nargs, next_instr): - # overridden by faster version in the standard object space. - # 'nargs' is the argument count excluding the implicit 'self' - w_callable = self.peekvalue(nargs) - try: - w_result = self.space.call_valuestack(w_callable, nargs, self) - finally: - self.dropvalues(nargs + 1) - self.pushvalue(w_result) + # overridden by faster version in the standard object space. + LOOKUP_METHOD = LOAD_ATTR + CALL_METHOD = CALL_FUNCTION def MISSING_OPCODE(self, oparg, next_instr): ofs = self.last_instr Modified: pypy/branch/fast-ctypes/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/interpreter/test/test_compiler.py (original) +++ pypy/branch/fast-ctypes/pypy/interpreter/test/test_compiler.py Tue Aug 10 20:07:15 2010 @@ -4,6 +4,7 @@ from pypy.interpreter.pycode import PyCode from pypy.interpreter.error import OperationError from pypy.interpreter.argument import Arguments +from pypy.conftest import gettestobjspace class BaseTestCompiler: def setup_method(self, method): @@ -848,14 +849,38 @@ import StringIO, sys, dis s = StringIO.StringIO() + out = sys.stdout sys.stdout = s try: dis.dis(code) finally: - sys.stdout = sys.__stdout__ + sys.stdout = out output = s.getvalue() assert "LOAD_GLOBAL" not in output +class AppTestCallMethod(object): + def setup_class(cls): + cls.space = gettestobjspace(**{'objspace.opcodes.CALL_METHOD': True}) + + def test_call_method_kwargs(self): + source = """def _f(a): + return a.f(a=a) + """ + exec source + code = _f.func_code + + import StringIO, sys, dis + s = StringIO.StringIO() + out = sys.stdout + sys.stdout = s + try: + dis.dis(code) + finally: + sys.stdout = out + output = s.getvalue() + assert "CALL_METHOD" in output + + class AppTestExceptions: def test_indentation_error(self): source = """if 1: Modified: pypy/branch/fast-ctypes/pypy/jit/backend/detect_cpu.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/detect_cpu.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/detect_cpu.py Tue Aug 10 20:07:15 2010 @@ -56,6 +56,8 @@ return "pypy.jit.backend.x86.runner", "CPU" elif backend_name == 'x86-without-sse2': return "pypy.jit.backend.x86.runner", "CPU386_NO_SSE2" + elif backend_name == 'x86_64': + return "pypy.jit.backend.x86.runner", "CPU_X86_64" elif backend_name == 'cli': return "pypy.jit.backend.cli.runner", "CliCPU" elif backend_name == 'llvm': Modified: pypy/branch/fast-ctypes/pypy/jit/backend/llsupport/regalloc.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/llsupport/regalloc.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/llsupport/regalloc.py Tue Aug 10 20:07:15 2010 @@ -22,18 +22,19 @@ def get(self, box): return self.frame_bindings.get(box, None) - def loc(self, box, size): + def loc(self, box): res = self.get(box) if res is not None: return res - newloc = self.frame_pos(self.frame_depth, size) + newloc = self.frame_pos(self.frame_depth, box.type) self.frame_bindings[box] = newloc - self.frame_depth += size + # Objects returned by frame_pos must support frame_size() + self.frame_depth += newloc.frame_size() return newloc # abstract methods that need to be overwritten for specific assemblers @staticmethod - def frame_pos(loc, size): + def frame_pos(loc, type): raise NotImplementedError("Purely abstract") class RegisterManager(object): @@ -43,7 +44,6 @@ all_regs = [] no_lower_byte_regs = [] save_around_call_regs = [] - reg_width = 1 # in terms of stack space eaten def __init__(self, longevity, frame_manager=None, assembler=None): self.free_regs = self.all_regs[:] @@ -148,7 +148,7 @@ loc = self.reg_bindings[v_to_spill] del self.reg_bindings[v_to_spill] if self.frame_manager.get(v_to_spill) is None: - newloc = self.frame_manager.loc(v_to_spill, self.reg_width) + newloc = self.frame_manager.loc(v_to_spill) self.assembler.regalloc_mov(loc, newloc) return loc @@ -204,7 +204,7 @@ try: return self.reg_bindings[box] except KeyError: - return self.frame_manager.loc(box, self.reg_width) + return self.frame_manager.loc(box) def return_constant(self, v, forbidden_vars=[], selected_reg=None, imm_fine=True): @@ -260,7 +260,7 @@ self.reg_bindings[v] = loc self.assembler.regalloc_mov(prev_loc, loc) else: - loc = self.frame_manager.loc(v, self.reg_width) + loc = self.frame_manager.loc(v) self.assembler.regalloc_mov(prev_loc, loc) def force_result_in_reg(self, result_v, v, forbidden_vars=[]): @@ -280,7 +280,7 @@ self.free_regs = [reg for reg in self.free_regs if reg is not loc] return loc if v not in self.reg_bindings: - prev_loc = self.frame_manager.loc(v, self.reg_width) + prev_loc = self.frame_manager.loc(v) loc = self.force_allocate_reg(v, forbidden_vars) self.assembler.regalloc_mov(prev_loc, loc) assert v in self.reg_bindings @@ -300,7 +300,7 @@ def _sync_var(self, v): if not self.frame_manager.get(v): reg = self.reg_bindings[v] - to = self.frame_manager.loc(v, self.reg_width) + to = self.frame_manager.loc(v) self.assembler.regalloc_mov(reg, to) # otherwise it's clean Modified: pypy/branch/fast-ctypes/pypy/jit/backend/llsupport/test/test_regalloc.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/llsupport/test/test_regalloc.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/llsupport/test/test_regalloc.py Tue Aug 10 20:07:15 2010 @@ -1,5 +1,5 @@ -from pypy.jit.metainterp.history import BoxInt, ConstInt, BoxFloat +from pypy.jit.metainterp.history import BoxInt, ConstInt, BoxFloat, INT, FLOAT from pypy.jit.backend.llsupport.regalloc import FrameManager from pypy.jit.backend.llsupport.regalloc import RegisterManager as BaseRegMan @@ -26,9 +26,20 @@ def convert_to_imm(self, v): return v +class FakeFramePos(object): + def __init__(self, pos, box_type): + self.pos = pos + self.box_type = box_type + + def frame_size(self): + if self.box_type == FLOAT: + return 2 + else: + return 1 + class TFrameManager(FrameManager): - def frame_pos(self, i, size): - return i + def frame_pos(self, i, box_type): + return FakeFramePos(i, box_type) class MockAsm(object): def __init__(self): @@ -146,8 +157,8 @@ rm.next_instruction() # allocate a stack position b0, b1, b2, b3, b4 = boxes - sp = fm.loc(b0, 1) - assert sp == 0 + sp = fm.loc(b0) + assert sp.pos == 0 loc = rm.make_sure_var_in_reg(b0) assert isinstance(loc, FakeReg) rm._check_invariants() @@ -207,13 +218,13 @@ asm = MockAsm() rm = RegisterManager(longevity, frame_manager=fm, assembler=asm) rm.next_instruction() - fm.loc(b0, 1) + fm.loc(b0) rm.force_result_in_reg(b1, b0) rm._check_invariants() loc = rm.loc(b1) assert isinstance(loc, FakeReg) loc = rm.loc(b0) - assert isinstance(loc, int) + assert isinstance(loc, FakeFramePos) assert len(asm.moves) == 1 def test_return_constant(self): @@ -304,7 +315,7 @@ def test_different_frame_width(self): class XRegisterManager(RegisterManager): - reg_width = 2 + pass fm = TFrameManager() b0 = BoxInt() Modified: pypy/branch/fast-ctypes/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/test/runner_test.py Tue Aug 10 20:07:15 2010 @@ -461,6 +461,25 @@ [funcbox] + args, 'float', descr=calldescr) assert abs(res.value - 4.6) < 0.0001 + + def test_call_many_arguments(self): + # Test calling a function with a large number of arguments (more than + # 6, which will force passing some arguments on the stack on 64-bit) + + def func(*args): + assert len(args) == 16 + # Try to sum up args in a way that would probably detect a + # transposed argument + return sum(arg * (2**i) for i, arg in enumerate(args)) + + FUNC = self.FuncType([lltype.Signed]*16, lltype.Signed) + FPTR = self.Ptr(FUNC) + calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + func_ptr = llhelper(FPTR, func) + args = range(16) + funcbox = self.get_funcbox(self.cpu, func_ptr) + res = self.execute_operation(rop.CALL, [funcbox] + map(BoxInt, args), 'int', descr=calldescr) + assert res.value == func(*args) def test_call_stack_alignment(self): # test stack alignment issues, notably for Mac OS/X. @@ -638,6 +657,21 @@ assert r.value == 1 def test_array_basic(self): + a_box, A = self.alloc_array_of(rffi.SHORT, 342) + arraydescr = self.cpu.arraydescrof(A) + assert not arraydescr.is_array_of_pointers() + # + r = self.execute_operation(rop.ARRAYLEN_GC, [a_box], + 'int', descr=arraydescr) + assert r.value == 342 + r = self.execute_operation(rop.SETARRAYITEM_GC, [a_box, BoxInt(310), + BoxInt(744)], + 'void', descr=arraydescr) + assert r is None + r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(310)], + 'int', descr=arraydescr) + assert r.value == 744 + a_box, A = self.alloc_array_of(lltype.Signed, 342) arraydescr = self.cpu.arraydescrof(A) assert not arraydescr.is_array_of_pointers() @@ -978,6 +1012,8 @@ else: assert 0 operations.append(ResOperation(opnum, boxargs, boxres)) + # Unique-ify inputargs + inputargs = list(set(inputargs)) faildescr = BasicFailDescr(1) operations.append(ResOperation(rop.FINISH, [], None, descr=faildescr)) @@ -1050,9 +1086,11 @@ descr=BasicFailDescr(5))] operations[1].fail_args = [] looptoken = LoopToken() - self.cpu.compile_loop(list(testcase), operations, + # Use "set" to unique-ify inputargs + unique_testcase_list = list(set(testcase)) + self.cpu.compile_loop(unique_testcase_list, operations, looptoken) - for i, box in enumerate(testcase): + for i, box in enumerate(unique_testcase_list): self.cpu.set_future_value_float(i, box.value) fail = self.cpu.execute_token(looptoken) if fail.identifier != 5 - (expected_id^expected): @@ -1695,7 +1733,7 @@ def test_assembler_call(self): called = [] def assembler_helper(failindex, virtualizable): - assert self.cpu.get_latest_value_int(0) == 10 + assert self.cpu.get_latest_value_int(0) == 97 called.append(failindex) return 4 + 9 @@ -1708,33 +1746,41 @@ _assembler_helper_ptr) ops = ''' - [i0, i1] - i2 = int_add(i0, i1) - finish(i2)''' + [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] + i10 = int_add(i0, i1) + i11 = int_add(i10, i2) + i12 = int_add(i11, i3) + i13 = int_add(i12, i4) + i14 = int_add(i13, i5) + i15 = int_add(i14, i6) + i16 = int_add(i15, i7) + i17 = int_add(i16, i8) + i18 = int_add(i17, i9) + finish(i18)''' loop = parse(ops) looptoken = LoopToken() looptoken.outermost_jitdriver_sd = FakeJitDriverSD() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) - ARGS = [lltype.Signed, lltype.Signed] + ARGS = [lltype.Signed] * 10 RES = lltype.Signed self.cpu.portal_calldescr = self.cpu.calldescrof( lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES) - self.cpu.set_future_value_int(0, 1) - self.cpu.set_future_value_int(1, 2) + for i in range(10): + self.cpu.set_future_value_int(i, i+1) res = self.cpu.execute_token(looptoken) - assert self.cpu.get_latest_value_int(0) == 3 + assert self.cpu.get_latest_value_int(0) == 55 ops = ''' - [i4, i5] - i6 = int_add(i4, 1) - i3 = call_assembler(i6, i5, descr=looptoken) + [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] + i10 = int_add(i0, 42) + i11 = call_assembler(i10, i1, i2, i3, i4, i5, i6, i7, i8, i9, descr=looptoken) guard_not_forced()[] - finish(i3) + finish(i11) ''' loop = parse(ops, namespace=locals()) othertoken = LoopToken() self.cpu.compile_loop(loop.inputargs, loop.operations, othertoken) - self.cpu.set_future_value_int(0, 4) - self.cpu.set_future_value_int(1, 5) + for i in range(10): + self.cpu.set_future_value_int(i, i+1) res = self.cpu.execute_token(othertoken) assert self.cpu.get_latest_value_int(0) == 13 assert called Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/assembler.py Tue Aug 10 20:07:15 2010 @@ -7,24 +7,35 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.annlowlevel import llhelper from pypy.tool.uid import fixid -from pypy.jit.backend.x86.regalloc import RegAlloc, WORD,\ - X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs, FRAME_FIXED_SIZE,\ - FORCE_INDEX_OFS +from pypy.jit.backend.x86.regalloc import RegAlloc, \ + X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs + +from pypy.jit.backend.x86.arch import FRAME_FIXED_SIZE, FORCE_INDEX_OFS, WORD, IS_X86_32, IS_X86_64 + +from pypy.jit.backend.x86.regloc import (eax, ecx, edx, ebx, + esp, ebp, esi, edi, + xmm0, xmm1, xmm2, xmm3, + xmm4, xmm5, xmm6, xmm7, + r8, r9, r10, r11, + r12, r13, r14, r15, + X86_64_SCRATCH_REG, + X86_64_XMM_SCRATCH_REG, + RegLoc, StackLoc, ConstFloatLoc, + ImmedLoc, AddressLoc, imm) + from pypy.rlib.objectmodel import we_are_translated, specialize -from pypy.jit.backend.x86 import codebuf -from pypy.jit.backend.x86.ri386 import * -from pypy.jit.metainterp.resoperation import rop +from pypy.jit.backend.x86 import rx86, regloc, codebuf +from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.backend.x86.support import values_array from pypy.rlib.debug import debug_print from pypy.rlib import rgc +from pypy.jit.backend.x86.jump import remap_frame_layout from pypy.rlib.streamio import open_file_as_stream - -# our calling convention - we pass first 6 args in registers -# and the rest stays on the stack +from pypy.jit.metainterp.history import ConstInt, BoxInt # darwin requires the stack to be 16 bytes aligned on calls. Same for gcc 4.5.0, # better safe than sorry -CALL_ALIGN = 4 +CALL_ALIGN = 16 // WORD def align_stack_words(words): return (words + CALL_ALIGN - 1) & ~(CALL_ALIGN-1) @@ -32,16 +43,36 @@ class MachineCodeBlockWrapper(object): MC_DEFAULT_SIZE = 1024*1024 - def __init__(self, bigsize, profile_agent=None): + def __init__(self, assembler, bigsize, profile_agent=None): + self.assembler = assembler self.old_mcs = [] # keepalive self.bigsize = bigsize self._mc = self._instantiate_mc() self.function_name = None self.profile_agent = profile_agent + self.reset_reserved_bytes() def _instantiate_mc(self): # hook for testing return codebuf.MachineCodeBlock(self.bigsize) + def ensure_bytes_available(self, num_bytes): + if self.bytes_free() <= (self._reserved_bytes + num_bytes): + self.make_new_mc() + + def reserve_bytes(self, num_bytes): + self.ensure_bytes_available(num_bytes) + self._reserved_bytes += num_bytes + + def reset_reserved_bytes(self): + # XXX er.... pretty random number, just to be sure + # not to write half-instruction + self._reserved_bytes = 64 + + def get_relative_pos(self): + return self._mc.get_relative_pos() + + def overwrite(self, pos, listofchars): + return self._mc.overwrite(pos, listofchars) def bytes_free(self): return self._mc._size - self._mc.get_relative_pos() @@ -63,12 +94,25 @@ def make_new_mc(self): new_mc = self._instantiate_mc() debug_print('[new machine code block at', new_mc.tell(), ']') - self._mc.JMP(rel32(new_mc.tell())) + + if IS_X86_64: + # The scratch register is sometimes used as a temporary + # register, but the JMP below might clobber it. Rather than risk + # subtle bugs, we preserve the scratch register across the jump. + self._mc.PUSH_r(X86_64_SCRATCH_REG.value) + + self._mc.JMP(imm(new_mc.tell())) + + if IS_X86_64: + # Restore scratch reg + new_mc.POP_r(X86_64_SCRATCH_REG.value) if self.function_name is not None: self.end_function(done=False) self.start_pos = new_mc.get_relative_pos() + self.assembler.write_pending_failure_recoveries() + self._mc.done() self.old_mcs.append(self._mc) self._mc = new_mc @@ -82,21 +126,35 @@ def _new_method(name): def method(self, *args): - # XXX er.... pretty random number, just to be sure - # not to write half-instruction - if self.bytes_free() < 64: + if self.bytes_free() < self._reserved_bytes: self.make_new_mc() getattr(self._mc, name)(*args) method.func_name = name return method +for _name in rx86.all_instructions + regloc.all_extra_instructions: + setattr(MachineCodeBlockWrapper, _name, _new_method(_name)) + for name in dir(codebuf.MachineCodeBlock): if name.upper() == name or name == "writechr": setattr(MachineCodeBlockWrapper, name, _new_method(name)) +class GuardToken(object): + def __init__(self, faildescr, failargs, fail_locs, exc, desc_bytes): + self.faildescr = faildescr + self.failargs = failargs + self.fail_locs = fail_locs + self.exc = exc + self.desc_bytes = desc_bytes + + def recovery_stub_size(self): + # XXX: 32 is pulled out of the air + return 32 + len(self.desc_bytes) + +DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed)) + class Assembler386(object): mc = None - mc2 = None mc_size = MachineCodeBlockWrapper.MC_DEFAULT_SIZE _float_constants = None _regalloc = None @@ -115,16 +173,16 @@ self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) self.fail_boxes_float = values_array(lltype.Float, failargs_limit) self.fail_ebp = 0 - self.loop_run_counter = values_array(lltype.Signed, 10000) - self.loop_names = [] + self.loop_run_counters = [] # if we have 10000 loops, we have some other problems I guess - self.loc_float_const_neg = None - self.loc_float_const_abs = None + self.float_const_neg_addr = 0 + self.float_const_abs_addr = 0 self.malloc_fixedsize_slowpath1 = 0 self.malloc_fixedsize_slowpath2 = 0 + self.pending_guard_tokens = None self.setup_failure_recovery() - self._loop_counter = 0 self._debug = False + self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') def leave_jitted_hook(self): ptrs = self.fail_boxes_ptr.ar @@ -134,7 +192,7 @@ def set_debug(self, v): self._debug = v - def make_sure_mc_exists(self): + def setup(self): if self.mc is None: # the address of the function called by 'new' gc_ll_descr = self.cpu.gc_ll_descr @@ -153,11 +211,7 @@ ll_new_unicode = gc_ll_descr.get_funcptr_for_newunicode() self.malloc_unicode_func_addr = rffi.cast(lltype.Signed, ll_new_unicode) - # done - # we generate the loop body in 'mc' - # 'mc2' is for guard recovery code - self.mc = MachineCodeBlockWrapper(self.mc_size, self.cpu.profile_agent) - self.mc2 = MachineCodeBlockWrapper(self.mc_size) + self.mc = MachineCodeBlockWrapper(self, self.mc_size, self.cpu.profile_agent) self._build_failure_recovery(False) self._build_failure_recovery(True) if self.cpu.supports_floats: @@ -173,56 +227,66 @@ s = s.split(':')[-1] self.set_debug(True) self._output_loop_log = s + ".count" + # Intialize here instead of __init__ to prevent + # pending_guard_tokens from being considered a prebuilt object, + # which sometimes causes memory leaks since the prebuilt list is + # still considered a GC root after we re-assign + # pending_guard_tokens in write_pending_failure_recoveries + self.pending_guard_tokens = [] def finish_once(self): if self._debug: output_log = self._output_loop_log assert output_log is not None f = open_file_as_stream(output_log, "w") - for i in range(self._loop_counter): - f.write(self.loop_names[i] + ":" + - str(self.loop_run_counter.getitem(i)) + "\n") + for i in range(len(self.loop_run_counters)): + name, struct = self.loop_run_counters[i] + f.write(name + ":" + str(struct.i) + "\n") f.close() def _build_float_constants(self): - # 11 words: 8 words for the data, and up to 3 words for alignment - addr = lltype.malloc(rffi.CArray(lltype.Signed), 11, flavor='raw') + # 44 bytes: 32 bytes for the data, and up to 12 bytes for alignment + addr = lltype.malloc(rffi.CArray(lltype.Char), 44, flavor='raw') if not we_are_translated(): self._keepalive_malloced_float_consts = addr float_constants = rffi.cast(lltype.Signed, addr) float_constants = (float_constants + 15) & ~15 # align to 16 bytes - addr = rffi.cast(rffi.CArrayPtr(lltype.Signed), float_constants) - addr[0] = 0 # \ - addr[1] = -2147483648 # / for neg - addr[2] = 0 # - addr[3] = 0 # - addr[4] = -1 # \ - addr[5] = 2147483647 # / for abs - addr[6] = 0 # - addr[7] = 0 # - self.loc_float_const_neg = heap64(float_constants) - self.loc_float_const_abs = heap64(float_constants + 16) + addr = rffi.cast(rffi.CArrayPtr(lltype.Char), float_constants) + qword_padding = '\x00\x00\x00\x00\x00\x00\x00\x00' + # 0x8000000000000000 + neg_const = '\x00\x00\x00\x00\x00\x00\x00\x80' + # 0x7FFFFFFFFFFFFFFF + abs_const = '\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x7F' + data = neg_const + qword_padding + abs_const + qword_padding + for i in range(len(data)): + addr[i] = data[i] + self.float_const_neg_addr = float_constants + self.float_const_abs_addr = float_constants + 16 def _build_malloc_fixedsize_slowpath(self): - mc = self.mc2._mc # ---------- first helper for the slow path of malloc ---------- - self.malloc_fixedsize_slowpath1 = mc.tell() + self.malloc_fixedsize_slowpath1 = self.mc.tell() if self.cpu.supports_floats: # save the XMM registers in - for i in range(8): # the *caller* frame, from esp+8 - mc.MOVSD(mem64(esp, 8+8*i), xmm_registers[i]) - mc.SUB(edx, eax) # compute the size we want - mc.MOV(mem(esp, 4), edx) # save it as the new argument + for i in range(self.cpu.NUM_REGS):# the *caller* frame, from esp+8 + self.mc.MOVSD_sx((WORD*2)+8*i, i) + self.mc.SUB_rr(edx.value, eax.value) # compute the size we want + if IS_X86_32: + self.mc.MOV_sr(WORD, edx.value) # save it as the new argument + elif IS_X86_64: + # FIXME: We can't just clobber rdi like this, can we? + self.mc.MOV_rr(edi.value, edx.value) + addr = self.cpu.gc_ll_descr.get_malloc_fixedsize_slowpath_addr() - mc.JMP(rel32(addr)) # tail call to the real malloc + self.mc.JMP(imm(addr)) # tail call to the real malloc # ---------- second helper for the slow path of malloc ---------- - self.malloc_fixedsize_slowpath2 = mc.tell() + self.malloc_fixedsize_slowpath2 = self.mc.tell() if self.cpu.supports_floats: # restore the XMM registers - for i in range(8): # from where they were saved - mc.MOVSD(xmm_registers[i], mem64(esp, 8+8*i)) + for i in range(self.cpu.NUM_REGS):# from where they were saved + self.mc.MOVSD_xs(i, (WORD*2)+8*i) nursery_free_adr = self.cpu.gc_ll_descr.get_nursery_free_addr() - mc.MOV(edx, heap(nursery_free_adr)) # load this in EDX - mc.RET() - self.mc2.done() + self.mc.MOV(edx, heap(nursery_free_adr)) # load this in EDX + self.mc.RET() + self.mc.done() def assemble_loop(self, inputargs, operations, looptoken): """adds the following attributes to looptoken: @@ -233,15 +297,18 @@ _x86_param_depth _x86_arglocs """ - self.make_sure_mc_exists() + if not we_are_translated(): + # Arguments should be unique + assert len(set(inputargs)) == len(inputargs) + + self.setup() funcname = self._find_debug_merge_point(operations) + regalloc = RegAlloc(self, self.cpu.translate_support_code) + operations = self._inject_debugging_code(operations) arglocs = regalloc.prepare_loop(inputargs, operations, looptoken) looptoken._x86_arglocs = arglocs - needed_mem = len(arglocs[0]) * 16 + 16 - if needed_mem >= self.mc.bytes_free(): - self.mc.make_new_mc() # profile support name = "Loop # %s: %s" % (looptoken.number, funcname) @@ -256,21 +323,21 @@ self._patch_stackadjust(adr_stackadjust, frame_depth+param_depth) looptoken._x86_frame_depth = frame_depth looptoken._x86_param_depth = param_depth - # we need to make sure here that we don't overload an mc badly. - # a safe estimate is that we need at most 16 bytes per arg - needed_mem = len(arglocs[0]) * 16 + 16 - if needed_mem >= self.mc.bytes_free(): - self.mc.make_new_mc() + looptoken._x86_direct_bootstrap_code = self.mc.tell() self._assemble_bootstrap_direct_call(arglocs, curadr, frame_depth+param_depth) debug_print("Loop #", looptoken.number, "has address", looptoken._x86_loop_code, "to", self.mc.tell()) self.mc.end_function() + self.write_pending_failure_recoveries() - def assemble_bridge(self, faildescr, inputargs, operations): - self.make_sure_mc_exists() + if not we_are_translated(): + # Arguments should be unique + assert len(set(inputargs)) == len(inputargs) + + self.setup() funcname = self._find_debug_merge_point(operations) arglocs = self.rebuild_faillocs_from_descr( @@ -279,6 +346,7 @@ assert ([loc.assembler() for loc in arglocs] == [loc.assembler() for loc in faildescr._x86_debug_faillocs]) regalloc = RegAlloc(self, self.cpu.translate_support_code) + operations = self._inject_debugging_code(operations) fail_depths = faildescr._x86_current_depths regalloc.prepare_bridge(fail_depths, inputargs, arglocs, operations) @@ -302,6 +370,19 @@ descr_number, "has address", adr_bridge, "to", self.mc.tell()) self.mc.end_function() + self.write_pending_failure_recoveries() + + def write_pending_failure_recoveries(self): + for tok in self.pending_guard_tokens: + # Okay to write to _mc because we've already made sure that + # there's enough space by "reserving" bytes. + addr = self.generate_quick_failure(self.mc._mc, tok.faildescr, tok.failargs, tok.fail_locs, tok.exc, tok.desc_bytes) + tok.faildescr._x86_adr_recovery_stub = addr + self.patch_jump_for_descr(tok.faildescr, addr) + + self.pending_guard_tokens = [] + self.mc.reset_reserved_bytes() + self.mc.done() def _find_debug_merge_point(self, operations): @@ -310,34 +391,60 @@ funcname = op.args[0]._get_str() break else: - funcname = "" % self._loop_counter + funcname = "" % len(self.loop_run_counters) # invent the counter, so we don't get too confused if self._debug: - self.loop_names.append(funcname) - self._loop_counter += 1 + struct = lltype.malloc(DEBUG_COUNTER, flavor='raw') + struct.i = 0 + self.loop_run_counters.append((funcname, struct)) return funcname def patch_jump_for_descr(self, faildescr, adr_new_target): adr_jump_offset = faildescr._x86_adr_jump_offset - mc = codebuf.InMemoryCodeBuilder(adr_jump_offset, adr_jump_offset + 4) - mc.write(packimm32(adr_new_target - adr_jump_offset - 4)) + adr_recovery_stub = faildescr._x86_adr_recovery_stub + offset = adr_new_target - (adr_jump_offset + 4) + # If the new target fits within a rel32 of the jump, just patch + # that. Otherwise, leave the original rel32 to the recovery stub in + # place, but clobber the recovery stub with a jump to the real + # target. + if rx86.fits_in_32bits(offset): + mc = codebuf.InMemoryCodeBuilder(adr_jump_offset, adr_jump_offset + 4) + mc.writeimm32(offset) + else: + # "mov r11, addr; jmp r11" is 13 bytes + mc = codebuf.InMemoryCodeBuilder(adr_recovery_stub, adr_recovery_stub + 13) + mc.MOV_ri(X86_64_SCRATCH_REG.value, adr_new_target) + mc.JMP_r(X86_64_SCRATCH_REG.value) + mc.valgrind_invalidated() mc.done() - def _assemble(self, regalloc, operations): - self._regalloc = regalloc + def _inject_debugging_code(self, operations): if self._debug: # before doing anything, let's increase a counter - # we need one register free (a bit of a hack, but whatever) - self.mc.PUSH(eax) - adr = self.loop_run_counter.get_addr_for_num(self._loop_counter - 1) - self.mc.MOV(eax, heap(adr)) - self.mc.ADD(eax, imm(1)) - self.mc.MOV(heap(adr), eax) - self.mc.POP(eax) + c_adr = ConstInt(rffi.cast(lltype.Signed, + self.loop_run_counters[-1][1])) + box = BoxInt() + box2 = BoxInt() + ops = [ResOperation(rop.GETFIELD_RAW, [c_adr], + box, descr=self.debug_counter_descr), + ResOperation(rop.INT_ADD, [box, ConstInt(1)], box2), + ResOperation(rop.SETFIELD_RAW, [c_adr, box2], + None, descr=self.debug_counter_descr)] + operations = ops + operations + # # we need one register free (a bit of a hack, but whatever) + # self.mc.PUSH(eax) + # adr = rffi.cast(lltype.Signed, self.loop_run_counters[-1][1]) + # self.mc.MOV(eax, heap(adr)) + # self.mc.ADD(eax, imm(1)) + # self.mc.MOV(heap(adr), eax) + # self.mc.POP(eax) + return operations + + def _assemble(self, regalloc, operations): + self._regalloc = regalloc regalloc.walk_operations(operations) self.mc.done() - self.mc2.done() if we_are_translated() or self.cpu.dont_keepalive_stuff: self._regalloc = None # else keep it around for debugging frame_depth = regalloc.fm.frame_depth @@ -352,7 +459,7 @@ def _patchable_stackadjust(self): # stack adjustment LEA - self.mc.LEA(esp, fixedsize_ebp_ofs(0)) + self.mc.LEA32_rb(esp.value, 0) return self.mc.tell() - 4 def _patch_stackadjust(self, adr_lea, reserved_depth): @@ -361,23 +468,34 @@ # Compute the correct offset for the instruction LEA ESP, [EBP-4*words]. # Given that [EBP] is where we saved EBP, i.e. in the last word # of our fixed frame, then the 'words' value is: - words = (FRAME_FIXED_SIZE - 1) + reserved_depth + words = (self.cpu.FRAME_FIXED_SIZE - 1) + reserved_depth # align, e.g. for Mac OS X aligned_words = align_stack_words(words+2)-2 # 2 = EIP+EBP - mc.write(packimm32(-WORD * aligned_words)) + mc.writeimm32(-WORD * aligned_words) mc.done() def _call_header(self): - self.mc.PUSH(ebp) - self.mc.MOV(ebp, esp) - self.mc.PUSH(ebx) - self.mc.PUSH(esi) - self.mc.PUSH(edi) + self.mc.PUSH_r(ebp.value) + self.mc.MOV_rr(ebp.value, esp.value) + for regloc in self.cpu.CALLEE_SAVE_REGISTERS: + self.mc.PUSH_r(regloc.value) + # NB. the shape of the frame is hard-coded in get_basic_shape() too. # Also, make sure this is consistent with FRAME_FIXED_SIZE. return self._patchable_stackadjust() + def _call_footer(self): + self.mc.LEA_rb(esp.value, -len(self.cpu.CALLEE_SAVE_REGISTERS) * WORD) + + for i in range(len(self.cpu.CALLEE_SAVE_REGISTERS)-1, -1, -1): + self.mc.POP_r(self.cpu.CALLEE_SAVE_REGISTERS[i].value) + + self.mc.POP_r(ebp.value) + self.mc.RET() + def _assemble_bootstrap_direct_call(self, arglocs, jmpadr, stackdepth): + if IS_X86_64: + return self._assemble_bootstrap_direct_call_64(arglocs, jmpadr, stackdepth) # XXX pushing ebx esi and edi is a bit pointless, since we store # all regsiters anyway, for the case of guard_not_forced # XXX this can be improved greatly. Right now it'll behave like @@ -388,23 +506,81 @@ self._patch_stackadjust(adr_stackadjust, stackdepth) for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] - if isinstance(loc, REG): - self.mc.MOV(loc, mem(ebp, (2 + i) * WORD)) + if isinstance(loc, RegLoc): + assert not loc.is_xmm + self.mc.MOV_rb(loc.value, (2 + i) * WORD) loc = floatlocs[i] - if isinstance(loc, XMMREG): - self.mc.MOVSD(loc, mem64(ebp, (1 + i) * 2 * WORD)) + if isinstance(loc, RegLoc): + assert loc.is_xmm + self.mc.MOVSD_xb(loc.value, (1 + i) * 2 * WORD) tmp = eax xmmtmp = xmm0 for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] - if loc is not None and not isinstance(loc, REG): - self.mc.MOV(tmp, mem(ebp, (2 + i) * WORD)) + if loc is not None and not isinstance(loc, RegLoc): + self.mc.MOV_rb(tmp.value, (2 + i) * WORD) self.mc.MOV(loc, tmp) loc = floatlocs[i] - if loc is not None and not isinstance(loc, XMMREG): - self.mc.MOVSD(xmmtmp, mem64(ebp, (1 + i) * 2 * WORD)) - self.mc.MOVSD(loc, xmmtmp) - self.mc.JMP(rel32(jmpadr)) + if loc is not None and not isinstance(loc, RegLoc): + self.mc.MOVSD_xb(xmmtmp.value, (1 + i) * 2 * WORD) + assert isinstance(loc, StackLoc) + self.mc.MOVSD_bx(loc.value, xmmtmp.value) + self.mc.JMP_l(jmpadr) + return adr_stackadjust + + def _assemble_bootstrap_direct_call_64(self, arglocs, jmpadr, stackdepth): + # XXX: Very similar to _emit_call_64 + + src_locs = [] + dst_locs = [] + xmm_src_locs = [] + xmm_dst_locs = [] + get_from_stack = [] + + # In reverse order for use with pop() + unused_gpr = [r9, r8, ecx, edx, esi, edi] + unused_xmm = [xmm7, xmm6, xmm5, xmm4, xmm3, xmm2, xmm1, xmm0] + + nonfloatlocs, floatlocs = arglocs + adr_stackadjust = self._call_header() + self._patch_stackadjust(adr_stackadjust, stackdepth) + + # The lists are padded with Nones + assert len(nonfloatlocs) == len(floatlocs) + + for i in range(len(nonfloatlocs)): + loc = nonfloatlocs[i] + if loc is not None: + if len(unused_gpr) > 0: + src_locs.append(unused_gpr.pop()) + dst_locs.append(loc) + else: + get_from_stack.append((loc, False)) + + floc = floatlocs[i] + if floc is not None: + if len(unused_xmm) > 0: + xmm_src_locs.append(unused_xmm.pop()) + xmm_dst_locs.append(floc) + else: + get_from_stack.append((floc, True)) + + remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG) + remap_frame_layout(self, xmm_src_locs, xmm_dst_locs, X86_64_XMM_SCRATCH_REG) + + for i in range(len(get_from_stack)): + loc, is_xmm = get_from_stack[i] + if is_xmm: + self.mc.MOVSD_xb(X86_64_XMM_SCRATCH_REG.value, (2 + i) * WORD) + self.mc.MOVSD(loc, X86_64_XMM_SCRATCH_REG) + else: + self.mc.MOV_rb(X86_64_SCRATCH_REG.value, (2 + i) * WORD) + # XXX: We're assuming that "loc" won't require regloc to + # clobber the scratch register + self.mc.MOV(loc, X86_64_SCRATCH_REG) + + self.mc.JMP(imm(jmpadr)) + return adr_stackadjust def _assemble_bootstrap_code(self, inputargs, arglocs): @@ -412,11 +588,12 @@ adr_stackadjust = self._call_header() tmp = X86RegisterManager.all_regs[0] xmmtmp = X86XMMRegisterManager.all_regs[0] + self.mc._mc.begin_reuse_scratch_register() for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] if loc is None: continue - if isinstance(loc, REG): + if isinstance(loc, RegLoc): target = loc else: target = tmp @@ -430,17 +607,20 @@ adr = self.fail_boxes_int.get_addr_for_num(i) self.mc.MOV(target, heap(adr)) if target is not loc: - self.mc.MOV(loc, target) + assert isinstance(loc, StackLoc) + self.mc.MOV_br(loc.value, target.value) for i in range(len(floatlocs)): loc = floatlocs[i] if loc is None: continue adr = self.fail_boxes_float.get_addr_for_num(i) - if isinstance(loc, REG): - self.mc.MOVSD(loc, heap64(adr)) + if isinstance(loc, RegLoc): + self.mc.MOVSD(loc, heap(adr)) else: - self.mc.MOVSD(xmmtmp, heap64(adr)) - self.mc.MOVSD(loc, xmmtmp) + self.mc.MOVSD(xmmtmp, heap(adr)) + assert isinstance(loc, StackLoc) + self.mc.MOVSD_bx(loc.value, xmmtmp.value) + self.mc._mc.end_reuse_scratch_register() return adr_stackadjust def dump(self, text): @@ -453,27 +633,10 @@ finally: Box._extended_display = _prev - def _start_block(self): - # Return a 'mc' that can be used to write an "atomic" block, - # i.e. one that will not contain any JMP. - mc = self.mc._mc - if not we_are_translated(): - self._block_started_mc = (self.mc, mc.tell()) - self.mc = "block started" - return mc - - def _stop_block(self): - if not we_are_translated(): - assert self.mc == "block started" - self.mc, orgpos = self._block_started_mc - assert 0 <= self.mc._mc.tell() - orgpos <= 58, ( - "too many bytes in _start_block/_stop_block pair") - del self._block_started_mc - # ------------------------------------------------------------ def mov(self, from_loc, to_loc): - if isinstance(from_loc, XMMREG) or isinstance(to_loc, XMMREG): + if (isinstance(from_loc, RegLoc) and from_loc.is_xmm) or (isinstance(to_loc, RegLoc) and to_loc.is_xmm): self.mc.MOVSD(to_loc, from_loc) else: self.mc.MOV(to_loc, from_loc) @@ -481,24 +644,24 @@ regalloc_mov = mov # legacy interface def regalloc_push(self, loc): - if isinstance(loc, XMMREG): - self.mc.SUB(esp, imm(2*WORD)) - self.mc.MOVSD(mem64(esp, 0), loc) - elif isinstance(loc, MODRM64): + if isinstance(loc, RegLoc) and loc.is_xmm: + self.mc.SUB_ri(esp.value, 2*WORD) + self.mc.MOVSD_sx(0, loc.value) + elif WORD == 4 and isinstance(loc, StackLoc) and loc.width == 8: # XXX evil trick - self.mc.PUSH(mem(ebp, get_ebp_ofs(loc.position))) - self.mc.PUSH(mem(ebp, get_ebp_ofs(loc.position + 1))) + self.mc.PUSH_b(get_ebp_ofs(loc.position)) + self.mc.PUSH_b(get_ebp_ofs(loc.position + 1)) else: self.mc.PUSH(loc) def regalloc_pop(self, loc): - if isinstance(loc, XMMREG): - self.mc.MOVSD(loc, mem64(esp, 0)) - self.mc.ADD(esp, imm(2*WORD)) - elif isinstance(loc, MODRM64): + if isinstance(loc, RegLoc) and loc.is_xmm: + self.mc.MOVSD_xs(loc.value, 0) + self.mc.ADD_ri(esp.value, 2*WORD) + elif WORD == 4 and isinstance(loc, StackLoc) and loc.width == 8: # XXX evil trick - self.mc.POP(mem(ebp, get_ebp_ofs(loc.position + 1))) - self.mc.POP(mem(ebp, get_ebp_ofs(loc.position))) + self.mc.POP_b(get_ebp_ofs(loc.position + 1)) + self.mc.POP_b(get_ebp_ofs(loc.position)) else: self.mc.POP(loc) @@ -515,14 +678,14 @@ faildescr._x86_current_depths = current_depths failargs = guard_op.fail_args guard_opnum = guard_op.opnum - failaddr = self.implement_guard_recovery(guard_opnum, - faildescr, failargs, - faillocs) + guard_token = self.implement_guard_recovery(guard_opnum, + faildescr, failargs, + faillocs) if op is None: dispatch_opnum = guard_opnum else: dispatch_opnum = op.opnum - res = genop_guard_list[dispatch_opnum](self, op, guard_op, failaddr, + res = genop_guard_list[dispatch_opnum](self, op, guard_op, guard_token, arglocs, resloc) faildescr._x86_adr_jump_offset = res @@ -549,103 +712,161 @@ rl = result_loc.lowest8bits() if isinstance(op.args[0], Const): self.mc.CMP(arglocs[1], arglocs[0]) - getattr(self.mc, 'SET' + rev_cond)(rl) + self.mc.SET_ir(rx86.Conditions[rev_cond], rl.value) else: self.mc.CMP(arglocs[0], arglocs[1]) - getattr(self.mc, 'SET' + cond)(rl) - self.mc.MOVZX(result_loc, rl) + self.mc.SET_ir(rx86.Conditions[cond], rl.value) + self.mc.MOVZX8_rr(result_loc.value, rl.value) return genop_cmp def _cmpop_float(cond, is_ne=False): def genop_cmp(self, op, arglocs, result_loc): self.mc.UCOMISD(arglocs[0], arglocs[1]) - rl = result_loc.lowest8bits() - rh = result_loc.higher8bits() - getattr(self.mc, 'SET' + cond)(rl) + tmp1 = result_loc.lowest8bits() + if IS_X86_32: + tmp2 = result_loc.higher8bits() + elif IS_X86_64: + tmp2 = X86_64_SCRATCH_REG.lowest8bits() + + self.mc.SET_ir(rx86.Conditions[cond], tmp1.value) if is_ne: - self.mc.SETP(rh) - self.mc.OR(rl, rh) + self.mc.SET_ir(rx86.Conditions['P'], tmp2.value) + self.mc.OR8_rr(tmp1.value, tmp2.value) else: - self.mc.SETNP(rh) - self.mc.AND(rl, rh) - self.mc.MOVZX(result_loc, rl) + self.mc.SET_ir(rx86.Conditions['NP'], tmp2.value) + self.mc.AND8_rr(tmp1.value, tmp2.value) + self.mc.MOVZX8_rr(result_loc.value, tmp1.value) return genop_cmp def _cmpop_guard(cond, rev_cond, false_cond, false_rev_cond): - def genop_cmp_guard(self, op, guard_op, addr, arglocs, result_loc): + def genop_cmp_guard(self, op, guard_op, guard_token, arglocs, result_loc): guard_opnum = guard_op.opnum if isinstance(op.args[0], Const): self.mc.CMP(arglocs[1], arglocs[0]) if guard_opnum == rop.GUARD_FALSE: - name = 'J' + rev_cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(guard_token, rev_cond) else: - name = 'J' + false_rev_cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(guard_token, false_rev_cond) else: self.mc.CMP(arglocs[0], arglocs[1]) if guard_opnum == rop.GUARD_FALSE: - name = 'J' + cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(guard_token, cond) else: - name = 'J' + false_cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(guard_token, false_cond) return genop_cmp_guard def _cmpop_guard_float(cond, false_cond, need_jp): - def genop_cmp_guard_float(self, op, guard_op, addr, arglocs, + def genop_cmp_guard_float(self, op, guard_op, guard_token, arglocs, result_loc): guard_opnum = guard_op.opnum self.mc.UCOMISD(arglocs[0], arglocs[1]) + # 16 is enough space for the rel8 jumps below and the rel32 + # jump in implement_guard + self.mc.ensure_bytes_available(16 + guard_token.recovery_stub_size()) if guard_opnum == rop.GUARD_FALSE: - mc = self.mc._mc - name = 'J' + cond if need_jp: - mc.JP(rel8(6)) - getattr(mc, name)(rel32(addr)) - return mc.tell() - 4 + self.mc.J_il8(rx86.Conditions['P'], 6) + return self.implement_guard(guard_token, cond) else: if need_jp: - mc = self.mc._mc - mc.JP(rel8(2)) - getattr(mc, 'J' + cond)(rel8(5)) - return self.implement_guard(addr, mc.JMP) - name = 'J' + false_cond - return self.implement_guard(addr, getattr(self.mc, name)) + self.mc.J_il8(rx86.Conditions['P'], 2) + self.mc.J_il8(rx86.Conditions[cond], 5) + return self.implement_guard(guard_token) + return self.implement_guard(guard_token, false_cond) return genop_cmp_guard_float - @specialize.arg(5) - def _emit_call(self, x, arglocs, start=0, tmp=eax, force_mc=False, - mc=None): - if not force_mc: - mc = self.mc + def _emit_call(self, x, arglocs, start=0, tmp=eax): + if IS_X86_64: + return self._emit_call_64(x, arglocs, start) + p = 0 n = len(arglocs) for i in range(start, n): loc = arglocs[i] - if isinstance(loc, REG): - if isinstance(loc, XMMREG): - mc.MOVSD(mem64(esp, p), loc) + if isinstance(loc, RegLoc): + if loc.is_xmm: + self.mc.MOVSD_sx(p, loc.value) else: - mc.MOV(mem(esp, p), loc) + self.mc.MOV_sr(p, loc.value) p += round_up_to_4(loc.width) p = 0 for i in range(start, n): loc = arglocs[i] - if not isinstance(loc, REG): - if isinstance(loc, MODRM64): - mc.MOVSD(xmm0, loc) - mc.MOVSD(mem64(esp, p), xmm0) + if not isinstance(loc, RegLoc): + if loc.width == 8: + self.mc.MOVSD(xmm0, loc) + self.mc.MOVSD_sx(p, xmm0.value) else: - mc.MOV(tmp, loc) - mc.MOV(mem(esp, p), tmp) + self.mc.MOV(tmp, loc) + self.mc.MOV_sr(p, tmp.value) p += round_up_to_4(loc.width) self._regalloc.reserve_param(p//WORD) - mc.CALL(x) + # x is a location + self.mc.CALL(x) self.mark_gc_roots() + + def _emit_call_64(self, x, arglocs, start=0): + src_locs = [] + dst_locs = [] + xmm_src_locs = [] + xmm_dst_locs = [] + pass_on_stack = [] + + # In reverse order for use with pop() + unused_gpr = [r9, r8, ecx, edx, esi, edi] + unused_xmm = [xmm7, xmm6, xmm5, xmm4, xmm3, xmm2, xmm1, xmm0] + + for i in range(start, len(arglocs)): + loc = arglocs[i] + # XXX: Should be much simplier to tell whether a location is a + # float! It's so ugly because we have to "guard" the access to + # .type with isinstance, since not all AssemblerLocation classes + # are "typed" + if ((isinstance(loc, RegLoc) and loc.is_xmm) or + (isinstance(loc, StackLoc) and loc.type == FLOAT) or + (isinstance(loc, ConstFloatLoc))): + if len(unused_xmm) > 0: + xmm_src_locs.append(loc) + xmm_dst_locs.append(unused_xmm.pop()) + else: + pass_on_stack.append(loc) + else: + if len(unused_gpr) > 0: + src_locs.append(loc) + dst_locs.append(unused_gpr.pop()) + else: + pass_on_stack.append(loc) + # Emit instructions to pass the stack arguments + # XXX: Would be nice to let remap_frame_layout take care of this, but + # we'd need to create something like StackLoc, but relative to esp, + # and I don't know if it's worth it. + for i in range(len(pass_on_stack)): + loc = pass_on_stack[i] + if not isinstance(loc, RegLoc): + if isinstance(loc, StackLoc) and loc.type == FLOAT: + self.mc.MOVSD(X86_64_XMM_SCRATCH_REG, loc) + self.mc.MOVSD_sx(i*WORD, X86_64_XMM_SCRATCH_REG.value) + else: + self.mc.MOV(X86_64_SCRATCH_REG, loc) + self.mc.MOV_sr(i*WORD, X86_64_SCRATCH_REG.value) + else: + # It's a register + if loc.is_xmm: + self.mc.MOVSD_sx(i*WORD, loc.value) + else: + self.mc.MOV_sr(i*WORD, loc.value) + + # Handle register arguments + remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG) + remap_frame_layout(self, xmm_src_locs, xmm_dst_locs, X86_64_XMM_SCRATCH_REG) + + self._regalloc.reserve_param(len(pass_on_stack)) + self.mc.CALL(x) + self.mark_gc_roots() + def call(self, addr, args, res): - self._emit_call(rel32(addr), args) + self._emit_call(imm(addr), args) assert res is eax genop_int_neg = _unaryop("NEG") @@ -656,6 +877,9 @@ genop_int_and = _binaryop("AND", True) genop_int_or = _binaryop("OR", True) genop_int_xor = _binaryop("XOR", True) + genop_int_lshift = _binaryop("SHL") + genop_int_rshift = _binaryop("SAR") + genop_uint_rshift = _binaryop("SHR") genop_float_add = _binaryop("ADDSD", True) genop_float_sub = _binaryop('SUBSD') genop_float_mul = _binaryop('MULSD', True) @@ -702,26 +926,27 @@ genop_guard_float_gt = _cmpop_guard_float("A", "BE", False) genop_guard_float_ge = _cmpop_guard_float("AE", "B", False) - def genop_guard_float_ne(self, op, guard_op, addr, arglocs, result_loc): + def genop_guard_float_ne(self, op, guard_op, guard_token, arglocs, result_loc): guard_opnum = guard_op.opnum self.mc.UCOMISD(arglocs[0], arglocs[1]) - mc = self.mc._mc + # 16 is enough space for the rel8 jumps below and the rel32 + # jump in implement_guard + self.mc.ensure_bytes_available(16 + guard_token.recovery_stub_size()) if guard_opnum == rop.GUARD_TRUE: - mc.JP(rel8(6)) - mc.JE(rel32(addr)) - return mc.tell() - 4 - else: - mc.JP(rel8(2)) - mc.JE(rel8(5)) - return self.implement_guard(addr, mc.JMP) + self.mc.J_il8(rx86.Conditions['P'], 6) + return self.implement_guard(guard_token, 'E') + else: + self.mc.J_il8(rx86.Conditions['P'], 2) + self.mc.J_il8(rx86.Conditions['E'], 5) + return self.implement_guard(guard_token) def genop_float_neg(self, op, arglocs, resloc): # Following what gcc does: res = x ^ 0x8000000000000000 - self.mc.XORPD(arglocs[0], self.loc_float_const_neg) + self.mc.XORPD(arglocs[0], heap(self.float_const_neg_addr)) def genop_float_abs(self, op, arglocs, resloc): # Following what gcc does: res = x & 0x7FFFFFFFFFFFFFFF - self.mc.ANDPD(arglocs[0], self.loc_float_const_abs) + self.mc.ANDPD(arglocs[0], heap(self.float_const_abs_addr)) def genop_cast_float_to_int(self, op, arglocs, resloc): self.mc.CVTTSD2SI(resloc, arglocs[0]) @@ -729,70 +954,56 @@ def genop_cast_int_to_float(self, op, arglocs, resloc): self.mc.CVTSI2SD(resloc, arglocs[0]) - def genop_int_lshift(self, op, arglocs, resloc): - loc, loc2 = arglocs - if loc2 is ecx: - loc2 = cl - self.mc.SHL(loc, loc2) - - def genop_int_rshift(self, op, arglocs, resloc): - loc, loc2 = arglocs - if loc2 is ecx: - loc2 = cl - self.mc.SAR(loc, loc2) - - def genop_uint_rshift(self, op, arglocs, resloc): - loc, loc2 = arglocs - if loc2 is ecx: - loc2 = cl - self.mc.SHR(loc, loc2) - - def genop_guard_int_is_true(self, op, guard_op, addr, arglocs, resloc): + def genop_guard_int_is_true(self, op, guard_op, guard_token, arglocs, resloc): guard_opnum = guard_op.opnum - self.mc.CMP(arglocs[0], imm8(0)) + self.mc.CMP(arglocs[0], imm(0)) if guard_opnum == rop.GUARD_TRUE: - return self.implement_guard(addr, self.mc.JZ) + return self.implement_guard(guard_token, 'Z') else: - return self.implement_guard(addr, self.mc.JNZ) + return self.implement_guard(guard_token, 'NZ') def genop_int_is_true(self, op, arglocs, resloc): - self.mc.CMP(arglocs[0], imm8(0)) + self.mc.CMP(arglocs[0], imm(0)) rl = resloc.lowest8bits() - self.mc.SETNE(rl) - self.mc.MOVZX(resloc, rl) + self.mc.SET_ir(rx86.Conditions['NE'], rl.value) + self.mc.MOVZX8(resloc, rl) - def genop_guard_int_is_zero(self, op, guard_op, addr, arglocs, resloc): + def genop_guard_int_is_zero(self, op, guard_op, guard_token, arglocs, resloc): guard_opnum = guard_op.opnum - self.mc.CMP(arglocs[0], imm8(0)) + self.mc.CMP(arglocs[0], imm(0)) if guard_opnum == rop.GUARD_TRUE: - return self.implement_guard(addr, self.mc.JNZ) + return self.implement_guard(guard_token, 'NZ') else: - return self.implement_guard(addr, self.mc.JZ) + return self.implement_guard(guard_token, 'Z') def genop_int_is_zero(self, op, arglocs, resloc): - self.mc.CMP(arglocs[0], imm8(0)) + self.mc.CMP(arglocs[0], imm(0)) rl = resloc.lowest8bits() - self.mc.SETE(rl) - self.mc.MOVZX(resloc, rl) + self.mc.SET_ir(rx86.Conditions['E'], rl.value) + self.mc.MOVZX8(resloc, rl) def genop_same_as(self, op, arglocs, resloc): self.mov(arglocs[0], resloc) #genop_cast_ptr_to_int = genop_same_as def genop_int_mod(self, op, arglocs, resloc): - self.mc.CDQ() - self.mc.IDIV(ecx) + if IS_X86_32: + self.mc.CDQ() + elif IS_X86_64: + self.mc.CQO() + + self.mc.IDIV_r(ecx.value) genop_int_floordiv = genop_int_mod def genop_uint_floordiv(self, op, arglocs, resloc): - self.mc.XOR(edx, edx) - self.mc.DIV(ecx) + self.mc.XOR_rr(edx.value, edx.value) + self.mc.DIV_r(ecx.value) def genop_new_with_vtable(self, op, arglocs, result_loc): assert result_loc is eax loc_vtable = arglocs[-1] - assert isinstance(loc_vtable, IMM32) + assert isinstance(loc_vtable, ImmedLoc) arglocs = arglocs[:-1] self.call(self.malloc_func_addr, arglocs, eax) # xxx ignore NULL returns for now @@ -800,7 +1011,9 @@ def set_vtable(self, loc, loc_vtable): if self.cpu.vtable_offset is not None: - self.mc.MOV(mem(loc, self.cpu.vtable_offset), loc_vtable) + assert isinstance(loc, RegLoc) + assert isinstance(loc_vtable, ImmedLoc) + self.mc.MOV_mi((loc.value, self.cpu.vtable_offset), loc_vtable.value) # XXX genop_new is abused for all varsized mallocs with Boehm, for now # (instead of genop_new_array, genop_newstr, genop_newunicode) @@ -822,16 +1035,19 @@ def genop_getfield_gc(self, op, arglocs, resloc): base_loc, ofs_loc, size_loc = arglocs - assert isinstance(size_loc, IMM32) + assert isinstance(size_loc, ImmedLoc) + assert isinstance(resloc, RegLoc) size = size_loc.value - if size == 1: - self.mc.MOVZX(resloc, addr8_add(base_loc, ofs_loc)) + + source_addr = AddressLoc(base_loc, ofs_loc) + if resloc.is_xmm: + self.mc.MOVSD(resloc, source_addr) + elif size == 1: + self.mc.MOVZX8(resloc, source_addr) elif size == 2: - self.mc.MOVZX(resloc, addr_add(base_loc, ofs_loc)) + self.mc.MOVZX16(resloc, source_addr) elif size == WORD: - self.mc.MOV(resloc, addr_add(base_loc, ofs_loc)) - elif size == 8: - self.mc.MOVSD(resloc, addr64_add(base_loc, ofs_loc)) + self.mc.MOV(resloc, source_addr) else: raise NotImplementedError("getfield size = %d" % size) @@ -841,20 +1057,23 @@ def genop_getarrayitem_gc(self, op, arglocs, resloc): base_loc, ofs_loc, scale, ofs = arglocs - assert isinstance(ofs, IMM32) - assert isinstance(scale, IMM32) + assert isinstance(ofs, ImmedLoc) + assert isinstance(scale, ImmedLoc) if op.result.type == FLOAT: - self.mc.MOVSD(resloc, addr64_add(base_loc, ofs_loc, ofs.value, + self.mc.MOVSD(resloc, addr_add(base_loc, ofs_loc, ofs.value, scale.value)) else: if scale.value == 0: - self.mc.MOVZX(resloc, addr8_add(base_loc, ofs_loc, ofs.value, + self.mc.MOVZX8(resloc, addr_add(base_loc, ofs_loc, ofs.value, + scale.value)) + elif scale.value == 1: + self.mc.MOVZX16(resloc, addr_add(base_loc, ofs_loc, ofs.value, scale.value)) - elif scale.value == 2: + elif (1 << scale.value) == WORD: self.mc.MOV(resloc, addr_add(base_loc, ofs_loc, ofs.value, scale.value)) else: - print "[asmgen]setarrayitem unsupported size: %d" % scale.value + print "[asmgen]getarrayitem unsupported size: %d" % scale.value raise NotImplementedError() genop_getarrayitem_gc_pure = genop_getarrayitem_gc @@ -862,34 +1081,35 @@ def genop_discard_setfield_gc(self, op, arglocs): base_loc, ofs_loc, size_loc, value_loc = arglocs - assert isinstance(size_loc, IMM32) + assert isinstance(size_loc, ImmedLoc) size = size_loc.value - if size == WORD * 2: - self.mc.MOVSD(addr64_add(base_loc, ofs_loc), value_loc) + dest_addr = AddressLoc(base_loc, ofs_loc) + if isinstance(value_loc, RegLoc) and value_loc.is_xmm: + self.mc.MOVSD(dest_addr, value_loc) elif size == WORD: - self.mc.MOV(addr_add(base_loc, ofs_loc), value_loc) + self.mc.MOV(dest_addr, value_loc) elif size == 2: - self.mc.MOV16(addr_add(base_loc, ofs_loc), value_loc) + self.mc.MOV16(dest_addr, value_loc) elif size == 1: - self.mc.MOV(addr8_add(base_loc, ofs_loc), value_loc.lowest8bits()) + self.mc.MOV8(dest_addr, value_loc.lowest8bits()) else: print "[asmgen]setfield addr size %d" % size raise NotImplementedError("Addr size %d" % size) def genop_discard_setarrayitem_gc(self, op, arglocs): base_loc, ofs_loc, value_loc, scale_loc, baseofs = arglocs - assert isinstance(baseofs, IMM32) - assert isinstance(scale_loc, IMM32) + assert isinstance(baseofs, ImmedLoc) + assert isinstance(scale_loc, ImmedLoc) + dest_addr = AddressLoc(base_loc, ofs_loc, scale_loc.value, baseofs.value) if op.args[2].type == FLOAT: - self.mc.MOVSD(addr64_add(base_loc, ofs_loc, baseofs.value, - scale_loc.value), value_loc) + self.mc.MOVSD(dest_addr, value_loc) else: - if scale_loc.value == 2: - self.mc.MOV(addr_add(base_loc, ofs_loc, baseofs.value, - scale_loc.value), value_loc) + if (1 << scale_loc.value) == WORD: + self.mc.MOV(dest_addr, value_loc) + elif scale_loc.value == 1: + self.mc.MOV16(dest_addr, value_loc) elif scale_loc.value == 0: - self.mc.MOV(addr8_add(base_loc, ofs_loc, baseofs.value, - scale_loc.value), value_loc.lowest8bits()) + self.mc.MOV8(dest_addr, value_loc.lowest8bits()) else: raise NotImplementedError("scale = %d" % scale_loc.value) @@ -898,17 +1118,17 @@ basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) assert itemsize == 1 - self.mc.MOV(addr8_add(base_loc, ofs_loc, basesize), - val_loc.lowest8bits()) + dest_addr = AddressLoc(base_loc, ofs_loc, 0, basesize) + self.mc.MOV8(dest_addr, val_loc.lowest8bits()) def genop_discard_unicodesetitem(self, op, arglocs): base_loc, ofs_loc, val_loc = arglocs basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) if itemsize == 4: - self.mc.MOV(addr_add(base_loc, ofs_loc, basesize, 2), val_loc) + self.mc.MOV32(AddressLoc(base_loc, ofs_loc, 2, basesize), val_loc) elif itemsize == 2: - self.mc.MOV16(addr_add(base_loc, ofs_loc, basesize, 1), val_loc) + self.mc.MOV16(AddressLoc(base_loc, ofs_loc, 1, basesize), val_loc) else: assert 0, itemsize @@ -929,7 +1149,7 @@ def genop_arraylen_gc(self, op, arglocs, resloc): base_loc, ofs_loc = arglocs - assert isinstance(ofs_loc, IMM32) + assert isinstance(ofs_loc, ImmedLoc) self.mc.MOV(resloc, addr_add_const(base_loc, ofs_loc.value)) def genop_strgetitem(self, op, arglocs, resloc): @@ -937,83 +1157,83 @@ basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) assert itemsize == 1 - self.mc.MOVZX(resloc, addr8_add(base_loc, ofs_loc, basesize)) + self.mc.MOVZX8(resloc, AddressLoc(base_loc, ofs_loc, 0, basesize)) def genop_unicodegetitem(self, op, arglocs, resloc): base_loc, ofs_loc = arglocs basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) if itemsize == 4: - self.mc.MOV(resloc, addr_add(base_loc, ofs_loc, basesize, 2)) + self.mc.MOV32(resloc, AddressLoc(base_loc, ofs_loc, 2, basesize)) elif itemsize == 2: - self.mc.MOVZX(resloc, addr_add(base_loc, ofs_loc, basesize, 1)) + self.mc.MOVZX16(resloc, AddressLoc(base_loc, ofs_loc, 1, basesize)) else: assert 0, itemsize - def genop_guard_guard_true(self, ign_1, guard_op, addr, locs, ign_2): + def genop_guard_guard_true(self, ign_1, guard_op, guard_token, locs, ign_2): loc = locs[0] self.mc.TEST(loc, loc) - return self.implement_guard(addr, self.mc.JZ) + return self.implement_guard(guard_token, 'Z') genop_guard_guard_nonnull = genop_guard_guard_true - def genop_guard_guard_no_exception(self, ign_1, guard_op, addr, + def genop_guard_guard_no_exception(self, ign_1, guard_op, guard_token, locs, ign_2): self.mc.CMP(heap(self.cpu.pos_exception()), imm(0)) - return self.implement_guard(addr, self.mc.JNZ) + return self.implement_guard(guard_token, 'NZ') - def genop_guard_guard_exception(self, ign_1, guard_op, addr, + def genop_guard_guard_exception(self, ign_1, guard_op, guard_token, locs, resloc): loc = locs[0] loc1 = locs[1] self.mc.MOV(loc1, heap(self.cpu.pos_exception())) self.mc.CMP(loc1, loc) - addr = self.implement_guard(addr, self.mc.JNE) + addr = self.implement_guard(guard_token, 'NE') if resloc is not None: self.mc.MOV(resloc, heap(self.cpu.pos_exc_value())) self.mc.MOV(heap(self.cpu.pos_exception()), imm(0)) self.mc.MOV(heap(self.cpu.pos_exc_value()), imm(0)) return addr - def _gen_guard_overflow(self, guard_op, addr): + def _gen_guard_overflow(self, guard_op, guard_token): guard_opnum = guard_op.opnum if guard_opnum == rop.GUARD_NO_OVERFLOW: - return self.implement_guard(addr, self.mc.JO) + return self.implement_guard(guard_token, 'O') elif guard_opnum == rop.GUARD_OVERFLOW: - return self.implement_guard(addr, self.mc.JNO) + return self.implement_guard(guard_token, 'NO') else: print "int_xxx_ovf followed by", guard_op.getopname() raise AssertionError - def genop_guard_int_add_ovf(self, op, guard_op, addr, arglocs, result_loc): + def genop_guard_int_add_ovf(self, op, guard_op, guard_token, arglocs, result_loc): self.genop_int_add(op, arglocs, result_loc) - return self._gen_guard_overflow(guard_op, addr) + return self._gen_guard_overflow(guard_op, guard_token) - def genop_guard_int_sub_ovf(self, op, guard_op, addr, arglocs, result_loc): + def genop_guard_int_sub_ovf(self, op, guard_op, guard_token, arglocs, result_loc): self.genop_int_sub(op, arglocs, result_loc) - return self._gen_guard_overflow(guard_op, addr) + return self._gen_guard_overflow(guard_op, guard_token) - def genop_guard_int_mul_ovf(self, op, guard_op, addr, arglocs, result_loc): + def genop_guard_int_mul_ovf(self, op, guard_op, guard_token, arglocs, result_loc): self.genop_int_mul(op, arglocs, result_loc) - return self._gen_guard_overflow(guard_op, addr) + return self._gen_guard_overflow(guard_op, guard_token) - def genop_guard_guard_false(self, ign_1, guard_op, addr, locs, ign_2): + def genop_guard_guard_false(self, ign_1, guard_op, guard_token, locs, ign_2): loc = locs[0] self.mc.TEST(loc, loc) - return self.implement_guard(addr, self.mc.JNZ) + return self.implement_guard(guard_token, 'NZ') genop_guard_guard_isnull = genop_guard_guard_false - def genop_guard_guard_value(self, ign_1, guard_op, addr, locs, ign_2): + def genop_guard_guard_value(self, ign_1, guard_op, guard_token, locs, ign_2): if guard_op.args[0].type == FLOAT: assert guard_op.args[1].type == FLOAT self.mc.UCOMISD(locs[0], locs[1]) else: self.mc.CMP(locs[0], locs[1]) - return self.implement_guard(addr, self.mc.JNE) + return self.implement_guard(guard_token, 'NE') - def _cmp_guard_class(self, mc, locs): + def _cmp_guard_class(self, locs): offset = self.cpu.vtable_offset if offset is not None: - mc.CMP(mem(locs[0], offset), locs[1]) + self.mc.CMP(mem(locs[0], offset), locs[1]) else: # XXX hard-coded assumption: to go from an object to its class # we use the following algorithm: @@ -1022,7 +1242,7 @@ # - multiply by 4 and use it as an offset in type_info_group # - add 16 bytes, to go past the TYPE_INFO structure loc = locs[1] - assert isinstance(loc, IMM32) + assert isinstance(loc, ImmedLoc) classptr = loc.value # here, we have to go back from 'classptr' to the value expected # from reading the 16 bits in the object header @@ -1031,37 +1251,37 @@ type_info_group = llop.gc_get_type_info_group(llmemory.Address) type_info_group = rffi.cast(lltype.Signed, type_info_group) expected_typeid = (classptr - sizeof_ti - type_info_group) >> 2 - mc.CMP16(mem(locs[0], 0), imm32(expected_typeid)) + self.mc.CMP16(mem(locs[0], 0), ImmedLoc(expected_typeid)) - def genop_guard_guard_class(self, ign_1, guard_op, addr, locs, ign_2): - mc = self._start_block() - self._cmp_guard_class(mc, locs) - self._stop_block() - return self.implement_guard(addr, self.mc.JNE) + def genop_guard_guard_class(self, ign_1, guard_op, guard_token, locs, ign_2): + self.mc.ensure_bytes_available(256) + self._cmp_guard_class(locs) + return self.implement_guard(guard_token, 'NE') def genop_guard_guard_nonnull_class(self, ign_1, guard_op, - addr, locs, ign_2): - mc = self._start_block() - mc.CMP(locs[0], imm8(1)) - mc.JB(rel8_patched_later) - jb_location = mc.get_relative_pos() - self._cmp_guard_class(mc, locs) + guard_token, locs, ign_2): + self.mc.ensure_bytes_available(256) + self.mc.CMP(locs[0], imm(1)) + # Patched below + self.mc.J_il8(rx86.Conditions['B'], 0) + jb_location = self.mc.get_relative_pos() + self._cmp_guard_class(locs) # patch the JB above - offset = mc.get_relative_pos() - jb_location + offset = self.mc.get_relative_pos() - jb_location assert 0 < offset <= 127 - mc.overwrite(jb_location-1, [chr(offset)]) - self._stop_block() + self.mc.overwrite(jb_location-1, [chr(offset)]) # - return self.implement_guard(addr, self.mc.JNE) + return self.implement_guard(guard_token, 'NE') def implement_guard_recovery(self, guard_opnum, faildescr, failargs, fail_locs): exc = (guard_opnum == rop.GUARD_EXCEPTION or guard_opnum == rop.GUARD_NO_EXCEPTION or guard_opnum == rop.GUARD_NOT_FORCED) - return self.generate_quick_failure(faildescr, failargs, fail_locs, exc) + desc_bytes = self.failure_recovery_description(failargs, fail_locs) + return GuardToken(faildescr, failargs, fail_locs, exc, desc_bytes) - def generate_quick_failure(self, faildescr, failargs, fail_locs, exc): + def generate_quick_failure(self, mc, faildescr, failargs, fail_locs, exc, desc_bytes): """Generate the initial code for handling a failure. We try to keep it as compact as possible. The idea is that this code is executed at most once (and very often, zero times); when @@ -1069,38 +1289,43 @@ really handle recovery from this particular failure. """ fail_index = self.cpu.get_fail_descr_number(faildescr) - bytes_needed = 20 + 5 * len(failargs) # conservative estimate - if self.mc2.bytes_free() < bytes_needed: - self.mc2.make_new_mc() - mc = self.mc2._mc addr = mc.tell() withfloats = False for box in failargs: if box is not None and box.type == FLOAT: withfloats = True break - mc.CALL(rel32(self.failure_recovery_code[exc + 2 * withfloats])) + mc.CALL(imm(self.failure_recovery_code[exc + 2 * withfloats])) # write tight data that describes the failure recovery faildescr._x86_failure_recovery_bytecode = mc.tell() - self.write_failure_recovery_description(mc, failargs, fail_locs) + for byte in desc_bytes: + mc.writechr(ord(byte)) # write the fail_index too - mc.write(packimm32(fail_index)) + mc.writeimm32(fail_index) # for testing the decoding, write a final byte 0xCC if not we_are_translated(): mc.writechr(0xCC) faildescr._x86_debug_faillocs = [loc for loc in fail_locs if loc is not None] + + # Make sure the recovery stub is at least 16 bytes long (for the + # case where we overwrite the recovery stub with a 64-bit absolute + # jump) + while mc.tell() - addr < 16: + mc.writechr(0x00) return addr DESCR_REF = 0x00 DESCR_INT = 0x01 DESCR_FLOAT = 0x02 DESCR_SPECIAL = 0x03 - CODE_FROMSTACK = 4*8 + # XXX: 4*8 works on i386, should we optimize for that case? + CODE_FROMSTACK = 4*16 CODE_STOP = 0 | DESCR_SPECIAL CODE_HOLE = 4 | DESCR_SPECIAL - def write_failure_recovery_description(self, mc, failargs, locs): + def failure_recovery_description(self, failargs, locs): + desc_bytes = [] for i in range(len(failargs)): arg = failargs[i] if arg is not None: @@ -1113,24 +1338,30 @@ else: raise AssertionError("bogus kind") loc = locs[i] - if isinstance(loc, MODRM): + if isinstance(loc, StackLoc): n = self.CODE_FROMSTACK//4 + loc.position else: - assert isinstance(loc, REG) - n = loc.op + assert isinstance(loc, RegLoc) + n = loc.value n = kind + 4*n while n > 0x7F: - mc.writechr((n & 0x7F) | 0x80) + desc_bytes.append(chr((n & 0x7F) | 0x80)) n >>= 7 else: n = self.CODE_HOLE - mc.writechr(n) - mc.writechr(self.CODE_STOP) + desc_bytes.append(chr(n)) + desc_bytes.append(chr(self.CODE_STOP)) # assert that the fail_boxes lists are big enough assert len(failargs) <= self.fail_boxes_int.SIZE + return desc_bytes + + def write_failure_recovery_description(self, mc, failargs, locs): + for byte in self.failure_recovery_description(failargs, locs): + mc.writechr(ord(byte)) def rebuild_faillocs_from_descr(self, bytecode): from pypy.jit.backend.x86.regalloc import X86FrameManager + descr_to_box_type = [REF, INT, FLOAT] bytecode = rffi.cast(rffi.UCHARP, bytecode) arglocs = [] while 1: @@ -1155,7 +1386,7 @@ size = 2 else: size = 1 - loc = X86FrameManager.frame_pos(code, size) + loc = X86FrameManager.frame_pos(code, descr_to_box_type[kind]) elif code == self.CODE_STOP: break elif code == self.CODE_HOLE: @@ -1165,16 +1396,16 @@ kind = code & 3 code >>= 2 if kind == self.DESCR_FLOAT: - loc = xmm_registers[code] + loc = regloc.XMMREGLOCS[code] else: - loc = registers[code] + loc = regloc.REGLOCS[code] arglocs.append(loc) return arglocs[:] @rgc.no_collect def grab_frame_values(self, bytecode, frame_addr, allregisters): # no malloc allowed here!! - self.fail_ebp = allregisters[16 + ebp.op] + self.fail_ebp = allregisters[16 + ebp.value] num = 0 value_hi = 0 while 1: @@ -1197,7 +1428,7 @@ code = (code - self.CODE_FROMSTACK) >> 2 stackloc = frame_addr + get_ebp_ofs(code) value = rffi.cast(rffi.LONGP, stackloc)[0] - if kind == self.DESCR_FLOAT: + if kind == self.DESCR_FLOAT and WORD == 4: value_hi = value value = rffi.cast(rffi.LONGP, stackloc - 4)[0] else: @@ -1211,8 +1442,11 @@ break code >>= 2 if kind == self.DESCR_FLOAT: - value = allregisters[2*code] - value_hi = allregisters[2*code + 1] + if WORD == 4: + value = allregisters[2*code] + value_hi = allregisters[2*code + 1] + else: + value = allregisters[code] else: value = allregisters[16 + code] @@ -1223,7 +1457,8 @@ tgt = self.fail_boxes_ptr.get_addr_for_num(num) elif kind == self.DESCR_FLOAT: tgt = self.fail_boxes_float.get_addr_for_num(num) - rffi.cast(rffi.LONGP, tgt)[1] = value_hi + if WORD == 4: + rffi.cast(rffi.LONGP, tgt)[1] = value_hi else: assert 0, "bogus kind" rffi.cast(rffi.LONGP, tgt)[0] = value @@ -1232,7 +1467,8 @@ if not we_are_translated(): assert bytecode[4] == 0xCC self.fail_boxes_count = num - fail_index = rffi.cast(rffi.LONGP, bytecode)[0] + fail_index = rffi.cast(rffi.INTP, bytecode)[0] + fail_index = rffi.cast(lltype.Signed, fail_index) return fail_index def setup_failure_recovery(self): @@ -1243,8 +1479,8 @@ # original value of the registers, optionally the original # value of XMM registers, and finally a reference to the # recovery bytecode. See _build_failure_recovery() for details. - stack_at_ebp = registers[ebp.op] - bytecode = rffi.cast(rffi.UCHARP, registers[8]) + stack_at_ebp = registers[ebp.value] + bytecode = rffi.cast(rffi.UCHARP, registers[self.cpu.NUM_REGS]) allregisters = rffi.ptradd(registers, -16) return self.grab_frame_values(bytecode, stack_at_ebp, allregisters) @@ -1259,23 +1495,23 @@ self.failure_recovery_func) failure_recovery_func = rffi.cast(lltype.Signed, failure_recovery_func) - mc = self.mc2._mc + mc = self.mc._mc # Assume that we are called at the beginning, when there is no risk # that 'mc' runs out of space. Checked by asserts in mc.write(). recovery_addr = mc.tell() - mc.PUSH(edi) - mc.PUSH(esi) - mc.PUSH(ebp) - mc.PUSH(esp) # <-- not really used, but needed to take up the space - mc.PUSH(ebx) - mc.PUSH(edx) - mc.PUSH(ecx) - mc.PUSH(eax) - mc.MOV(esi, esp) + + # Push all general purpose registers + for gpr in range(self.cpu.NUM_REGS-1, -1, -1): + mc.PUSH_r(gpr) + + # ebx/rbx is callee-save in both i386 and x86-64 + mc.MOV_rr(ebx.value, esp.value) + if withfloats: - mc.SUB(esp, imm(8*8)) - for i in range(8): - mc.MOVSD(mem64(esp, 8*i), xmm_registers[i]) + # Push all float registers + mc.SUB_ri(esp.value, self.cpu.NUM_REGS*8) + for i in range(self.cpu.NUM_REGS): + mc.MOVSD_sx(8*i, i) # we call a provided function that will # - call our on_leave_jitted_hook which will mark @@ -1283,7 +1519,7 @@ # avoid unwarranted freeing # - optionally save exception depending on the flag addr = self.cpu.get_on_leave_jitted_int(save_exception=exc) - mc.CALL(rel32(addr)) + mc.CALL(imm(addr)) # the following call saves all values from the stack and from # registers to the right 'fail_boxes_' location. @@ -1293,50 +1529,58 @@ # bytecode, pushed just before by the CALL instruction written by # generate_quick_failure(). XXX misaligned stack in the call, but # it's ok because failure_recovery_func is not calling anything more - mc.PUSH(esi) - mc.CALL(rel32(failure_recovery_func)) + + # XXX + if IS_X86_32: + mc.PUSH_r(ebx.value) + elif IS_X86_64: + mc.MOV_rr(edi.value, ebx.value) + # XXX: Correct to only align the stack on 64-bit? + mc.AND_ri(esp.value, -16) + else: + raise AssertionError("Shouldn't happen") + + mc.CALL(imm(failure_recovery_func)) # returns in eax the fail_index # now we return from the complete frame, which starts from - # _assemble_bootstrap_code(). The LEA below throws away most - # of the frame, including all the PUSHes that we did just above. - mc.LEA(esp, addr_add(ebp, imm(-3 * WORD))) - mc.POP(edi) # [ebp-12] - mc.POP(esi) # [ebp-8] - mc.POP(ebx) # [ebp-4] - mc.POP(ebp) # [ebp] - mc.RET() - self.mc2.done() + # _assemble_bootstrap_code(). The LEA in _call_footer below throws + # away most of the frame, including all the PUSHes that we did just + # above. + + self._call_footer() + self.mc.done() self.failure_recovery_code[exc + 2 * withfloats] = recovery_addr def generate_failure(self, fail_index, locs, exc, locs_are_ref): - mc = self.mc + self.mc._mc.begin_reuse_scratch_register() for i in range(len(locs)): loc = locs[i] - if isinstance(loc, REG): - if loc.width == 8: + if isinstance(loc, RegLoc): + if loc.is_xmm: adr = self.fail_boxes_float.get_addr_for_num(i) - mc.MOVSD(heap64(adr), loc) + self.mc.MOVSD(heap(adr), loc) else: if locs_are_ref[i]: adr = self.fail_boxes_ptr.get_addr_for_num(i) else: adr = self.fail_boxes_int.get_addr_for_num(i) - mc.MOV(heap(adr), loc) + self.mc.MOV(heap(adr), loc) for i in range(len(locs)): loc = locs[i] - if not isinstance(loc, REG): - if loc.width == 8: - mc.MOVSD(xmm0, loc) + if not isinstance(loc, RegLoc): + if isinstance(loc, StackLoc) and loc.type == FLOAT: + self.mc.MOVSD_xb(xmm0.value, loc.value) adr = self.fail_boxes_float.get_addr_for_num(i) - mc.MOVSD(heap64(adr), xmm0) + self.mc.MOVSD(heap(adr), xmm0) else: if locs_are_ref[i]: adr = self.fail_boxes_ptr.get_addr_for_num(i) else: adr = self.fail_boxes_int.get_addr_for_num(i) - mc.MOV(eax, loc) - mc.MOV(heap(adr), eax) + self.mc.MOV(eax, loc) + self.mc.MOV(heap(adr), eax) + self.mc._mc.end_reuse_scratch_register() # we call a provided function that will # - call our on_leave_jitted_hook which will mark @@ -1344,28 +1588,31 @@ # avoid unwarranted freeing # - optionally save exception depending on the flag addr = self.cpu.get_on_leave_jitted_int(save_exception=exc) - mc.CALL(rel32(addr)) + self.mc.CALL(imm(addr)) - mc.LEA(esp, addr_add(ebp, imm(-3 * WORD))) - mc.MOV(eax, imm(fail_index)) - mc.POP(edi) # [ebp-12] - mc.POP(esi) # [ebp-8] - mc.POP(ebx) # [ebp-4] - mc.POP(ebp) # [ebp] - mc.RET() - - @specialize.arg(2) - def implement_guard(self, addr, emit_jump): - emit_jump(rel32(addr)) + self.mc.MOV_ri(eax.value, fail_index) + + # exit function + self._call_footer() + + def implement_guard(self, guard_token, condition=None): + self.mc.reserve_bytes(guard_token.recovery_stub_size()) + self.pending_guard_tokens.append(guard_token) + # XXX: These jumps are patched later, the self.mc.tell() are just + # dummy values + if condition: + self.mc.J_il(rx86.Conditions[condition], self.mc.tell()) + else: + self.mc.JMP_l(self.mc.tell()) return self.mc.tell() - 4 def genop_call(self, op, arglocs, resloc): sizeloc = arglocs[0] - assert isinstance(sizeloc, IMM32) + assert isinstance(sizeloc, ImmedLoc) size = sizeloc.value if isinstance(op.args[0], Const): - x = rel32(op.args[0].getint()) + x = imm(op.args[0].getint()) else: x = arglocs[1] if x is eax: @@ -1375,35 +1622,35 @@ self._emit_call(x, arglocs, 2, tmp=tmp) - if isinstance(resloc, MODRM64): - self.mc.FSTP(resloc) + if isinstance(resloc, StackLoc) and resloc.width == 8 and IS_X86_32: + self.mc.FSTP_b(resloc.value) elif size == 1: - self.mc.AND(eax, imm(0xff)) + self.mc.AND_ri(eax.value, 0xff) elif size == 2: - self.mc.AND(eax, imm(0xffff)) + self.mc.AND_ri(eax.value, 0xffff) - def genop_guard_call_may_force(self, op, guard_op, addr, + def genop_guard_call_may_force(self, op, guard_op, guard_token, arglocs, result_loc): faildescr = guard_op.descr fail_index = self.cpu.get_fail_descr_number(faildescr) - self.mc.MOV(mem(ebp, FORCE_INDEX_OFS), imm(fail_index)) + self.mc.MOV_bi(FORCE_INDEX_OFS, fail_index) self.genop_call(op, arglocs, result_loc) - self.mc.CMP(mem(ebp, FORCE_INDEX_OFS), imm(0)) - return self.implement_guard(addr, self.mc.JL) + self.mc.CMP_bi(FORCE_INDEX_OFS, 0) + return self.implement_guard(guard_token, 'L') - def genop_guard_call_assembler(self, op, guard_op, addr, + def genop_guard_call_assembler(self, op, guard_op, guard_token, arglocs, result_loc): faildescr = guard_op.descr fail_index = self.cpu.get_fail_descr_number(faildescr) - self.mc.MOV(mem(ebp, FORCE_INDEX_OFS), imm(fail_index)) + self.mc.MOV_bi(FORCE_INDEX_OFS, fail_index) descr = op.descr assert isinstance(descr, LoopToken) assert len(arglocs) - 2 == len(descr._x86_arglocs[0]) # # Write a call to the direct_bootstrap_code of the target assembler - self._emit_call(rel32(descr._x86_direct_bootstrap_code), arglocs, 2, + self._emit_call(imm(descr._x86_direct_bootstrap_code), arglocs, 2, tmp=eax) - mc = self._start_block() + self.mc.ensure_bytes_available(256) if op.result is None: assert result_loc is None value = self.cpu.done_with_this_frame_void_v @@ -1419,26 +1666,27 @@ value = self.cpu.done_with_this_frame_float_v else: raise AssertionError(kind) - mc.CMP(eax, imm(value)) - mc.JE(rel8_patched_later) # goto B if we get 'done_with_this_frame' - je_location = mc.get_relative_pos() + self.mc.CMP_ri(eax.value, value) + # patched later + self.mc.J_il8(rx86.Conditions['E'], 0) # goto B if we get 'done_with_this_frame' + je_location = self.mc.get_relative_pos() # # Path A: use assembler_helper_adr jd = descr.outermost_jitdriver_sd assert jd is not None asm_helper_adr = self.cpu.cast_adr_to_int(jd.assembler_helper_adr) - self._emit_call(rel32(asm_helper_adr), [eax, arglocs[1]], 0, - tmp=ecx, force_mc=True, mc=mc) - if isinstance(result_loc, MODRM64): - mc.FSTP(result_loc) + self._emit_call(imm(asm_helper_adr), [eax, arglocs[1]], 0, + tmp=ecx) + if IS_X86_32 and isinstance(result_loc, StackLoc) and result_loc.type == FLOAT: + self.mc.FSTP_b(result_loc.value) #else: result_loc is already either eax or None, checked below - mc.JMP(rel8_patched_later) # done - jmp_location = mc.get_relative_pos() + self.mc.JMP_l8(0) # jump to done, patched later + jmp_location = self.mc.get_relative_pos() # # Path B: fast path. Must load the return value, and reset the token offset = jmp_location - je_location assert 0 < offset <= 127 - mc.overwrite(je_location - 1, [chr(offset)]) + self.mc.overwrite(je_location - 1, [chr(offset)]) # # Reset the vable token --- XXX really too much special logic here:-( if jd.index_of_virtualizable >= 0: @@ -1446,8 +1694,8 @@ fielddescr = jd.vable_token_descr assert isinstance(fielddescr, BaseFieldDescr) ofs = fielddescr.offset - mc.MOV(eax, arglocs[1]) - mc.MOV(addr_add(eax, imm(ofs)), imm(0)) + self.mc.MOV(eax, arglocs[1]) + self.mc.MOV_mi((eax.value, ofs), 0) # in the line above, TOKEN_NONE = 0 # if op.result is not None: @@ -1456,27 +1704,26 @@ if kind == FLOAT: xmmtmp = X86XMMRegisterManager.all_regs[0] adr = self.fail_boxes_float.get_addr_for_num(0) - mc.MOVSD(xmmtmp, heap64(adr)) - mc.MOVSD(result_loc, xmmtmp) + self.mc.MOVSD(xmmtmp, heap(adr)) + self.mc.MOVSD(result_loc, xmmtmp) else: assert result_loc is eax if kind == INT: adr = self.fail_boxes_int.get_addr_for_num(0) - mc.MOV(eax, heap(adr)) + self.mc.MOV(eax, heap(adr)) elif kind == REF: adr = self.fail_boxes_ptr.get_addr_for_num(0) - mc.XOR(eax, eax) - mc.XCHG(eax, heap(adr)) + self.mc.XOR_rr(eax.value, eax.value) + self.mc.XCHG(eax, heap(adr)) else: raise AssertionError(kind) # # Here we join Path A and Path B again - offset = mc.get_relative_pos() - jmp_location + offset = self.mc.get_relative_pos() - jmp_location assert 0 <= offset <= 127 - mc.overwrite(jmp_location - 1, [chr(offset)]) - self._stop_block() - self.mc.CMP(mem(ebp, FORCE_INDEX_OFS), imm(0)) - return self.implement_guard(addr, self.mc.JL) + self.mc.overwrite(jmp_location - 1, [chr(offset)]) + self.mc.CMP_bi(FORCE_INDEX_OFS, 0) + return self.implement_guard(guard_token, 'L') def genop_discard_cond_call_gc_wb(self, op, arglocs): # use 'mc._mc' directly instead of 'mc', to avoid @@ -1486,31 +1733,41 @@ cls = self.cpu.gc_ll_descr.has_write_barrier_class() assert cls is not None and isinstance(descr, cls) loc_base = arglocs[0] - mc = self._start_block() - mc.TEST(mem8(loc_base, descr.jit_wb_if_flag_byteofs), - imm8(descr.jit_wb_if_flag_singlebyte)) - mc.JZ(rel8_patched_later) - jz_location = mc.get_relative_pos() + self.mc.ensure_bytes_available(256) + self.mc.TEST8_mi((loc_base.value, descr.jit_wb_if_flag_byteofs), + descr.jit_wb_if_flag_singlebyte) + self.mc.J_il8(rx86.Conditions['Z'], 0) # patched later + jz_location = self.mc.get_relative_pos() # the following is supposed to be the slow path, so whenever possible # we choose the most compact encoding over the most efficient one. for i in range(len(arglocs)-1, -1, -1): - mc.PUSH(arglocs[i]) + self.mc.PUSH(arglocs[i]) + + if IS_X86_64: + # We clobber these registers to pass the arguments, but that's + # okay, because consider_cond_call_gc_wb makes sure that any + # caller-save registers with values in them are present in arglocs, + # so they are saved on the stack above and restored below + self.mc.MOV_rs(edi.value, 0) + self.mc.MOV_rs(esi.value, 8) + # misaligned stack in the call, but it's ok because the write barrier # is not going to call anything more. Also, this assumes that the # write barrier does not touch the xmm registers. - mc.CALL(rel32(descr.get_write_barrier_fn(self.cpu))) + self.mc.CALL(imm(descr.get_write_barrier_fn(self.cpu))) for i in range(len(arglocs)): loc = arglocs[i] - assert isinstance(loc, REG) - mc.POP(loc) + assert isinstance(loc, RegLoc) + self.mc.POP(loc) # patch the JZ above - offset = mc.get_relative_pos() - jz_location + offset = self.mc.get_relative_pos() - jz_location assert 0 < offset <= 127 - mc.overwrite(jz_location-1, [chr(offset)]) - self._stop_block() + self.mc.overwrite(jz_location-1, [chr(offset)]) def genop_force_token(self, op, arglocs, resloc): - self.mc.LEA(resloc, mem(ebp, FORCE_INDEX_OFS)) + # RegAlloc.consider_force_token ensures this: + assert isinstance(resloc, RegLoc) + self.mc.LEA_rb(resloc.value, FORCE_INDEX_OFS) def not_implemented_op_discard(self, op, arglocs): msg = "not implemented operation: %s" % op.getopname() @@ -1538,17 +1795,16 @@ return loop_token._x86_arglocs def closing_jump(self, loop_token): - self.mc.JMP(rel32(loop_token._x86_loop_code)) + self.mc.JMP(imm(loop_token._x86_loop_code)) def malloc_cond_fixedsize(self, nursery_free_adr, nursery_top_adr, size, tid): - # don't use self.mc - mc = self._start_block() - mc.MOV(eax, heap(nursery_free_adr)) - mc.LEA(edx, addr_add(eax, imm(size))) - mc.CMP(edx, heap(nursery_top_adr)) - mc.JNA(rel8_patched_later) - jmp_adr = mc.get_relative_pos() + self.mc.ensure_bytes_available(256) + self.mc.MOV(eax, heap(nursery_free_adr)) + self.mc.LEA_rm(edx.value, (eax.value, size)) + self.mc.CMP(edx, heap(nursery_top_adr)) + self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later + jmp_adr = self.mc.get_relative_pos() # See comments in _build_malloc_fixedsize_slowpath for the # details of the two helper functions that we are calling below. @@ -1564,17 +1820,16 @@ # reserve room for the argument to the real malloc and the # 8 saved XMM regs self._regalloc.reserve_param(1+16) - mc.CALL(rel32(slowpath_addr1)) + self.mc.CALL(imm(slowpath_addr1)) self.mark_gc_roots() slowpath_addr2 = self.malloc_fixedsize_slowpath2 - mc.CALL(rel32(slowpath_addr2)) + self.mc.CALL(imm(slowpath_addr2)) - offset = mc.get_relative_pos() - jmp_adr + offset = self.mc.get_relative_pos() - jmp_adr assert 0 < offset <= 127 - mc.overwrite(jmp_adr-1, [chr(offset)]) - mc.MOV(addr_add(eax, imm(0)), imm(tid)) - mc.MOV(heap(nursery_free_adr), edx) - self._stop_block() + self.mc.overwrite(jmp_adr-1, [chr(offset)]) + self.mc.MOV_mi((eax.value, 0), tid) + self.mc.MOV(heap(nursery_free_adr), edx) genop_discard_list = [Assembler386.not_implemented_op_discard] * rop._LAST genop_list = [Assembler386.not_implemented_op] * rop._LAST @@ -1594,32 +1849,20 @@ num = getattr(rop, opname.upper()) genop_list[num] = value -def new_addr_add(heap, mem, memsib): - def addr_add(reg_or_imm1, reg_or_imm2, offset=0, scale=0): - if isinstance(reg_or_imm1, IMM32): - if isinstance(reg_or_imm2, IMM32): - return heap(reg_or_imm1.value + offset + - (reg_or_imm2.value << scale)) - else: - return memsib(None, reg_or_imm2, scale, reg_or_imm1.value + offset) - else: - if isinstance(reg_or_imm2, IMM32): - return mem(reg_or_imm1, offset + (reg_or_imm2.value << scale)) - else: - return memsib(reg_or_imm1, reg_or_imm2, scale, offset) - return addr_add - -addr8_add = new_addr_add(heap8, mem8, memSIB8) -addr_add = new_addr_add(heap, mem, memSIB) -addr64_add = new_addr_add(heap64, mem64, memSIB64) - -def addr_add_const(reg_or_imm1, offset): - if isinstance(reg_or_imm1, IMM32): - return heap(reg_or_imm1.value + offset) - else: - return mem(reg_or_imm1, offset) - def round_up_to_4(size): if size < 4: return 4 return size + +# XXX: ri386 migration shims: +def addr_add(reg_or_imm1, reg_or_imm2, offset=0, scale=0): + return AddressLoc(reg_or_imm1, reg_or_imm2, scale, offset) + +def addr_add_const(reg_or_imm1, offset): + return AddressLoc(reg_or_imm1, ImmedLoc(0), 0, offset) + +def mem(loc, offset): + return AddressLoc(loc, ImmedLoc(0), 0, offset) + +def heap(addr): + return AddressLoc(ImmedLoc(addr), ImmedLoc(0), 0, 0) Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/codebuf.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/codebuf.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/codebuf.py Tue Aug 10 20:07:15 2010 @@ -2,12 +2,20 @@ import os, sys from pypy.rpython.lltypesystem import lltype, rffi from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.jit.backend.x86.ri386 import I386CodeBuilder +from pypy.jit.backend.x86.rx86 import X86_32_CodeBuilder, X86_64_CodeBuilder +from pypy.jit.backend.x86.regloc import LocationCodeBuilder from pypy.rlib.rmmap import PTR, alloc, free from pypy.rlib.debug import make_sure_not_resized +from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 +from pypy.rlib.objectmodel import we_are_translated +# XXX: Seems nasty to change the superclass of InMemoryCodeBuilder like this +if IS_X86_32: + codebuilder_cls = X86_32_CodeBuilder +elif IS_X86_64: + codebuilder_cls = X86_64_CodeBuilder -class InMemoryCodeBuilder(I386CodeBuilder): +class InMemoryCodeBuilder(codebuilder_cls, LocationCodeBuilder): _last_dump_start = 0 def __init__(self, start, end): @@ -31,13 +39,15 @@ def write(self, listofchars): self._pos = self.overwrite(self._pos, listofchars) - def writechr(self, n): - # purely for performance: don't make the one-element list [chr(n)] + def writechar(self, char): pos = self._pos assert pos + 1 <= self._size - self._data[pos] = chr(n) + self._data[pos] = char self._pos = pos + 1 + def writechr(self, n): + self.writechar(chr(n)) + def get_relative_pos(self): return self._pos @@ -50,11 +60,6 @@ self._pos = pos self._last_dump_start = pos - def execute(self, arg1, arg2): - # XXX old testing stuff - fnptr = rffi.cast(lltype.Ptr(BINARYFN), self._data) - return fnptr(arg1, arg2) - def done(self): # normally, no special action is needed here if machine_code_dumper.enabled: @@ -77,9 +82,6 @@ valgrind.discard_translations(self._data, self._size) -BINARYFN = lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed) - - class MachineCodeDumper: enabled = True log_fd = -1 @@ -107,7 +109,10 @@ return False # log the executable name from pypy.jit.backend.hlinfo import highleveljitinfo - os.write(self.log_fd, 'BACKEND i386\n') + if IS_X86_32: + os.write(self.log_fd, 'BACKEND x86\n') + elif IS_X86_64: + os.write(self.log_fd, 'BACKEND x86_64\n') if highleveljitinfo.sys_executable: os.write(self.log_fd, 'SYS_EXECUTABLE %s\n' % ( highleveljitinfo.sys_executable,)) @@ -137,6 +142,12 @@ def __init__(self, map_size): data = alloc(map_size) + if IS_X86_64 and not we_are_translated(): + # Hack to make sure that mcs are not within 32-bits of one + # another for testing purposes + from pypy.rlib.rmmap import hint + hint.pos += 0xFFFFFFFF + self._init(data, map_size) def __del__(self): Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/jump.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/jump.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/jump.py Tue Aug 10 20:07:15 2010 @@ -1,23 +1,6 @@ import sys from pypy.tool.pairtype import extendabletype -from pypy.jit.backend.x86.ri386 import * - -class __extend__(OPERAND): - __metaclass__ = extendabletype - def _getregkey(self): - raise AssertionError("should only happen to registers and frame " - "positions") - -class __extend__(REG): - __metaclass__ = extendabletype - def _getregkey(self): - return ~self.op - -class __extend__(MODRM): - __metaclass__ = extendabletype - def _getregkey(self): - return self.position - +from pypy.jit.backend.x86.regloc import ImmedLoc, StackLoc def remap_frame_layout(assembler, src_locations, dst_locations, tmpreg): pending_dests = len(dst_locations) @@ -27,7 +10,7 @@ srccount[dst._getregkey()] = 0 for i in range(len(dst_locations)): src = src_locations[i] - if isinstance(src, IMM32): + if isinstance(src, ImmedLoc): continue key = src._getregkey() if key in srccount: @@ -46,7 +29,7 @@ srccount[key] = -1 # means "it's done" pending_dests -= 1 src = src_locations[i] - if not isinstance(src, IMM32): + if not isinstance(src, ImmedLoc): key = src._getregkey() if key in srccount: srccount[key] -= 1 @@ -80,7 +63,7 @@ assert pending_dests == 0 def _move(assembler, src, dst, tmpreg): - if isinstance(dst, MODRM) and isinstance(src, MODRM): + if dst.is_memory_reference() and src.is_memory_reference(): assembler.regalloc_mov(src, tmpreg) src = tmpreg assembler.regalloc_mov(src, dst) Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/regalloc.py Tue Aug 10 20:07:15 2010 @@ -5,7 +5,7 @@ from pypy.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr, ResOperation, BoxPtr, LoopToken, INT, REF, FLOAT) -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.regloc import * from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi, rstr from pypy.rlib.objectmodel import we_are_translated from pypy.rlib import rgc @@ -17,16 +17,7 @@ from pypy.jit.backend.llsupport.descr import BaseCallDescr, BaseSizeDescr from pypy.jit.backend.llsupport.regalloc import FrameManager, RegisterManager,\ TempBox - -WORD = 4 -FRAME_FIXED_SIZE = 5 # ebp + ebx + esi + edi + force_index = 5 words -FORCE_INDEX_OFS = -4*WORD - -width_of_type = { - INT : 1, - REF : 1, - FLOAT : 2, - } +from pypy.jit.backend.x86.arch import WORD, FRAME_FIXED_SIZE, IS_X86_32, IS_X86_64 class X86RegisterManager(RegisterManager): @@ -50,12 +41,19 @@ print "convert_to_imm: got a %s" % c raise AssertionError +class X86_64_RegisterManager(X86RegisterManager): + # r11 omitted because it's used as scratch + all_regs = [eax, ecx, edx, ebx, esi, edi, r8, r9, r10, r12, r13, r14, r15] + no_lower_byte_regs = [] + save_around_call_regs = [eax, ecx, edx, esi, edi, r8, r9, r10] + class FloatConstants(object): BASE_CONSTANT_SIZE = 1000 def __init__(self): self.cur_array_free = 0 + self.const_id = 0 def _get_new_array(self): n = self.BASE_CONSTANT_SIZE @@ -71,7 +69,8 @@ n = self.cur_array_free - 1 arr[n] = floatval self.cur_array_free = n - return rffi.cast(lltype.Signed, arr) + n * 8 + self.const_id += 1 + return (self.const_id, rffi.cast(lltype.Signed, arr) + n * 8) class X86XMMRegisterManager(RegisterManager): @@ -80,7 +79,6 @@ all_regs = [xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7] # we never need lower byte I hope save_around_call_regs = all_regs - reg_width = 2 def __init__(self, longevity, frame_manager=None, assembler=None): RegisterManager.__init__(self, longevity, frame_manager=frame_manager, @@ -93,28 +91,36 @@ self.float_constants = assembler._float_constants def convert_to_imm(self, c): - adr = self.float_constants.record_float(c.getfloat()) - return heap64(adr) + const_id, adr = self.float_constants.record_float(c.getfloat()) + return ConstFloatLoc(adr, const_id) def after_call(self, v): # the result is stored in st0, but we don't have this around, # so genop_call will move it to some frame location immediately # after the call - return self.frame_manager.loc(v, 2) + return self.frame_manager.loc(v) -class X86FrameManager(FrameManager): +class X86_64_XMMRegisterManager(X86XMMRegisterManager): + # xmm15 reserved for scratch use + all_regs = [xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14] + save_around_call_regs = all_regs + + def call_result_location(self, v): + return xmm0 + + def after_call(self, v): + # We use RegisterManager's implementation, since X86XMMRegisterManager + # places the result on the stack, which we don't need to do when the + # calling convention places the result in xmm0 + return RegisterManager.after_call(self, v) +class X86FrameManager(FrameManager): @staticmethod - def frame_pos(i, size): - if size == 1: - res = mem(ebp, get_ebp_ofs(i)) - elif size == 2: - res = mem64(ebp, get_ebp_ofs(i + 1)) - else: - print "Unimplemented size %d" % i - raise NotImplementedError("unimplemented size %d" % i) - res.position = i - return res + def frame_pos(i, box_type): + if IS_X86_32 and box_type == FLOAT: + return StackLoc(i, get_ebp_ofs(i+1), 2, box_type) + else: + return StackLoc(i, get_ebp_ofs(i), 1, box_type) class RegAlloc(object): exc = False @@ -135,11 +141,21 @@ # compute longevity of variables longevity = self._compute_vars_longevity(inputargs, operations) self.longevity = longevity - self.rm = X86RegisterManager(longevity, - frame_manager = self.fm, - assembler = self.assembler) - self.xrm = X86XMMRegisterManager(longevity, frame_manager = self.fm, - assembler = self.assembler) + # XXX + if cpu.WORD == 4: + gpr_reg_mgr_cls = X86RegisterManager + xmm_reg_mgr_cls = X86XMMRegisterManager + elif cpu.WORD == 8: + gpr_reg_mgr_cls = X86_64_RegisterManager + xmm_reg_mgr_cls = X86_64_XMMRegisterManager + else: + raise AssertionError("Word size should be 4 or 8") + + self.rm = gpr_reg_mgr_cls(longevity, + frame_manager = self.fm, + assembler = self.assembler) + self.xrm = xmm_reg_mgr_cls(longevity, frame_manager = self.fm, + assembler = self.assembler) def prepare_loop(self, inputargs, operations, looptoken): self._prepare(inputargs, operations) @@ -184,7 +200,7 @@ if reg: loc = reg else: - loc = self.fm.loc(arg, width_of_type[arg.type]) + loc = self.fm.loc(arg) if arg.type == FLOAT: floatlocs[i] = loc else: @@ -252,23 +268,23 @@ arg = inputargs[i] i += 1 if arg.type == FLOAT: - if isinstance(loc, REG): + if isinstance(loc, RegLoc): self.xrm.reg_bindings[arg] = loc used[loc] = None else: self.fm.frame_bindings[arg] = loc else: - if isinstance(loc, REG): + if isinstance(loc, RegLoc): self.rm.reg_bindings[arg] = loc used[loc] = None else: self.fm.frame_bindings[arg] = loc self.rm.free_regs = [] - for reg in X86RegisterManager.all_regs: + for reg in self.rm.all_regs: if reg not in used: self.rm.free_regs.append(reg) self.xrm.free_regs = [] - for reg in X86XMMRegisterManager.all_regs: + for reg in self.xrm.all_regs: if reg not in used: self.xrm.free_regs.append(reg) # note: we need to make a copy of inputargs because possibly_free_vars @@ -647,7 +663,7 @@ vable_index = jd.index_of_virtualizable if vable_index >= 0: self.rm._sync_var(op.args[vable_index]) - vable = self.fm.loc(op.args[vable_index], 1) + vable = self.fm.loc(op.args[vable_index]) else: vable = imm(0) self._call(op, [imm(size), vable] + @@ -671,7 +687,7 @@ # function, a GC write barrier, is known not to touch them. # See remember_young_pointer() in rpython/memory/gc/generation.py. for v, reg in self.rm.reg_bindings.items(): - if ((reg is eax or reg is ecx or reg is edx) + if (reg in self.rm.save_around_call_regs and self.rm.stays_alive(v)): arglocs.append(reg) self.PerformDiscard(op, arglocs) @@ -810,7 +826,7 @@ def consider_setfield_gc(self, op): ofs_loc, size_loc, ptr = self._unpack_fielddescr(op.descr) - assert isinstance(size_loc, IMM32) + assert isinstance(size_loc, ImmedLoc) if size_loc.value == 1: need_lower_byte = True else: @@ -951,7 +967,7 @@ shape = gcrootmap.get_basic_shape() for v, val in self.fm.frame_bindings.items(): if (isinstance(v, BoxPtr) and self.rm.stays_alive(v)): - assert isinstance(val, MODRM) + assert isinstance(val, StackLoc) gcrootmap.add_ebp_offset(shape, get_ebp_ofs(val.position)) for v, reg in self.rm.reg_bindings.items(): if reg is eax: Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/runner.py Tue Aug 10 20:07:15 2010 @@ -4,11 +4,13 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.jit.metainterp import history, compile from pypy.jit.backend.x86.assembler import Assembler386 -from pypy.jit.backend.x86.regalloc import FORCE_INDEX_OFS +from pypy.jit.backend.x86.arch import FORCE_INDEX_OFS from pypy.jit.backend.x86.profagent import ProfileAgent from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU +from pypy.jit.backend.x86 import regloc +import sys -class CPU386(AbstractLLCPU): +class AbstractX86CPU(AbstractLLCPU): debug = True supports_floats = True @@ -132,10 +134,28 @@ assert fail_index == fail_index_2 return faildescr +class CPU386(AbstractX86CPU): + WORD = 4 + NUM_REGS = 8 + CALLEE_SAVE_REGISTERS = [regloc.ebx, regloc.esi, regloc.edi] + FRAME_FIXED_SIZE = len(CALLEE_SAVE_REGISTERS) + 2 + + def __init__(self, *args, **kwargs): + assert sys.maxint == (2**31 - 1) + super(CPU386, self).__init__(*args, **kwargs) class CPU386_NO_SSE2(CPU386): supports_floats = False +class CPU_X86_64(AbstractX86CPU): + WORD = 8 + NUM_REGS = 16 + CALLEE_SAVE_REGISTERS = [regloc.ebx, regloc.r12, regloc.r13, regloc.r14, regloc.r15] + FRAME_FIXED_SIZE = len(CALLEE_SAVE_REGISTERS) + 2 + + def __init__(self, *args, **kwargs): + assert sys.maxint == (2**63 - 1) + super(CPU_X86_64, self).__init__(*args, **kwargs) CPU = CPU386 Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/conftest.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/conftest.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/conftest.py Tue Aug 10 20:07:15 2010 @@ -3,6 +3,5 @@ cpu = detect_cpu.autodetect() def pytest_runtest_setup(item): - if cpu != 'x86': - py.test.skip("x86 directory skipped: cpu is %r" % (cpu,)) - + if cpu not in ('x86', 'x86_64'): + py.test.skip("x86/x86_64 tests skipped: cpu is %r" % (cpu,)) Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_assembler.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_assembler.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_assembler.py Tue Aug 10 20:07:15 2010 @@ -1,14 +1,22 @@ -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.regloc import * from pypy.jit.backend.x86.assembler import Assembler386, MachineCodeBlockWrapper from pypy.jit.backend.x86.regalloc import X86FrameManager, get_ebp_ofs -from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat +from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, INT, REF, FLOAT from pypy.rlib.rarithmetic import intmask from pypy.rpython.lltypesystem import lltype, llmemory, rffi +from pypy.jit.backend.x86.arch import WORD, IS_X86_32, IS_X86_64 +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.regalloc import X86RegisterManager, X86_64_RegisterManager, X86XMMRegisterManager, X86_64_XMMRegisterManager +ACTUAL_CPU = getcpuclass() class FakeCPU: rtyper = None supports_floats = True + NUM_REGS = ACTUAL_CPU.NUM_REGS + + def fielddescrof(self, STRUCT, name): + return 42 class FakeMC: def __init__(self, base_address=0): @@ -25,7 +33,14 @@ self.content.append(("JMP", args)) def done(self): pass + def PUSH_r(self, reg): + pass + def POP_r(self, reg): + pass +class FakeAssembler: + def write_pending_failure_recoveries(self): + pass def test_write_failure_recovery_description(): assembler = Assembler386(FakeCPU()) @@ -33,12 +48,12 @@ failargs = [BoxInt(), BoxPtr(), BoxFloat()] * 3 failargs.insert(6, None) failargs.insert(7, None) - locs = [X86FrameManager.frame_pos(0, 1), - X86FrameManager.frame_pos(1, 1), - X86FrameManager.frame_pos(10, 2), - X86FrameManager.frame_pos(100, 1), - X86FrameManager.frame_pos(101, 1), - X86FrameManager.frame_pos(110, 2), + locs = [X86FrameManager.frame_pos(0, INT), + X86FrameManager.frame_pos(1, REF), + X86FrameManager.frame_pos(10, FLOAT), + X86FrameManager.frame_pos(100, INT), + X86FrameManager.frame_pos(101, REF), + X86FrameManager.frame_pos(110, FLOAT), None, None, ebx, @@ -46,17 +61,17 @@ xmm2] assert len(failargs) == len(locs) assembler.write_failure_recovery_description(mc, failargs, locs) - nums = [Assembler386.DESCR_INT + 4*(8+0), - Assembler386.DESCR_REF + 4*(8+1), - Assembler386.DESCR_FLOAT + 4*(8+10), - Assembler386.DESCR_INT + 4*(8+100), - Assembler386.DESCR_REF + 4*(8+101), - Assembler386.DESCR_FLOAT + 4*(8+110), + nums = [Assembler386.DESCR_INT + 4*(16+0), + Assembler386.DESCR_REF + 4*(16+1), + Assembler386.DESCR_FLOAT + 4*(16+10), + Assembler386.DESCR_INT + 4*(16+100), + Assembler386.DESCR_REF + 4*(16+101), + Assembler386.DESCR_FLOAT + 4*(16+110), Assembler386.CODE_HOLE, Assembler386.CODE_HOLE, - Assembler386.DESCR_INT + 4*ebx.op, - Assembler386.DESCR_REF + 4*esi.op, - Assembler386.DESCR_FLOAT + 4*xmm2.op] + Assembler386.DESCR_INT + 4*ebx.value, + Assembler386.DESCR_REF + 4*esi.value, + Assembler386.DESCR_FLOAT + 4*xmm2.value] double_byte_nums = [] for num in nums[3:6]: double_byte_nums.append((num & 0x7F) | 0x80) @@ -94,6 +109,9 @@ return lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S)) def get_random_float(): + # Returns , , + # NB: on 64-bit, will be the entire float and + # will be random garbage from malloc! assert withfloats value = random.random() - 0.5 # make sure it fits into 64 bits @@ -101,9 +119,16 @@ rffi.cast(rffi.DOUBLEP, tmp)[0] = value return rffi.cast(rffi.DOUBLEP, tmp)[0], tmp[0], tmp[1] + if IS_X86_32: + main_registers = X86RegisterManager.all_regs + xmm_registers = X86XMMRegisterManager.all_regs + elif IS_X86_64: + main_registers = X86_64_RegisterManager.all_regs + xmm_registers = X86_64_XMMRegisterManager.all_regs + # memory locations: 26 integers, 26 pointers, 26 floats # main registers: half of them as signed and the other half as ptrs - # xmm registers: all floats, from xmm0 to xmm7 + # xmm registers: all floats, from xmm0 to xmm(7|15) # holes: 8 locations = [] baseloc = 4 @@ -117,18 +142,17 @@ content = ([('int', locations.pop()) for _ in range(26)] + [('ptr', locations.pop()) for _ in range(26)] + [(['int', 'ptr'][random.randrange(0, 2)], reg) - for reg in [eax, ecx, edx, ebx, esi, edi]]) + for reg in main_registers]) if withfloats: content += ([('float', locations.pop()) for _ in range(26)] + - [('float', reg) for reg in [xmm0, xmm1, xmm2, xmm3, - xmm4, xmm5, xmm6, xmm7]]) + [('float', reg) for reg in xmm_registers]) for i in range(8): content.append(('hole', None)) random.shuffle(content) # prepare the expected target arrays, the descr_bytecode, # the 'registers' and the 'stack' arrays according to 'content' - xmmregisters = lltype.malloc(rffi.LONGP.TO, 16+9, flavor='raw') + xmmregisters = lltype.malloc(rffi.LONGP.TO, 16+ACTUAL_CPU.NUM_REGS+1, flavor='raw') registers = rffi.ptradd(xmmregisters, 16) stacklen = baseloc + 10 stack = lltype.malloc(rffi.LONGP.TO, stacklen, flavor='raw') @@ -140,8 +164,8 @@ assert loc >= 0 ofs = get_ebp_ofs(loc) assert ofs < 0 - assert (ofs % 4) == 0 - stack[stacklen + ofs//4] = value + assert (ofs % WORD) == 0 + stack[stacklen + ofs//WORD] = value descr_bytecode = [] for i, (kind, loc) in enumerate(content): @@ -152,12 +176,18 @@ value, lo, hi = get_random_float() expected_floats[i] = value kind = Assembler386.DESCR_FLOAT - if isinstance(loc, REG): - xmmregisters[2*loc.op] = lo - xmmregisters[2*loc.op+1] = hi + if isinstance(loc, RegLoc): + if WORD == 4: + xmmregisters[2*loc.value] = lo + xmmregisters[2*loc.value+1] = hi + elif WORD == 8: + xmmregisters[loc.value] = lo else: - write_in_stack(loc, hi) - write_in_stack(loc+1, lo) + if WORD == 4: + write_in_stack(loc, hi) + write_in_stack(loc+1, lo) + elif WORD == 8: + write_in_stack(loc, lo) else: if kind == 'int': value = get_random_int() @@ -170,15 +200,15 @@ value = rffi.cast(rffi.LONG, value) else: assert 0, kind - if isinstance(loc, REG): - registers[loc.op] = value + if isinstance(loc, RegLoc): + registers[loc.value] = value else: write_in_stack(loc, value) - if isinstance(loc, REG): - num = kind + 4*loc.op + if isinstance(loc, RegLoc): + num = kind + 4*loc.value else: - num = kind + 4*(8+loc) + num = kind + Assembler386.CODE_FROMSTACK + (4*loc) while num >= 0x80: descr_bytecode.append((num & 0x7F) | 0x80) num >>= 7 @@ -195,8 +225,8 @@ for i in range(len(descr_bytecode)): assert 0 <= descr_bytecode[i] <= 255 descr_bytes[i] = rffi.cast(rffi.UCHAR, descr_bytecode[i]) - registers[8] = rffi.cast(rffi.LONG, descr_bytes) - registers[ebp.op] = rffi.cast(rffi.LONG, stack) + 4*stacklen + registers[ACTUAL_CPU.NUM_REGS] = rffi.cast(rffi.LONG, descr_bytes) + registers[ebp.value] = rffi.cast(rffi.LONG, stack) + WORD*stacklen # run! assembler = Assembler386(FakeCPU()) @@ -237,7 +267,8 @@ def test_mc_wrapper_profile_agent(): agent = FakeProfileAgent() - mc = FakeMCWrapper(100, agent) + assembler = FakeAssembler() + mc = FakeMCWrapper(assembler, 100, agent) mc.start_function("abc") mc.writechr("x") mc.writechr("x") Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_basic.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_basic.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_basic.py Tue Aug 10 20:07:15 2010 @@ -1,5 +1,5 @@ import py -from pypy.jit.backend.x86.runner import CPU386 +from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.metainterp.warmspot import ll_meta_interp from pypy.jit.metainterp.test import test_basic from pypy.jit.codewriter.policy import StopAtXPolicy @@ -7,7 +7,7 @@ class Jit386Mixin(test_basic.LLJitMixin): type_system = 'lltype' - CPUClass = CPU386 + CPUClass = getcpuclass() def check_jumps(self, maxcount): pass Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_gc_integration.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_gc_integration.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_gc_integration.py Tue Aug 10 20:07:15 2010 @@ -9,13 +9,13 @@ from pypy.jit.codewriter import heaptracker from pypy.jit.backend.llsupport.descr import GcCache from pypy.jit.backend.llsupport.gc import GcLLDescription -from pypy.jit.backend.x86.runner import CPU -from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, FRAME_FIXED_SIZE +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.regalloc import RegAlloc +from pypy.jit.backend.x86.arch import WORD, FRAME_FIXED_SIZE from pypy.jit.metainterp.test.oparser import parse from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.annlowlevel import llhelper from pypy.rpython.lltypesystem import rclass, rstr -from pypy.jit.backend.x86.ri386 import * from pypy.jit.backend.llsupport.gc import GcLLDescr_framework, GcRefList, GcPtrFieldDescr from pypy.jit.backend.x86.test.test_regalloc import MockAssembler @@ -23,6 +23,8 @@ from pypy.jit.backend.x86.regalloc import X86RegisterManager, X86FrameManager,\ X86XMMRegisterManager +CPU = getcpuclass() + class MockGcRootMap(object): def get_basic_shape(self): return ['shape'] @@ -84,7 +86,7 @@ mark = regalloc.get_mark_gc_roots(cpu.gc_ll_descr.gcrootmap) assert mark[0] == 'compressed' base = -WORD * FRAME_FIXED_SIZE - expected = ['ebx', 'esi', 'edi', base, base-4, base-8] + expected = ['ebx', 'esi', 'edi', base, base-WORD, base-WORD*2] assert dict.fromkeys(mark[1:]) == dict.fromkeys(expected) class TestRegallocGcIntegration(BaseTestRegalloc): @@ -175,7 +177,7 @@ self.addrs[1] = self.addrs[0] + 64 # 64 bytes def malloc_slowpath(size): - assert size == 8 + assert size == WORD*2 nadr = rffi.cast(lltype.Signed, self.nursery) self.addrs[0] = nadr + size return nadr @@ -199,7 +201,7 @@ return rffi.cast(lltype.Signed, self.addrs) def get_nursery_top_addr(self): - return rffi.cast(lltype.Signed, self.addrs) + 4 + return rffi.cast(lltype.Signed, self.addrs) + WORD def get_malloc_fixedsize_slowpath_addr(self): fptr = llhelper(lltype.Ptr(self.MALLOC_SLOWPATH), self.malloc_slowpath) @@ -213,7 +215,7 @@ def setup_method(self, method): cpu = CPU(None, None) - cpu.vtable_offset = 4 + cpu.vtable_offset = WORD cpu.gc_ll_descr = GCDescrFastpathMalloc() NODE = lltype.Struct('node', ('tid', lltype.Signed), @@ -249,7 +251,7 @@ assert gc_ll_descr.nursery[0] == self.nodedescr.tid assert gc_ll_descr.nursery[1] == 42 nurs_adr = rffi.cast(lltype.Signed, gc_ll_descr.nursery) - assert gc_ll_descr.addrs[0] == nurs_adr + 8 + assert gc_ll_descr.addrs[0] == nurs_adr + (WORD*2) def test_malloc_slowpath(self): ops = ''' @@ -269,7 +271,7 @@ # this should call slow path once gc_ll_descr = self.cpu.gc_ll_descr nadr = rffi.cast(lltype.Signed, gc_ll_descr.nursery) - assert gc_ll_descr.addrs[0] == nadr + 8 + assert gc_ll_descr.addrs[0] == nadr + (WORD*2) def test_new_with_vtable(self): ops = ''' @@ -284,4 +286,4 @@ assert gc_ll_descr.nursery[0] == self.descrsize.tid assert gc_ll_descr.nursery[1] == self.vtable_int nurs_adr = rffi.cast(lltype.Signed, gc_ll_descr.nursery) - assert gc_ll_descr.addrs[0] == nurs_adr + 12 + assert gc_ll_descr.addrs[0] == nurs_adr + (WORD*3) Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_jump.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_jump.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_jump.py Tue Aug 10 20:07:15 2010 @@ -1,6 +1,7 @@ -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.regloc import * from pypy.jit.backend.x86.regalloc import X86FrameManager from pypy.jit.backend.x86.jump import remap_frame_layout +from pypy.jit.metainterp.history import INT frame_pos = X86FrameManager.frame_pos @@ -25,7 +26,7 @@ continue assert len(op1) == len(op2) for x, y in zip(op1, op2): - if isinstance(x, MODRM) and isinstance(y, MODRM): + if isinstance(x, StackLoc) and isinstance(y, MODRM): assert x.byte == y.byte assert x.extradata == y.extradata else: @@ -41,9 +42,9 @@ remap_frame_layout(assembler, [eax, ebx, ecx, edx, esi, edi], [eax, ebx, ecx, edx, esi, edi], '?') assert assembler.ops == [] - s8 = frame_pos(1, 1) - s12 = frame_pos(31, 1) - s20 = frame_pos(6, 1) + s8 = frame_pos(1, INT) + s12 = frame_pos(31, INT) + s20 = frame_pos(6, INT) remap_frame_layout(assembler, [eax, ebx, ecx, s20, s8, edx, s12, esi, edi], [eax, ebx, ecx, s20, s8, edx, s12, esi, edi], '?') @@ -58,10 +59,10 @@ def test_simple_framelocs(): assembler = MockAssembler() - s8 = frame_pos(0, 1) - s12 = frame_pos(13, 1) - s20 = frame_pos(20, 1) - s24 = frame_pos(221, 1) + s8 = frame_pos(0, INT) + s12 = frame_pos(13, INT) + s20 = frame_pos(20, INT) + s24 = frame_pos(221, INT) remap_frame_layout(assembler, [s8, eax, s12], [s20, s24, edi], edx) assert assembler.ops == [('mov', s8, edx), ('mov', edx, s20), @@ -70,10 +71,10 @@ def test_reordering(): assembler = MockAssembler() - s8 = frame_pos(8, 1) - s12 = frame_pos(12, 1) - s20 = frame_pos(19, 1) - s24 = frame_pos(1, 1) + s8 = frame_pos(8, INT) + s12 = frame_pos(12, INT) + s20 = frame_pos(19, INT) + s24 = frame_pos(1, INT) remap_frame_layout(assembler, [eax, s8, s20, ebx], [s8, ebx, eax, edi], '?') assert assembler.got([('mov', ebx, edi), @@ -83,10 +84,10 @@ def test_cycle(): assembler = MockAssembler() - s8 = frame_pos(8, 1) - s12 = frame_pos(12, 1) - s20 = frame_pos(19, 1) - s24 = frame_pos(1, 1) + s8 = frame_pos(8, INT) + s12 = frame_pos(12, INT) + s20 = frame_pos(19, INT) + s24 = frame_pos(1, INT) remap_frame_layout(assembler, [eax, s8, s20, ebx], [s8, ebx, eax, s20], '?') assert assembler.got([('push', s8), @@ -97,12 +98,12 @@ def test_cycle_2(): assembler = MockAssembler() - s8 = frame_pos(8, 1) - s12 = frame_pos(12, 1) - s20 = frame_pos(19, 1) - s24 = frame_pos(1, 1) - s2 = frame_pos(2, 1) - s3 = frame_pos(3, 1) + s8 = frame_pos(8, INT) + s12 = frame_pos(12, INT) + s20 = frame_pos(19, INT) + s24 = frame_pos(1, INT) + s2 = frame_pos(2, INT) + s3 = frame_pos(3, INT) remap_frame_layout(assembler, [eax, s8, edi, s20, eax, s20, s24, esi, s2, s3], [s8, s20, edi, eax, edx, s24, ebx, s12, s3, s2], @@ -127,14 +128,14 @@ remap_frame_layout(assembler, [c3], [eax], '?') assert assembler.ops == [('mov', c3, eax)] assembler = MockAssembler() - s12 = frame_pos(12, 1) + s12 = frame_pos(12, INT) remap_frame_layout(assembler, [c3], [s12], '?') assert assembler.ops == [('mov', c3, s12)] def test_constants_and_cycle(): assembler = MockAssembler() c3 = imm(3) - s12 = frame_pos(13, 1) + s12 = frame_pos(13, INT) remap_frame_layout(assembler, [ebx, c3, s12], [s12, eax, ebx], edi) assert assembler.ops == [('mov', c3, eax), Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_recompilation.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_recompilation.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_recompilation.py Tue Aug 10 20:07:15 2010 @@ -1,6 +1,5 @@ - -from pypy.jit.backend.x86.runner import CPU from pypy.jit.backend.x86.test.test_regalloc import BaseTestRegalloc +from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 class TestRecompilation(BaseTestRegalloc): def test_compile_bridge_not_deeper(self): @@ -51,7 +50,9 @@ descr = loop.operations[2].descr new = descr._x86_bridge_frame_depth assert descr._x86_bridge_param_depth == 0 - assert new > previous + # XXX: Maybe add enough ops to force stack on 64-bit as well? + if IS_X86_32: + assert new > previous self.cpu.set_future_value_int(0, 0) fail = self.run(loop) assert fail.identifier == 2 @@ -111,7 +112,9 @@ guard_op = loop.operations[5] loop_frame_depth = loop.token._x86_frame_depth assert loop.token._x86_param_depth == 0 - assert guard_op.descr._x86_bridge_frame_depth > loop_frame_depth + # XXX: Maybe add enough ops to force stack on 64-bit as well? + if IS_X86_32: + assert guard_op.descr._x86_bridge_frame_depth > loop_frame_depth assert guard_op.descr._x86_bridge_param_depth == 0 self.cpu.set_future_value_int(0, 0) self.cpu.set_future_value_int(1, 0) Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_regalloc.py Tue Aug 10 20:07:15 2010 @@ -7,15 +7,17 @@ BoxPtr, ConstPtr, LoopToken, BasicFailDescr from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.backend.llsupport.descr import GcCache -from pypy.jit.backend.x86.runner import CPU -from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, X86RegisterManager,\ +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.regalloc import RegAlloc, X86RegisterManager,\ FloatConstants +from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 from pypy.jit.metainterp.test.oparser import parse from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.annlowlevel import llhelper from pypy.rpython.lltypesystem import rclass, rstr -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.rx86 import * +CPU = getcpuclass() class MockGcDescr(GcCache): def get_funcptr_for_new(self): return 123 @@ -92,13 +94,20 @@ def f2(x, y): return x*y + def f10(*args): + assert len(args) == 10 + return sum(args) + F1PTR = lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Signed)) F2PTR = lltype.Ptr(lltype.FuncType([lltype.Signed]*2, lltype.Signed)) + F10PTR = lltype.Ptr(lltype.FuncType([lltype.Signed]*10, lltype.Signed)) f1ptr = llhelper(F1PTR, f1) f2ptr = llhelper(F2PTR, f2) + f10ptr = llhelper(F10PTR, f10) f1_calldescr = cpu.calldescrof(F1PTR.TO, F1PTR.TO.ARGS, F1PTR.TO.RESULT) f2_calldescr = cpu.calldescrof(F2PTR.TO, F2PTR.TO.ARGS, F2PTR.TO.RESULT) + f10_calldescr = cpu.calldescrof(F10PTR.TO, F10PTR.TO.ARGS, F10PTR.TO.RESULT) namespace = locals().copy() type_system = 'lltype' @@ -541,6 +550,12 @@ assert self.getints(9) == [0, 1, 1, 1, 1, 1, 1, 1, 1] class TestRegAllocCallAndStackDepth(BaseTestRegalloc): + def expected_param_depth(self, num_args): + # Assumes the arguments are all non-float + if IS_X86_32: + return num_args + elif IS_X86_64: + return max(num_args - 6, 0) def test_one_call(self): ops = ''' @@ -550,7 +565,7 @@ ''' loop = self.interpret(ops, [4, 7, 9, 9 ,9, 9, 9, 9, 9, 9, 9]) assert self.getints(11) == [5, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9] - assert loop.token._x86_param_depth == 1 + assert loop.token._x86_param_depth == self.expected_param_depth(1) def test_two_calls(self): ops = ''' @@ -561,8 +576,21 @@ ''' loop = self.interpret(ops, [4, 7, 9, 9 ,9, 9, 9, 9, 9, 9, 9]) assert self.getints(11) == [5*7, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9] - assert loop.token._x86_param_depth == 2 - + assert loop.token._x86_param_depth == self.expected_param_depth(2) + + def test_call_many_arguments(self): + # NB: The first and last arguments in the call are constants. This + # is primarily for x86-64, to ensure that loading a constant to an + # argument register or to the stack works correctly + ops = ''' + [i0, i1, i2, i3, i4, i5, i6, i7] + i8 = call(ConstClass(f10ptr), 1, i0, i1, i2, i3, i4, i5, i6, i7, 10, descr=f10_calldescr) + finish(i8) + ''' + loop = self.interpret(ops, [2, 3, 4, 5, 6, 7, 8, 9]) + assert self.getint(0) == 55 + assert loop.token._x86_param_depth == self.expected_param_depth(10) + def test_bridge_calls_1(self): ops = ''' [i0, i1] @@ -579,7 +607,7 @@ ''' bridge = self.attach_bridge(ops, loop, -2) - assert loop.operations[-2].descr._x86_bridge_param_depth == 2 + assert loop.operations[-2].descr._x86_bridge_param_depth == self.expected_param_depth(2) self.cpu.set_future_value_int(0, 4) self.cpu.set_future_value_int(1, 7) @@ -602,7 +630,7 @@ ''' bridge = self.attach_bridge(ops, loop, -2) - assert loop.operations[-2].descr._x86_bridge_param_depth == 2 + assert loop.operations[-2].descr._x86_bridge_param_depth == self.expected_param_depth(2) self.cpu.set_future_value_int(0, 4) self.cpu.set_future_value_int(1, 7) Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_regalloc2.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_regalloc2.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_regalloc2.py Tue Aug 10 20:07:15 2010 @@ -2,7 +2,9 @@ from pypy.jit.metainterp.history import ResOperation, BoxInt, ConstInt,\ BoxPtr, ConstPtr, BasicFailDescr, LoopToken from pypy.jit.metainterp.resoperation import rop -from pypy.jit.backend.x86.runner import CPU +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.arch import WORD +CPU = getcpuclass() def test_bug_rshift(): v1 = BoxInt() @@ -281,5 +283,8 @@ assert cpu.get_latest_value_int(16) == -57344 assert cpu.get_latest_value_int(17) == 1 assert cpu.get_latest_value_int(18) == -1 - assert cpu.get_latest_value_int(19) == -2147483648 + if WORD == 4: + assert cpu.get_latest_value_int(19) == -2147483648 + elif WORD == 8: + assert cpu.get_latest_value_int(19) == 19327352832 assert cpu.get_latest_value_int(20) == -49 Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_runner.py Tue Aug 10 20:07:15 2010 @@ -1,10 +1,11 @@ import py from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rstr, rclass +from pypy.rpython.annlowlevel import llhelper from pypy.jit.metainterp.history import ResOperation, LoopToken -from pypy.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstPtr, - Box, BasicFailDescr) -from pypy.jit.backend.x86.runner import CPU -from pypy.jit.backend.x86.regalloc import WORD +from pypy.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstFloat, + ConstPtr, Box, BoxFloat, BasicFailDescr) +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.arch import WORD from pypy.jit.backend.llsupport import symbolic from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.executor import execute @@ -15,6 +16,8 @@ import sys import os +CPU = getcpuclass() + class FakeStats(object): pass @@ -59,7 +62,7 @@ assert u.chars[3] == u'd' @staticmethod - def _resbuf(res, item_tp=ctypes.c_int): + def _resbuf(res, item_tp=ctypes.c_long): return ctypes.cast(res.value._obj.intval, ctypes.POINTER(item_tp)) def test_allocations(self): @@ -74,8 +77,11 @@ return ctypes.cast(buf, ctypes.c_void_p).value func = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int)(f) addr = ctypes.cast(func, ctypes.c_void_p).value + # ctypes produces an unsigned value. We need it to be signed for, eg, + # relative addressing to work properly. + addr = rffi.cast(lltype.Signed, addr) - self.cpu.assembler.make_sure_mc_exists() + self.cpu.assembler.setup() self.cpu.assembler.malloc_func_addr = addr ofs = symbolic.get_field_token(rstr.STR, 'chars', False)[0] @@ -360,7 +366,9 @@ self.cpu.compile_bridge(faildescr1, [i1b], bridge) name, address, size = agent.functions[1] assert name == "Bridge # 0: bye" - assert address == loopaddress + loopsize + # Would be exactly ==, but there are some guard failure recovery + # stubs in-between + assert address >= loopaddress + loopsize assert size >= 10 # randomish number self.cpu.set_future_value_int(0, 2) @@ -369,6 +377,19 @@ res = self.cpu.get_latest_value_int(0) assert res == 20 + def test_call_with_const_floats(self): + def func(f1, f2): + return f1 + f2 + + FUNC = self.FuncType([lltype.Float, lltype.Float], lltype.Float) + FPTR = self.Ptr(FUNC) + calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + func_ptr = llhelper(FPTR, func) + funcbox = self.get_funcbox(self.cpu, func_ptr) + res = self.execute_operation(rop.CALL, [funcbox, ConstFloat(1.5), ConstFloat(2.5)], 'float', descr=calldescr) + assert res.value == 4.0 + + class TestX86OverflowMC(TestX86): def setup_method(self, meth): @@ -386,7 +407,7 @@ ops.append(ResOperation(rop.FINISH, [v], None, descr=BasicFailDescr())) looptoken = LoopToken() - self.cpu.assembler.make_sure_mc_exists() + self.cpu.assembler.setup() old_mc_mc = self.cpu.assembler.mc._mc self.cpu.compile_loop([base_v], ops, looptoken) assert self.cpu.assembler.mc._mc != old_mc_mc # overflowed @@ -394,6 +415,63 @@ self.cpu.execute_token(looptoken) assert self.cpu.get_latest_value_int(0) == 1024 + def test_overflow_guard_float_cmp(self): + # The float comparisons on x86 tend to use small relative jumps, + # which may run into trouble if they fall on the edge of a + # MachineCodeBlock change. + a = BoxFloat(1.0) + b = BoxFloat(2.0) + failed = BoxInt(41) + finished = BoxInt(42) + + # We select guards that will always succeed, so that execution will + # continue through the entire set of comparisions + ops_to_test = ( + (rop.FLOAT_LT, [a, b], rop.GUARD_TRUE), + (rop.FLOAT_LT, [b, a], rop.GUARD_FALSE), + + (rop.FLOAT_LE, [a, a], rop.GUARD_TRUE), + (rop.FLOAT_LE, [a, b], rop.GUARD_TRUE), + (rop.FLOAT_LE, [b, a], rop.GUARD_FALSE), + + (rop.FLOAT_EQ, [a, a], rop.GUARD_TRUE), + (rop.FLOAT_EQ, [a, b], rop.GUARD_FALSE), + + (rop.FLOAT_NE, [a, b], rop.GUARD_TRUE), + (rop.FLOAT_NE, [a, a], rop.GUARD_FALSE), + + (rop.FLOAT_GT, [b, a], rop.GUARD_TRUE), + (rop.FLOAT_GT, [a, b], rop.GUARD_FALSE), + + (rop.FLOAT_GE, [a, a], rop.GUARD_TRUE), + (rop.FLOAT_GE, [b, a], rop.GUARD_TRUE), + (rop.FLOAT_GE, [a, b], rop.GUARD_FALSE), + ) + + for float_op, args, guard_op in ops_to_test: + ops = [] + + for i in range(200): + cmp_result = BoxInt() + ops.append(ResOperation(float_op, args, cmp_result)) + ops.append(ResOperation(guard_op, [cmp_result], None, descr=BasicFailDescr())) + ops[-1].fail_args = [failed] + + ops.append(ResOperation(rop.FINISH, [finished], None, descr=BasicFailDescr())) + + looptoken = LoopToken() + self.cpu.compile_loop([a, b, failed, finished], ops, looptoken) + self.cpu.set_future_value_float(0, a.value) + self.cpu.set_future_value_float(1, b.value) + self.cpu.set_future_value_int(2, failed.value) + self.cpu.set_future_value_int(3, finished.value) + self.cpu.execute_token(looptoken) + + # Really just a sanity check. We're actually interested in + # whether the test segfaults. + assert self.cpu.get_latest_value_int(0) == finished.value + + class TestDebuggingAssembler(object): def setup_method(self, meth): self.pypylog = os.environ.get('PYPYLOG', None) @@ -420,8 +498,9 @@ self.cpu.set_future_value_int(0, 0) self.cpu.execute_token(ops.token) # check debugging info - assert self.cpu.assembler.loop_names == ["xyz"] - assert self.cpu.assembler.loop_run_counter.getitem(0) == 10 + name, struct = self.cpu.assembler.loop_run_counters[0] + assert name == 'xyz' + assert struct.i == 10 self.cpu.finish_once() lines = py.path.local(self.logfile + ".count").readlines() assert lines[0] == 'xyz:10\n' Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_symbolic_x86.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_symbolic_x86.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_symbolic_x86.py Tue Aug 10 20:07:15 2010 @@ -1,6 +1,7 @@ import py from pypy.jit.backend.llsupport.symbolic import * from pypy.rpython.lltypesystem import lltype, rffi +from pypy.jit.backend.x86.arch import WORD # This test file is here and not in llsupport/test/ because it checks # that we get correct numbers for a 32-bit machine. @@ -19,32 +20,32 @@ ofs_z, size_z = get_field_token(S, 'z', False) # ofs_x might be 0 or not, depending on how we count the headers # but the rest should be as expected for a 386 machine - assert size_x == size_y == size_z == 4 + assert size_x == size_y == size_z == WORD assert ofs_x >= 0 - assert ofs_y == ofs_x + 4 - assert ofs_z == ofs_x + 8 + assert ofs_y == ofs_x + WORD + assert ofs_z == ofs_x + (WORD*2) def test_struct_size(): ofs_z, size_z = get_field_token(S, 'z', False) totalsize = get_size(S, False) - assert totalsize == ofs_z + 4 + assert totalsize == ofs_z + WORD def test_primitive_size(): - assert get_size(lltype.Signed, False) == 4 + assert get_size(lltype.Signed, False) == WORD assert get_size(lltype.Char, False) == 1 - assert get_size(lltype.Ptr(S), False) == 4 + assert get_size(lltype.Ptr(S), False) == WORD def test_array_token(): A = lltype.GcArray(lltype.Char) basesize, itemsize, ofs_length = get_array_token(A, False) - assert basesize >= 4 # at least the 'length', maybe some gc headers + assert basesize >= WORD # at least the 'length', maybe some gc headers assert itemsize == 1 - assert ofs_length == basesize - 4 + assert ofs_length == basesize - WORD A = lltype.GcArray(lltype.Signed) basesize, itemsize, ofs_length = get_array_token(A, False) - assert basesize >= 4 # at least the 'length', maybe some gc headers - assert itemsize == 4 - assert ofs_length == basesize - 4 + assert basesize >= WORD # at least the 'length', maybe some gc headers + assert itemsize == WORD + assert ofs_length == basesize - WORD def test_varsized_struct_size(): S1 = lltype.GcStruct('S1', ('parent', S), @@ -54,9 +55,9 @@ ofs_extra, size_extra = get_field_token(S1, 'extra', False) basesize, itemsize, ofs_length = get_array_token(S1, False) assert size_parent == ofs_extra - assert size_extra == 4 - assert ofs_length == ofs_extra + 4 - assert basesize == ofs_length + 4 + assert size_extra == WORD + assert ofs_length == ofs_extra + WORD + assert basesize == ofs_length + WORD assert itemsize == 1 def test_string(): Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_zll_random.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_zll_random.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_zll_random.py Tue Aug 10 20:07:15 2010 @@ -1,9 +1,11 @@ from pypy.jit.backend.test.test_random import check_random_function, Random from pypy.jit.backend.test.test_ll_random import LLtypeOperationBuilder -from pypy.jit.backend.x86.runner import CPU386 +from pypy.jit.backend.detect_cpu import getcpuclass + +CPU = getcpuclass() def test_stress(): - cpu = CPU386(None, None) + cpu = CPU(None, None) r = Random() for i in range(1000): check_random_function(cpu, LLtypeOperationBuilder, r, i, 1000) Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_zrpy_gc.py Tue Aug 10 20:07:15 2010 @@ -17,6 +17,8 @@ from pypy.jit.backend.llsupport.gc import GcRefList, GcRootMap_asmgcc from pypy.jit.backend.llsupport.gc import GcLLDescr_framework from pypy.tool.udir import udir +from pypy.jit.backend.x86.arch import IS_X86_64 +import py.test class X(object): def __init__(self, x=0): @@ -126,6 +128,10 @@ class TestCompileHybrid(object): def setup_class(cls): + if IS_X86_64: + # No hybrid GC on 64-bit for the time being + py.test.skip() + funcs = [] name_to_func = {} for fullname in dir(cls): Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_ztranslation.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_ztranslation.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_ztranslation.py Tue Aug 10 20:07:15 2010 @@ -3,13 +3,14 @@ from pypy.rlib.jit import JitDriver, OPTIMIZER_FULL, unroll_parameters from pypy.rlib.jit import PARAMETERS, dont_look_inside from pypy.jit.metainterp.jitprof import Profiler -from pypy.jit.backend.x86.runner import CPU386 +from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.backend.test.support import CCompiledMixin from pypy.jit.codewriter.policy import StopAtXPolicy from pypy.translator.translator import TranslationContext +from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 class TestTranslationX86(CCompiledMixin): - CPUClass = CPU386 + CPUClass = getcpuclass() def _check_cbuilder(self, cbuilder): # We assume here that we have sse2. If not, the CPUClass @@ -114,7 +115,7 @@ class TestTranslationRemoveTypePtrX86(CCompiledMixin): - CPUClass = CPU386 + CPUClass = getcpuclass() def _get_TranslationContext(self): t = TranslationContext() @@ -125,6 +126,10 @@ return t def test_external_exception_handling_translates(self): + # FIXME + if IS_X86_64: + import py.test; py.test.skip() + jitdriver = JitDriver(greens = [], reds = ['n', 'total']) class ImDone(Exception): Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/tool/viewcode.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/tool/viewcode.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/tool/viewcode.py Tue Aug 10 20:07:15 2010 @@ -31,16 +31,23 @@ if sys.platform == "win32": XXX # lots more in Psyco -def machine_code_dump(data, originaddr): - # the disassembler to use. 'objdump' writes GNU-style instructions. - # 'ndisasm' would use Intel syntax, but you need to fix the output parsing. - objdump = ('objdump -M intel -b binary -m i386 ' +def machine_code_dump(data, originaddr, backend_name): + objdump_backend_option = { + 'x86': 'i386', + 'x86_64': 'x86-64', + 'i386': 'i386', + } + objdump = ('objdump -M intel,%(backend)s -b binary -m i386 ' '--adjust-vma=%(origin)d -D %(file)s') # f = open(tmpfile, 'wb') f.write(data) f.close() - g = os.popen(objdump % {'file': tmpfile, 'origin': originaddr}, 'r') + g = os.popen(objdump % { + 'file': tmpfile, + 'origin': originaddr, + 'backend': objdump_backend_option[backend_name], + }, 'r') result = g.readlines() g.close() return result[6:] # drop some objdump cruft @@ -126,7 +133,7 @@ def disassemble(self): if not hasattr(self, 'text'): - lines = machine_code_dump(self.data, self.addr) + lines = machine_code_dump(self.data, self.addr, self.world.backend_name) # instead of adding symbol names in the dumps we could # also make the 0xNNNNNNNN addresses be red and show the # symbol name when the mouse is over them @@ -171,10 +178,13 @@ self.jumps = {} self.symbols = {} self.logentries = {} + self.backend_name = None def parse(self, f, textonly=True): for line in f: - if line.startswith('CODE_DUMP '): + if line.startswith('BACKEND '): + self.backend_name = line.split(' ')[1].strip() + elif line.startswith('CODE_DUMP '): pieces = line.split() assert pieces[1].startswith('@') assert pieces[2].startswith('+') Modified: pypy/branch/fast-ctypes/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/metainterp/optimizeopt.py Tue Aug 10 20:07:15 2010 @@ -1,7 +1,7 @@ from pypy.jit.metainterp.history import Box, BoxInt, LoopToken, BoxFloat,\ ConstFloat from pypy.jit.metainterp.history import Const, ConstInt, ConstPtr, ConstObj, REF -from pypy.jit.metainterp.resoperation import rop, ResOperation +from pypy.jit.metainterp.resoperation import rop, ResOperation, opboolinvers, opboolreflex from pypy.jit.metainterp import jitprof from pypy.jit.metainterp.executor import execute_nonspec from pypy.jit.metainterp.specnode import SpecNode, NotSpecNode, ConstantSpecNode @@ -612,12 +612,59 @@ assert oldop.opnum == op.opnum self.make_equal_to(op.result, self.getvalue(oldop.result)) return + elif self.find_rewriteable_bool(op, args): + return else: self.pure_operations[args] = op # otherwise, the operation remains self.emit_operation(op) + + def try_boolinvers(self, op, targs): + oldop = self.pure_operations.get(targs, None) + if oldop is not None and oldop.descr is op.descr: + value = self.getvalue(oldop.result) + if value.is_constant(): + if value.box is CONST_1: + self.make_constant(op.result, CONST_0) + return True + elif value.box is CONST_0: + self.make_constant(op.result, CONST_1) + return True + return False + + + def find_rewriteable_bool(self, op, args): + try: + oldopnum = opboolinvers[op.opnum] + targs = [args[0], args[1], ConstInt(oldopnum)] + if self.try_boolinvers(op, targs): + return True + except KeyError: + pass + + try: + oldopnum = opboolreflex[op.opnum] + targs = [args[1], args[0], ConstInt(oldopnum)] + oldop = self.pure_operations.get(targs, None) + if oldop is not None and oldop.descr is op.descr: + self.make_equal_to(op.result, self.getvalue(oldop.result)) + return True + except KeyError: + pass + + try: + oldopnum = opboolinvers[opboolreflex[op.opnum]] + targs = [args[1], args[0], ConstInt(oldopnum)] + if self.try_boolinvers(op, targs): + return True + except KeyError: + pass + + return False + + def optimize_JUMP(self, op): orgop = self.loop.operations[-1] exitargs = [] Modified: pypy/branch/fast-ctypes/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/metainterp/resoperation.py Tue Aug 10 20:07:15 2010 @@ -274,3 +274,51 @@ setup(__name__ == '__main__') # print out the table when run directly del _oplist + +opboolinvers = { + rop.INT_EQ: rop.INT_NE, + rop.INT_NE: rop.INT_EQ, + rop.INT_LT: rop.INT_GE, + rop.INT_GE: rop.INT_LT, + rop.INT_GT: rop.INT_LE, + rop.INT_LE: rop.INT_GT, + + rop.UINT_LT: rop.UINT_GE, + rop.UINT_GE: rop.UINT_LT, + rop.UINT_GT: rop.UINT_LE, + rop.UINT_LE: rop.UINT_GT, + + rop.FLOAT_EQ: rop.FLOAT_NE, + rop.FLOAT_NE: rop.FLOAT_EQ, + rop.FLOAT_LT: rop.FLOAT_GE, + rop.FLOAT_GE: rop.FLOAT_LT, + rop.FLOAT_GT: rop.FLOAT_LE, + rop.FLOAT_LE: rop.FLOAT_GT, + + rop.PTR_EQ: rop.PTR_NE, + rop.PTR_NE: rop.PTR_EQ, + } + +opboolreflex = { + rop.INT_EQ: rop.INT_EQ, + rop.INT_NE: rop.INT_NE, + rop.INT_LT: rop.INT_GT, + rop.INT_GE: rop.INT_LE, + rop.INT_GT: rop.INT_LT, + rop.INT_LE: rop.INT_GE, + + rop.UINT_LT: rop.UINT_GT, + rop.UINT_GE: rop.UINT_LE, + rop.UINT_GT: rop.UINT_LT, + rop.UINT_LE: rop.UINT_GE, + + rop.FLOAT_EQ: rop.FLOAT_EQ, + rop.FLOAT_NE: rop.FLOAT_NE, + rop.FLOAT_LT: rop.FLOAT_GT, + rop.FLOAT_GE: rop.FLOAT_LE, + rop.FLOAT_GT: rop.FLOAT_LT, + rop.FLOAT_LE: rop.FLOAT_GE, + + rop.PTR_EQ: rop.PTR_EQ, + rop.PTR_NE: rop.PTR_NE, + } Modified: pypy/branch/fast-ctypes/pypy/jit/metainterp/test/test_executor.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/metainterp/test/test_executor.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/metainterp/test/test_executor.py Tue Aug 10 20:07:15 2010 @@ -4,14 +4,14 @@ from pypy.rpython.lltypesystem import lltype, llmemory from pypy.jit.metainterp.executor import execute from pypy.jit.metainterp.executor import execute_varargs, execute_nonspec -from pypy.jit.metainterp.resoperation import rop +from pypy.jit.metainterp.resoperation import rop, opboolinvers, opboolreflex, opname from pypy.jit.metainterp.history import BoxInt, ConstInt from pypy.jit.metainterp.history import BoxPtr, ConstPtr from pypy.jit.metainterp.history import BoxFloat, ConstFloat from pypy.jit.metainterp.history import AbstractDescr, Box from pypy.jit.metainterp import history from pypy.jit.backend.model import AbstractCPU - +from pypy.rpython.lltypesystem import llmemory, rffi class FakeDescr(AbstractDescr): pass @@ -312,3 +312,40 @@ assert box.getint() == retvalue else: assert 0, "rettype is %r" % (rettype,) + +def make_args_for_op(op, a, b): + n=opname[op] + if n[0:3] == 'INT' or n[0:4] == 'UINT': + arg1 = ConstInt(a) + arg2 = ConstInt(b) + elif n[0:5] == 'FLOAT': + arg1 = ConstFloat(float(a)) + arg2 = ConstFloat(float(b)) + elif n[0:3] == 'PTR': + arg1 = ConstPtr(rffi.cast(llmemory.GCREF, a)) + arg2 = ConstPtr(rffi.cast(llmemory.GCREF, b)) + else: + raise NotImplementedError( + "Don't know how to make args for " + n) + return arg1, arg2 + + +def test_opboolinvers(): + cpu = FakeCPU() + for op1, op2 in opboolinvers.items(): + for a in (1,2,3): + for b in (1,2,3): + arg1, arg2 = make_args_for_op(op1, a, b) + box1 = execute(cpu, None, op1, None, arg1, arg2) + box2 = execute(cpu, None, op2, None, arg1, arg2) + assert box1.value == (not box2.value) + +def test_opboolreflex(): + cpu = FakeCPU() + for op1, op2 in opboolreflex.items(): + for a in (1,2,3): + for b in (1,2,3): + arg1, arg2 = make_args_for_op(op1, a, b) + box1 = execute(cpu, None, op1, None, arg1, arg2) + box2 = execute(cpu, None, op2, None, arg2, arg1) + assert box1.value == box2.value Modified: pypy/branch/fast-ctypes/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/metainterp/test/test_optimizeopt.py Tue Aug 10 20:07:15 2010 @@ -364,6 +364,74 @@ """ self.optimize_loop(ops, 'Not', expected) + def test_constant_boolrewrite_lt(self): + ops = """ + [i0] + i1 = int_lt(i0, 0) + guard_true(i1) [] + i2 = int_ge(i0, 0) + guard_false(i2) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_lt(i0, 0) + guard_true(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_constant_boolrewrite_gt(self): + ops = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + i2 = int_le(i0, 0) + guard_false(i2) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_constant_boolrewrite_reflex(self): + ops = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + i2 = int_lt(0, i0) + guard_true(i2) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_constant_boolrewrite_reflex_invers(self): + ops = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + i2 = int_ge(0, i0) + guard_false(i2) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + def test_remove_consecutive_guard_value_constfold(self): ops = """ [] @@ -411,7 +479,6 @@ self.optimize_loop(ops, 'Not', expected) def test_int_is_true_1(self): - py.test.skip("XXX implement me") ops = """ [i0] i1 = int_is_true(i0) @@ -806,16 +873,10 @@ guard_nonnull(p0) [] i7 = ptr_ne(p0, p1) guard_true(i7) [] - i8 = ptr_eq(p0, p1) - guard_false(i8) [] i9 = ptr_ne(p0, p2) guard_true(i9) [] - i10 = ptr_eq(p0, p2) - guard_false(i10) [] i11 = ptr_ne(p2, p1) guard_true(i11) [] - i12 = ptr_eq(p2, p1) - guard_false(i12) [] jump(p0, p1, p2) """ self.optimize_loop(ops, 'Not, Not, Not', expected2) Modified: pypy/branch/fast-ctypes/pypy/jit/tl/pypyjit.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/tl/pypyjit.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/tl/pypyjit.py Tue Aug 10 20:07:15 2010 @@ -37,6 +37,7 @@ set_opt_level(config, level='jit') config.objspace.allworkingmodules = False config.objspace.usemodules.pypyjit = True +config.objspace.usemodules.array = True config.objspace.usemodules._weakref = False config.objspace.usemodules._sre = False set_pypy_opt_level(config, level='jit') Modified: pypy/branch/fast-ctypes/pypy/jit/tl/pypyjit_demo.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/tl/pypyjit_demo.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/tl/pypyjit_demo.py Tue Aug 10 20:07:15 2010 @@ -1,38 +1,64 @@ -base = object +## base = object -class Number(base): - __slots__ = ('val', ) - def __init__(self, val=0): - self.val = val - - def __add__(self, other): - if not isinstance(other, int): - other = other.val - return Number(val=self.val + other) +## class Number(base): +## __slots__ = ('val', ) +## def __init__(self, val=0): +## self.val = val + +## def __add__(self, other): +## if not isinstance(other, int): +## other = other.val +## return Number(val=self.val + other) - def __cmp__(self, other): - val = self.val - if not isinstance(other, int): - other = other.val - return cmp(val, other) - - def __nonzero__(self): - return bool(self.val) - -def g(x, inc=2): - return x + inc - -def f(n, x, inc): - while x < n: - x = g(x, inc=1) - return x - -import time -#t1 = time.time() -#f(10000000, Number(), 1) -#t2 = time.time() -#print t2 - t1 -t1 = time.time() -f(10000000, 0, 1) -t2 = time.time() -print t2 - t1 +## def __cmp__(self, other): +## val = self.val +## if not isinstance(other, int): +## other = other.val +## return cmp(val, other) + +## def __nonzero__(self): +## return bool(self.val) + +## def g(x, inc=2): +## return x + inc + +## def f(n, x, inc): +## while x < n: +## x = g(x, inc=1) +## return x + +## import time +## #t1 = time.time() +## #f(10000000, Number(), 1) +## #t2 = time.time() +## #print t2 - t1 +## t1 = time.time() +## f(10000000, 0, 1) +## t2 = time.time() +## print t2 - t1 + +try: + from array import array + def f(img): + i=0 + sa=0 + while i < img.__len__(): + sa+=img[i] + i+=1 + return sa + + img=array('h',(1,2,3,4)) + print f(img) +except Exception, e: + print "Exception: ", type(e) + print e + +## def f(): +## a=7 +## i=0 +## while i<4: +## if i<0: break +## if i<0: break +## i+=1 + +## f() Modified: pypy/branch/fast-ctypes/pypy/module/_demo/demo.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/_demo/demo.py (original) +++ pypy/branch/fast-ctypes/pypy/module/_demo/demo.py Tue Aug 10 20:07:15 2010 @@ -4,11 +4,14 @@ from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.tool import rffi_platform +from pypy.translator.tool.cbuild import ExternalCompilationInfo import sys, math time_t = rffi_platform.getsimpletype('time_t', '#include ', rffi.LONG) -time = rffi.llexternal('time', [rffi.VOIDP], time_t, includes=['time.h']) +eci = ExternalCompilationInfo(includes=['time.h']) +time = rffi.llexternal('time', [int], time_t, + compilation_info=eci) def get(space, name): w_module = space.getbuiltinmodule('_demo') @@ -20,10 +23,10 @@ w_DemoError = get(space, 'DemoError') msg = "repetition count must be > 0" raise OperationError(w_DemoError, space.wrap(msg)) - starttime = time(None) + starttime = time(0) for i in range(repetitions): space.call_function(w_callable) - endtime = time(None) + endtime = time(0) return space.wrap(endtime - starttime) measuretime.unwrap_spec = [ObjSpace, int, W_Root] @@ -62,11 +65,16 @@ self.x = space.int_w(w_value) def mytype_new(space, w_subtype, x): + if x == 3: + return space.wrap(MySubType(space, x)) return space.wrap(W_MyType(space, x)) mytype_new.unwrap_spec = [ObjSpace, W_Root, int] getset_x = GetSetProperty(W_MyType.fget_x, W_MyType.fset_x, cls=W_MyType) +class MySubType(W_MyType): + pass + W_MyType.typedef = TypeDef('MyType', __new__ = interp2app(mytype_new), x = getset_x, Modified: pypy/branch/fast-ctypes/pypy/module/_stackless/interp_coroutine.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/_stackless/interp_coroutine.py (original) +++ pypy/branch/fast-ctypes/pypy/module/_stackless/interp_coroutine.py Tue Aug 10 20:07:15 2010 @@ -265,10 +265,14 @@ instr += 1 oparg = ord(code[instr]) | ord(code[instr + 1]) << 8 nargs = oparg & 0xff + nkwds = (oparg >> 8) & 0xff if space.config.objspace.opcodes.CALL_METHOD and opcode == map['CALL_METHOD']: - chain = resume_state_create(chain, 'CALL_METHOD', frame, - nargs) - elif opcode == map['CALL_FUNCTION'] and (oparg >> 8) & 0xff == 0: + if nkwds == 0: # only positional arguments + chain = resume_state_create(chain, 'CALL_METHOD', frame, + nargs) + else: # includes keyword arguments + chain = resume_state_create(chain, 'CALL_METHOD_KW', frame) + elif opcode == map['CALL_FUNCTION'] and nkwds == 0: # Only positional arguments # case1: ("CALL_FUNCTION", f, nargs, returns=w_result) chain = resume_state_create(chain, 'CALL_FUNCTION', frame, Modified: pypy/branch/fast-ctypes/pypy/module/pypyjit/policy.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/pypyjit/policy.py (original) +++ pypy/branch/fast-ctypes/pypy/module/pypyjit/policy.py Tue Aug 10 20:07:15 2010 @@ -11,7 +11,7 @@ if '.' in modname: modname, _ = modname.split('.', 1) if modname in ['pypyjit', 'signal', 'micronumpy', 'math', 'exceptions', - 'imp', 'sys']: + 'imp', 'sys', 'array']: return True return False Modified: pypy/branch/fast-ctypes/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/fast-ctypes/pypy/module/pypyjit/test/test_pypy_c.py Tue Aug 10 20:07:15 2010 @@ -615,6 +615,237 @@ return total ''', 170, ([], 4999450000L)) + def test_boolrewrite_invers(self): + for a, b, res, ops in (('2000', '2000', 20001000, 51), + ( '500', '500', 15001500, 81), + ( '300', '600', 16001700, 83), + ( 'a', 'b', 16001700, 89), + ( 'a', 'a', 13001700, 85)): + + self.run_source(''' + def main(): + sa = 0 + a = 300 + b = 600 + for i in range(1000): + if i < %s: sa += 1 + else: sa += 2 + if i >= %s: sa += 10000 + else: sa += 20000 + return sa + '''%(a, b), ops, ([], res)) + + def test_boolrewrite_reflex(self): + for a, b, res, ops in (('2000', '2000', 10001000, 51), + ( '500', '500', 15001500, 81), + ( '300', '600', 14001700, 83), + ( 'a', 'b', 14001700, 89), + ( 'a', 'a', 17001700, 85)): + + self.run_source(''' + def main(): + sa = 0 + a = 300 + b = 600 + for i in range(1000): + if i < %s: sa += 1 + else: sa += 2 + if %s > i: sa += 10000 + else: sa += 20000 + return sa + '''%(a, b), ops, ([], res)) + + + def test_boolrewrite_correct_invers(self): + def opval(i, op, a): + if eval('%d %s %d' % (i, op, a)): return 1 + return 2 + + ops = ('<', '>', '<=', '>=', '==', '!=') + for op1 in ops: + for op2 in ops: + for a,b in ((500, 500), (300, 600)): + res = 0 + res += opval(a-1, op1, a) * (a) + res += opval( a, op1, a) + res += opval(a+1, op1, a) * (1000 - a - 1) + res += opval(b-1, op2, b) * 10000 * (b) + res += opval( b, op2, b) * 10000 + res += opval(b+1, op2, b) * 10000 * (1000 - b - 1) + + self.run_source(''' + def main(): + sa = 0 + for i in range(1000): + if i %s %d: sa += 1 + else: sa += 2 + if i %s %d: sa += 10000 + else: sa += 20000 + return sa + '''%(op1, a, op2, b), 83, ([], res)) + + self.run_source(''' + def main(): + sa = 0 + i = 0.0 + while i < 250.0: + if i %s %f: sa += 1 + else: sa += 2 + if i %s %f: sa += 10000 + else: sa += 20000 + i += 0.25 + return sa + '''%(op1, float(a)/4.0, op2, float(b)/4.0), 109, ([], res)) + + + def test_boolrewrite_correct_reflex(self): + def opval(i, op, a): + if eval('%d %s %d' % (i, op, a)): return 1 + return 2 + + ops = ('<', '>', '<=', '>=', '==', '!=') + for op1 in ops: + for op2 in ops: + for a,b in ((500, 500), (300, 600)): + res = 0 + res += opval(a-1, op1, a) * (a) + res += opval( a, op1, a) + res += opval(a+1, op1, a) * (1000 - a - 1) + res += opval(b, op2, b-1) * 10000 * (b) + res += opval(b, op2, b) * 10000 + res += opval(b, op2, b+1) * 10000 * (1000 - b - 1) + + self.run_source(''' + def main(): + sa = 0 + for i in range(1000): + if i %s %d: sa += 1 + else: sa += 2 + if %d %s i: sa += 10000 + else: sa += 20000 + return sa + '''%(op1, a, b, op2), 83, ([], res)) + + self.run_source(''' + def main(): + sa = 0 + i = 0.0 + while i < 250.0: + if i %s %f: sa += 1 + else: sa += 2 + if %f %s i: sa += 10000 + else: sa += 20000 + i += 0.25 + return sa + '''%(op1, float(a)/4.0, float(b)/4.0, op2), 109, ([], res)) + + def test_boolrewrite_ptr(self): + compares = ('a == b', 'b == a', 'a != b', 'b != a', 'a == c', 'c != b') + for e1 in compares: + for e2 in compares: + a, b, c = 1, 2, 3 + if eval(e1): res = 752 * 1 + else: res = 752 * 2 + if eval(e2): res += 752 * 10000 + else: res += 752 * 20000 + a = b + if eval(e1): res += 248 * 1 + else: res += 248 * 2 + if eval(e2): res += 248 * 10000 + else: res += 248 * 20000 + + + if 'c' in e1 or 'c' in e2: + n = 337 + else: + n = 215 + + self.run_source(''' + class tst: + pass + def main(): + a = tst() + b = tst() + c = tst() + sa = 0 + for i in range(1000): + if %s: sa += 1 + else: sa += 2 + if %s: sa += 10000 + else: sa += 20000 + if i > 750: a = b + return sa + '''%(e1, e2), n, ([], res)) + + def test_array_sum(self): + for tc, maxops in zip('bhilBHILfd', (38,) * 6 + (40, 40, 41, 38)): + res = 19352859 + if tc in 'IL': + res = long(res) + elif tc in 'fd': + res = float(res) + + self.run_source(''' + from array import array + + def main(): + img = array("%s", range(127) * 5) * 484 + l, i = 0, 0 + while i < 640 * 480: + l += img[i] + i += 1 + return l + ''' % tc, maxops, ([], res)) + + def test_array_sum_char(self): + self.run_source(''' + from array import array + + def main(): + img = array("c", "Hello") * 130 * 480 + l, i = 0, 0 + while i < 640 * 480: + l += ord(img[i]) + i += 1 + return l + ''', 60, ([], 30720000)) + + def test_array_sum_unicode(self): + self.run_source(''' + from array import array + + def main(): + img = array("u", u"Hello") * 130 * 480 + l, i = 0, 0 + while i < 640 * 480: + if img[i] == u"l": + l += 1 + i += 1 + return l + ''', 65, ([], 122880)) + + def test_array_intimg(self): + for tc, maxops in zip('ilILd', (67, 67, 69, 69, 61)): + res = 73574560 + if tc in 'IL': + res = long(res) + elif tc in 'fd': + res = float(res) + + self.run_source(''' + from array import array + + def main(tc): + img = array(tc, range(3)) * (350 * 480) + intimg = array(tc, (0,)) * (640 * 480) + l, i = 0, 640 + while i < 640 * 480: + l = l + img[i] + intimg[i] = (intimg[i-640] + l) + i += 1 + return intimg[i - 1] + ''', maxops, ([tc], res)) + class AppTestJIT(PyPyCJITTests): def setup_class(cls): if not option.runappdirect: @@ -637,6 +868,7 @@ cls.counter = 0 cls.pypy_c = option.pypy_c + def has_info(pypy_c, option): g = os.popen('"%s" --info' % pypy_c, 'r') lines = g.readlines() Modified: pypy/branch/fast-ctypes/pypy/module/signal/interp_signal.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/signal/interp_signal.py (original) +++ pypy/branch/fast-ctypes/pypy/module/signal/interp_signal.py Tue Aug 10 20:07:15 2010 @@ -7,6 +7,7 @@ from pypy.translator.tool.cbuild import ExternalCompilationInfo import py from pypy.tool import autopath +from pypy.rlib import jit def setup(): for key, value in cpy_signal.__dict__.items(): @@ -159,10 +160,12 @@ return space.wrap(SIG_DFL) getsignal.unwrap_spec = [ObjSpace, int] + at jit.dont_look_inside def alarm(space, timeout): return space.wrap(c_alarm(timeout)) alarm.unwrap_spec = [ObjSpace, int] + at jit.dont_look_inside def pause(space): c_pause() return space.w_None @@ -173,6 +176,7 @@ raise OperationError(space.w_ValueError, space.wrap("signal number out of range")) + at jit.dont_look_inside def signal(space, signum, w_handler): """ signal(sig, action) -> action Modified: pypy/branch/fast-ctypes/pypy/objspace/std/callmethod.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/objspace/std/callmethod.py (original) +++ pypy/branch/fast-ctypes/pypy/objspace/std/callmethod.py Tue Aug 10 20:07:15 2010 @@ -12,7 +12,7 @@ from pypy.interpreter import function from pypy.objspace.descroperation import object_getattribute -from pypy.rlib import rstack # for resume points +from pypy.rlib import jit, rstack # for resume points # This module exports two extra methods for StdObjSpaceFrame implementing # the LOOKUP_METHOD and CALL_METHOD opcodes in an efficient way, as well @@ -56,16 +56,41 @@ f.pushvalue(w_value) f.pushvalue(None) -def CALL_METHOD(f, nargs, *ignored): - # 'nargs' is the argument count excluding the implicit 'self' - w_self = f.peekvalue(nargs) - w_callable = f.peekvalue(nargs + 1) - n = nargs + (w_self is not None) - try: - w_result = f.space.call_valuestack(w_callable, n, f) - rstack.resume_point("CALL_METHOD", f, nargs, returns=w_result) - finally: - f.dropvalues(nargs + 2) + at jit.unroll_safe +def CALL_METHOD(f, oparg, *ignored): + # opargs contains the arg, and kwarg count, excluding the implicit 'self' + n_args = oparg & 0xff + n_kwargs = (oparg >> 8) & 0xff + w_self = f.peekvalue(n_args + (2 * n_kwargs)) + n = n_args + (w_self is not None) + + if not n_kwargs: + w_callable = f.peekvalue(n_args + (2 * n_kwargs) + 1) + try: + w_result = f.space.call_valuestack(w_callable, n, f) + rstack.resume_point("CALL_METHOD", f, n_args, returns=w_result) + finally: + f.dropvalues(n_args + 2) + else: + keywords = [None] * n_kwargs + keywords_w = [None] * n_kwargs + while True: + n_kwargs -= 1 + if n_kwargs < 0: + break + w_value = f.popvalue() + w_key = f.popvalue() + key = f.space.str_w(w_key) + keywords[n_kwargs] = key + keywords_w[n_kwargs] = w_value + + arguments = f.popvalues(n) # includes w_self if it is not None + args = f.argument_factory(arguments, keywords, keywords_w, None, None) + if w_self is None: + f.popvalue() # removes w_self, which is None + w_callable = f.popvalue() + w_result = f.space.call_args(w_callable, args) + rstack.resume_point("CALL_METHOD_KW", f, returns=w_result) f.pushvalue(w_result) Modified: pypy/branch/fast-ctypes/pypy/objspace/std/itertype.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/objspace/std/itertype.py (original) +++ pypy/branch/fast-ctypes/pypy/objspace/std/itertype.py Tue Aug 10 20:07:15 2010 @@ -1,5 +1,6 @@ from pypy.interpreter import gateway from pypy.objspace.std.stdtypedef import StdTypeDef +from pypy.interpreter.error import OperationError # ____________________________________________________________ @@ -8,6 +9,11 @@ XXX to do: remove this __reduce__ method and do a registration with copy_reg, instead. """ + + # cpython does not support pickling iterators + msg = 'Pickling for iterators dissabled as cpython does not support it' + raise OperationError(space.w_TypeError, space.wrap(msg)) + from pypy.objspace.std.iterobject import W_AbstractSeqIterObject assert isinstance(w_self, W_AbstractSeqIterObject) from pypy.interpreter.mixedmodule import MixedModule Modified: pypy/branch/fast-ctypes/pypy/objspace/std/model.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/objspace/std/model.py (original) +++ pypy/branch/fast-ctypes/pypy/objspace/std/model.py Tue Aug 10 20:07:15 2010 @@ -88,6 +88,7 @@ import pypy.objspace.std.default # register a few catch-all multimethods import pypy.objspace.std.marshal_impl # install marshal multimethods + import pypy.module.array # the set of implementation types self.typeorder = { @@ -140,6 +141,8 @@ # check if we missed implementations for implcls in _registered_implementations: + if hasattr(implcls, 'register'): + implcls.register(self.typeorder) assert (implcls in self.typeorder or implcls in self.imported_but_not_registered), ( "please add %r in StdTypeModel.typeorder" % (implcls,)) Modified: pypy/branch/fast-ctypes/pypy/objspace/std/test/test_callmethod.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/objspace/std/test/test_callmethod.py (original) +++ pypy/branch/fast-ctypes/pypy/objspace/std/test/test_callmethod.py Tue Aug 10 20:07:15 2010 @@ -106,6 +106,15 @@ else: raise Exception("did not raise?") """ + + def test_kwargs(self): + exec """if 1: + class C(object): + def f(self, a): + return a + 2 + + assert C().f(a=3) == 5 + """ class AppTestCallMethodWithGetattributeShortcut(AppTestCallMethod): Modified: pypy/branch/fast-ctypes/pypy/rlib/objectmodel.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/objectmodel.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/objectmodel.py Tue Aug 10 20:07:15 2010 @@ -82,6 +82,17 @@ specialize = _Specialize() +def enforceargs(*args): + """ Decorate a function with forcing of RPython-level types on arguments. + None means no enforcing. + + XXX shouldn't we also add asserts in function body? + """ + def decorator(f): + f._annenforceargs_ = args + return f + return decorator + # ____________________________________________________________ class Symbolic(object): Modified: pypy/branch/fast-ctypes/pypy/rlib/rmmap.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rmmap.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rmmap.py Tue Aug 10 20:07:15 2010 @@ -646,8 +646,14 @@ hintp = rffi.cast(PTR, hint.pos) res = c_mmap_safe(hintp, map_size, prot, flags, -1, 0) if res == rffi.cast(PTR, -1): - raise MemoryError - hint.pos += map_size + # some systems (some versions of OS/X?) complain if they + # are passed a non-zero address. Try again. + hintp = rffi.cast(PTR, 0) + res = c_mmap_safe(hintp, map_size, prot, flags, -1, 0) + if res == rffi.cast(PTR, -1): + raise MemoryError + else: + hint.pos += map_size return res alloc._annenforceargs_ = (int,) Modified: pypy/branch/fast-ctypes/pypy/rlib/test/test_objectmodel.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/test/test_objectmodel.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/test/test_objectmodel.py Tue Aug 10 20:07:15 2010 @@ -404,6 +404,13 @@ assert f._annspecialcase_ == 'specialize:arg(1)' +def test_enforceargs_decorator(): + @enforceargs(int, str, None) + def f(a, b, c): + pass + + assert f._annenforceargs_ == (int, str, None) + def getgraph(f, argtypes): from pypy.translator.translator import TranslationContext, graphof from pypy.translator.backendopt.all import backend_optimizations Modified: pypy/branch/fast-ctypes/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/fast-ctypes/pypy/rpython/lltypesystem/rstr.py Tue Aug 10 20:07:15 2010 @@ -2,7 +2,7 @@ from pypy.tool.pairtype import pairtype from pypy.rpython.error import TyperError from pypy.rlib.objectmodel import malloc_zero_filled, we_are_translated -from pypy.rlib.objectmodel import _hash_string +from pypy.rlib.objectmodel import _hash_string, enforceargs from pypy.rlib.debug import ll_assert from pypy.rlib.jit import purefunction from pypy.rpython.robject import PyObjRepr, pyobj_repr @@ -56,6 +56,7 @@ llmemory.itemoffsetof(TP.chars, 0) + llmemory.sizeof(CHAR_TP) * item) + @enforceargs(None, None, int, int, int) def copy_string_contents(src, dst, srcstart, dststart, length): assert srcstart >= 0 assert dststart >= 0 Modified: pypy/branch/fast-ctypes/pypy/rpython/rstr.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rpython/rstr.py (original) +++ pypy/branch/fast-ctypes/pypy/rpython/rstr.py Tue Aug 10 20:07:15 2010 @@ -288,8 +288,8 @@ if not hop.args_s[1].is_constant(): raise TyperError("encoding must be constant") encoding = hop.args_s[1].const - if encoding == "ascii": - expect = self.lowleveltype # can be a UniChar + if encoding == "ascii" and self.lowleveltype == UniChar: + expect = UniChar # only for unichar.encode('ascii') else: expect = self.repr # must be a regular unicode string v_self = hop.inputarg(expect, 0) Modified: pypy/branch/fast-ctypes/pypy/rpython/tool/rfficache.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rpython/tool/rfficache.py (original) +++ pypy/branch/fast-ctypes/pypy/rpython/tool/rfficache.py Tue Aug 10 20:07:15 2010 @@ -29,7 +29,7 @@ } ''' % (include_string, add_source, str(question))) c_file = udir.join("gcctest.c") - c_file.write(c_source) + c_file.write(str(c_source) + '\n') eci = ExternalCompilationInfo() return build_executable_cache([c_file], eci) Modified: pypy/branch/fast-ctypes/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/translator/c/genc.py (original) +++ pypy/branch/fast-ctypes/pypy/translator/c/genc.py Tue Aug 10 20:07:15 2010 @@ -1,7 +1,6 @@ import autopath import py import sys, os -from pypy.translator.c.node import PyObjectNode, FuncNode from pypy.translator.c.database import LowLevelDatabase from pypy.translator.c.extfunc import pre_include_code_lines from pypy.translator.llsupport.wrapper import new_wrapper @@ -196,7 +195,7 @@ all = [] for node in self.db.globalcontainers(): - eci = getattr(node, 'compilation_info', None) + eci = node.compilation_info() if eci: all.append(eci) self.merge_eci(*all) @@ -222,7 +221,7 @@ graphs = db.all_graphs() db.gctransformer.prepare_inline_helpers(graphs) for node in db.containerlist: - if isinstance(node, FuncNode): + if hasattr(node, 'funcgens'): for funcgen in node.funcgens: funcgen.patch_graph(copy_graph=False) return db Modified: pypy/branch/fast-ctypes/pypy/translator/c/node.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/translator/c/node.py (original) +++ pypy/branch/fast-ctypes/pypy/translator/c/node.py Tue Aug 10 20:07:15 2010 @@ -466,15 +466,15 @@ class ContainerNode(object): - if USESLOTS: - __slots__ = """db T obj + if USESLOTS: # keep the number of slots down! + __slots__ = """db obj typename implementationtypename - name ptrname compilation_info + name ptrname globalcontainer""".split() + eci_name = '_compilation_info' def __init__(self, db, T, obj): self.db = db - self.T = T self.obj = obj #self.dependencies = {} self.typename = db.gettype(T) #, who_asks=self) @@ -489,16 +489,22 @@ else: self.globalcontainer = False parentnode = db.getcontainernode(parent) - defnode = db.gettypedefnode(parentnode.T) + defnode = db.gettypedefnode(parentnode.getTYPE()) self.name = defnode.access_expr(parentnode.name, parentindex) if self.typename != self.implementationtypename: if db.gettypedefnode(T).extra_union_for_varlength: self.name += '.b' - self.compilation_info = getattr(obj, '_compilation_info', None) self.ptrname = '(&%s)' % self.name + def getTYPE(self): + return typeOf(self.obj) + def is_thread_local(self): - return hasattr(self.T, "_hints") and self.T._hints.get('thread_local') + T = self.getTYPE() + return hasattr(T, "_hints") and T._hints.get('thread_local') + + def compilation_info(self): + return getattr(self.obj, self.eci_name, None) def get_declaration(self): if self.name[-2:] == '.b': @@ -546,27 +552,31 @@ __slots__ = () def basename(self): - return self.T._name + T = self.getTYPE() + return T._name def enum_dependencies(self): - for name in self.T._names: + T = self.getTYPE() + for name in T._names: yield getattr(self.obj, name) def getlength(self): - if self.T._arrayfld is None: + T = self.getTYPE() + if T._arrayfld is None: return 1 else: - array = getattr(self.obj, self.T._arrayfld) + array = getattr(self.obj, T._arrayfld) return len(array.items) def initializationexpr(self, decoration=''): + T = self.getTYPE() is_empty = True yield '{' - defnode = self.db.gettypedefnode(self.T) + defnode = self.db.gettypedefnode(T) data = [] - if needs_gcheader(self.T): + if needs_gcheader(T): gc_init = self.db.gcpolicy.struct_gcheader_initdata(self) data.append(('gcheader', gc_init)) @@ -578,16 +588,16 @@ # '.fieldname = value'. But here we don't know which of the # fields need initialization, so XXX we pick the first one # arbitrarily. - if hasattr(self.T, "_hints") and self.T._hints.get('union'): + if hasattr(T, "_hints") and T._hints.get('union'): data = data[0:1] - if 'get_padding_drop' in self.T._hints: + if 'get_padding_drop' in T._hints: d = {} for name, _ in data: - T = defnode.c_struct_field_type(name) - typename = self.db.gettype(T) + T1 = defnode.c_struct_field_type(name) + typename = self.db.gettype(T1) d[name] = cdecl(typename, '') - padding_drop = self.T._hints['get_padding_drop'](d) + padding_drop = T._hints['get_padding_drop'](d) else: padding_drop = [] @@ -617,9 +627,10 @@ return 'struct _hashT_%s @' % self.name def forward_declaration(self): + T = self.getTYPE() assert self.typename == self.implementationtypename # no array part hash_typename = self.get_hash_typename() - hash_offset = self.db.gctransformer.get_hash_offset(self.T) + hash_offset = self.db.gctransformer.get_hash_offset(T) yield '%s {' % cdecl(hash_typename, '') yield '\tunion {' yield '\t\t%s;' % cdecl(self.implementationtypename, 'head') @@ -671,22 +682,23 @@ return len(self.obj.items) def initializationexpr(self, decoration=''): - defnode = self.db.gettypedefnode(self.T) + T = self.getTYPE() + defnode = self.db.gettypedefnode(T) yield '{' - if needs_gcheader(self.T): + if needs_gcheader(T): gc_init = self.db.gcpolicy.array_gcheader_initdata(self) lines = generic_initializationexpr(self.db, gc_init, 'gcheader', '%sgcheader' % (decoration,)) for line in lines: yield line - if self.T._hints.get('nolength', False): + if T._hints.get('nolength', False): length = '' else: length = '%d, ' % len(self.obj.items) - if self.T.OF is Void or len(self.obj.items) == 0: + if T.OF is Void or len(self.obj.items) == 0: yield '\t%s' % length.rstrip(', ') yield '}' - elif self.T.OF == Char: + elif T.OF == Char: if len(self.obj.items) and self.obj.items[0] is None: s = ''.join([self.obj.getitem(i) for i in range(len(self.obj.items))]) else: @@ -694,7 +706,7 @@ yield '\t%s%s' % (length, c_char_array_constant(s)) yield '}' else: - barebone = barebonearray(self.T) + barebone = barebonearray(T) if not barebone: yield '\t%s{' % length for j in range(len(self.obj.items)): @@ -722,7 +734,8 @@ self.ptrname = self.name def basename(self): - return self.T._name + T = self.getTYPE() + return T._name def enum_dependencies(self): for i in range(self.obj.getlength()): @@ -732,11 +745,12 @@ return 1 # not variable-sized! def initializationexpr(self, decoration=''): + T = self.getTYPE() assert self.typename == self.implementationtypename # not var-sized is_empty = True yield '{' # _names == ['item0', 'item1', ...] - for j, name in enumerate(self.T._names): + for j, name in enumerate(T._names): value = getattr(self.obj, name) lines = generic_initializationexpr(self.db, value, '%s[%d]' % (self.name, j), @@ -777,6 +791,7 @@ class FuncNode(ContainerNode): nodekind = 'func' + eci_name = 'compilation_info' # there not so many node of this kind, slots should not # be necessary @@ -794,7 +809,6 @@ else: self.name = (forcename or db.namespace.uniquename('g_' + self.basename())) - self.compilation_info = getattr(obj, 'compilation_info', None) self.make_funcgens() #self.dependencies = {} self.typename = db.gettype(T) #, who_asks=self) @@ -939,18 +953,20 @@ return [] def initializationexpr(self, decoration=''): - yield 'RPyOpaque_INITEXPR_%s' % (self.T.tag,) + T = self.getTYPE() + yield 'RPyOpaque_INITEXPR_%s' % (T.tag,) def startupcode(self): + T = self.getTYPE() args = [self.ptrname] # XXX how to make this code more generic? - if self.T.tag == 'ThreadLock': + if T.tag == 'ThreadLock': lock = self.obj.externalobj if lock.locked(): args.append('1') else: args.append('0') - yield 'RPyOpaque_SETUP_%s(%s);' % (self.T.tag, ', '.join(args)) + yield 'RPyOpaque_SETUP_%s(%s);' % (T.tag, ', '.join(args)) def opaquenode_factory(db, T, obj): Modified: pypy/branch/fast-ctypes/pypy/translator/platform/posix.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/translator/platform/posix.py (original) +++ pypy/branch/fast-ctypes/pypy/translator/platform/posix.py Tue Aug 10 20:07:15 2010 @@ -14,7 +14,10 @@ def __init__(self, cc=None): if cc is None: - cc = 'gcc' + try: + cc = os.environ['CC'] + except KeyError: + cc = 'gcc' self.cc = cc def _libs(self, libraries): From jcreigh at codespeak.net Tue Aug 10 20:09:04 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Tue, 10 Aug 2010 20:09:04 +0200 (CEST) Subject: [pypy-svn] r76576 - in pypy/trunk/pypy/jit/backend/x86: . test Message-ID: <20100810180904.13F61282BAD@codespeak.net> Author: jcreigh Date: Tue Aug 10 20:09:03 2010 New Revision: 76576 Modified: pypy/trunk/pypy/jit/backend/x86/rx86.py pypy/trunk/pypy/jit/backend/x86/test/test_regloc.py Log: rx86: add tests to expose bug in MOV16/CMP16, and a hack to fix it Modified: pypy/trunk/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/rx86.py (original) +++ pypy/trunk/pypy/jit/backend/x86/rx86.py Tue Aug 10 20:09:03 2010 @@ -290,6 +290,9 @@ def encode_rex(mc, rexbyte, basevalue, orbyte): if mc.WORD == 8: assert 0 <= rexbyte < 8 + # XXX: Hack. Ignore REX.W if we are using 16-bit operands + if mc._use_16_bit_immediate: + basevalue &= ~REX_W if basevalue != 0x40 or rexbyte != 0: mc.writechar(chr(basevalue | rexbyte)) else: Modified: pypy/trunk/pypy/jit/backend/x86/test/test_regloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_regloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_regloc.py Tue Aug 10 20:09:03 2010 @@ -16,13 +16,27 @@ cb64 = LocationCodeBuilder64 def test_mov_16(): + # 32-bit assert_encodes_as(cb32, "MOV16", (ecx, ebx), '\x66\x89\xD9') assert_encodes_as(cb32, "MOV16", (ecx, ImmedLoc(12345)), '\x66\xB9\x39\x30') + # 64-bit + assert_encodes_as(cb64, "MOV16", (ecx, ebx), '\x66\x89\xD9') + # XXX: What we are testing for here is actually not the most compact + # encoding. + assert_encodes_as(cb64, "MOV16", (ecx, ImmedLoc(12345)), '\x66\xC7\xC1\x39\x30') + assert_encodes_as(cb64, "MOV16", (AddressLoc(r13, ImmedLoc(0), 0, 0), ImmedLoc(12345)), '\x66\x41\xC7\x45\x00\x39\x30') + def test_cmp_16(): + # 32-bit assert_encodes_as(cb32, "CMP16", (ecx, ebx), '\x66\x39\xD9') assert_encodes_as(cb32, "CMP16", (ecx, ImmedLoc(12345)), '\x66\x81\xF9\x39\x30') + # 64-bit + assert_encodes_as(cb64, "CMP16", (ecx, ebx), '\x66\x39\xD9') + assert_encodes_as(cb64, "CMP16", (ecx, ImmedLoc(12345)), '\x66\x81\xF9\x39\x30') + assert_encodes_as(cb64, "CMP16", (AddressLoc(r13, ImmedLoc(0), 0, 0), ImmedLoc(12345)), '\x66\x41\x81\x7D\x00\x39\x30') + def test_jmp_wraparound(): if not IS_X86_32: py.test.skip() From jcreigh at codespeak.net Tue Aug 10 20:15:09 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Tue, 10 Aug 2010 20:15:09 +0200 (CEST) Subject: [pypy-svn] r76577 - in pypy/branch/asmgcc-64: . pypy/jit/backend/x86 pypy/jit/backend/x86/test pypy/module/array/benchmark pypy/module/array/test Message-ID: <20100810181509.614CD36C21E@codespeak.net> Author: jcreigh Date: Tue Aug 10 20:15:03 2010 New Revision: 76577 Modified: pypy/branch/asmgcc-64/ (props changed) pypy/branch/asmgcc-64/pypy/jit/backend/x86/rx86.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_regloc.py pypy/branch/asmgcc-64/pypy/module/array/benchmark/Makefile (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/intimg.c (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/intimgtst.c (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/intimgtst.py (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/loop.c (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/sum.c (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/sumtst.c (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/sumtst.py (props changed) pypy/branch/asmgcc-64/pypy/module/array/test/test_array_old.py (props changed) Log: merged changes from trunk through r76576 Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/rx86.py Tue Aug 10 20:15:03 2010 @@ -290,6 +290,9 @@ def encode_rex(mc, rexbyte, basevalue, orbyte): if mc.WORD == 8: assert 0 <= rexbyte < 8 + # XXX: Hack. Ignore REX.W if we are using 16-bit operands + if mc._use_16_bit_immediate: + basevalue &= ~REX_W if basevalue != 0x40 or rexbyte != 0: mc.writechar(chr(basevalue | rexbyte)) else: Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_regloc.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_regloc.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_regloc.py Tue Aug 10 20:15:03 2010 @@ -16,13 +16,27 @@ cb64 = LocationCodeBuilder64 def test_mov_16(): + # 32-bit assert_encodes_as(cb32, "MOV16", (ecx, ebx), '\x66\x89\xD9') assert_encodes_as(cb32, "MOV16", (ecx, ImmedLoc(12345)), '\x66\xB9\x39\x30') + # 64-bit + assert_encodes_as(cb64, "MOV16", (ecx, ebx), '\x66\x89\xD9') + # XXX: What we are testing for here is actually not the most compact + # encoding. + assert_encodes_as(cb64, "MOV16", (ecx, ImmedLoc(12345)), '\x66\xC7\xC1\x39\x30') + assert_encodes_as(cb64, "MOV16", (AddressLoc(r13, ImmedLoc(0), 0, 0), ImmedLoc(12345)), '\x66\x41\xC7\x45\x00\x39\x30') + def test_cmp_16(): + # 32-bit assert_encodes_as(cb32, "CMP16", (ecx, ebx), '\x66\x39\xD9') assert_encodes_as(cb32, "CMP16", (ecx, ImmedLoc(12345)), '\x66\x81\xF9\x39\x30') + # 64-bit + assert_encodes_as(cb64, "CMP16", (ecx, ebx), '\x66\x39\xD9') + assert_encodes_as(cb64, "CMP16", (ecx, ImmedLoc(12345)), '\x66\x81\xF9\x39\x30') + assert_encodes_as(cb64, "CMP16", (AddressLoc(r13, ImmedLoc(0), 0, 0), ImmedLoc(12345)), '\x66\x41\x81\x7D\x00\x39\x30') + def test_jmp_wraparound(): if not IS_X86_32: py.test.skip() From getxsick at codespeak.net Tue Aug 10 20:38:57 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Tue, 10 Aug 2010 20:38:57 +0200 (CEST) Subject: [pypy-svn] r76578 - pypy/trunk/pypy/module/thread/test Message-ID: <20100810183857.B0DB3282BAD@codespeak.net> Author: getxsick Date: Tue Aug 10 20:38:53 2010 New Revision: 76578 Modified: pypy/trunk/pypy/module/thread/test/test_gil.py Log: kill redundant import Modified: pypy/trunk/pypy/module/thread/test/test_gil.py ============================================================================== --- pypy/trunk/pypy/module/thread/test/test_gil.py (original) +++ pypy/trunk/pypy/module/thread/test/test_gil.py Tue Aug 10 20:38:53 2010 @@ -1,7 +1,6 @@ import time from pypy.module.thread import gil from pypy.module.thread.test import test_ll_thread -from pypy.rpython.lltypesystem import rffi from pypy.module.thread import ll_thread as thread from pypy.rlib.objectmodel import we_are_translated From benjamin at codespeak.net Wed Aug 11 04:43:51 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 11 Aug 2010 04:43:51 +0200 (CEST) Subject: [pypy-svn] r76579 - pypy/trunk/pypy/jit/metainterp Message-ID: <20100811024351.25F64282BAD@codespeak.net> Author: benjamin Date: Wed Aug 11 04:43:49 2010 New Revision: 76579 Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py Log: english spelling fix Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Wed Aug 11 04:43:49 2010 @@ -612,7 +612,7 @@ assert oldop.opnum == op.opnum self.make_equal_to(op.result, self.getvalue(oldop.result)) return - elif self.find_rewriteable_bool(op, args): + elif self.find_rewritable_bool(op, args): return else: self.pure_operations[args] = op @@ -635,7 +635,7 @@ return False - def find_rewriteable_bool(self, op, args): + def find_rewritable_bool(self, op, args): try: oldopnum = opboolinvers[op.opnum] targs = [args[0], args[1], ConstInt(oldopnum)] From fijal at codespeak.net Wed Aug 11 09:51:40 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 11 Aug 2010 09:51:40 +0200 (CEST) Subject: [pypy-svn] r76580 - pypy/branch/interplevel-array Message-ID: <20100811075140.D9465282BF3@codespeak.net> Author: fijal Date: Wed Aug 11 09:51:39 2010 New Revision: 76580 Added: pypy/branch/interplevel-array/ (props changed) - copied from r76579, pypy/trunk/ Log: Recreate this branch, it broke quite a few tests From fijal at codespeak.net Wed Aug 11 09:55:00 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 11 Aug 2010 09:55:00 +0200 (CEST) Subject: [pypy-svn] r76581 - pypy/trunk/pypy/config Message-ID: <20100811075500.716BC282BF3@codespeak.net> Author: fijal Date: Wed Aug 11 09:54:58 2010 New Revision: 76581 Modified: pypy/trunk/pypy/config/pypyoption.py Log: Kill array from allworkingmodules. In theory this should fix everything for now (fix tests first on interplevel array branch) Modified: pypy/trunk/pypy/config/pypyoption.py ============================================================================== --- pypy/trunk/pypy/config/pypyoption.py (original) +++ pypy/trunk/pypy/config/pypyoption.py Wed Aug 11 09:54:58 2010 @@ -30,7 +30,7 @@ "rctime" , "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios", "zlib", "struct", "md5", "sha", "bz2", "_minimal_curses", "cStringIO", - "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array"] + "thread", "itertools", "pyexpat", "_ssl", "cpyext"] )) working_oo_modules = default_modules.copy() From arigo at codespeak.net Wed Aug 11 13:31:23 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 11 Aug 2010 13:31:23 +0200 (CEST) Subject: [pypy-svn] r76582 - in pypy/trunk/pypy/jit/backend/x86: . test Message-ID: <20100811113123.BCC1F282BF3@codespeak.net> Author: arigo Date: Wed Aug 11 13:31:20 2010 New Revision: 76582 Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/regalloc.py pypy/trunk/pypy/jit/backend/x86/rx86.py pypy/trunk/pypy/jit/backend/x86/test/test_rx86.py Log: Fix for a rare case. No test; I still think that the API of llsupport.regalloc should be redesigned to get rid of these kinds of rare cases altogether... Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Wed Aug 11 13:31:20 2010 @@ -1741,7 +1741,15 @@ # the following is supposed to be the slow path, so whenever possible # we choose the most compact encoding over the most efficient one. for i in range(len(arglocs)-1, -1, -1): - self.mc.PUSH(arglocs[i]) + loc = arglocs[i] + if isinstance(loc, RegLoc): + self.mc.PUSH_r(loc.value) + else: + if IS_X86_64: + self.mc.MOV_ri(X86_64_SCRATCH_REG.value, loc.getint()) + self.mc.PUSH_r(X86_64_SCRATCH_REG.value) + else: + self.mc.PUSH_i32(loc.getint()) if IS_X86_64: # We clobber these registers to pass the arguments, but that's @@ -1757,8 +1765,10 @@ self.mc.CALL(imm(descr.get_write_barrier_fn(self.cpu))) for i in range(len(arglocs)): loc = arglocs[i] - assert isinstance(loc, RegLoc) - self.mc.POP(loc) + if isinstance(loc, RegLoc): + self.mc.POP_r(loc.value) + else: + self.mc.ADD_ri(esp.value, WORD) # ignore the pushed constant # patch the JZ above offset = self.mc.get_relative_pos() - jz_location assert 0 < offset <= 127 Modified: pypy/trunk/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regalloc.py Wed Aug 11 13:31:20 2010 @@ -671,13 +671,12 @@ def consider_cond_call_gc_wb(self, op): assert op.result is None + loc_newvalue = self.rm.make_sure_var_in_reg(op.args[1], op.args) + # ^^^ we force loc_newvalue in a reg (unless it's a Const), + # because it will be needed anyway by the following setfield_gc. + # It avoids loading it twice from the memory. loc_base = self.rm.make_sure_var_in_reg(op.args[0], op.args, imm_fine=False) - loc_newvalue = self.rm.make_sure_var_in_reg(op.args[1], op.args, - imm_fine=False) - # ^^^ we also force loc_newvalue in a reg, because it will be needed - # anyway by the following setfield_gc. It avoids loading it twice - # from the memory. arglocs = [loc_base, loc_newvalue] # add eax, ecx and edx as extra "arguments" to ensure they are # saved and restored. Fish in self.rm to know which of these Modified: pypy/trunk/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/rx86.py (original) +++ pypy/trunk/pypy/jit/backend/x86/rx86.py Wed Aug 11 13:31:20 2010 @@ -495,6 +495,7 @@ PUSH_r = insn(rex_nw, register(1), '\x50') PUSH_b = insn(rex_nw, '\xFF', orbyte(6<<3), stack_bp(1)) + PUSH_i32 = insn('\x68', immediate(1, 'i')) POP_r = insn(rex_nw, register(1), '\x58') POP_b = insn(rex_nw, '\x8F', orbyte(0<<3), stack_bp(1)) Modified: pypy/trunk/pypy/jit/backend/x86/test/test_rx86.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_rx86.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_rx86.py Wed Aug 11 13:31:20 2010 @@ -188,6 +188,10 @@ assert_encodes_as(cb, 'MOV8_mi', ((edx, 16), 99), '\xC6\x42\x10\x63') assert_encodes_as(cb, 'MOV8_ai', ((ebx, ecx, 2, 16), 99), '\xC6\x44\x8B\x10\x63') +def test_push32(): + cb = CodeBuilder32 + assert_encodes_as(cb, 'PUSH_i32', (9,), '\x68\x09\x00\x00\x00') + class CodeBuilder64(CodeBuilderMixin, X86_64_CodeBuilder): pass From hakanardo at codespeak.net Wed Aug 11 14:04:56 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Wed, 11 Aug 2010 14:04:56 +0200 (CEST) Subject: [pypy-svn] r76583 - in pypy/trunk/pypy: module/__builtin__/test module/_codecs/test module/_file/test module/_socket/test module/_sre/test module/array module/array/test module/fcntl/test module/marshal/test objspace/std Message-ID: <20100811120456.9FCEE282BF3@codespeak.net> Author: hakanardo Date: Wed Aug 11 14:04:54 2010 New Revision: 76583 Modified: pypy/trunk/pypy/module/__builtin__/test/test_buffer.py pypy/trunk/pypy/module/_codecs/test/test_codecs.py pypy/trunk/pypy/module/_file/test/test_file_extra.py pypy/trunk/pypy/module/_socket/test/test_sock_app.py pypy/trunk/pypy/module/_sre/test/test_app_sre.py pypy/trunk/pypy/module/array/interp_array.py pypy/trunk/pypy/module/array/test/test_array.py pypy/trunk/pypy/module/fcntl/test/test_fcntl.py pypy/trunk/pypy/module/marshal/test/test_marshalimpl.py pypy/trunk/pypy/objspace/std/itertype.py Log: * Pickling iterators reenabled * Array buffer now writeable * Enabling array module in tests using "import array" * Supporting multiplication with negative numbers Modified: pypy/trunk/pypy/module/__builtin__/test/test_buffer.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/test/test_buffer.py (original) +++ pypy/trunk/pypy/module/__builtin__/test/test_buffer.py Wed Aug 11 14:04:54 2010 @@ -1,8 +1,11 @@ """Tests some behaviour of the buffer type that is not tested in lib-python/2.5.2/test/test_types.py where the stdlib buffer tests live.""" import autopath +from pypy.conftest import gettestobjspace class AppTestBuffer: + def setup_class(cls): + cls.space = gettestobjspace(usemodules=('array',)) def test_unicode_buffer(self): import sys Modified: pypy/trunk/pypy/module/_codecs/test/test_codecs.py ============================================================================== --- pypy/trunk/pypy/module/_codecs/test/test_codecs.py (original) +++ pypy/trunk/pypy/module/_codecs/test/test_codecs.py Wed Aug 11 14:04:54 2010 @@ -4,7 +4,7 @@ class AppTestCodecs: def setup_class(cls): - space = gettestobjspace(usemodules=('unicodedata',)) + space = gettestobjspace(usemodules=('unicodedata')) cls.space = space def test_register_noncallable(self): @@ -123,6 +123,10 @@ class AppTestPartialEvaluation: + def setup_class(cls): + space = gettestobjspace(usemodules=('array',)) + cls.space = space + def test_partial_utf8(self): import _codecs encoding = 'utf-8' Modified: pypy/trunk/pypy/module/_file/test/test_file_extra.py ============================================================================== --- pypy/trunk/pypy/module/_file/test/test_file_extra.py (original) +++ pypy/trunk/pypy/module/_file/test/test_file_extra.py Wed Aug 11 14:04:54 2010 @@ -353,6 +353,10 @@ class AppTestAFewExtra: + def setup_class(cls): + space = gettestobjspace(usemodules=('array',)) + cls.space = space + def setup_method(self, method): fn = str(udir.join('temptestfile')) self.w_temptestfile = self.space.wrap(fn) Modified: pypy/trunk/pypy/module/_socket/test/test_sock_app.py ============================================================================== --- pypy/trunk/pypy/module/_socket/test/test_sock_app.py (original) +++ pypy/trunk/pypy/module/_socket/test/test_sock_app.py Wed Aug 11 14:04:54 2010 @@ -4,7 +4,7 @@ from pypy.tool.udir import udir def setup_module(mod): - mod.space = gettestobjspace(usemodules=['_socket']) + mod.space = gettestobjspace(usemodules=['_socket', 'array']) global socket import socket mod.w_socket = space.appexec([], "(): import _socket as m; return m") Modified: pypy/trunk/pypy/module/_sre/test/test_app_sre.py ============================================================================== --- pypy/trunk/pypy/module/_sre/test/test_app_sre.py (original) +++ pypy/trunk/pypy/module/_sre/test/test_app_sre.py Wed Aug 11 14:04:54 2010 @@ -87,7 +87,9 @@ class AppTestSreMatch: - + def setup_class(cls): + cls.space = gettestobjspace(usemodules=('array', )) + def test_copy(self): import re # copy support is disabled by default in _sre.c Modified: pypy/trunk/pypy/module/array/interp_array.py ============================================================================== --- pypy/trunk/pypy/module/array/interp_array.py (original) +++ pypy/trunk/pypy/module/array/interp_array.py Wed Aug 11 14:04:54 2010 @@ -14,6 +14,7 @@ from pypy.objspace.std.model import W_Object from pypy.interpreter.argument import Arguments, Signature from pypy.module._file.interp_file import W_File +from pypy.interpreter.buffer import RWBuffer def w_array(space, w_cls, typecode, w_initializer=None, w_args=None): if len(w_args.arguments_w) > 0: @@ -142,6 +143,22 @@ v.typecode = k unroll_typecodes = unrolling_iterable(types.keys()) +class ArrayBuffer(RWBuffer): + def __init__(self, data, bytes): + self.data = data + self.len = bytes + + def getlength(self): + return self.len + + def getitem(self, index): + return self.data[index] + + def setitem(self, index, char): + self.data[index] = char + + + def make_array(mytype): class W_Array(W_ArrayBase): @@ -460,6 +477,7 @@ def mul__Array_ANY(space, self, w_repeat): repeat = space.int_w(w_repeat) a = mytype.w_class(space) + repeat = max(repeat, 0) a.setlen(self.len * repeat) for r in range(repeat): for i in range(self.len): @@ -472,6 +490,7 @@ def inplace_mul__Array_ANY(space, self, w_repeat): repeat = space.int_w(w_repeat) oldlen = self.len + repeat = max(repeat, 0) self.setlen(self.len * repeat) for r in range(1, repeat): for i in range(oldlen): @@ -565,9 +584,8 @@ # Misc methods def buffer__Array(space, self): - from pypy.interpreter.buffer import StringLikeBuffer - w_s = array_tostring__Array(space, self) - return space.wrap(StringLikeBuffer(space, w_s)) + b = ArrayBuffer(self.charbuf(), self.len * mytype.bytes) + return space.wrap(b) def array_buffer_info__Array(space, self): w_ptr = space.wrap(rffi.cast(lltype.Unsigned, self.buffer)) Modified: pypy/trunk/pypy/module/array/test/test_array.py ============================================================================== --- pypy/trunk/pypy/module/array/test/test_array.py (original) +++ pypy/trunk/pypy/module/array/test/test_array.py Wed Aug 11 14:04:54 2010 @@ -421,7 +421,7 @@ a = self.array('h', 'Hi') buf = buffer(a) assert buf[1] == 'i' - raises(TypeError, buf.__setitem__, 1, 'o') + #raises(TypeError, buf.__setitem__, 1, 'o') def test_list_methods(self): assert repr(self.array('i')) == "array('i')" @@ -531,10 +531,8 @@ assert len(b) == 0 and b.typecode == 'l' a = self.array('i', [1, 2, 4]) - print "itter" i = iter(a) - print "ok" - raises(TypeError, pickle.dumps, i, 1) + #raises(TypeError, pickle.dumps, i, 1) def test_copy_swap(self): a = self.array('i', [1, 2, 3]) @@ -599,6 +597,13 @@ assert addable() + self.array('i') == 'add' assert self.array('i') + addable() == 'radd' + a = self.array('i', [1, 2]) + assert a * -1 == self.array('i') + b = a + a *= -1 + assert a == self.array('i') + assert b == self.array('i') + def test_delitem(self): a = self.array('i', [1, 2, 3]) del a[1] Modified: pypy/trunk/pypy/module/fcntl/test/test_fcntl.py ============================================================================== --- pypy/trunk/pypy/module/fcntl/test/test_fcntl.py (original) +++ pypy/trunk/pypy/module/fcntl/test/test_fcntl.py Wed Aug 11 14:04:54 2010 @@ -13,7 +13,7 @@ class AppTestFcntl: def setup_class(cls): - space = gettestobjspace(usemodules=('fcntl',)) + space = gettestobjspace(usemodules=('fcntl', 'array')) cls.space = space tmpprefix = str(udir.ensure('test_fcntl', dir=1).join('tmp_')) cls.w_tmp = space.wrap(tmpprefix) Modified: pypy/trunk/pypy/module/marshal/test/test_marshalimpl.py ============================================================================== --- pypy/trunk/pypy/module/marshal/test/test_marshalimpl.py (original) +++ pypy/trunk/pypy/module/marshal/test/test_marshalimpl.py Wed Aug 11 14:04:54 2010 @@ -1,9 +1,13 @@ from pypy.module.marshal import interp_marshal from pypy.interpreter.error import OperationError +from pypy.conftest import gettestobjspace import sys class AppTestMarshalMore: + def setup_class(cls): + space = gettestobjspace(usemodules=('array',)) + cls.space = space def test_long_0(self): import marshal Modified: pypy/trunk/pypy/objspace/std/itertype.py ============================================================================== --- pypy/trunk/pypy/objspace/std/itertype.py (original) +++ pypy/trunk/pypy/objspace/std/itertype.py Wed Aug 11 14:04:54 2010 @@ -10,9 +10,9 @@ a registration with copy_reg, instead. """ - # cpython does not support pickling iterators - msg = 'Pickling for iterators dissabled as cpython does not support it' - raise OperationError(space.w_TypeError, space.wrap(msg)) + # cpython does not support pickling iterators but stackless python do + #msg = 'Pickling for iterators dissabled as cpython does not support it' + #raise OperationError(space.w_TypeError, space.wrap(msg)) from pypy.objspace.std.iterobject import W_AbstractSeqIterObject assert isinstance(w_self, W_AbstractSeqIterObject) From hakanardo at codespeak.net Wed Aug 11 14:09:49 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Wed, 11 Aug 2010 14:09:49 +0200 (CEST) Subject: [pypy-svn] r76584 - pypy/trunk/pypy/module/_codecs/test Message-ID: <20100811120949.6CF5C282BF3@codespeak.net> Author: hakanardo Date: Wed Aug 11 14:09:47 2010 New Revision: 76584 Modified: pypy/trunk/pypy/module/_codecs/test/test_codecs.py Log: typo Modified: pypy/trunk/pypy/module/_codecs/test/test_codecs.py ============================================================================== --- pypy/trunk/pypy/module/_codecs/test/test_codecs.py (original) +++ pypy/trunk/pypy/module/_codecs/test/test_codecs.py Wed Aug 11 14:09:47 2010 @@ -4,7 +4,7 @@ class AppTestCodecs: def setup_class(cls): - space = gettestobjspace(usemodules=('unicodedata')) + space = gettestobjspace(usemodules=('unicodedata',)) cls.space = space def test_register_noncallable(self): From jcreigh at codespeak.net Wed Aug 11 15:47:44 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Wed, 11 Aug 2010 15:47:44 +0200 (CEST) Subject: [pypy-svn] r76585 - in pypy/branch/asmgcc-64: . pypy/config pypy/jit/backend/x86 pypy/jit/backend/x86/test pypy/jit/metainterp pypy/module/__builtin__/test pypy/module/_codecs/test pypy/module/_file/test pypy/module/_socket/test pypy/module/_sre/test pypy/module/array pypy/module/array/benchmark pypy/module/array/test pypy/module/fcntl/test pypy/module/marshal/test pypy/module/thread/test pypy/objspace/std Message-ID: <20100811134744.42C72282BF3@codespeak.net> Author: jcreigh Date: Wed Aug 11 15:47:41 2010 New Revision: 76585 Modified: pypy/branch/asmgcc-64/ (props changed) pypy/branch/asmgcc-64/pypy/config/pypyoption.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/assembler.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/regalloc.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/rx86.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_rx86.py pypy/branch/asmgcc-64/pypy/jit/metainterp/optimizeopt.py pypy/branch/asmgcc-64/pypy/module/__builtin__/test/test_buffer.py pypy/branch/asmgcc-64/pypy/module/_codecs/test/test_codecs.py pypy/branch/asmgcc-64/pypy/module/_file/test/test_file_extra.py pypy/branch/asmgcc-64/pypy/module/_socket/test/test_sock_app.py pypy/branch/asmgcc-64/pypy/module/_sre/test/test_app_sre.py pypy/branch/asmgcc-64/pypy/module/array/benchmark/Makefile (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/intimg.c (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/intimgtst.c (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/intimgtst.py (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/loop.c (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/sum.c (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/sumtst.c (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/sumtst.py (props changed) pypy/branch/asmgcc-64/pypy/module/array/interp_array.py pypy/branch/asmgcc-64/pypy/module/array/test/test_array.py pypy/branch/asmgcc-64/pypy/module/array/test/test_array_old.py (props changed) pypy/branch/asmgcc-64/pypy/module/fcntl/test/test_fcntl.py pypy/branch/asmgcc-64/pypy/module/marshal/test/test_marshalimpl.py pypy/branch/asmgcc-64/pypy/module/thread/test/test_gil.py pypy/branch/asmgcc-64/pypy/objspace/std/itertype.py Log: merged changes from trunk through r76584 Modified: pypy/branch/asmgcc-64/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/config/pypyoption.py (original) +++ pypy/branch/asmgcc-64/pypy/config/pypyoption.py Wed Aug 11 15:47:41 2010 @@ -30,7 +30,7 @@ "rctime" , "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios", "zlib", "struct", "md5", "sha", "bz2", "_minimal_curses", "cStringIO", - "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array"] + "thread", "itertools", "pyexpat", "_ssl", "cpyext"] )) working_oo_modules = default_modules.copy() Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/assembler.py Wed Aug 11 15:47:41 2010 @@ -1741,7 +1741,15 @@ # the following is supposed to be the slow path, so whenever possible # we choose the most compact encoding over the most efficient one. for i in range(len(arglocs)-1, -1, -1): - self.mc.PUSH(arglocs[i]) + loc = arglocs[i] + if isinstance(loc, RegLoc): + self.mc.PUSH_r(loc.value) + else: + if IS_X86_64: + self.mc.MOV_ri(X86_64_SCRATCH_REG.value, loc.getint()) + self.mc.PUSH_r(X86_64_SCRATCH_REG.value) + else: + self.mc.PUSH_i32(loc.getint()) if IS_X86_64: # We clobber these registers to pass the arguments, but that's @@ -1757,8 +1765,10 @@ self.mc.CALL(imm(descr.get_write_barrier_fn(self.cpu))) for i in range(len(arglocs)): loc = arglocs[i] - assert isinstance(loc, RegLoc) - self.mc.POP(loc) + if isinstance(loc, RegLoc): + self.mc.POP_r(loc.value) + else: + self.mc.ADD_ri(esp.value, WORD) # ignore the pushed constant # patch the JZ above offset = self.mc.get_relative_pos() - jz_location assert 0 < offset <= 127 Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/regalloc.py Wed Aug 11 15:47:41 2010 @@ -684,13 +684,12 @@ def consider_cond_call_gc_wb(self, op): assert op.result is None + loc_newvalue = self.rm.make_sure_var_in_reg(op.args[1], op.args) + # ^^^ we force loc_newvalue in a reg (unless it's a Const), + # because it will be needed anyway by the following setfield_gc. + # It avoids loading it twice from the memory. loc_base = self.rm.make_sure_var_in_reg(op.args[0], op.args, imm_fine=False) - loc_newvalue = self.rm.make_sure_var_in_reg(op.args[1], op.args, - imm_fine=False) - # ^^^ we also force loc_newvalue in a reg, because it will be needed - # anyway by the following setfield_gc. It avoids loading it twice - # from the memory. arglocs = [loc_base, loc_newvalue] # add eax, ecx and edx as extra "arguments" to ensure they are # saved and restored. Fish in self.rm to know which of these Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/rx86.py Wed Aug 11 15:47:41 2010 @@ -495,6 +495,7 @@ PUSH_r = insn(rex_nw, register(1), '\x50') PUSH_b = insn(rex_nw, '\xFF', orbyte(6<<3), stack_bp(1)) + PUSH_i32 = insn('\x68', immediate(1, 'i')) POP_r = insn(rex_nw, register(1), '\x58') POP_b = insn(rex_nw, '\x8F', orbyte(0<<3), stack_bp(1)) Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_rx86.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_rx86.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_rx86.py Wed Aug 11 15:47:41 2010 @@ -188,6 +188,10 @@ assert_encodes_as(cb, 'MOV8_mi', ((edx, 16), 99), '\xC6\x42\x10\x63') assert_encodes_as(cb, 'MOV8_ai', ((ebx, ecx, 2, 16), 99), '\xC6\x44\x8B\x10\x63') +def test_push32(): + cb = CodeBuilder32 + assert_encodes_as(cb, 'PUSH_i32', (9,), '\x68\x09\x00\x00\x00') + class CodeBuilder64(CodeBuilderMixin, X86_64_CodeBuilder): pass Modified: pypy/branch/asmgcc-64/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/metainterp/optimizeopt.py Wed Aug 11 15:47:41 2010 @@ -612,7 +612,7 @@ assert oldop.opnum == op.opnum self.make_equal_to(op.result, self.getvalue(oldop.result)) return - elif self.find_rewriteable_bool(op, args): + elif self.find_rewritable_bool(op, args): return else: self.pure_operations[args] = op @@ -635,7 +635,7 @@ return False - def find_rewriteable_bool(self, op, args): + def find_rewritable_bool(self, op, args): try: oldopnum = opboolinvers[op.opnum] targs = [args[0], args[1], ConstInt(oldopnum)] Modified: pypy/branch/asmgcc-64/pypy/module/__builtin__/test/test_buffer.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/__builtin__/test/test_buffer.py (original) +++ pypy/branch/asmgcc-64/pypy/module/__builtin__/test/test_buffer.py Wed Aug 11 15:47:41 2010 @@ -1,8 +1,11 @@ """Tests some behaviour of the buffer type that is not tested in lib-python/2.5.2/test/test_types.py where the stdlib buffer tests live.""" import autopath +from pypy.conftest import gettestobjspace class AppTestBuffer: + def setup_class(cls): + cls.space = gettestobjspace(usemodules=('array',)) def test_unicode_buffer(self): import sys Modified: pypy/branch/asmgcc-64/pypy/module/_codecs/test/test_codecs.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/_codecs/test/test_codecs.py (original) +++ pypy/branch/asmgcc-64/pypy/module/_codecs/test/test_codecs.py Wed Aug 11 15:47:41 2010 @@ -123,6 +123,10 @@ class AppTestPartialEvaluation: + def setup_class(cls): + space = gettestobjspace(usemodules=('array',)) + cls.space = space + def test_partial_utf8(self): import _codecs encoding = 'utf-8' Modified: pypy/branch/asmgcc-64/pypy/module/_file/test/test_file_extra.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/_file/test/test_file_extra.py (original) +++ pypy/branch/asmgcc-64/pypy/module/_file/test/test_file_extra.py Wed Aug 11 15:47:41 2010 @@ -353,6 +353,10 @@ class AppTestAFewExtra: + def setup_class(cls): + space = gettestobjspace(usemodules=('array',)) + cls.space = space + def setup_method(self, method): fn = str(udir.join('temptestfile')) self.w_temptestfile = self.space.wrap(fn) Modified: pypy/branch/asmgcc-64/pypy/module/_socket/test/test_sock_app.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/_socket/test/test_sock_app.py (original) +++ pypy/branch/asmgcc-64/pypy/module/_socket/test/test_sock_app.py Wed Aug 11 15:47:41 2010 @@ -4,7 +4,7 @@ from pypy.tool.udir import udir def setup_module(mod): - mod.space = gettestobjspace(usemodules=['_socket']) + mod.space = gettestobjspace(usemodules=['_socket', 'array']) global socket import socket mod.w_socket = space.appexec([], "(): import _socket as m; return m") Modified: pypy/branch/asmgcc-64/pypy/module/_sre/test/test_app_sre.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/_sre/test/test_app_sre.py (original) +++ pypy/branch/asmgcc-64/pypy/module/_sre/test/test_app_sre.py Wed Aug 11 15:47:41 2010 @@ -87,7 +87,9 @@ class AppTestSreMatch: - + def setup_class(cls): + cls.space = gettestobjspace(usemodules=('array', )) + def test_copy(self): import re # copy support is disabled by default in _sre.c Modified: pypy/branch/asmgcc-64/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/array/interp_array.py (original) +++ pypy/branch/asmgcc-64/pypy/module/array/interp_array.py Wed Aug 11 15:47:41 2010 @@ -14,6 +14,7 @@ from pypy.objspace.std.model import W_Object from pypy.interpreter.argument import Arguments, Signature from pypy.module._file.interp_file import W_File +from pypy.interpreter.buffer import RWBuffer def w_array(space, w_cls, typecode, w_initializer=None, w_args=None): if len(w_args.arguments_w) > 0: @@ -142,6 +143,22 @@ v.typecode = k unroll_typecodes = unrolling_iterable(types.keys()) +class ArrayBuffer(RWBuffer): + def __init__(self, data, bytes): + self.data = data + self.len = bytes + + def getlength(self): + return self.len + + def getitem(self, index): + return self.data[index] + + def setitem(self, index, char): + self.data[index] = char + + + def make_array(mytype): class W_Array(W_ArrayBase): @@ -460,6 +477,7 @@ def mul__Array_ANY(space, self, w_repeat): repeat = space.int_w(w_repeat) a = mytype.w_class(space) + repeat = max(repeat, 0) a.setlen(self.len * repeat) for r in range(repeat): for i in range(self.len): @@ -472,6 +490,7 @@ def inplace_mul__Array_ANY(space, self, w_repeat): repeat = space.int_w(w_repeat) oldlen = self.len + repeat = max(repeat, 0) self.setlen(self.len * repeat) for r in range(1, repeat): for i in range(oldlen): @@ -565,9 +584,8 @@ # Misc methods def buffer__Array(space, self): - from pypy.interpreter.buffer import StringLikeBuffer - w_s = array_tostring__Array(space, self) - return space.wrap(StringLikeBuffer(space, w_s)) + b = ArrayBuffer(self.charbuf(), self.len * mytype.bytes) + return space.wrap(b) def array_buffer_info__Array(space, self): w_ptr = space.wrap(rffi.cast(lltype.Unsigned, self.buffer)) Modified: pypy/branch/asmgcc-64/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/array/test/test_array.py (original) +++ pypy/branch/asmgcc-64/pypy/module/array/test/test_array.py Wed Aug 11 15:47:41 2010 @@ -421,7 +421,7 @@ a = self.array('h', 'Hi') buf = buffer(a) assert buf[1] == 'i' - raises(TypeError, buf.__setitem__, 1, 'o') + #raises(TypeError, buf.__setitem__, 1, 'o') def test_list_methods(self): assert repr(self.array('i')) == "array('i')" @@ -531,10 +531,8 @@ assert len(b) == 0 and b.typecode == 'l' a = self.array('i', [1, 2, 4]) - print "itter" i = iter(a) - print "ok" - raises(TypeError, pickle.dumps, i, 1) + #raises(TypeError, pickle.dumps, i, 1) def test_copy_swap(self): a = self.array('i', [1, 2, 3]) @@ -599,6 +597,13 @@ assert addable() + self.array('i') == 'add' assert self.array('i') + addable() == 'radd' + a = self.array('i', [1, 2]) + assert a * -1 == self.array('i') + b = a + a *= -1 + assert a == self.array('i') + assert b == self.array('i') + def test_delitem(self): a = self.array('i', [1, 2, 3]) del a[1] Modified: pypy/branch/asmgcc-64/pypy/module/fcntl/test/test_fcntl.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/fcntl/test/test_fcntl.py (original) +++ pypy/branch/asmgcc-64/pypy/module/fcntl/test/test_fcntl.py Wed Aug 11 15:47:41 2010 @@ -13,7 +13,7 @@ class AppTestFcntl: def setup_class(cls): - space = gettestobjspace(usemodules=('fcntl',)) + space = gettestobjspace(usemodules=('fcntl', 'array')) cls.space = space tmpprefix = str(udir.ensure('test_fcntl', dir=1).join('tmp_')) cls.w_tmp = space.wrap(tmpprefix) Modified: pypy/branch/asmgcc-64/pypy/module/marshal/test/test_marshalimpl.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/marshal/test/test_marshalimpl.py (original) +++ pypy/branch/asmgcc-64/pypy/module/marshal/test/test_marshalimpl.py Wed Aug 11 15:47:41 2010 @@ -1,9 +1,13 @@ from pypy.module.marshal import interp_marshal from pypy.interpreter.error import OperationError +from pypy.conftest import gettestobjspace import sys class AppTestMarshalMore: + def setup_class(cls): + space = gettestobjspace(usemodules=('array',)) + cls.space = space def test_long_0(self): import marshal Modified: pypy/branch/asmgcc-64/pypy/module/thread/test/test_gil.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/thread/test/test_gil.py (original) +++ pypy/branch/asmgcc-64/pypy/module/thread/test/test_gil.py Wed Aug 11 15:47:41 2010 @@ -1,7 +1,6 @@ import time from pypy.module.thread import gil from pypy.module.thread.test import test_ll_thread -from pypy.rpython.lltypesystem import rffi from pypy.module.thread import ll_thread as thread from pypy.rlib.objectmodel import we_are_translated Modified: pypy/branch/asmgcc-64/pypy/objspace/std/itertype.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/objspace/std/itertype.py (original) +++ pypy/branch/asmgcc-64/pypy/objspace/std/itertype.py Wed Aug 11 15:47:41 2010 @@ -10,9 +10,9 @@ a registration with copy_reg, instead. """ - # cpython does not support pickling iterators - msg = 'Pickling for iterators dissabled as cpython does not support it' - raise OperationError(space.w_TypeError, space.wrap(msg)) + # cpython does not support pickling iterators but stackless python do + #msg = 'Pickling for iterators dissabled as cpython does not support it' + #raise OperationError(space.w_TypeError, space.wrap(msg)) from pypy.objspace.std.iterobject import W_AbstractSeqIterObject assert isinstance(w_self, W_AbstractSeqIterObject) From arigo at codespeak.net Wed Aug 11 15:54:54 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 11 Aug 2010 15:54:54 +0200 (CEST) Subject: [pypy-svn] r76586 - pypy/trunk/pypy/module/cpyext Message-ID: <20100811135454.9D744282BF3@codespeak.net> Author: arigo Date: Wed Aug 11 15:54:53 2010 New Revision: 76586 Modified: pypy/trunk/pypy/module/cpyext/api.py Log: "xyz.so" and "./xyz.so" mean different things to dlopen(). We always want the latter here. Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Wed Aug 11 15:54:53 2010 @@ -1,5 +1,5 @@ import ctypes -import sys +import sys, os import atexit import py @@ -896,6 +896,8 @@ initfunctype = lltype.Ptr(lltype.FuncType([], lltype.Void)) @unwrap_spec(ObjSpace, str, str) def load_extension_module(space, path, name): + if os.sep not in path: + path = os.curdir + os.sep + path # force a '/' in the path state = space.fromcache(State) state.package_context = name try: From agaynor at codespeak.net Wed Aug 11 16:08:34 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Wed, 11 Aug 2010 16:08:34 +0200 (CEST) Subject: [pypy-svn] r76587 - pypy/extradoc/planning Message-ID: <20100811140834.C58E3282BF3@codespeak.net> Author: agaynor Date: Wed Aug 11 16:08:33 2010 New Revision: 76587 Modified: pypy/extradoc/planning/jit.txt Log: CALL_METHOD no support keyword arguments. Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Wed Aug 11 16:08:33 2010 @@ -95,9 +95,6 @@ Python interpreter: -- CALL_METHOD optimization only works for calls with no keyword args. This - is silly and slows down things quite a bit. - - goal: on average <=5 guards per original bytecode. Almost achieved :-) pypy/jit/tool/traceviewer.py can view where we're failing (red boxes out of jit-log-opt logging) From getxsick at codespeak.net Wed Aug 11 16:40:15 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Wed, 11 Aug 2010 16:40:15 +0200 (CEST) Subject: [pypy-svn] r76588 - pypy/branch/fast-ctypes/pypy/rlib Message-ID: <20100811144015.1DEFF282BF3@codespeak.net> Author: getxsick Date: Wed Aug 11 16:40:14 2010 New Revision: 76588 Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: raise OSError in correct way Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Wed Aug 11 16:40:14 2010 @@ -32,7 +32,7 @@ try: self.handler = rdynload.dlopen(name_ptr) except rdynload.DLOpenError, e: - raise OSError('%s: %s', name, e.msg or 'unspecified error') + raise OSError(-1, '%s: %s' % (name, e.msg or 'unspecified error')) finally: rffi.free_charp(name_ptr) From hakanardo at codespeak.net Wed Aug 11 16:53:21 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Wed, 11 Aug 2010 16:53:21 +0200 (CEST) Subject: [pypy-svn] r76589 - in pypy/trunk/pypy: module/array module/array/test rpython/lltypesystem Message-ID: <20100811145321.AC91F282BAD@codespeak.net> Author: hakanardo Date: Wed Aug 11 16:53:20 2010 New Revision: 76589 Modified: pypy/trunk/pypy/module/array/interp_array.py pypy/trunk/pypy/module/array/test/test_array.py pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Log: cpython test fixes Modified: pypy/trunk/pypy/module/array/interp_array.py ============================================================================== --- pypy/trunk/pypy/module/array/interp_array.py (original) +++ pypy/trunk/pypy/module/array/interp_array.py Wed Aug 11 16:53:20 2010 @@ -16,8 +16,8 @@ from pypy.module._file.interp_file import W_File from pypy.interpreter.buffer import RWBuffer -def w_array(space, w_cls, typecode, w_initializer=None, w_args=None): - if len(w_args.arguments_w) > 0: +def w_array(space, w_cls, typecode, w_args=None): + if len(w_args.arguments_w) > 1: msg = 'array() takes at most 2 arguments' raise OperationError(space.w_TypeError, space.wrap(msg)) if len(typecode) != 1: @@ -30,23 +30,23 @@ a = space.allocate_instance(types[tc].w_class, w_cls) a.__init__(space) - if w_initializer is not None: - if not space.is_w(w_initializer, space.w_None): - if space.type(w_initializer) is space.w_str: - a.fromstring(w_initializer) - elif space.type(w_initializer) is space.w_unicode: - a.fromsequence(w_initializer) - elif space.type(w_initializer) is space.w_list: - a.fromlist(w_initializer) - else: - a.extend(w_initializer) + if len(w_args.arguments_w) > 0: + w_initializer = w_args.arguments_w[0] + if space.type(w_initializer) is space.w_str: + a.fromstring(w_initializer) + elif space.type(w_initializer) is space.w_unicode: + a.fromsequence(w_initializer) + elif space.type(w_initializer) is space.w_list: + a.fromlist(w_initializer) + else: + a.extend(w_initializer) break else: msg = 'bad typecode (must be c, b, B, u, h, H, i, I, l, L, f or d)' raise OperationError(space.w_ValueError, space.wrap(msg)) return a -w_array.unwrap_spec = (ObjSpace, W_Root, str, W_Root, Arguments) +w_array.unwrap_spec = (ObjSpace, W_Root, str, Arguments) array_append = SMM('append', 2) @@ -579,7 +579,10 @@ w_lst2 = space.call_method(other, 'tolist') return space.cmp(w_lst1, w_lst2) else: - raise OperationError(space.w_NotImplementedError, space.wrap('')) + try: + return space.call_method(other, '__cmp__', self) + except OperationError: + return space.wrap(-1) # Misc methods @@ -651,8 +654,21 @@ from pypy.objspace.std.sliceobject import W_SliceObject from pypy.objspace.std.listobject import W_ListObject from pypy.objspace.std.unicodeobject import W_UnicodeObject + #localdict = locals() + #localdict['W_ArrayBase'] = W_ArrayBase register_all(locals(), globals()) for mytype in types.values(): make_array(mytype) + +#def cmp__ArrayBase_ArrayBase(space, self, other): +# print 78 +# w_lst1 = space.call_method(self, 'tolist') +# w_lst2 = space.call_method(other, 'tolist') +# return space.cmp(w_lst1, w_lst2) +# +#def cmp__ArrayBase_ANY(space, self, other): +# return space.wrap(1) + +register_all(locals(), globals()) Modified: pypy/trunk/pypy/module/array/test/test_array.py ============================================================================== --- pypy/trunk/pypy/module/array/test/test_array.py (original) +++ pypy/trunk/pypy/module/array/test/test_array.py Wed Aug 11 16:53:20 2010 @@ -61,6 +61,7 @@ for tc in 'bhilBHILfd': assert self.array(tc).typecode == tc + raises(TypeError, self.array, tc, None) def test_value_range(self): values = (-129, 128, -128, 127, 0, 255, -1, 256, @@ -468,6 +469,12 @@ assert repr(a) == "array('i', [20, 8, 2, 9, 7, 10])" def test_compare(self): + class comparable(object): + def __cmp__(self, other): + return 0 + class incomparable(object): + pass + for v1, v2, tt in (([1, 2, 3], [1, 3, 2], 'bhilBHIL'), ('abc', 'acb', 'c'), (unicode('abc'), unicode('acb'), 'u')): @@ -476,6 +483,13 @@ b = self.array(t, v1) c = self.array(t, v2) + print (a==7) + assert (a == 7) is False + assert (comparable() == a) is True + assert (a == comparable()) is True + assert (a == incomparable()) is False + assert (incomparable() == a) is False + assert (a == a) is True assert (a == b) is True assert (b == a) is True @@ -731,6 +745,12 @@ assert repr(mya('i', [1, 2, 3])) == "array('i', [1, 2, 3])" assert repr(mya('i', (1, 2, 3))) == "array('i', [1, 2, 3])" + def test_unicode_outofrange(self): + a = self.array('u', unicode(r'\x01\u263a\x00\ufeff', 'unicode-escape')) + b = self.array('u', unicode(r'\x01\u263a\x00\ufeff', 'unicode-escape')) + b.byteswap() + assert a != b + class TestCPythonsOwnArray(BaseArrayTests): Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Wed Aug 11 16:53:20 2010 @@ -757,6 +757,10 @@ llobj = chr(cobj) elif T is lltype.UniChar: llobj = unichr(cobj) + #try: + # llobj = unichr(cobj) + #except ValueError: + # llobj = u'\x00' # FIXME: Hack elif T is lltype.Signed: llobj = cobj elif T is lltype.Bool: From hakanardo at codespeak.net Wed Aug 11 17:07:53 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Wed, 11 Aug 2010 17:07:53 +0200 (CEST) Subject: [pypy-svn] r76590 - pypy/trunk/pypy/module/array Message-ID: <20100811150753.4E570282BF3@codespeak.net> Author: hakanardo Date: Wed Aug 11 17:07:51 2010 New Revision: 76590 Modified: pypy/trunk/pypy/module/array/interp_array.py Log: clean up cmp mess Modified: pypy/trunk/pypy/module/array/interp_array.py ============================================================================== --- pypy/trunk/pypy/module/array/interp_array.py (original) +++ pypy/trunk/pypy/module/array/interp_array.py Wed Aug 11 17:07:51 2010 @@ -579,10 +579,7 @@ w_lst2 = space.call_method(other, 'tolist') return space.cmp(w_lst1, w_lst2) else: - try: - return space.call_method(other, '__cmp__', self) - except OperationError: - return space.wrap(-1) + return space.w_NotImplemented # Misc methods @@ -654,21 +651,10 @@ from pypy.objspace.std.sliceobject import W_SliceObject from pypy.objspace.std.listobject import W_ListObject from pypy.objspace.std.unicodeobject import W_UnicodeObject - #localdict = locals() - #localdict['W_ArrayBase'] = W_ArrayBase register_all(locals(), globals()) for mytype in types.values(): make_array(mytype) -#def cmp__ArrayBase_ArrayBase(space, self, other): -# print 78 -# w_lst1 = space.call_method(self, 'tolist') -# w_lst2 = space.call_method(other, 'tolist') -# return space.cmp(w_lst1, w_lst2) -# -#def cmp__ArrayBase_ANY(space, self, other): -# return space.wrap(1) - register_all(locals(), globals()) From arigo at codespeak.net Wed Aug 11 18:05:06 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 11 Aug 2010 18:05:06 +0200 (CEST) Subject: [pypy-svn] r76591 - pypy/trunk/pypy/jit/metainterp/doc Message-ID: <20100811160506.6BC9D282BF3@codespeak.net> Author: arigo Date: Wed Aug 11 18:05:04 2010 New Revision: 76591 Removed: pypy/trunk/pypy/jit/metainterp/doc/ Log: Kill old directory. See the docs in pypy/doc/jit/. From fijal at codespeak.net Wed Aug 11 18:12:32 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 11 Aug 2010 18:12:32 +0200 (CEST) Subject: [pypy-svn] r76592 - pypy/extradoc/planning Message-ID: <20100811161232.76382282BEC@codespeak.net> Author: fijal Date: Wed Aug 11 18:12:31 2010 New Revision: 76592 Modified: pypy/extradoc/planning/jit.txt Log: (anto, fijal, arigo) Clean up the jit-related tasks. Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Wed Aug 11 18:12:31 2010 @@ -1,21 +1,20 @@ NEW TASKS --------- -- look at assembler-assembler calls again: if the inner function is traced - after the outer one, the call is slow. other cases could be faster too, - probably. +- look at assembler-assembler calls again: if the inner function is + traced after the outer one, the call is slow. Might be solved + easily if we implement full out-of-line guards (e.g. by invalidating + the outer function when the inner one gets compiled) - have benchmarks for jit compile time and jit memory usage - trace into functions even if they have a loop. only if the loop is actually - hit, a residual portal call is produced + hit, a residual portal call is produced (status: kill-caninline branch, + buggy) - generators are not really fast ? maybe add a JUMP_ABSOLUTE_GENERATOR that does not call can_enter_jit after an iteration in which there was a yield. - obviously. - -- think about handling hidden registers - mayb we can use couple of - first spots on the stack as registers + obviously. (status: kill-caninline branch) - think again about perfect specialization. Check if we loose anything if we turn it off. Another approach to specialization: specialize things @@ -28,60 +27,25 @@ current exception from the struct in memory, followed by a regular GUARD_CLASS. -- getfields which result is never used never get removed (probably cause - - they used to be as livevars in removed guards). also getfields which result - is only used as a livevar in a guard should be removed and encoded in - the guard recovert code. +- write a document that says what you cannot expect the jit to optimize. + E.g. http://paste.pocoo.org/show/181319/ with B being old-style and + C being new-style, or vice-versa. -- think about strings more. since string are immutable, unnecessary copies - does not make any sense (sometimes people construct strings through - arrays, which is harder to track and then not use the result) -TASKS ------ +OPTIMIZATIONS +------------- -- think about code memory management - -- forcing virtualizables should only force fields affected, not everything +Things we can do mostly by editing optimizeopt.py: -- think out looking into functions or not, based on arguments, - for example contains__Tuple should be unrolled if tuple is of constant - length. HARD, blocked by the fact that we don't know constants soon enough - -- look at example of storing small strings in large lists (any sane templating - engine would do it) and not spend all the time in - _trace_and_drag_out_of_nursery. - Requires thinking about card-marking GC, which is hard, postpone - - Card marking GC is hard in our model. Think about solutions to chunked - list (a list that if big enough is a list of lists of stuff). Experiments - show that it helps a lot for some examples. - -- improve tracing/blackholing speed - Essential, especially blackholing in translate.py, as well as html5lib. - -- some guards will always fail if they ever start failing - (e.g. the class version tag). Do something more clever about it. - (already done a hack that helps: don't compile more guard_value's - if the value guarded for keeps changing fast enough: r71527) +- getfields which result is never used never get removed (probably cause - + they used to be as livevars in removed guards). also getfields which result + is only used as a livevar in a guard should be removed and encoded in + the guard recovert code (only if we are sure that the stored field cannot + change) - int_add_ovf(x, 0) guard_overflow is 20% of all int_add_ovf, not much overall, but probably worth attacking -- think about such example: - - http://paste.pocoo.org/show/188520/ - - this will compile new assembler path for each new type, even though that's - overspecialization since in this particular case it's not relevant. - This is treated as a megamorphic call (promotion of w_self in typeobject.py) - while in fact it is not. - -- a suggestion - if we call some code via call_assembler that raises an - exception, in theory we could do something smarter in case our frames - don't escape and call simplified version that does not allocate all - frames. Sounds hard - - if we move a promotion up the chain, some arguments don't get replaced with constants (those between current and previous locations). So we get like @@ -93,102 +57,38 @@ maybe we should move promote even higher, before the first use and we could possibly remove more stuff? -Python interpreter: - -- goal: on average <=5 guards per original bytecode. - Almost achieved :-) pypy/jit/tool/traceviewer.py can view where we're - failing (red boxes out of jit-log-opt logging) - -- put the class into the structure to get only one promote when using an - instance - -- this example: http://paste.pocoo.org/show/181319/ - showcases a problem that works fine as long as you not present a - combination of oldstyle and newstyle classes. If you however present - a combination of old and newstyle classes (try modifying) things go - far slower and traces look bad. - DON'T DO THAT? - -Benchmark Notes ----------------------------- - - spitfire: - - it's an issue with GC, that's probably won't fix. On spitfire's small - benchmarks we're either matching CPython or are faster (if using cStringIO) +PYTHON EXAMPLES +--------------- - - html5lib: - - we're faster (a bit) than CPython on a long enough run. blackholing is - an issue there. - - slowness seems to be mostly the fault of PyUnicode_DecodeCharmap in - module/_codecs/app_codecs.py. Are such things not jitted? - - the tokenizer uses regular expressions and generators, which probably - doesn't help +Extracted from some real-life Python programs, examples that don't give +nice code at all so far: - - spambayes - - uses regular expressions and generators a lot - - regexes are 80% of runtime of long-enough run - - - ai - - the slowness is the fault of generators and generator expressions - - many of the generator expressions are a bit stupid (like tuple()) - WON'T FIX, maybe? - - -JIT-related Release Tasks ---------------------------- - -(there are other release tasks, specifically about packaging, documentation, -website and stability that need sorting out too. However, they are beyond the -scope of this section) - -wishlist: -- the checks that look whether profiling/tracing in the Python interpreter is - enabled look expensive. Do we want to do something about them? - - - - -META ------ - -- stability! - -- keep test coverage in check - -- prevent too much method and fields demoting in the jit - -- the tragedy of the skipped tests - -- update things in metainterp/doc - -inlining discussion --------------------- - -- at some point we need to merge the tails of loops, to avoid exponential - explosion -- tracing aggressively will put pressure on the speed of tracing -- what should we do about recursive calls? +- http://paste.pocoo.org/show/188520/ + this will compile new assembler path for each new type, even though that's + overspecialization since in this particular case it's not relevant. + This is treated as a megamorphic call (promotion of w_self in typeobject.py) + while in fact it is not. +- pypy/objspace/std/inlinedict: put the class into the structure to get + only one promote when using an instance, instead of two: the promotion + of the '.w__class__' and the promotion of the '.structure' -things we know are missing ---------------------------- +- guard_true(frame.is_being_profiled) all over the place -tests: -- find a test for r64742 (JitException capture) +- xxx (find more examples :-) -Goals/Benchmarks ------------------ -Goal: be somehow faster than CPython in real programs - actually, DONE! +LATER (maybe) TASKS +------------------- -Benchmarks: - they live at svn+ssh://codespeak.net/svn/pypy/benchmarks +- think about code memory management +- think out looking into functions or not, based on arguments, + for example contains__Tuple should be unrolled if tuple is of constant + length. HARD, blocked by the fact that we don't know constants soon enough -ootype discussion ------------------- +- out-of-line guards (when an external change would invalidate existing + pieces of assembler) -- try to unify interfaces to make doing the right thing for ootype easier -- different constraints for different groups of people -- what to do with ootype jit support after Anto finished his PhD? +- merge tails of loops-and-bridges? From hakanardo at codespeak.net Wed Aug 11 19:10:41 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Wed, 11 Aug 2010 19:10:41 +0200 (CEST) Subject: [pypy-svn] r76593 - in pypy/trunk/pypy: module/array rpython/lltypesystem Message-ID: <20100811171041.03054282BEC@codespeak.net> Author: hakanardo Date: Wed Aug 11 19:10:40 2010 New Revision: 76593 Modified: pypy/trunk/pypy/module/array/interp_array.py pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Log: support pickling of subclasses of arrays Modified: pypy/trunk/pypy/module/array/interp_array.py ============================================================================== --- pypy/trunk/pypy/module/array/interp_array.py (original) +++ pypy/trunk/pypy/module/array/interp_array.py Wed Aug 11 19:10:40 2010 @@ -598,7 +598,11 @@ args = [space.wrap(mytype.typecode), w_s] else: args = [space.wrap(mytype.typecode)] - return space.newtuple([space.type(self), space.newtuple(args)]) + try: + dct = space.getattr(self, space.wrap('__dict__')) + except OperationError: + dct = space.w_None + return space.newtuple([space.type(self), space.newtuple(args), dct]) def array_byteswap__Array(space, self): if mytype.bytes not in [1, 2, 4, 8]: Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Wed Aug 11 19:10:40 2010 @@ -24,6 +24,7 @@ from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE from pypy.rpython import raddress from pypy.translator.platform import platform +from array import array def uaddressof(obj): return fixid(ctypes.addressof(obj)) @@ -756,11 +757,15 @@ elif T is lltype.Char: llobj = chr(cobj) elif T is lltype.UniChar: - llobj = unichr(cobj) - #try: - # llobj = unichr(cobj) - #except ValueError: - # llobj = u'\x00' # FIXME: Hack + try: + llobj = unichr(cobj) + except ValueError: + for tc in 'HIL': + if array(tc).itemsize == array('u').itemsize: + llobj = array('u', array(tc, (cobj,)).tostring())[0] + break + else: + raise elif T is lltype.Signed: llobj = cobj elif T is lltype.Bool: From hakanardo at codespeak.net Wed Aug 11 19:23:52 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Wed, 11 Aug 2010 19:23:52 +0200 (CEST) Subject: [pypy-svn] r76594 - pypy/trunk/lib_pypy Message-ID: <20100811172352.6AC04282BEC@codespeak.net> Author: hakanardo Date: Wed Aug 11 19:23:50 2010 New Revision: 76594 Modified: pypy/trunk/lib_pypy/array.py Log: old applevel implementation Modified: pypy/trunk/lib_pypy/array.py ============================================================================== --- pypy/trunk/lib_pypy/array.py (original) +++ pypy/trunk/lib_pypy/array.py Wed Aug 11 19:23:50 2010 @@ -1,24 +1,531 @@ -from array import array as _array +"""This module defines an object type which can efficiently represent +an array of basic values: characters, integers, floating point +numbers. Arrays are sequence types and behave very much like lists, +except that the type of objects stored in them is constrained. The +type is specified at object creation time by using a type code, which +is a single character. The following type codes are defined: + + Type code C Type Minimum size in bytes + 'c' character 1 + 'b' signed integer 1 + 'B' unsigned integer 1 + 'u' Unicode character 2 + 'h' signed integer 2 + 'H' unsigned integer 2 + 'i' signed integer 2 + 'I' unsigned integer 2 + 'l' signed integer 4 + 'L' unsigned integer 4 + 'f' floating point 4 + 'd' floating point 8 + +The constructor is: + +array(typecode [, initializer]) -- create a new array +""" + +from struct import calcsize, pack, pack_into, unpack_from +import operator + +# the buffer-like object to use internally: trying from +# various places in order... +try: + import _rawffi # a reasonable implementation based + _RAWARRAY = _rawffi.Array('c') # on raw_malloc, and providing a + def bytebuffer(size): # real address + return _RAWARRAY(size, autofree=True) + def getbufaddress(buf): + return buf.buffer +except ImportError: + try: + from __pypy__ import bytebuffer # a reasonable implementation + def getbufaddress(buf): # compatible with oo backends, + return 0 # but no address + except ImportError: + # not running on PyPy. Fall back to ctypes... + import ctypes + bytebuffer = ctypes.create_string_buffer + def getbufaddress(buf): + voidp = ctypes.cast(ctypes.pointer(buf), ctypes.c_void_p) + return voidp.value + +# ____________________________________________________________ + +TYPECODES = "cbBuhHiIlLfd" class array(object): - def __init__(self, typecode, initializer=None): - self._array = _array(typecode) - if initializer is not None: + """array(typecode [, initializer]) -> array + + Return a new array whose items are restricted by typecode, and + initialized from the optional initializer value, which must be a list, + string. or iterable over elements of the appropriate type. + + Arrays represent basic values and behave very much like lists, except + the type of objects stored in them is constrained. + + Methods: + + append() -- append a new item to the end of the array + buffer_info() -- return information giving the current memory info + byteswap() -- byteswap all the items of the array + count() -- return number of occurences of an object + extend() -- extend array by appending multiple elements from an iterable + fromfile() -- read items from a file object + fromlist() -- append items from the list + fromstring() -- append items from the string + index() -- return index of first occurence of an object + insert() -- insert a new item into the array at a provided position + pop() -- remove and return item (default last) + read() -- DEPRECATED, use fromfile() + remove() -- remove first occurence of an object + reverse() -- reverse the order of the items in the array + tofile() -- write all items to a file object + tolist() -- return the array converted to an ordinary list + tostring() -- return the array converted to a string + write() -- DEPRECATED, use tofile() + + Attributes: + + typecode -- the typecode character used to create the array + itemsize -- the length in bytes of one array item + """ + __slots__ = ["typecode", "itemsize", "_data", "_descriptor", "__weakref__"] + + def __new__(cls, typecode, initializer=[], **extrakwds): + self = object.__new__(cls) + if cls is array and extrakwds: + raise TypeError("array() does not take keyword arguments") + if not isinstance(typecode, str) or len(typecode) != 1: + raise TypeError( + "array() argument 1 must be char, not %s" % type(typecode)) + if typecode not in TYPECODES: + raise ValueError( + "bad typecode (must be one of %s)" % ', '.join(TYPECODES)) + self._data = bytebuffer(0) + self.typecode = typecode + self.itemsize = calcsize(typecode) + if isinstance(initializer, list): + self.fromlist(initializer) + elif isinstance(initializer, str): + self.fromstring(initializer) + elif isinstance(initializer, unicode) and self.typecode == "u": + self.fromunicode(initializer) + else: self.extend(initializer) + return self + def _clear(self): + self._data = bytebuffer(0) - def append(self ,x): - self._array.append(x) - def __getitem__(self, idx): - return self._array[idx] - def __setitem__(self, idx, val): - self._array[idx]=val - def __len__(self): - return len(self._array) + ##### array-specific operations + def fromfile(self, f, n): + """Read n objects from the file object f and append them to the end of + the array. Also called as read.""" + if not isinstance(f, file): + raise TypeError("arg1 must be open file") + size = self.itemsize * n + item = f.read(size) + if len(item) < size: + raise EOFError("not enough items in file") + self.fromstring(item) + + def fromlist(self, l): + """Append items to array from list.""" + if not isinstance(l, list): + raise TypeError("arg must be list") + self._fromiterable(l) + + def fromstring(self, s): + """Appends items from the string, interpreting it as an array of machine + values, as if it had been read from a file using the fromfile() + method.""" + if isinstance(s, unicode): + s = str(s) + self._frombuffer(s) + + def _frombuffer(self, s): + length = len(s) + if length % self.itemsize != 0: + raise ValueError("string length not a multiple of item size") + boundary = len(self._data) + newdata = bytebuffer(boundary + length) + newdata[:boundary] = self._data + newdata[boundary:] = s + self._data = newdata + + def fromunicode(self, ustr): + """Extends this array with data from the unicode string ustr. The array + must be a type 'u' array; otherwise a ValueError is raised. Use + array.fromstring(ustr.encode(...)) to append Unicode data to an array of + some other type.""" + if not self.typecode == "u": + raise ValueError( + "fromunicode() may only be called on type 'u' arrays") + # XXX the following probable bug is not emulated: + # CPython accepts a non-unicode string or a buffer, and then + # behaves just like fromstring(), except that it strangely truncates + # string arguments at multiples of the unicode byte size. + # Let's only accept unicode arguments for now. + if not isinstance(ustr, unicode): + raise TypeError("fromunicode() argument should probably be " + "a unicode string") + # _frombuffer() does the currect thing using + # the buffer behavior of unicode objects + self._frombuffer(buffer(ustr)) + + def tofile(self, f): + """Write all items (as machine values) to the file object f. Also + called as write.""" + if not isinstance(f, file): + raise TypeError("arg must be open file") + f.write(self.tostring()) + + def tolist(self): + """Convert array to an ordinary list with the same items.""" + count = len(self._data) // self.itemsize + return list(unpack_from('%d%s' % (count, self.typecode), self._data)) + + def tostring(self): + return self._data[:] + + def __buffer__(self): + return buffer(self._data) + + def tounicode(self): + """Convert the array to a unicode string. The array must be a type 'u' + array; otherwise a ValueError is raised. Use array.tostring().decode() + to obtain a unicode string from an array of some other type.""" + if self.typecode != "u": + raise ValueError("tounicode() may only be called on type 'u' arrays") + # XXX performance is not too good + return u"".join(self.tolist()) + + def byteswap(self): + """Byteswap all items of the array. If the items in the array are not + 1, 2, 4, or 8 bytes in size, RuntimeError is raised.""" + if self.itemsize not in [1, 2, 4, 8]: + raise RuntimeError("byteswap not supported for this array") + # XXX slowish + itemsize = self.itemsize + bytes = self._data + for start in range(0, len(bytes), itemsize): + stop = start + itemsize + bytes[start:stop] = bytes[start:stop][::-1] + + def buffer_info(self): + """Return a tuple (address, length) giving the current memory address + and the length in items of the buffer used to hold array's contents. The + length should be multiplied by the itemsize attribute to calculate the + buffer length in bytes. On PyPy the address might be meaningless + (returned as 0), depending on the available modules.""" + return (getbufaddress(self._data), len(self)) + read = fromfile + + write = tofile + + ##### general object protocol + + def __repr__(self): + if len(self._data) == 0: + return "array('%s')" % self.typecode + elif self.typecode == "c": + return "array('%s', %s)" % (self.typecode, repr(self.tostring())) + elif self.typecode == "u": + return "array('%s', %s)" % (self.typecode, repr(self.tounicode())) + else: + return "array('%s', %s)" % (self.typecode, repr(self.tolist())) + + def __copy__(self): + a = array(self.typecode) + a._data = bytebuffer(len(self._data)) + a._data[:] = self._data + return a + + def __eq__(self, other): + if not isinstance(other, array): + return NotImplemented + if self.typecode == 'c': + return buffer(self._data) == buffer(other._data) + else: + return self.tolist() == other.tolist() + + def __ne__(self, other): + if not isinstance(other, array): + return NotImplemented + if self.typecode == 'c': + return buffer(self._data) != buffer(other._data) + else: + return self.tolist() != other.tolist() + + def __lt__(self, other): + if not isinstance(other, array): + return NotImplemented + if self.typecode == 'c': + return buffer(self._data) < buffer(other._data) + else: + return self.tolist() < other.tolist() + + def __gt__(self, other): + if not isinstance(other, array): + return NotImplemented + if self.typecode == 'c': + return buffer(self._data) > buffer(other._data) + else: + return self.tolist() > other.tolist() + + def __le__(self, other): + if not isinstance(other, array): + return NotImplemented + if self.typecode == 'c': + return buffer(self._data) <= buffer(other._data) + else: + return self.tolist() <= other.tolist() + + def __ge__(self, other): + if not isinstance(other, array): + return NotImplemented + if self.typecode == 'c': + return buffer(self._data) >= buffer(other._data) + else: + return self.tolist() >= other.tolist() + + def __reduce__(self): + dict = getattr(self, '__dict__', None) + data = self.tostring() + if data: + initargs = (self.typecode, data) + else: + initargs = (self.typecode,) + return (type(self), initargs, dict) + + ##### list methods + + def append(self, x): + """Append new value x to the end of the array.""" + self._frombuffer(pack(self.typecode, x)) + + def count(self, x): + """Return number of occurences of x in the array.""" + return operator.countOf(self, x) + def extend(self, iterable): - for i in iterable: self.append(i) + """Append items to the end of the array.""" + if isinstance(iterable, array) \ + and not self.typecode == iterable.typecode: + raise TypeError("can only extend with array of same kind") + self._fromiterable(iterable) + + def index(self, x): + """Return index of first occurence of x in the array.""" + return operator.indexOf(self, x) + def insert(self, i, x): + """Insert a new item x into the array before position i.""" + seqlength = len(self) + if i < 0: + i += seqlength + if i < 0: + i = 0 + elif i > seqlength: + i = seqlength + boundary = i * self.itemsize + data = pack(self.typecode, x) + newdata = bytebuffer(len(self._data) + len(data)) + newdata[:boundary] = self._data[:boundary] + newdata[boundary:boundary+self.itemsize] = data + newdata[boundary+self.itemsize:] = self._data[boundary:] + self._data = newdata + + def pop(self, i=-1): + """Return the i-th element and delete it from the array. i defaults to + -1.""" + seqlength = len(self) + if i < 0: + i += seqlength + if not (0 <= i < seqlength): + raise IndexError(i) + boundary = i * self.itemsize + result = unpack_from(self.typecode, self._data, boundary)[0] + newdata = bytebuffer(len(self._data) - self.itemsize) + newdata[:boundary] = self._data[:boundary] + newdata[boundary:] = self._data[boundary+self.itemsize:] + self._data = newdata + return result + + def remove(self, x): + """Remove the first occurence of x in the array.""" + self.pop(self.index(x)) + + def reverse(self): + """Reverse the order of the items in the array.""" + lst = self.tolist() + lst.reverse() + self._clear() + self.fromlist(lst) + ##### list protocol + def __len__(self): + return len(self._data) // self.itemsize + + def __add__(self, other): + if not isinstance(other, array): + raise TypeError("can only append array to array") + if self.typecode != other.typecode: + raise TypeError("bad argument type for built-in operation") + return array(self.typecode, buffer(self._data) + buffer(other._data)) + + def __mul__(self, repeat): + return array(self.typecode, buffer(self._data) * repeat) + + __rmul__ = __mul__ + + def __getitem__(self, i): + seqlength = len(self) + if isinstance(i, slice): + start, stop, step = i.indices(seqlength) + if step != 1: + sublist = self.tolist()[i] # fall-back + return array(self.typecode, sublist) + if start < 0: + start = 0 + if stop < start: + stop = start + assert stop <= seqlength + return array(self.typecode, self._data[start * self.itemsize : + stop * self.itemsize]) + else: + if i < 0: + i += seqlength + if self.typecode == 'c': # speed trick + return self._data[i] + if not (0 <= i < seqlength): + raise IndexError(i) + boundary = i * self.itemsize + return unpack_from(self.typecode, self._data, boundary)[0] + + def __getslice__(self, i, j): + return self.__getitem__(slice(i, j)) + + def __setitem__(self, i, x): + if isinstance(i, slice): + if (not isinstance(x, array) + or self.typecode != x.typecode): + raise TypeError("can only assign array of same kind" + " to array slice") + seqlength = len(self) + start, stop, step = i.indices(seqlength) + if step != 1: + sublist = self.tolist() # fall-back + sublist[i] = x.tolist() + self._clear() + self.fromlist(sublist) + return + if start < 0: + start = 0 + if stop < start: + stop = start + assert stop <= seqlength + boundary1 = start * self.itemsize + boundary2 = stop * self.itemsize + boundary2new = boundary1 + len(x._data) + if boundary2 == boundary2new: + self._data[boundary1:boundary2] = x._data + else: + newdata = bytebuffer(len(self._data) + boundary2new-boundary2) + newdata[:boundary1] = self._data[:boundary1] + newdata[boundary1:boundary2new] = x._data + newdata[boundary2new:] = self._data[boundary2:] + self._data = newdata + else: + seqlength = len(self) + if i < 0: + i += seqlength + if self.typecode == 'c': # speed trick + self._data[i] = x + return + if not (0 <= i < seqlength): + raise IndexError(i) + boundary = i * self.itemsize + pack_into(self.typecode, self._data, boundary, x) + + def __setslice__(self, i, j, x): + self.__setitem__(slice(i, j), x) + + def __delitem__(self, i): + if isinstance(i, slice): + seqlength = len(self) + start, stop, step = i.indices(seqlength) + if start < 0: + start = 0 + if stop < start: + stop = start + assert stop <= seqlength + if step != 1: + sublist = self.tolist() # fall-back + del sublist[i] + self._clear() + self.fromlist(sublist) + return + dellength = stop - start + boundary1 = start * self.itemsize + boundary2 = stop * self.itemsize + newdata = bytebuffer(len(self._data) - (boundary2-boundary1)) + newdata[:boundary1] = self._data[:boundary1] + newdata[boundary1:] = self._data[boundary2:] + self._data = newdata + else: + seqlength = len(self) + if i < 0: + i += seqlength + if not (0 <= i < seqlength): + raise IndexError(i) + boundary = i * self.itemsize + newdata = bytebuffer(len(self._data) - self.itemsize) + newdata[:boundary] = self._data[:boundary] + newdata[boundary:] = self._data[boundary+self.itemsize:] + self._data = newdata + + def __delslice__(self, i, j): + self.__delitem__(slice(i, j)) + + def __contains__(self, item): + for x in self: + if x == item: + return True + return False + + def __iadd__(self, other): + if not isinstance(other, array): + raise TypeError("can only extend array with array") + self.extend(other) + return self + + def __imul__(self, repeat): + newdata = buffer(self._data) * repeat + self._data = bytebuffer(len(newdata)) + self._data[:] = newdata + return self + + def __iter__(self): + p = 0 + typecode = self.typecode + itemsize = self.itemsize + while p < len(self._data): + yield unpack_from(typecode, self._data, p)[0] + p += itemsize + + ##### internal methods + + def _fromiterable(self, iterable): + iterable = tuple(iterable) + n = len(iterable) + boundary = len(self._data) + newdata = bytebuffer(boundary + n * self.itemsize) + newdata[:boundary] = self._data + pack_into('%d%s' % (n, self.typecode), newdata, boundary, *iterable) + self._data = newdata + +ArrayType = array From hakanardo at codespeak.net Wed Aug 11 19:29:02 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Wed, 11 Aug 2010 19:29:02 +0200 (CEST) Subject: [pypy-svn] r76595 - in pypy/trunk: lib-python pypy/config Message-ID: <20100811172902.3A8C8282BEC@codespeak.net> Author: hakanardo Date: Wed Aug 11 19:28:56 2010 New Revision: 76595 Modified: pypy/trunk/lib-python/conftest.py pypy/trunk/pypy/config/pypyoption.py Log: reenabled array Modified: pypy/trunk/lib-python/conftest.py ============================================================================== --- pypy/trunk/lib-python/conftest.py (original) +++ pypy/trunk/lib-python/conftest.py Wed Aug 11 19:28:56 2010 @@ -132,7 +132,7 @@ RegrTest('test_ast.py', core=True), RegrTest('test_anydbm.py'), RegrTest('test_applesingle.py', skip=True), - RegrTest('test_array.py', core=True, usemodules='struct'), + RegrTest('test_array.py', core=True, usemodules='struct array'), RegrTest('test_asynchat.py', usemodules='thread'), RegrTest('test_atexit.py', core=True), RegrTest('test_audioop.py', skip=True), Modified: pypy/trunk/pypy/config/pypyoption.py ============================================================================== --- pypy/trunk/pypy/config/pypyoption.py (original) +++ pypy/trunk/pypy/config/pypyoption.py Wed Aug 11 19:28:56 2010 @@ -30,7 +30,7 @@ "rctime" , "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios", "zlib", "struct", "md5", "sha", "bz2", "_minimal_curses", "cStringIO", - "thread", "itertools", "pyexpat", "_ssl", "cpyext"] + "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array"] )) working_oo_modules = default_modules.copy() From arigo at codespeak.net Wed Aug 11 19:38:08 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 11 Aug 2010 19:38:08 +0200 (CEST) Subject: [pypy-svn] r76596 - pypy/branch/memrecord Message-ID: <20100811173808.E6E71282BEC@codespeak.net> Author: arigo Date: Wed Aug 11 19:38:06 2010 New Revision: 76596 Removed: pypy/branch/memrecord/ Log: Kill the branch, now that 'undodb' eventually worked. From hakanardo at codespeak.net Wed Aug 11 20:26:09 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Wed, 11 Aug 2010 20:26:09 +0200 (CEST) Subject: [pypy-svn] r76597 - pypy/trunk/pypy/rpython/lltypesystem Message-ID: <20100811182609.48189282BEC@codespeak.net> Author: hakanardo Date: Wed Aug 11 20:26:06 2010 New Revision: 76597 Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Log: This overflows 32bit Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Wed Aug 11 20:26:06 2010 @@ -759,7 +759,7 @@ elif T is lltype.UniChar: try: llobj = unichr(cobj) - except ValueError: + except (ValueError, OverflowError): for tc in 'HIL': if array(tc).itemsize == array('u').itemsize: llobj = array('u', array(tc, (cobj,)).tostring())[0] From getxsick at codespeak.net Thu Aug 12 00:14:01 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Thu, 12 Aug 2010 00:14:01 +0200 (CEST) Subject: [pypy-svn] r76598 - in pypy/branch/fast-ctypes/pypy: module/jitffi module/jitffi/test rlib rlib/test Message-ID: <20100811221401.570D9282BEC@codespeak.net> Author: getxsick Date: Thu Aug 12 00:13:59 2010 New Revision: 76598 Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/__init__.py pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Log: add support for C pointers in module/jitffi one test is commented as it fails and i don't know why Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/__init__.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/__init__.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/__init__.py Thu Aug 12 00:13:59 2010 @@ -3,6 +3,7 @@ class Module(MixedModule): interpleveldefs = { 'CDLL' : 'interp_jitffi.W_CDLL', + 'Test' : 'interp_jitffi.W_Test', } appleveldefs = {} Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Thu Aug 12 00:13:59 2010 @@ -3,6 +3,7 @@ from pypy.interpreter.error import OperationError, wrap_oserror from pypy.interpreter.gateway import interp2app from pypy.interpreter.typedef import TypeDef +from pypy.rpython.lltypesystem import rffi, lltype class W_CDLL(Wrappable): def __init__(self, space, name): @@ -69,7 +70,8 @@ def __init__(self, space, cpu, lib, func, args_type, res_type='v'): self.space = space - wrap_func = (self.wrap_int_w, self.wrap_float_w, self.wrap_void_w) + wrap_func = (self.wrap_int_w, self.wrap_float_w, + self.wrap_ref_w, self.wrap_void_w) self.rget = rjitffi._Get(cpu, lib, func, args_type, res_type, wrap_func, cache=True) @@ -99,6 +101,8 @@ self.rget.push_int(space.int_w(w_arg)) elif self.rget.args_type[i] == 'f': self.rget.push_float(space.float_w(w_arg)) + elif self.rget.args_type[i] == 'p': + self.rget.push_ref(space.int_w(w_arg)) else: # should never happen (raised earlier) raise OperationError( @@ -114,6 +118,9 @@ def wrap_float_w(self, value): return self.space.wrap(value) + def wrap_ref_w(self, value): + return self.space.wrap(value) + def wrap_void_w(self, w_value): return w_value @@ -131,3 +138,42 @@ wrap_float = interp2app(W_Get.wrap_float_w, unwrap_spec=['self', float]), wrap_void = interp2app(W_Get.wrap_void_w, unwrap_spec=['self', W_Root]) ) + +# for tests only +class W_Test(Wrappable): + def __init__(self, space): + self.space = space + + def get_intp_w(self, space, n, w_values): + values_w = [ space.int_w(w_x) for w_x in space.listview(w_values) ] + intp = lltype.malloc(rffi.INTP.TO, n, flavor='raw') # XXX free it! + for i in xrange(n): + intp[i] = values_w[i] + return space.wrap(rffi.cast(lltype.Signed, intp)) + + def get_charp_w(self, space, txt): + charp = rffi.str2charp(txt) + return space.wrap(rffi.cast(lltype.Signed, charp)) # XXX free it! + + def get_str_w(self, space, addr): + charp = rffi.cast(rffi.CCHARP, addr) + return space.wrap(rffi.charp2str(charp)) # XXX free it? + + def adr_to_intp_w(self, space, addr): + return space.wrap(rffi.cast(rffi.INTP, addr)) + +def W_Test___new__(space, w_x): + return space.wrap(W_Test(space)) + +W_Test.typedef = TypeDef( + 'Test', + __new__ = interp2app(W_Test___new__, unwrap_spec=[ObjSpace, W_Root]), + get_intp = interp2app(W_Test.get_intp_w, + unwrap_spec=['self', ObjSpace, int, W_Root]), + get_charp = interp2app(W_Test.get_charp_w, + unwrap_spec=['self', ObjSpace, str]), + get_str = interp2app(W_Test.get_str_w, + unwrap_spec=['self', ObjSpace, int]), + adr_to_intp = interp2app(W_Test.adr_to_intp_w, + unwrap_spec=['self', ObjSpace, int]) +) Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py Thu Aug 12 00:13:59 2010 @@ -49,11 +49,40 @@ int c; c = a + b; } + int return_ptrvalue(int a, int *b) + { + return a+(*b); + } + int sum_intarray(int *a) + { + int i; + int sum = 0; + for(i=0; i<5; i++) + { + sum += *a+i; + } + return sum; + } + void a2b(char *txt) + { + int i; + for(i=0; txt[i] != '\0'; i++) + { + if (txt[i] == 'a') txt[i] = 'b'; + } + } + int *return_intptr(int a) + { + int *x = malloc(sizeof(int)); + *x = a; + return x; + } ''' )) symbols = ['add_integers', 'add_floats', 'add_intfloat', - 'return_float', 'max3', 'fvoid', 'return_void'] + 'return_float', 'max3', 'fvoid', 'return_void', + 'return_ptrvalue', 'sum_intarray', 'a2b', 'return_intptr'] eci = ExternalCompilationInfo(export_symbols=symbols) return str(platform.compile([c_file], eci, 'x', standalone=False)) @@ -104,6 +133,34 @@ assert func.call([1, 2.9]) == 3 assert func.call([0, 1.3]) == 1 + def test_ptrargs(self): + import jitffi + t = jitffi.Test() + lib = jitffi.CDLL(self.lib_name) + + func = lib.get('return_ptrvalue', ['i', 'p'], 'i') + intp = t.get_intp(1, [10]) + assert func.call([20, intp]) == 30 + + func = lib.get('sum_intarray', ['p'], 'i') + intp = t.get_intp(5, [ i for i in xrange(5) ]) + assert func.call([intp]) == 10 + + func = lib.get('a2b', ['p']) + charp = t.get_charp('xaxaxa') + func.call([charp]) + assert t.get_str(charp) == 'xbxbxb' + + #def test_get_ptr(self): + # import jitffi + # t = jitffi.Test() + # lib = jitffi.CDLL(self.lib_name) + + # func = lib.get('return_intptr', ['i'], 'p') + # addr = func.call([22]) + # ret = t.adr_to_intp(addr) + # assert ret[0] == 22 + def test_undefined_func(self): import jitffi lib = jitffi.CDLL(self.lib_name) Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Thu Aug 12 00:13:59 2010 @@ -111,9 +111,9 @@ elif self.res_type == 'f': r = self.push_result[1](self.cpu.get_latest_value_float(0)) elif self.res_type == 'p': - r = self.push_result[0](self.cpu.get_latest_value_int(0)) + r = self.push_result[2](self.cpu.get_latest_value_int(0)) elif self.res_type == 'v': - r = self.push_result[2](None) + r = self.push_result[3](None) else: raise ValueError(self.res_type) Modified: pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/test/test_rjitffi.py Thu Aug 12 00:13:59 2010 @@ -92,7 +92,7 @@ def setup_class(cls): cls.lib_name = cls.preprare_c_example() - cls.push_result = [ lambda x: x for i in xrange(3) ] # mock function + cls.push_result = [ lambda x: x for i in xrange(4) ] # mock function def fromcache(self, f, args_type, res_type): if not hasattr(self, 'cache'): From getxsick at codespeak.net Thu Aug 12 00:21:23 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Thu, 12 Aug 2010 00:21:23 +0200 (CEST) Subject: [pypy-svn] r76599 - in pypy/branch/fast-ctypes: . lib-python lib_pypy pypy/jit/backend/x86 pypy/jit/backend/x86/test pypy/jit/metainterp pypy/jit/metainterp/doc pypy/module/__builtin__/test pypy/module/_codecs/test pypy/module/_file/test pypy/module/_socket/test pypy/module/_sre/test pypy/module/array pypy/module/array/benchmark pypy/module/array/test pypy/module/cpyext pypy/module/fcntl/test pypy/module/marshal/test pypy/module/thread/test pypy/objspace/std pypy/rpython/lltypesystem Message-ID: <20100811222123.96090282BEC@codespeak.net> Author: getxsick Date: Thu Aug 12 00:21:21 2010 New Revision: 76599 Removed: pypy/branch/fast-ctypes/pypy/jit/metainterp/doc/ Modified: pypy/branch/fast-ctypes/ (props changed) pypy/branch/fast-ctypes/lib-python/conftest.py pypy/branch/fast-ctypes/lib_pypy/array.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/assembler.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/regalloc.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/rx86.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_regloc.py pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_rx86.py pypy/branch/fast-ctypes/pypy/jit/metainterp/optimizeopt.py pypy/branch/fast-ctypes/pypy/module/__builtin__/test/test_buffer.py pypy/branch/fast-ctypes/pypy/module/_codecs/test/test_codecs.py pypy/branch/fast-ctypes/pypy/module/_file/test/test_file_extra.py pypy/branch/fast-ctypes/pypy/module/_socket/test/test_sock_app.py pypy/branch/fast-ctypes/pypy/module/_sre/test/test_app_sre.py pypy/branch/fast-ctypes/pypy/module/array/benchmark/Makefile (props changed) pypy/branch/fast-ctypes/pypy/module/array/benchmark/intimg.c (props changed) pypy/branch/fast-ctypes/pypy/module/array/benchmark/intimgtst.c (props changed) pypy/branch/fast-ctypes/pypy/module/array/benchmark/intimgtst.py (props changed) pypy/branch/fast-ctypes/pypy/module/array/benchmark/loop.c (props changed) pypy/branch/fast-ctypes/pypy/module/array/benchmark/sum.c (props changed) pypy/branch/fast-ctypes/pypy/module/array/benchmark/sumtst.c (props changed) pypy/branch/fast-ctypes/pypy/module/array/benchmark/sumtst.py (props changed) pypy/branch/fast-ctypes/pypy/module/array/interp_array.py pypy/branch/fast-ctypes/pypy/module/array/test/test_array.py pypy/branch/fast-ctypes/pypy/module/array/test/test_array_old.py (props changed) pypy/branch/fast-ctypes/pypy/module/cpyext/api.py pypy/branch/fast-ctypes/pypy/module/fcntl/test/test_fcntl.py pypy/branch/fast-ctypes/pypy/module/marshal/test/test_marshalimpl.py pypy/branch/fast-ctypes/pypy/module/thread/test/test_gil.py pypy/branch/fast-ctypes/pypy/objspace/std/itertype.py pypy/branch/fast-ctypes/pypy/rpython/lltypesystem/ll2ctypes.py Log: merge from trunk Modified: pypy/branch/fast-ctypes/lib-python/conftest.py ============================================================================== --- pypy/branch/fast-ctypes/lib-python/conftest.py (original) +++ pypy/branch/fast-ctypes/lib-python/conftest.py Thu Aug 12 00:21:21 2010 @@ -132,7 +132,7 @@ RegrTest('test_ast.py', core=True), RegrTest('test_anydbm.py'), RegrTest('test_applesingle.py', skip=True), - RegrTest('test_array.py', core=True, usemodules='struct'), + RegrTest('test_array.py', core=True, usemodules='struct array'), RegrTest('test_asynchat.py', usemodules='thread'), RegrTest('test_atexit.py', core=True), RegrTest('test_audioop.py', skip=True), Modified: pypy/branch/fast-ctypes/lib_pypy/array.py ============================================================================== --- pypy/branch/fast-ctypes/lib_pypy/array.py (original) +++ pypy/branch/fast-ctypes/lib_pypy/array.py Thu Aug 12 00:21:21 2010 @@ -1,24 +1,531 @@ -from array import array as _array +"""This module defines an object type which can efficiently represent +an array of basic values: characters, integers, floating point +numbers. Arrays are sequence types and behave very much like lists, +except that the type of objects stored in them is constrained. The +type is specified at object creation time by using a type code, which +is a single character. The following type codes are defined: + + Type code C Type Minimum size in bytes + 'c' character 1 + 'b' signed integer 1 + 'B' unsigned integer 1 + 'u' Unicode character 2 + 'h' signed integer 2 + 'H' unsigned integer 2 + 'i' signed integer 2 + 'I' unsigned integer 2 + 'l' signed integer 4 + 'L' unsigned integer 4 + 'f' floating point 4 + 'd' floating point 8 + +The constructor is: + +array(typecode [, initializer]) -- create a new array +""" + +from struct import calcsize, pack, pack_into, unpack_from +import operator + +# the buffer-like object to use internally: trying from +# various places in order... +try: + import _rawffi # a reasonable implementation based + _RAWARRAY = _rawffi.Array('c') # on raw_malloc, and providing a + def bytebuffer(size): # real address + return _RAWARRAY(size, autofree=True) + def getbufaddress(buf): + return buf.buffer +except ImportError: + try: + from __pypy__ import bytebuffer # a reasonable implementation + def getbufaddress(buf): # compatible with oo backends, + return 0 # but no address + except ImportError: + # not running on PyPy. Fall back to ctypes... + import ctypes + bytebuffer = ctypes.create_string_buffer + def getbufaddress(buf): + voidp = ctypes.cast(ctypes.pointer(buf), ctypes.c_void_p) + return voidp.value + +# ____________________________________________________________ + +TYPECODES = "cbBuhHiIlLfd" class array(object): - def __init__(self, typecode, initializer=None): - self._array = _array(typecode) - if initializer is not None: + """array(typecode [, initializer]) -> array + + Return a new array whose items are restricted by typecode, and + initialized from the optional initializer value, which must be a list, + string. or iterable over elements of the appropriate type. + + Arrays represent basic values and behave very much like lists, except + the type of objects stored in them is constrained. + + Methods: + + append() -- append a new item to the end of the array + buffer_info() -- return information giving the current memory info + byteswap() -- byteswap all the items of the array + count() -- return number of occurences of an object + extend() -- extend array by appending multiple elements from an iterable + fromfile() -- read items from a file object + fromlist() -- append items from the list + fromstring() -- append items from the string + index() -- return index of first occurence of an object + insert() -- insert a new item into the array at a provided position + pop() -- remove and return item (default last) + read() -- DEPRECATED, use fromfile() + remove() -- remove first occurence of an object + reverse() -- reverse the order of the items in the array + tofile() -- write all items to a file object + tolist() -- return the array converted to an ordinary list + tostring() -- return the array converted to a string + write() -- DEPRECATED, use tofile() + + Attributes: + + typecode -- the typecode character used to create the array + itemsize -- the length in bytes of one array item + """ + __slots__ = ["typecode", "itemsize", "_data", "_descriptor", "__weakref__"] + + def __new__(cls, typecode, initializer=[], **extrakwds): + self = object.__new__(cls) + if cls is array and extrakwds: + raise TypeError("array() does not take keyword arguments") + if not isinstance(typecode, str) or len(typecode) != 1: + raise TypeError( + "array() argument 1 must be char, not %s" % type(typecode)) + if typecode not in TYPECODES: + raise ValueError( + "bad typecode (must be one of %s)" % ', '.join(TYPECODES)) + self._data = bytebuffer(0) + self.typecode = typecode + self.itemsize = calcsize(typecode) + if isinstance(initializer, list): + self.fromlist(initializer) + elif isinstance(initializer, str): + self.fromstring(initializer) + elif isinstance(initializer, unicode) and self.typecode == "u": + self.fromunicode(initializer) + else: self.extend(initializer) + return self + def _clear(self): + self._data = bytebuffer(0) - def append(self ,x): - self._array.append(x) - def __getitem__(self, idx): - return self._array[idx] - def __setitem__(self, idx, val): - self._array[idx]=val - def __len__(self): - return len(self._array) + ##### array-specific operations + def fromfile(self, f, n): + """Read n objects from the file object f and append them to the end of + the array. Also called as read.""" + if not isinstance(f, file): + raise TypeError("arg1 must be open file") + size = self.itemsize * n + item = f.read(size) + if len(item) < size: + raise EOFError("not enough items in file") + self.fromstring(item) + + def fromlist(self, l): + """Append items to array from list.""" + if not isinstance(l, list): + raise TypeError("arg must be list") + self._fromiterable(l) + + def fromstring(self, s): + """Appends items from the string, interpreting it as an array of machine + values, as if it had been read from a file using the fromfile() + method.""" + if isinstance(s, unicode): + s = str(s) + self._frombuffer(s) + + def _frombuffer(self, s): + length = len(s) + if length % self.itemsize != 0: + raise ValueError("string length not a multiple of item size") + boundary = len(self._data) + newdata = bytebuffer(boundary + length) + newdata[:boundary] = self._data + newdata[boundary:] = s + self._data = newdata + + def fromunicode(self, ustr): + """Extends this array with data from the unicode string ustr. The array + must be a type 'u' array; otherwise a ValueError is raised. Use + array.fromstring(ustr.encode(...)) to append Unicode data to an array of + some other type.""" + if not self.typecode == "u": + raise ValueError( + "fromunicode() may only be called on type 'u' arrays") + # XXX the following probable bug is not emulated: + # CPython accepts a non-unicode string or a buffer, and then + # behaves just like fromstring(), except that it strangely truncates + # string arguments at multiples of the unicode byte size. + # Let's only accept unicode arguments for now. + if not isinstance(ustr, unicode): + raise TypeError("fromunicode() argument should probably be " + "a unicode string") + # _frombuffer() does the currect thing using + # the buffer behavior of unicode objects + self._frombuffer(buffer(ustr)) + + def tofile(self, f): + """Write all items (as machine values) to the file object f. Also + called as write.""" + if not isinstance(f, file): + raise TypeError("arg must be open file") + f.write(self.tostring()) + + def tolist(self): + """Convert array to an ordinary list with the same items.""" + count = len(self._data) // self.itemsize + return list(unpack_from('%d%s' % (count, self.typecode), self._data)) + + def tostring(self): + return self._data[:] + + def __buffer__(self): + return buffer(self._data) + + def tounicode(self): + """Convert the array to a unicode string. The array must be a type 'u' + array; otherwise a ValueError is raised. Use array.tostring().decode() + to obtain a unicode string from an array of some other type.""" + if self.typecode != "u": + raise ValueError("tounicode() may only be called on type 'u' arrays") + # XXX performance is not too good + return u"".join(self.tolist()) + + def byteswap(self): + """Byteswap all items of the array. If the items in the array are not + 1, 2, 4, or 8 bytes in size, RuntimeError is raised.""" + if self.itemsize not in [1, 2, 4, 8]: + raise RuntimeError("byteswap not supported for this array") + # XXX slowish + itemsize = self.itemsize + bytes = self._data + for start in range(0, len(bytes), itemsize): + stop = start + itemsize + bytes[start:stop] = bytes[start:stop][::-1] + + def buffer_info(self): + """Return a tuple (address, length) giving the current memory address + and the length in items of the buffer used to hold array's contents. The + length should be multiplied by the itemsize attribute to calculate the + buffer length in bytes. On PyPy the address might be meaningless + (returned as 0), depending on the available modules.""" + return (getbufaddress(self._data), len(self)) + read = fromfile + + write = tofile + + ##### general object protocol + + def __repr__(self): + if len(self._data) == 0: + return "array('%s')" % self.typecode + elif self.typecode == "c": + return "array('%s', %s)" % (self.typecode, repr(self.tostring())) + elif self.typecode == "u": + return "array('%s', %s)" % (self.typecode, repr(self.tounicode())) + else: + return "array('%s', %s)" % (self.typecode, repr(self.tolist())) + + def __copy__(self): + a = array(self.typecode) + a._data = bytebuffer(len(self._data)) + a._data[:] = self._data + return a + + def __eq__(self, other): + if not isinstance(other, array): + return NotImplemented + if self.typecode == 'c': + return buffer(self._data) == buffer(other._data) + else: + return self.tolist() == other.tolist() + + def __ne__(self, other): + if not isinstance(other, array): + return NotImplemented + if self.typecode == 'c': + return buffer(self._data) != buffer(other._data) + else: + return self.tolist() != other.tolist() + + def __lt__(self, other): + if not isinstance(other, array): + return NotImplemented + if self.typecode == 'c': + return buffer(self._data) < buffer(other._data) + else: + return self.tolist() < other.tolist() + + def __gt__(self, other): + if not isinstance(other, array): + return NotImplemented + if self.typecode == 'c': + return buffer(self._data) > buffer(other._data) + else: + return self.tolist() > other.tolist() + + def __le__(self, other): + if not isinstance(other, array): + return NotImplemented + if self.typecode == 'c': + return buffer(self._data) <= buffer(other._data) + else: + return self.tolist() <= other.tolist() + + def __ge__(self, other): + if not isinstance(other, array): + return NotImplemented + if self.typecode == 'c': + return buffer(self._data) >= buffer(other._data) + else: + return self.tolist() >= other.tolist() + + def __reduce__(self): + dict = getattr(self, '__dict__', None) + data = self.tostring() + if data: + initargs = (self.typecode, data) + else: + initargs = (self.typecode,) + return (type(self), initargs, dict) + + ##### list methods + + def append(self, x): + """Append new value x to the end of the array.""" + self._frombuffer(pack(self.typecode, x)) + + def count(self, x): + """Return number of occurences of x in the array.""" + return operator.countOf(self, x) + def extend(self, iterable): - for i in iterable: self.append(i) + """Append items to the end of the array.""" + if isinstance(iterable, array) \ + and not self.typecode == iterable.typecode: + raise TypeError("can only extend with array of same kind") + self._fromiterable(iterable) + + def index(self, x): + """Return index of first occurence of x in the array.""" + return operator.indexOf(self, x) + def insert(self, i, x): + """Insert a new item x into the array before position i.""" + seqlength = len(self) + if i < 0: + i += seqlength + if i < 0: + i = 0 + elif i > seqlength: + i = seqlength + boundary = i * self.itemsize + data = pack(self.typecode, x) + newdata = bytebuffer(len(self._data) + len(data)) + newdata[:boundary] = self._data[:boundary] + newdata[boundary:boundary+self.itemsize] = data + newdata[boundary+self.itemsize:] = self._data[boundary:] + self._data = newdata + + def pop(self, i=-1): + """Return the i-th element and delete it from the array. i defaults to + -1.""" + seqlength = len(self) + if i < 0: + i += seqlength + if not (0 <= i < seqlength): + raise IndexError(i) + boundary = i * self.itemsize + result = unpack_from(self.typecode, self._data, boundary)[0] + newdata = bytebuffer(len(self._data) - self.itemsize) + newdata[:boundary] = self._data[:boundary] + newdata[boundary:] = self._data[boundary+self.itemsize:] + self._data = newdata + return result + + def remove(self, x): + """Remove the first occurence of x in the array.""" + self.pop(self.index(x)) + + def reverse(self): + """Reverse the order of the items in the array.""" + lst = self.tolist() + lst.reverse() + self._clear() + self.fromlist(lst) + ##### list protocol + def __len__(self): + return len(self._data) // self.itemsize + + def __add__(self, other): + if not isinstance(other, array): + raise TypeError("can only append array to array") + if self.typecode != other.typecode: + raise TypeError("bad argument type for built-in operation") + return array(self.typecode, buffer(self._data) + buffer(other._data)) + + def __mul__(self, repeat): + return array(self.typecode, buffer(self._data) * repeat) + + __rmul__ = __mul__ + + def __getitem__(self, i): + seqlength = len(self) + if isinstance(i, slice): + start, stop, step = i.indices(seqlength) + if step != 1: + sublist = self.tolist()[i] # fall-back + return array(self.typecode, sublist) + if start < 0: + start = 0 + if stop < start: + stop = start + assert stop <= seqlength + return array(self.typecode, self._data[start * self.itemsize : + stop * self.itemsize]) + else: + if i < 0: + i += seqlength + if self.typecode == 'c': # speed trick + return self._data[i] + if not (0 <= i < seqlength): + raise IndexError(i) + boundary = i * self.itemsize + return unpack_from(self.typecode, self._data, boundary)[0] + + def __getslice__(self, i, j): + return self.__getitem__(slice(i, j)) + + def __setitem__(self, i, x): + if isinstance(i, slice): + if (not isinstance(x, array) + or self.typecode != x.typecode): + raise TypeError("can only assign array of same kind" + " to array slice") + seqlength = len(self) + start, stop, step = i.indices(seqlength) + if step != 1: + sublist = self.tolist() # fall-back + sublist[i] = x.tolist() + self._clear() + self.fromlist(sublist) + return + if start < 0: + start = 0 + if stop < start: + stop = start + assert stop <= seqlength + boundary1 = start * self.itemsize + boundary2 = stop * self.itemsize + boundary2new = boundary1 + len(x._data) + if boundary2 == boundary2new: + self._data[boundary1:boundary2] = x._data + else: + newdata = bytebuffer(len(self._data) + boundary2new-boundary2) + newdata[:boundary1] = self._data[:boundary1] + newdata[boundary1:boundary2new] = x._data + newdata[boundary2new:] = self._data[boundary2:] + self._data = newdata + else: + seqlength = len(self) + if i < 0: + i += seqlength + if self.typecode == 'c': # speed trick + self._data[i] = x + return + if not (0 <= i < seqlength): + raise IndexError(i) + boundary = i * self.itemsize + pack_into(self.typecode, self._data, boundary, x) + + def __setslice__(self, i, j, x): + self.__setitem__(slice(i, j), x) + + def __delitem__(self, i): + if isinstance(i, slice): + seqlength = len(self) + start, stop, step = i.indices(seqlength) + if start < 0: + start = 0 + if stop < start: + stop = start + assert stop <= seqlength + if step != 1: + sublist = self.tolist() # fall-back + del sublist[i] + self._clear() + self.fromlist(sublist) + return + dellength = stop - start + boundary1 = start * self.itemsize + boundary2 = stop * self.itemsize + newdata = bytebuffer(len(self._data) - (boundary2-boundary1)) + newdata[:boundary1] = self._data[:boundary1] + newdata[boundary1:] = self._data[boundary2:] + self._data = newdata + else: + seqlength = len(self) + if i < 0: + i += seqlength + if not (0 <= i < seqlength): + raise IndexError(i) + boundary = i * self.itemsize + newdata = bytebuffer(len(self._data) - self.itemsize) + newdata[:boundary] = self._data[:boundary] + newdata[boundary:] = self._data[boundary+self.itemsize:] + self._data = newdata + + def __delslice__(self, i, j): + self.__delitem__(slice(i, j)) + + def __contains__(self, item): + for x in self: + if x == item: + return True + return False + + def __iadd__(self, other): + if not isinstance(other, array): + raise TypeError("can only extend array with array") + self.extend(other) + return self + + def __imul__(self, repeat): + newdata = buffer(self._data) * repeat + self._data = bytebuffer(len(newdata)) + self._data[:] = newdata + return self + + def __iter__(self): + p = 0 + typecode = self.typecode + itemsize = self.itemsize + while p < len(self._data): + yield unpack_from(typecode, self._data, p)[0] + p += itemsize + + ##### internal methods + + def _fromiterable(self, iterable): + iterable = tuple(iterable) + n = len(iterable) + boundary = len(self._data) + newdata = bytebuffer(boundary + n * self.itemsize) + newdata[:boundary] = self._data + pack_into('%d%s' % (n, self.typecode), newdata, boundary, *iterable) + self._data = newdata + +ArrayType = array Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/assembler.py Thu Aug 12 00:21:21 2010 @@ -1741,7 +1741,15 @@ # the following is supposed to be the slow path, so whenever possible # we choose the most compact encoding over the most efficient one. for i in range(len(arglocs)-1, -1, -1): - self.mc.PUSH(arglocs[i]) + loc = arglocs[i] + if isinstance(loc, RegLoc): + self.mc.PUSH_r(loc.value) + else: + if IS_X86_64: + self.mc.MOV_ri(X86_64_SCRATCH_REG.value, loc.getint()) + self.mc.PUSH_r(X86_64_SCRATCH_REG.value) + else: + self.mc.PUSH_i32(loc.getint()) if IS_X86_64: # We clobber these registers to pass the arguments, but that's @@ -1757,8 +1765,10 @@ self.mc.CALL(imm(descr.get_write_barrier_fn(self.cpu))) for i in range(len(arglocs)): loc = arglocs[i] - assert isinstance(loc, RegLoc) - self.mc.POP(loc) + if isinstance(loc, RegLoc): + self.mc.POP_r(loc.value) + else: + self.mc.ADD_ri(esp.value, WORD) # ignore the pushed constant # patch the JZ above offset = self.mc.get_relative_pos() - jz_location assert 0 < offset <= 127 Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/regalloc.py Thu Aug 12 00:21:21 2010 @@ -672,13 +672,12 @@ def consider_cond_call_gc_wb(self, op): assert op.result is None + loc_newvalue = self.rm.make_sure_var_in_reg(op.args[1], op.args) + # ^^^ we force loc_newvalue in a reg (unless it's a Const), + # because it will be needed anyway by the following setfield_gc. + # It avoids loading it twice from the memory. loc_base = self.rm.make_sure_var_in_reg(op.args[0], op.args, imm_fine=False) - loc_newvalue = self.rm.make_sure_var_in_reg(op.args[1], op.args, - imm_fine=False) - # ^^^ we also force loc_newvalue in a reg, because it will be needed - # anyway by the following setfield_gc. It avoids loading it twice - # from the memory. arglocs = [loc_base, loc_newvalue] # add eax, ecx and edx as extra "arguments" to ensure they are # saved and restored. Fish in self.rm to know which of these Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/rx86.py Thu Aug 12 00:21:21 2010 @@ -290,6 +290,9 @@ def encode_rex(mc, rexbyte, basevalue, orbyte): if mc.WORD == 8: assert 0 <= rexbyte < 8 + # XXX: Hack. Ignore REX.W if we are using 16-bit operands + if mc._use_16_bit_immediate: + basevalue &= ~REX_W if basevalue != 0x40 or rexbyte != 0: mc.writechar(chr(basevalue | rexbyte)) else: @@ -492,6 +495,7 @@ PUSH_r = insn(rex_nw, register(1), '\x50') PUSH_b = insn(rex_nw, '\xFF', orbyte(6<<3), stack_bp(1)) + PUSH_i32 = insn('\x68', immediate(1, 'i')) POP_r = insn(rex_nw, register(1), '\x58') POP_b = insn(rex_nw, '\x8F', orbyte(0<<3), stack_bp(1)) Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_regloc.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_regloc.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_regloc.py Thu Aug 12 00:21:21 2010 @@ -16,13 +16,27 @@ cb64 = LocationCodeBuilder64 def test_mov_16(): + # 32-bit assert_encodes_as(cb32, "MOV16", (ecx, ebx), '\x66\x89\xD9') assert_encodes_as(cb32, "MOV16", (ecx, ImmedLoc(12345)), '\x66\xB9\x39\x30') + # 64-bit + assert_encodes_as(cb64, "MOV16", (ecx, ebx), '\x66\x89\xD9') + # XXX: What we are testing for here is actually not the most compact + # encoding. + assert_encodes_as(cb64, "MOV16", (ecx, ImmedLoc(12345)), '\x66\xC7\xC1\x39\x30') + assert_encodes_as(cb64, "MOV16", (AddressLoc(r13, ImmedLoc(0), 0, 0), ImmedLoc(12345)), '\x66\x41\xC7\x45\x00\x39\x30') + def test_cmp_16(): + # 32-bit assert_encodes_as(cb32, "CMP16", (ecx, ebx), '\x66\x39\xD9') assert_encodes_as(cb32, "CMP16", (ecx, ImmedLoc(12345)), '\x66\x81\xF9\x39\x30') + # 64-bit + assert_encodes_as(cb64, "CMP16", (ecx, ebx), '\x66\x39\xD9') + assert_encodes_as(cb64, "CMP16", (ecx, ImmedLoc(12345)), '\x66\x81\xF9\x39\x30') + assert_encodes_as(cb64, "CMP16", (AddressLoc(r13, ImmedLoc(0), 0, 0), ImmedLoc(12345)), '\x66\x41\x81\x7D\x00\x39\x30') + def test_jmp_wraparound(): if not IS_X86_32: py.test.skip() Modified: pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_rx86.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_rx86.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/backend/x86/test/test_rx86.py Thu Aug 12 00:21:21 2010 @@ -188,6 +188,10 @@ assert_encodes_as(cb, 'MOV8_mi', ((edx, 16), 99), '\xC6\x42\x10\x63') assert_encodes_as(cb, 'MOV8_ai', ((ebx, ecx, 2, 16), 99), '\xC6\x44\x8B\x10\x63') +def test_push32(): + cb = CodeBuilder32 + assert_encodes_as(cb, 'PUSH_i32', (9,), '\x68\x09\x00\x00\x00') + class CodeBuilder64(CodeBuilderMixin, X86_64_CodeBuilder): pass Modified: pypy/branch/fast-ctypes/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/fast-ctypes/pypy/jit/metainterp/optimizeopt.py Thu Aug 12 00:21:21 2010 @@ -612,7 +612,7 @@ assert oldop.opnum == op.opnum self.make_equal_to(op.result, self.getvalue(oldop.result)) return - elif self.find_rewriteable_bool(op, args): + elif self.find_rewritable_bool(op, args): return else: self.pure_operations[args] = op @@ -635,7 +635,7 @@ return False - def find_rewriteable_bool(self, op, args): + def find_rewritable_bool(self, op, args): try: oldopnum = opboolinvers[op.opnum] targs = [args[0], args[1], ConstInt(oldopnum)] Modified: pypy/branch/fast-ctypes/pypy/module/__builtin__/test/test_buffer.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/__builtin__/test/test_buffer.py (original) +++ pypy/branch/fast-ctypes/pypy/module/__builtin__/test/test_buffer.py Thu Aug 12 00:21:21 2010 @@ -1,8 +1,11 @@ """Tests some behaviour of the buffer type that is not tested in lib-python/2.5.2/test/test_types.py where the stdlib buffer tests live.""" import autopath +from pypy.conftest import gettestobjspace class AppTestBuffer: + def setup_class(cls): + cls.space = gettestobjspace(usemodules=('array',)) def test_unicode_buffer(self): import sys Modified: pypy/branch/fast-ctypes/pypy/module/_codecs/test/test_codecs.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/_codecs/test/test_codecs.py (original) +++ pypy/branch/fast-ctypes/pypy/module/_codecs/test/test_codecs.py Thu Aug 12 00:21:21 2010 @@ -123,6 +123,10 @@ class AppTestPartialEvaluation: + def setup_class(cls): + space = gettestobjspace(usemodules=('array',)) + cls.space = space + def test_partial_utf8(self): import _codecs encoding = 'utf-8' Modified: pypy/branch/fast-ctypes/pypy/module/_file/test/test_file_extra.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/_file/test/test_file_extra.py (original) +++ pypy/branch/fast-ctypes/pypy/module/_file/test/test_file_extra.py Thu Aug 12 00:21:21 2010 @@ -353,6 +353,10 @@ class AppTestAFewExtra: + def setup_class(cls): + space = gettestobjspace(usemodules=('array',)) + cls.space = space + def setup_method(self, method): fn = str(udir.join('temptestfile')) self.w_temptestfile = self.space.wrap(fn) Modified: pypy/branch/fast-ctypes/pypy/module/_socket/test/test_sock_app.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/_socket/test/test_sock_app.py (original) +++ pypy/branch/fast-ctypes/pypy/module/_socket/test/test_sock_app.py Thu Aug 12 00:21:21 2010 @@ -4,7 +4,7 @@ from pypy.tool.udir import udir def setup_module(mod): - mod.space = gettestobjspace(usemodules=['_socket']) + mod.space = gettestobjspace(usemodules=['_socket', 'array']) global socket import socket mod.w_socket = space.appexec([], "(): import _socket as m; return m") Modified: pypy/branch/fast-ctypes/pypy/module/_sre/test/test_app_sre.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/_sre/test/test_app_sre.py (original) +++ pypy/branch/fast-ctypes/pypy/module/_sre/test/test_app_sre.py Thu Aug 12 00:21:21 2010 @@ -87,7 +87,9 @@ class AppTestSreMatch: - + def setup_class(cls): + cls.space = gettestobjspace(usemodules=('array', )) + def test_copy(self): import re # copy support is disabled by default in _sre.c Modified: pypy/branch/fast-ctypes/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/array/interp_array.py (original) +++ pypy/branch/fast-ctypes/pypy/module/array/interp_array.py Thu Aug 12 00:21:21 2010 @@ -14,9 +14,10 @@ from pypy.objspace.std.model import W_Object from pypy.interpreter.argument import Arguments, Signature from pypy.module._file.interp_file import W_File +from pypy.interpreter.buffer import RWBuffer -def w_array(space, w_cls, typecode, w_initializer=None, w_args=None): - if len(w_args.arguments_w) > 0: +def w_array(space, w_cls, typecode, w_args=None): + if len(w_args.arguments_w) > 1: msg = 'array() takes at most 2 arguments' raise OperationError(space.w_TypeError, space.wrap(msg)) if len(typecode) != 1: @@ -29,23 +30,23 @@ a = space.allocate_instance(types[tc].w_class, w_cls) a.__init__(space) - if w_initializer is not None: - if not space.is_w(w_initializer, space.w_None): - if space.type(w_initializer) is space.w_str: - a.fromstring(w_initializer) - elif space.type(w_initializer) is space.w_unicode: - a.fromsequence(w_initializer) - elif space.type(w_initializer) is space.w_list: - a.fromlist(w_initializer) - else: - a.extend(w_initializer) + if len(w_args.arguments_w) > 0: + w_initializer = w_args.arguments_w[0] + if space.type(w_initializer) is space.w_str: + a.fromstring(w_initializer) + elif space.type(w_initializer) is space.w_unicode: + a.fromsequence(w_initializer) + elif space.type(w_initializer) is space.w_list: + a.fromlist(w_initializer) + else: + a.extend(w_initializer) break else: msg = 'bad typecode (must be c, b, B, u, h, H, i, I, l, L, f or d)' raise OperationError(space.w_ValueError, space.wrap(msg)) return a -w_array.unwrap_spec = (ObjSpace, W_Root, str, W_Root, Arguments) +w_array.unwrap_spec = (ObjSpace, W_Root, str, Arguments) array_append = SMM('append', 2) @@ -142,6 +143,22 @@ v.typecode = k unroll_typecodes = unrolling_iterable(types.keys()) +class ArrayBuffer(RWBuffer): + def __init__(self, data, bytes): + self.data = data + self.len = bytes + + def getlength(self): + return self.len + + def getitem(self, index): + return self.data[index] + + def setitem(self, index, char): + self.data[index] = char + + + def make_array(mytype): class W_Array(W_ArrayBase): @@ -460,6 +477,7 @@ def mul__Array_ANY(space, self, w_repeat): repeat = space.int_w(w_repeat) a = mytype.w_class(space) + repeat = max(repeat, 0) a.setlen(self.len * repeat) for r in range(repeat): for i in range(self.len): @@ -472,6 +490,7 @@ def inplace_mul__Array_ANY(space, self, w_repeat): repeat = space.int_w(w_repeat) oldlen = self.len + repeat = max(repeat, 0) self.setlen(self.len * repeat) for r in range(1, repeat): for i in range(oldlen): @@ -560,14 +579,13 @@ w_lst2 = space.call_method(other, 'tolist') return space.cmp(w_lst1, w_lst2) else: - raise OperationError(space.w_NotImplementedError, space.wrap('')) + return space.w_NotImplemented # Misc methods def buffer__Array(space, self): - from pypy.interpreter.buffer import StringLikeBuffer - w_s = array_tostring__Array(space, self) - return space.wrap(StringLikeBuffer(space, w_s)) + b = ArrayBuffer(self.charbuf(), self.len * mytype.bytes) + return space.wrap(b) def array_buffer_info__Array(space, self): w_ptr = space.wrap(rffi.cast(lltype.Unsigned, self.buffer)) @@ -580,7 +598,11 @@ args = [space.wrap(mytype.typecode), w_s] else: args = [space.wrap(mytype.typecode)] - return space.newtuple([space.type(self), space.newtuple(args)]) + try: + dct = space.getattr(self, space.wrap('__dict__')) + except OperationError: + dct = space.w_None + return space.newtuple([space.type(self), space.newtuple(args), dct]) def array_byteswap__Array(space, self): if mytype.bytes not in [1, 2, 4, 8]: @@ -638,3 +660,5 @@ for mytype in types.values(): make_array(mytype) + +register_all(locals(), globals()) Modified: pypy/branch/fast-ctypes/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/array/test/test_array.py (original) +++ pypy/branch/fast-ctypes/pypy/module/array/test/test_array.py Thu Aug 12 00:21:21 2010 @@ -61,6 +61,7 @@ for tc in 'bhilBHILfd': assert self.array(tc).typecode == tc + raises(TypeError, self.array, tc, None) def test_value_range(self): values = (-129, 128, -128, 127, 0, 255, -1, 256, @@ -421,7 +422,7 @@ a = self.array('h', 'Hi') buf = buffer(a) assert buf[1] == 'i' - raises(TypeError, buf.__setitem__, 1, 'o') + #raises(TypeError, buf.__setitem__, 1, 'o') def test_list_methods(self): assert repr(self.array('i')) == "array('i')" @@ -468,6 +469,12 @@ assert repr(a) == "array('i', [20, 8, 2, 9, 7, 10])" def test_compare(self): + class comparable(object): + def __cmp__(self, other): + return 0 + class incomparable(object): + pass + for v1, v2, tt in (([1, 2, 3], [1, 3, 2], 'bhilBHIL'), ('abc', 'acb', 'c'), (unicode('abc'), unicode('acb'), 'u')): @@ -476,6 +483,13 @@ b = self.array(t, v1) c = self.array(t, v2) + print (a==7) + assert (a == 7) is False + assert (comparable() == a) is True + assert (a == comparable()) is True + assert (a == incomparable()) is False + assert (incomparable() == a) is False + assert (a == a) is True assert (a == b) is True assert (b == a) is True @@ -531,10 +545,8 @@ assert len(b) == 0 and b.typecode == 'l' a = self.array('i', [1, 2, 4]) - print "itter" i = iter(a) - print "ok" - raises(TypeError, pickle.dumps, i, 1) + #raises(TypeError, pickle.dumps, i, 1) def test_copy_swap(self): a = self.array('i', [1, 2, 3]) @@ -599,6 +611,13 @@ assert addable() + self.array('i') == 'add' assert self.array('i') + addable() == 'radd' + a = self.array('i', [1, 2]) + assert a * -1 == self.array('i') + b = a + a *= -1 + assert a == self.array('i') + assert b == self.array('i') + def test_delitem(self): a = self.array('i', [1, 2, 3]) del a[1] @@ -726,6 +745,12 @@ assert repr(mya('i', [1, 2, 3])) == "array('i', [1, 2, 3])" assert repr(mya('i', (1, 2, 3))) == "array('i', [1, 2, 3])" + def test_unicode_outofrange(self): + a = self.array('u', unicode(r'\x01\u263a\x00\ufeff', 'unicode-escape')) + b = self.array('u', unicode(r'\x01\u263a\x00\ufeff', 'unicode-escape')) + b.byteswap() + assert a != b + class TestCPythonsOwnArray(BaseArrayTests): Modified: pypy/branch/fast-ctypes/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/cpyext/api.py (original) +++ pypy/branch/fast-ctypes/pypy/module/cpyext/api.py Thu Aug 12 00:21:21 2010 @@ -1,5 +1,5 @@ import ctypes -import sys +import sys, os import atexit import py @@ -896,6 +896,8 @@ initfunctype = lltype.Ptr(lltype.FuncType([], lltype.Void)) @unwrap_spec(ObjSpace, str, str) def load_extension_module(space, path, name): + if os.sep not in path: + path = os.curdir + os.sep + path # force a '/' in the path state = space.fromcache(State) state.package_context = name try: Modified: pypy/branch/fast-ctypes/pypy/module/fcntl/test/test_fcntl.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/fcntl/test/test_fcntl.py (original) +++ pypy/branch/fast-ctypes/pypy/module/fcntl/test/test_fcntl.py Thu Aug 12 00:21:21 2010 @@ -13,7 +13,7 @@ class AppTestFcntl: def setup_class(cls): - space = gettestobjspace(usemodules=('fcntl',)) + space = gettestobjspace(usemodules=('fcntl', 'array')) cls.space = space tmpprefix = str(udir.ensure('test_fcntl', dir=1).join('tmp_')) cls.w_tmp = space.wrap(tmpprefix) Modified: pypy/branch/fast-ctypes/pypy/module/marshal/test/test_marshalimpl.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/marshal/test/test_marshalimpl.py (original) +++ pypy/branch/fast-ctypes/pypy/module/marshal/test/test_marshalimpl.py Thu Aug 12 00:21:21 2010 @@ -1,9 +1,13 @@ from pypy.module.marshal import interp_marshal from pypy.interpreter.error import OperationError +from pypy.conftest import gettestobjspace import sys class AppTestMarshalMore: + def setup_class(cls): + space = gettestobjspace(usemodules=('array',)) + cls.space = space def test_long_0(self): import marshal Modified: pypy/branch/fast-ctypes/pypy/module/thread/test/test_gil.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/thread/test/test_gil.py (original) +++ pypy/branch/fast-ctypes/pypy/module/thread/test/test_gil.py Thu Aug 12 00:21:21 2010 @@ -1,7 +1,6 @@ import time from pypy.module.thread import gil from pypy.module.thread.test import test_ll_thread -from pypy.rpython.lltypesystem import rffi from pypy.module.thread import ll_thread as thread from pypy.rlib.objectmodel import we_are_translated Modified: pypy/branch/fast-ctypes/pypy/objspace/std/itertype.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/objspace/std/itertype.py (original) +++ pypy/branch/fast-ctypes/pypy/objspace/std/itertype.py Thu Aug 12 00:21:21 2010 @@ -10,9 +10,9 @@ a registration with copy_reg, instead. """ - # cpython does not support pickling iterators - msg = 'Pickling for iterators dissabled as cpython does not support it' - raise OperationError(space.w_TypeError, space.wrap(msg)) + # cpython does not support pickling iterators but stackless python do + #msg = 'Pickling for iterators dissabled as cpython does not support it' + #raise OperationError(space.w_TypeError, space.wrap(msg)) from pypy.objspace.std.iterobject import W_AbstractSeqIterObject assert isinstance(w_self, W_AbstractSeqIterObject) Modified: pypy/branch/fast-ctypes/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/fast-ctypes/pypy/rpython/lltypesystem/ll2ctypes.py Thu Aug 12 00:21:21 2010 @@ -24,6 +24,7 @@ from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE from pypy.rpython import raddress from pypy.translator.platform import platform +from array import array def uaddressof(obj): return fixid(ctypes.addressof(obj)) @@ -756,7 +757,15 @@ elif T is lltype.Char: llobj = chr(cobj) elif T is lltype.UniChar: - llobj = unichr(cobj) + try: + llobj = unichr(cobj) + except (ValueError, OverflowError): + for tc in 'HIL': + if array(tc).itemsize == array('u').itemsize: + llobj = array('u', array(tc, (cobj,)).tostring())[0] + break + else: + raise elif T is lltype.Signed: llobj = cobj elif T is lltype.Bool: From getxsick at codespeak.net Thu Aug 12 01:07:44 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Thu, 12 Aug 2010 01:07:44 +0200 (CEST) Subject: [pypy-svn] r76600 - pypy/branch/fast-ctypes/pypy/module/jitffi Message-ID: <20100811230744.6831F282BEC@codespeak.net> Author: getxsick Date: Thu Aug 12 01:07:42 2010 New Revision: 76600 Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Log: forgot to push it to the app-level Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Thu Aug 12 01:07:42 2010 @@ -136,6 +136,7 @@ call = interp2app(W_Get.call_w, unwrap_spec=['self', ObjSpace, W_Root]), wrap_int = interp2app(W_Get.wrap_int_w, unwrap_spec=['self', int]), wrap_float = interp2app(W_Get.wrap_float_w, unwrap_spec=['self', float]), + wrap_ref = interp2app(W_Get.wrap_ref_w, unwrap_spec=['self', int]), wrap_void = interp2app(W_Get.wrap_void_w, unwrap_spec=['self', W_Root]) ) From hakanardo at codespeak.net Thu Aug 12 11:31:57 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Thu, 12 Aug 2010 11:31:57 +0200 (CEST) Subject: [pypy-svn] r76601 - in pypy/trunk: lib-python/modified-2.5.2/test pypy/module/array pypy/module/array/test Message-ID: <20100812093157.42364282BEC@codespeak.net> Author: hakanardo Date: Thu Aug 12 11:31:54 2010 New Revision: 76601 Modified: pypy/trunk/lib-python/modified-2.5.2/test/test_array.py pypy/trunk/pypy/module/array/interp_array.py pypy/trunk/pypy/module/array/test/test_array.py Log: better add and mul Modified: pypy/trunk/lib-python/modified-2.5.2/test/test_array.py ============================================================================== --- pypy/trunk/lib-python/modified-2.5.2/test/test_array.py (original) +++ pypy/trunk/lib-python/modified-2.5.2/test/test_array.py Thu Aug 12 11:31:54 2010 @@ -269,9 +269,11 @@ ) b = array.array(self.badtypecode()) - self.assertRaises(TypeError, a.__add__, b) + #self.assertRaises(TypeError, a.__add__, b) + #self.assertRaises(TypeError, a.__add__, "bad") + self.assertRaises(TypeError, lambda i, j: i + j, a, b) + self.assertRaises(TypeError, lambda i, j: i + j, a, "bad") - self.assertRaises(TypeError, a.__add__, "bad") def test_iadd(self): a = array.array(self.typecode, self.example[::-1]) @@ -284,9 +286,12 @@ ) b = array.array(self.badtypecode()) - self.assertRaises(TypeError, a.__add__, b) - - self.assertRaises(TypeError, a.__iadd__, "bad") + #self.assertRaises(TypeError, a.__add__, b) + #self.assertRaises(TypeError, a.__iadd__, "bad") + def f(i, j): + i += j + self.assertRaises(TypeError, f, a, b) + self.assertRaises(TypeError, f, a, "bad") def test_mul(self): a = 5*array.array(self.typecode, self.example) @@ -313,7 +318,8 @@ array.array(self.typecode) ) - self.assertRaises(TypeError, a.__mul__, "bad") + #self.assertRaises(TypeError, a.__mul__, "bad") + self.assertRaises(TypeError, lambda i, j: i * j, a, "bad") def test_imul(self): a = array.array(self.typecode, self.example) @@ -342,7 +348,10 @@ a *= -1 self.assertEqual(a, array.array(self.typecode)) - self.assertRaises(TypeError, a.__imul__, "bad") + #self.assertRaises(TypeError, a.__imul__, "bad") + def f(i, j): + i *= j + self.assertRaises(TypeError, f, a, "bad") def test_getitem(self): a = array.array(self.typecode, self.example) Modified: pypy/trunk/pypy/module/array/interp_array.py ============================================================================== --- pypy/trunk/pypy/module/array/interp_array.py (original) +++ pypy/trunk/pypy/module/array/interp_array.py Thu Aug 12 11:31:54 2010 @@ -475,7 +475,10 @@ return self def mul__Array_ANY(space, self, w_repeat): - repeat = space.int_w(w_repeat) + try: + repeat = space.int_w(w_repeat) + except OperationError: + return space.w_NotImplemented a = mytype.w_class(space) repeat = max(repeat, 0) a.setlen(self.len * repeat) @@ -488,7 +491,10 @@ return mul__Array_ANY(space, self, w_repeat) def inplace_mul__Array_ANY(space, self, w_repeat): - repeat = space.int_w(w_repeat) + try: + repeat = space.int_w(w_repeat) + except OperationError: + return space.w_NotImplemented oldlen = self.len repeat = max(repeat, 0) self.setlen(self.len * repeat) Modified: pypy/trunk/pypy/module/array/test/test_array.py ============================================================================== --- pypy/trunk/pypy/module/array/test/test_array.py (original) +++ pypy/trunk/pypy/module/array/test/test_array.py Thu Aug 12 11:31:54 2010 @@ -593,13 +593,16 @@ raises(TypeError, "a = self.array('i') + 2") raises(TypeError, "self.array('i') + self.array('b')") + a = self.array('i') + raises(TypeError, "a += 7") # Calling __add__ directly raises TypeError in cpython but # returns NotImplemented in pypy if placed within a # try: except TypeError: construction. # - # raises(TypeError, self.array('i').__add__, (2,)) - # raises(TypeError, self.array('i').__add__, self.array('b')) + #raises(TypeError, self.array('i').__add__, (2,)) + #raises(TypeError, self.array('i').__iadd__, (2,)) + #raises(TypeError, self.array('i').__add__, self.array('b')) class addable(object): def __add__(self, other): @@ -611,6 +614,10 @@ assert addable() + self.array('i') == 'add' assert self.array('i') + addable() == 'radd' + a = self.array('i') + a += addable() + assert a == 'radd' + a = self.array('i', [1, 2]) assert a * -1 == self.array('i') b = a @@ -618,6 +625,24 @@ assert a == self.array('i') assert b == self.array('i') + a = self.array('i') + raises(TypeError, "a * 'hi'") + raises(TypeError, "'hi' * a") + + class mulable(object): + def __mul__(self, other): + return "mul" + + def __rmul__(self, other): + return "rmul" + + assert mulable() * self.array('i') == 'mul' + assert self.array('i') * mulable() == 'rmul' + + a = self.array('i') + a *= mulable() + assert a == 'rmul' + def test_delitem(self): a = self.array('i', [1, 2, 3]) del a[1] From hakanardo at codespeak.net Thu Aug 12 13:23:26 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Thu, 12 Aug 2010 13:23:26 +0200 (CEST) Subject: [pypy-svn] r76602 - pypy/trunk/pypy/module/array/benchmark Message-ID: <20100812112326.0BE81282B9E@codespeak.net> Author: hakanardo Date: Thu Aug 12 13:23:25 2010 New Revision: 76602 Modified: pypy/trunk/pypy/module/array/benchmark/intimgtst.py pypy/trunk/pypy/module/array/benchmark/sumtst.py Log: cleanups Modified: pypy/trunk/pypy/module/array/benchmark/intimgtst.py ============================================================================== --- pypy/trunk/pypy/module/array/benchmark/intimgtst.py (original) +++ pypy/trunk/pypy/module/array/benchmark/intimgtst.py Thu Aug 12 13:23:25 2010 @@ -1,7 +1,5 @@ #!/usr/bin/python -from time import time - -from array import array, simple_array +from array import array def f(img, intimg): l=0 @@ -14,13 +12,7 @@ -if True: - img=array('d','\x00'*640*480*8) - intimg=array('d','\x00'*640*480*8) -else: - img=simple_array(640*480) - intimg=simple_array(640*480) +img=array('d','\x00'*640*480*8) +intimg=array('d','\x00'*640*480*8) -start=time() for l in range(500): f(img, intimg) -print time()-start Modified: pypy/trunk/pypy/module/array/benchmark/sumtst.py ============================================================================== --- pypy/trunk/pypy/module/array/benchmark/sumtst.py (original) +++ pypy/trunk/pypy/module/array/benchmark/sumtst.py Thu Aug 12 13:23:25 2010 @@ -1,7 +1,6 @@ #!/usr/bin/python from array import array -#img=array('d',(0,)*640*480); def f(img): l=0 i=0; @@ -11,12 +10,9 @@ return l img=array('d', (0,)) * (640*480) -#img=array('d', [0]*640*480) -#img=array('d', (0,))*(640*480) for l in range(500): f(img) -#print f(img) # C pypy-simple pypy cpython -# sumtst: 0m0.630s 0m0.659s 0m0.762s 0m33.447s -# intimg: 0m0.646s 0m1.078s 0m1.357s 1m0.279s +# sumtst: 0m0.630s 0m0.659s 0m0.851s 0m33.447s +# intimg: 0m0.646s 0m1.078s 0m1.446s 1m0.279s From arigo at codespeak.net Thu Aug 12 13:47:48 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 12 Aug 2010 13:47:48 +0200 (CEST) Subject: [pypy-svn] r76603 - in pypy/branch/kill-caninline: . lib-python lib-python/modified-2.5.2 lib-python/modified-2.5.2/ctypes lib_pypy lib_pypy/_ctypes lib_pypy/pypy_test pypy/config pypy/doc pypy/doc/config pypy/interpreter pypy/interpreter/astcompiler pypy/interpreter/astcompiler/tools pypy/interpreter/test pypy/jit/backend pypy/jit/backend/llsupport pypy/jit/backend/llsupport/test pypy/jit/backend/test pypy/jit/backend/x86 pypy/jit/backend/x86/test pypy/jit/backend/x86/tool pypy/jit/codewriter pypy/jit/codewriter/test pypy/jit/metainterp pypy/jit/metainterp/doc pypy/jit/metainterp/test pypy/jit/tl pypy/jit/tool pypy/module/__builtin__ pypy/module/__builtin__/test pypy/module/_ast pypy/module/_ast/test pypy/module/_codecs/test pypy/module/_demo pypy/module/_file pypy/module/_file/test pypy/module/_rawffi/test pypy/module/_socket/test pypy/module/_sre/test pypy/module/_stackless pypy/module/array pypy/module/cpyext pypy/module/fcntl/test pypy/module/marshal/test pypy/module/posix pypy/module/posix/test pypy/module/pypyjit pypy/module/pypyjit/test pypy/module/signal pypy/module/test_lib_pypy pypy/module/test_lib_pypy/ctypes_tests pypy/module/thread/test pypy/objspace/std pypy/objspace/std/test pypy/rlib pypy/rlib/test pypy/rpython pypy/rpython/lltypesystem pypy/rpython/module pypy/rpython/module/test pypy/rpython/test pypy/rpython/tool pypy/translator/c pypy/translator/c/test pypy/translator/goal pypy/translator/goal/test2 pypy/translator/platform Message-ID: <20100812114748.0E19C282B9E@codespeak.net> Author: arigo Date: Thu Aug 12 13:47:36 2010 New Revision: 76603 Added: pypy/branch/kill-caninline/lib_pypy/array.py - copied unchanged from r76600, pypy/trunk/lib_pypy/array.py pypy/branch/kill-caninline/pypy/doc/config/objspace.usemodules.array.txt - copied unchanged from r76600, pypy/trunk/pypy/doc/config/objspace.usemodules.array.txt pypy/branch/kill-caninline/pypy/jit/backend/x86/arch.py - copied unchanged from r76600, pypy/trunk/pypy/jit/backend/x86/arch.py pypy/branch/kill-caninline/pypy/jit/backend/x86/regloc.py - copied unchanged from r76600, pypy/trunk/pypy/jit/backend/x86/regloc.py pypy/branch/kill-caninline/pypy/jit/backend/x86/rx86.py - copied unchanged from r76600, pypy/trunk/pypy/jit/backend/x86/rx86.py pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_regloc.py - copied unchanged from r76600, pypy/trunk/pypy/jit/backend/x86/test/test_regloc.py pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_rx86.py - copied unchanged from r76600, pypy/trunk/pypy/jit/backend/x86/test/test_rx86.py pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py - copied unchanged from r76600, pypy/trunk/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py - copied unchanged from r76600, pypy/trunk/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py pypy/branch/kill-caninline/pypy/jit/backend/x86/tool/instruction_encoding.sh - copied unchanged from r76600, pypy/trunk/pypy/jit/backend/x86/tool/instruction_encoding.sh pypy/branch/kill-caninline/pypy/jit/tool/gen-trace-mode-keywords.py - copied unchanged from r76600, pypy/trunk/pypy/jit/tool/gen-trace-mode-keywords.py pypy/branch/kill-caninline/pypy/jit/tool/pypytrace-mode.el - copied unchanged from r76600, pypy/trunk/pypy/jit/tool/pypytrace-mode.el pypy/branch/kill-caninline/pypy/module/array/ - copied from r76600, pypy/trunk/pypy/module/array/ pypy/branch/kill-caninline/pypy/rlib/test/test_rposix.py - copied unchanged from r76600, pypy/trunk/pypy/rlib/test/test_rposix.py pypy/branch/kill-caninline/pypy/rpython/module/ll_win32file.py - copied unchanged from r76600, pypy/trunk/pypy/rpython/module/ll_win32file.py Removed: pypy/branch/kill-caninline/lib_pypy/greenlet.py pypy/branch/kill-caninline/pypy/jit/backend/x86/ri386.py pypy/branch/kill-caninline/pypy/jit/backend/x86/ri386setup.py pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_ri386.py pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py pypy/branch/kill-caninline/pypy/jit/metainterp/doc/ pypy/branch/kill-caninline/pypy/module/test_lib_pypy/test_array.py Modified: pypy/branch/kill-caninline/ (props changed) pypy/branch/kill-caninline/lib-python/ (props changed) pypy/branch/kill-caninline/lib-python/conftest.py pypy/branch/kill-caninline/lib-python/modified-2.5.2/ctypes/__init__.py pypy/branch/kill-caninline/lib-python/modified-2.5.2/site.py pypy/branch/kill-caninline/lib_pypy/ (props changed) pypy/branch/kill-caninline/lib_pypy/_ctypes/__init__.py pypy/branch/kill-caninline/lib_pypy/_ctypes/array.py pypy/branch/kill-caninline/lib_pypy/_ctypes/builtin.py pypy/branch/kill-caninline/lib_pypy/_ctypes/primitive.py pypy/branch/kill-caninline/lib_pypy/datetime.py pypy/branch/kill-caninline/lib_pypy/dbm.py (props changed) pypy/branch/kill-caninline/lib_pypy/pypy_test/test_coroutine.py pypy/branch/kill-caninline/lib_pypy/pypy_test/test_ctypes_support.py pypy/branch/kill-caninline/lib_pypy/pypy_test/test_datetime.py pypy/branch/kill-caninline/lib_pypy/pypy_test/test_functools.py (props changed) pypy/branch/kill-caninline/lib_pypy/stackless.py pypy/branch/kill-caninline/pypy/config/pypyoption.py pypy/branch/kill-caninline/pypy/config/translationoption.py pypy/branch/kill-caninline/pypy/doc/faq.txt pypy/branch/kill-caninline/pypy/interpreter/astcompiler/assemble.py pypy/branch/kill-caninline/pypy/interpreter/astcompiler/ast.py pypy/branch/kill-caninline/pypy/interpreter/astcompiler/codegen.py pypy/branch/kill-caninline/pypy/interpreter/astcompiler/consts.py pypy/branch/kill-caninline/pypy/interpreter/astcompiler/symtable.py pypy/branch/kill-caninline/pypy/interpreter/astcompiler/tools/asdl_py.py pypy/branch/kill-caninline/pypy/interpreter/baseobjspace.py pypy/branch/kill-caninline/pypy/interpreter/error.py pypy/branch/kill-caninline/pypy/interpreter/gateway.py pypy/branch/kill-caninline/pypy/interpreter/pyopcode.py pypy/branch/kill-caninline/pypy/interpreter/test/test_compiler.py pypy/branch/kill-caninline/pypy/interpreter/test/test_gateway.py pypy/branch/kill-caninline/pypy/jit/backend/detect_cpu.py pypy/branch/kill-caninline/pypy/jit/backend/llsupport/descr.py pypy/branch/kill-caninline/pypy/jit/backend/llsupport/gc.py pypy/branch/kill-caninline/pypy/jit/backend/llsupport/regalloc.py pypy/branch/kill-caninline/pypy/jit/backend/llsupport/test/test_descr.py pypy/branch/kill-caninline/pypy/jit/backend/llsupport/test/test_regalloc.py pypy/branch/kill-caninline/pypy/jit/backend/test/runner_test.py pypy/branch/kill-caninline/pypy/jit/backend/x86/assembler.py pypy/branch/kill-caninline/pypy/jit/backend/x86/codebuf.py pypy/branch/kill-caninline/pypy/jit/backend/x86/jump.py pypy/branch/kill-caninline/pypy/jit/backend/x86/regalloc.py pypy/branch/kill-caninline/pypy/jit/backend/x86/runner.py pypy/branch/kill-caninline/pypy/jit/backend/x86/test/conftest.py pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_assembler.py pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_basic.py pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_gc_integration.py pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_jump.py pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_recompilation.py pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_regalloc.py pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_regalloc2.py pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_runner.py pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_symbolic_x86.py pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_zll_random.py pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_zrpy_gc.py pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_ztranslation.py pypy/branch/kill-caninline/pypy/jit/backend/x86/tool/viewcode.py pypy/branch/kill-caninline/pypy/jit/codewriter/codewriter.py pypy/branch/kill-caninline/pypy/jit/codewriter/jtransform.py pypy/branch/kill-caninline/pypy/jit/codewriter/test/test_flatten.py pypy/branch/kill-caninline/pypy/jit/metainterp/optimizeopt.py pypy/branch/kill-caninline/pypy/jit/metainterp/resoperation.py pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_executor.py pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_loop.py pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_loop_spec.py pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/kill-caninline/pypy/jit/tl/pypyjit.py pypy/branch/kill-caninline/pypy/jit/tl/pypyjit_demo.py pypy/branch/kill-caninline/pypy/module/__builtin__/compiling.py pypy/branch/kill-caninline/pypy/module/__builtin__/test/test_buffer.py pypy/branch/kill-caninline/pypy/module/_ast/__init__.py pypy/branch/kill-caninline/pypy/module/_ast/test/test_ast.py pypy/branch/kill-caninline/pypy/module/_codecs/test/test_codecs.py pypy/branch/kill-caninline/pypy/module/_demo/demo.py pypy/branch/kill-caninline/pypy/module/_file/interp_file.py pypy/branch/kill-caninline/pypy/module/_file/test/test_file.py pypy/branch/kill-caninline/pypy/module/_file/test/test_file_extra.py pypy/branch/kill-caninline/pypy/module/_rawffi/test/test__rawffi.py pypy/branch/kill-caninline/pypy/module/_socket/test/test_sock_app.py pypy/branch/kill-caninline/pypy/module/_sre/test/test_app_sre.py pypy/branch/kill-caninline/pypy/module/_stackless/interp_coroutine.py pypy/branch/kill-caninline/pypy/module/cpyext/api.py pypy/branch/kill-caninline/pypy/module/cpyext/methodobject.py pypy/branch/kill-caninline/pypy/module/fcntl/test/test_fcntl.py pypy/branch/kill-caninline/pypy/module/marshal/test/test_marshalimpl.py pypy/branch/kill-caninline/pypy/module/posix/interp_posix.py pypy/branch/kill-caninline/pypy/module/posix/test/test_posix2.py pypy/branch/kill-caninline/pypy/module/pypyjit/policy.py pypy/branch/kill-caninline/pypy/module/pypyjit/test/test_pypy_c.py pypy/branch/kill-caninline/pypy/module/signal/interp_signal.py pypy/branch/kill-caninline/pypy/module/test_lib_pypy/ctypes_tests/ (props changed) pypy/branch/kill-caninline/pypy/module/thread/test/test_gil.py pypy/branch/kill-caninline/pypy/objspace/std/callmethod.py pypy/branch/kill-caninline/pypy/objspace/std/itertype.py pypy/branch/kill-caninline/pypy/objspace/std/model.py pypy/branch/kill-caninline/pypy/objspace/std/test/test_callmethod.py pypy/branch/kill-caninline/pypy/rlib/objectmodel.py pypy/branch/kill-caninline/pypy/rlib/rarithmetic.py pypy/branch/kill-caninline/pypy/rlib/rdynload.py pypy/branch/kill-caninline/pypy/rlib/rmmap.py pypy/branch/kill-caninline/pypy/rlib/rposix.py pypy/branch/kill-caninline/pypy/rlib/rwin32.py pypy/branch/kill-caninline/pypy/rlib/streamio.py pypy/branch/kill-caninline/pypy/rlib/test/test_objectmodel.py pypy/branch/kill-caninline/pypy/rlib/test/test_rarithmetic.py pypy/branch/kill-caninline/pypy/rpython/extfunc.py pypy/branch/kill-caninline/pypy/rpython/lltypesystem/ll2ctypes.py pypy/branch/kill-caninline/pypy/rpython/lltypesystem/rffi.py pypy/branch/kill-caninline/pypy/rpython/lltypesystem/rstr.py pypy/branch/kill-caninline/pypy/rpython/module/ll_os.py pypy/branch/kill-caninline/pypy/rpython/module/ll_os_stat.py pypy/branch/kill-caninline/pypy/rpython/module/test/test_ll_os_stat.py pypy/branch/kill-caninline/pypy/rpython/rstr.py pypy/branch/kill-caninline/pypy/rpython/test/test_extfunc.py pypy/branch/kill-caninline/pypy/rpython/test/test_rstr.py pypy/branch/kill-caninline/pypy/rpython/tool/rfficache.py pypy/branch/kill-caninline/pypy/translator/c/genc.py pypy/branch/kill-caninline/pypy/translator/c/node.py pypy/branch/kill-caninline/pypy/translator/c/test/test_typed.py pypy/branch/kill-caninline/pypy/translator/goal/app_main.py pypy/branch/kill-caninline/pypy/translator/goal/test2/test_app_main.py pypy/branch/kill-caninline/pypy/translator/platform/__init__.py pypy/branch/kill-caninline/pypy/translator/platform/darwin.py pypy/branch/kill-caninline/pypy/translator/platform/posix.py pypy/branch/kill-caninline/pypy/translator/platform/windows.py Log: Merge trunk: svn merge -r75997:76600 svn+ssh://codespeak.net/svn/pypy/trunk . Modified: pypy/branch/kill-caninline/lib-python/conftest.py ============================================================================== --- pypy/branch/kill-caninline/lib-python/conftest.py (original) +++ pypy/branch/kill-caninline/lib-python/conftest.py Thu Aug 12 13:47:36 2010 @@ -132,7 +132,7 @@ RegrTest('test_ast.py', core=True), RegrTest('test_anydbm.py'), RegrTest('test_applesingle.py', skip=True), - RegrTest('test_array.py', core=True, usemodules='struct'), + RegrTest('test_array.py', core=True, usemodules='struct array'), RegrTest('test_asynchat.py', usemodules='thread'), RegrTest('test_atexit.py', core=True), RegrTest('test_audioop.py', skip=True), @@ -464,11 +464,7 @@ RegrTest('test_coding.py'), RegrTest('test_complex_args.py'), RegrTest('test_contextlib.py', usemodules="thread"), - # we skip test ctypes, since we adapted it massively in order - # to test what we want to support. There are real failures, - # but it's about missing features that we don't want to support - # now - RegrTest('test_ctypes.py', skip="we have a replacement"), + RegrTest('test_ctypes.py', usemodules="_rawffi"), RegrTest('test_defaultdict.py'), RegrTest('test_email_renamed.py'), RegrTest('test_exception_variations.py'), Modified: pypy/branch/kill-caninline/lib-python/modified-2.5.2/ctypes/__init__.py ============================================================================== --- pypy/branch/kill-caninline/lib-python/modified-2.5.2/ctypes/__init__.py (original) +++ pypy/branch/kill-caninline/lib-python/modified-2.5.2/ctypes/__init__.py Thu Aug 12 13:47:36 2010 @@ -471,7 +471,7 @@ # functions -from _ctypes import _memmove_addr, _memset_addr, _string_at_addr, _cast_addr +from _ctypes import _memmove_addr, _memset_addr, _cast_addr ## void *memmove(void *, const void *, size_t); memmove = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)(_memmove_addr) @@ -490,24 +490,34 @@ def cast(obj, typ): return _cast(obj, obj, typ) -_string_at = CFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr) +try: + from _ctypes import _string_at_addr +except ImportError: + from _ctypes import _string_at +else: + _string_at = CFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr) + def string_at(ptr, size=-1): """string_at(addr[, size]) -> string Return the string at addr.""" return _string_at(ptr, size) +def wstring_at(ptr, size=-1): + """wstring_at(addr[, size]) -> string + + Return the string at addr.""" + return _wstring_at(ptr, size) + try: from _ctypes import _wstring_at_addr except ImportError: - pass + try: + from _ctypes import _wstring_at + except ImportError: + del wstring_at else: _wstring_at = CFUNCTYPE(py_object, c_void_p, c_int)(_wstring_at_addr) - def wstring_at(ptr, size=-1): - """wstring_at(addr[, size]) -> string - - Return the string at addr.""" - return _wstring_at(ptr, size) if _os.name in ("nt", "ce"): # COM stuff Modified: pypy/branch/kill-caninline/lib-python/modified-2.5.2/site.py ============================================================================== --- pypy/branch/kill-caninline/lib-python/modified-2.5.2/site.py (original) +++ pypy/branch/kill-caninline/lib-python/modified-2.5.2/site.py Thu Aug 12 13:47:36 2010 @@ -175,7 +175,7 @@ def addsitepackages(known_paths): """Add site-packages to sys.path, in a PyPy-specific way.""" - if hasattr(sys, 'pypy_version_info'): + if hasattr(sys, 'pypy_version_info') and hasattr(sys, 'prefix'): from distutils.sysconfig import get_python_lib sitedir = get_python_lib(standard_lib=False) if os.path.isdir(sitedir): Modified: pypy/branch/kill-caninline/lib_pypy/_ctypes/__init__.py ============================================================================== --- pypy/branch/kill-caninline/lib_pypy/_ctypes/__init__.py (original) +++ pypy/branch/kill-caninline/lib_pypy/_ctypes/__init__.py Thu Aug 12 13:47:36 2010 @@ -7,8 +7,8 @@ from _ctypes.dll import dlopen from _ctypes.structure import Structure from _ctypes.array import Array -from _ctypes.builtin import _memmove_addr, _string_at_addr, _memset_addr,\ - set_conversion_mode, _wstring_at_addr +from _ctypes.builtin import _memmove_addr, _string_at, _memset_addr,\ + set_conversion_mode, _wstring_at from _ctypes.union import Union import os as _os Modified: pypy/branch/kill-caninline/lib_pypy/_ctypes/array.py ============================================================================== --- pypy/branch/kill-caninline/lib_pypy/_ctypes/array.py (original) +++ pypy/branch/kill-caninline/lib_pypy/_ctypes/array.py Thu Aug 12 13:47:36 2010 @@ -4,7 +4,6 @@ from _ctypes.basics import _CData, cdata_from_address, _CDataMeta, sizeof from _ctypes.basics import keepalive_key, store_reference, ensure_objects from _ctypes.basics import CArgObject -from _ctypes.builtin import _string_at_addr, _wstring_at_addr def _create_unicode(buffer, maxlength): res = [] Modified: pypy/branch/kill-caninline/lib_pypy/_ctypes/builtin.py ============================================================================== --- pypy/branch/kill-caninline/lib_pypy/_ctypes/builtin.py (original) +++ pypy/branch/kill-caninline/lib_pypy/_ctypes/builtin.py Thu Aug 12 13:47:36 2010 @@ -8,10 +8,10 @@ _memmove_addr = _rawffi.get_libc().getaddressindll('memmove') _memset_addr = _rawffi.get_libc().getaddressindll('memset') -def _string_at_addr(addr, lgt): +def _string_at(addr, lgt): # address here can be almost anything import ctypes - arg = ctypes.c_char_p._CData_value(addr) + arg = ctypes.c_void_p._CData_value(addr) return _rawffi.charp2rawstring(arg, lgt) def set_conversion_mode(encoding, errors): @@ -20,9 +20,9 @@ ConvMode.encoding = encoding return old_cm -def _wstring_at_addr(addr, lgt): +def _wstring_at(addr, lgt): import ctypes - arg = ctypes.c_wchar_p._CData_value(addr) + arg = ctypes.c_void_p._CData_value(addr) # XXX purely applevel if lgt == -1: lgt = sys.maxint Modified: pypy/branch/kill-caninline/lib_pypy/_ctypes/primitive.py ============================================================================== --- pypy/branch/kill-caninline/lib_pypy/_ctypes/primitive.py (original) +++ pypy/branch/kill-caninline/lib_pypy/_ctypes/primitive.py Thu Aug 12 13:47:36 2010 @@ -86,6 +86,8 @@ return value if isinstance(value, _Pointer): return cls.from_address(value._buffer.buffer) + if isinstance(value, (int, long)): + return cls(value) FROM_PARAM_BY_TYPE = { 'z': from_param_char_p, @@ -141,13 +143,13 @@ result.value = property(_getvalue, _setvalue) elif tp == 'Z': # c_wchar_p - from _ctypes import Array, _Pointer, _wstring_at_addr + from _ctypes import Array, _Pointer, _wstring_at def _getvalue(self): addr = self._buffer[0] if addr == 0: return None else: - return _wstring_at_addr(addr, -1) + return _wstring_at(addr, -1) def _setvalue(self, value): if isinstance(value, basestring): @@ -216,14 +218,14 @@ SysAllocStringLen = windll.oleaut32.SysAllocStringLen SysStringLen = windll.oleaut32.SysStringLen SysFreeString = windll.oleaut32.SysFreeString - from _ctypes import _wstring_at_addr + from _ctypes import _wstring_at def _getvalue(self): addr = self._buffer[0] if addr == 0: return None else: size = SysStringLen(addr) - return _wstring_at_addr(addr, size) + return _wstring_at(addr, size) def _setvalue(self, value): if isinstance(value, basestring): @@ -254,18 +256,21 @@ from_address = cdata_from_address def from_param(self, value): + if isinstance(value, self): + return value + from_param_f = FROM_PARAM_BY_TYPE.get(self._type_) if from_param_f: res = from_param_f(self, value) if res is not None: return res - - if isinstance(value, self): - return value - try: - return self(value) - except (TypeError, ValueError): - return super(SimpleType, self).from_param(value) + else: + try: + return self(value) + except (TypeError, ValueError): + pass + + return super(SimpleType, self).from_param(value) def _CData_output(self, resbuffer, base=None, index=-1): output = super(SimpleType, self)._CData_output(resbuffer, base, index) Modified: pypy/branch/kill-caninline/lib_pypy/datetime.py ============================================================================== --- pypy/branch/kill-caninline/lib_pypy/datetime.py (original) +++ pypy/branch/kill-caninline/lib_pypy/datetime.py Thu Aug 12 13:47:36 2010 @@ -1412,7 +1412,7 @@ def utcfromtimestamp(cls, t): "Construct a UTC datetime from a POSIX timestamp (like time.time())." - if 1 - (t % 1.0) < 0.000001: + if 1 - (t % 1.0) < 0.0000005: t = float(int(t)) + 1 if t < 0: t -= 1 Modified: pypy/branch/kill-caninline/lib_pypy/pypy_test/test_coroutine.py ============================================================================== --- pypy/branch/kill-caninline/lib_pypy/pypy_test/test_coroutine.py (original) +++ pypy/branch/kill-caninline/lib_pypy/pypy_test/test_coroutine.py Thu Aug 12 13:47:36 2010 @@ -2,7 +2,7 @@ from py.test import skip, raises try: - from ..stackless import coroutine + from ..stackless import coroutine, CoroutineExit except ImportError, e: skip('cannot import stackless: %s' % (e,)) Modified: pypy/branch/kill-caninline/lib_pypy/pypy_test/test_ctypes_support.py ============================================================================== --- pypy/branch/kill-caninline/lib_pypy/pypy_test/test_ctypes_support.py (original) +++ pypy/branch/kill-caninline/lib_pypy/pypy_test/test_ctypes_support.py Thu Aug 12 13:47:36 2010 @@ -20,3 +20,14 @@ assert get_errno() != 0 set_errno(0) assert get_errno() == 0 + +def test_argument_conversion_and_checks(): + import ctypes + libc = ctypes.cdll.LoadLibrary("libc.so.6") + libc.strlen.argtypes = ctypes.c_char_p, + libc.strlen.restype = ctypes.c_size_t + assert libc.strlen("eggs") == 4 + + # Should raise ArgumentError, not segfault + py.test.raises(ctypes.ArgumentError, libc.strlen, False) + Modified: pypy/branch/kill-caninline/lib_pypy/pypy_test/test_datetime.py ============================================================================== --- pypy/branch/kill-caninline/lib_pypy/pypy_test/test_datetime.py (original) +++ pypy/branch/kill-caninline/lib_pypy/pypy_test/test_datetime.py Thu Aug 12 13:47:36 2010 @@ -15,4 +15,18 @@ expected = datetime.datetime(*(time.strptime(string, format)[0:6])) got = datetime.datetime.strptime(string, format) assert expected == got + +def test_datetime_rounding(): + b = 0.0000001 + a = 0.9999994 + + assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 + assert datetime.datetime.utcfromtimestamp(a).second == 0 + a += b + assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 + assert datetime.datetime.utcfromtimestamp(a).second == 0 + a += b + assert datetime.datetime.utcfromtimestamp(a).microsecond == 0 + assert datetime.datetime.utcfromtimestamp(a).second == 1 + Modified: pypy/branch/kill-caninline/lib_pypy/stackless.py ============================================================================== --- pypy/branch/kill-caninline/lib_pypy/stackless.py (original) +++ pypy/branch/kill-caninline/lib_pypy/stackless.py Thu Aug 12 13:47:36 2010 @@ -14,9 +14,13 @@ import traceback import sys try: + # If _stackless can be imported then TaskletExit and CoroutineExit are + # automatically added to the builtins. from _stackless import coroutine, greenlet except ImportError: # we are running from CPython - from greenlet import greenlet + from greenlet import greenlet, GreenletExit + TaskletExit = CoroutineExit = GreenletExit + del GreenletExit try: from functools import partial except ImportError: # we are not running python 2.5 Modified: pypy/branch/kill-caninline/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/kill-caninline/pypy/config/pypyoption.py (original) +++ pypy/branch/kill-caninline/pypy/config/pypyoption.py Thu Aug 12 13:47:36 2010 @@ -30,7 +30,7 @@ "rctime" , "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios", "zlib", "struct", "md5", "sha", "bz2", "_minimal_curses", "cStringIO", - "thread", "itertools", "pyexpat", "_ssl", "cpyext"] + "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array"] )) working_oo_modules = default_modules.copy() Modified: pypy/branch/kill-caninline/pypy/config/translationoption.py ============================================================================== --- pypy/branch/kill-caninline/pypy/config/translationoption.py (original) +++ pypy/branch/kill-caninline/pypy/config/translationoption.py Thu Aug 12 13:47:36 2010 @@ -342,6 +342,10 @@ 'jit': 'hybrid extraopts jit', } +# For now, 64-bit JIT requires boehm +if IS_64_BITS: + OPT_TABLE['jit'] = OPT_TABLE['jit'].replace('hybrid', 'boehm') + def set_opt_level(config, level): """Apply optimization suggestions on the 'config'. The optimizations depend on the selected level and possibly on the backend. Modified: pypy/branch/kill-caninline/pypy/doc/faq.txt ============================================================================== --- pypy/branch/kill-caninline/pypy/doc/faq.txt (original) +++ pypy/branch/kill-caninline/pypy/doc/faq.txt Thu Aug 12 13:47:36 2010 @@ -47,8 +47,8 @@ There is also an experimental support for CPython extension modules, so they'll run without change (from current observation, rather with little -change) on trunk. It has not been released yet, although it should be a major -point of the next pypy release. +change) on trunk. It has been a part of 1.3 release, but support is still +in alpha phase. .. _`extension modules`: cpython_differences.html#extension-modules .. _`cpython_differences`: cpython_differences.html @@ -373,7 +373,7 @@ -------------------------------------------- No. PyPy always runs your code in its own interpreter, which is a -full and compliant Python 2.4 interpreter. RPython_ is only the +full and compliant Python 2.5 interpreter. RPython_ is only the language in which parts of PyPy itself are written and extension modules for it. The answer to whether something needs to be written as an extension module, apart from the "gluing to external libraries" reason, will Modified: pypy/branch/kill-caninline/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/kill-caninline/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/kill-caninline/pypy/interpreter/astcompiler/assemble.py Thu Aug 12 13:47:36 2010 @@ -566,22 +566,22 @@ return (oparg % 256) + 2 * (oparg / 256) def _compute_CALL_FUNCTION(arg): - return _num_args(arg) + return -_num_args(arg) def _compute_CALL_FUNCTION_VAR(arg): - return _num_args(arg) - 1 + return -_num_args(arg) - 1 def _compute_CALL_FUNCTION_KW(arg): - return _num_args(arg) - 1 + return -_num_args(arg) - 1 def _compute_CALL_FUNCTION_VAR_KW(arg): - return _num_args(arg) - 2 + return -_num_args(arg) - 2 def _compute_CALL_LIKELY_BUILTIN(arg): return -(arg & 0xFF) + 1 def _compute_CALL_METHOD(arg): - return -arg - 1 + return -_num_args(arg) - 1 _stack_effect_computers = {} Modified: pypy/branch/kill-caninline/pypy/interpreter/astcompiler/ast.py ============================================================================== --- pypy/branch/kill-caninline/pypy/interpreter/astcompiler/ast.py (original) +++ pypy/branch/kill-caninline/pypy/interpreter/astcompiler/ast.py Thu Aug 12 13:47:36 2010 @@ -230,6 +230,7 @@ visitor.visit_FunctionDef(self) def mutate_over(self, visitor): + self.args = self.args.mutate_over(visitor) if self.body: visitor._mutate_sequence(self.body) if self.decorators: @@ -784,6 +785,8 @@ def mutate_over(self, visitor): if self.body: visitor._mutate_sequence(self.body) + if self.handlers: + visitor._mutate_sequence(self.handlers) if self.orelse: visitor._mutate_sequence(self.orelse) return visitor.visit_TryExcept(self) @@ -927,6 +930,8 @@ visitor.visit_Import(self) def mutate_over(self, visitor): + if self.names: + visitor._mutate_sequence(self.names) return visitor.visit_Import(self) def sync_app_attrs(self, space): @@ -965,6 +970,8 @@ visitor.visit_ImportFrom(self) def mutate_over(self, visitor): + if self.names: + visitor._mutate_sequence(self.names) return visitor.visit_ImportFrom(self) def sync_app_attrs(self, space): @@ -1282,6 +1289,7 @@ visitor.visit_Lambda(self) def mutate_over(self, visitor): + self.args = self.args.mutate_over(visitor) self.body = self.body.mutate_over(visitor) return visitor.visit_Lambda(self) @@ -1398,6 +1406,8 @@ def mutate_over(self, visitor): self.elt = self.elt.mutate_over(visitor) + if self.generators: + visitor._mutate_sequence(self.generators) return visitor.visit_ListComp(self) def sync_app_attrs(self, space): @@ -1437,6 +1447,8 @@ def mutate_over(self, visitor): self.elt = self.elt.mutate_over(visitor) + if self.generators: + visitor._mutate_sequence(self.generators) return visitor.visit_GeneratorExp(self) def sync_app_attrs(self, space): @@ -1562,6 +1574,8 @@ self.func = self.func.mutate_over(visitor) if self.args: visitor._mutate_sequence(self.args) + if self.keywords: + visitor._mutate_sequence(self.keywords) if self.starargs: self.starargs = self.starargs.mutate_over(visitor) if self.kwargs: @@ -2293,6 +2307,13 @@ self.w_ifs = None self.initialization_state = 7 + def mutate_over(self, visitor): + self.target = self.target.mutate_over(visitor) + self.iter = self.iter.mutate_over(visitor) + if self.ifs: + visitor._mutate_sequence(self.ifs) + return visitor.visit_comprehension(self) + def walkabout(self, visitor): visitor.visit_comprehension(self) @@ -2327,6 +2348,15 @@ self.col_offset = col_offset self.initialization_state = 31 + def mutate_over(self, visitor): + if self.type: + self.type = self.type.mutate_over(visitor) + if self.name: + self.name = self.name.mutate_over(visitor) + if self.body: + visitor._mutate_sequence(self.body) + return visitor.visit_excepthandler(self) + def walkabout(self, visitor): visitor.visit_excepthandler(self) @@ -2366,6 +2396,13 @@ self.w_defaults = None self.initialization_state = 15 + def mutate_over(self, visitor): + if self.args: + visitor._mutate_sequence(self.args) + if self.defaults: + visitor._mutate_sequence(self.defaults) + return visitor.visit_arguments(self) + def walkabout(self, visitor): visitor.visit_arguments(self) @@ -2407,6 +2444,10 @@ self.value = value self.initialization_state = 3 + def mutate_over(self, visitor): + self.value = self.value.mutate_over(visitor) + return visitor.visit_keyword(self) + def walkabout(self, visitor): visitor.visit_keyword(self) @@ -2426,6 +2467,9 @@ self.asname = asname self.initialization_state = 3 + def mutate_over(self, visitor): + return visitor.visit_alias(self) + def walkabout(self, visitor): visitor.visit_alias(self) Modified: pypy/branch/kill-caninline/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/kill-caninline/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/kill-caninline/pypy/interpreter/astcompiler/codegen.py Thu Aug 12 13:47:36 2010 @@ -258,9 +258,11 @@ # Load decorators first, but apply them after the function is created. if func.decorators: self.visit_sequence(func.decorators) - if func.args.defaults: - self.visit_sequence(func.args.defaults) - num_defaults = len(func.args.defaults) + args = func.args + assert isinstance(args, ast.arguments) + if args.defaults: + self.visit_sequence(args.defaults) + num_defaults = len(args.defaults) else: num_defaults = 0 code = self.sub_scope(FunctionCodeGenerator, func.name, func, @@ -274,9 +276,11 @@ def visit_Lambda(self, lam): self.update_position(lam.lineno) - if lam.args.defaults: - self.visit_sequence(lam.args.defaults) - default_count = len(lam.args.defaults) + args = lam.args + assert isinstance(args, ast.arguments) + if args.defaults: + self.visit_sequence(args.defaults) + default_count = len(args.defaults) else: default_count = 0 code = self.sub_scope(LambdaCodeGenerator, "", lam, lam.lineno) @@ -956,9 +960,12 @@ elif call_type == 3: op = ops.CALL_FUNCTION_VAR_KW self.emit_op_arg(op, arg) + + def _call_has_no_star_args(self, call): + return not call.starargs and not call.kwargs def _call_has_simple_args(self, call): - return not call.starargs and not call.kwargs and not call.keywords + return self._call_has_no_star_args(call) and not call.keywords def _optimize_builtin_call(self, call): if not self.space.config.objspace.opcodes.CALL_LIKELY_BUILTIN or \ @@ -984,7 +991,7 @@ def _optimize_method_call(self, call): if not self.space.config.objspace.opcodes.CALL_METHOD or \ - not self._call_has_simple_args(call) or \ + not self._call_has_no_star_args(call) or \ not isinstance(call.func, ast.Attribute): return False attr_lookup = call.func @@ -996,7 +1003,12 @@ arg_count = len(call.args) else: arg_count = 0 - self.emit_op_arg(ops.CALL_METHOD, arg_count) + if call.keywords: + self.visit_sequence(call.keywords) + kwarg_count = len(call.keywords) + else: + kwarg_count = 0 + self.emit_op_arg(ops.CALL_METHOD, (kwarg_count << 8) | arg_count) return True def _listcomp_generator(self, list_name, gens, gen_index, elt): @@ -1275,9 +1287,11 @@ else: self.add_const(self.space.w_None) start = 0 - if func.args.args: - self._handle_nested_args(func.args.args) - self.argcount = len(func.args.args) + args = func.args + assert isinstance(args, ast.arguments) + if args.args: + self._handle_nested_args(args.args) + self.argcount = len(args.args) for i in range(start, len(func.body)): func.body[i].walkabout(self) @@ -1286,9 +1300,11 @@ def _compile(self, lam): assert isinstance(lam, ast.Lambda) - if lam.args.args: - self._handle_nested_args(lam.args.args) - self.argcount = len(lam.args.args) + args = lam.args + assert isinstance(args, ast.arguments) + if args.args: + self._handle_nested_args(args.args) + self.argcount = len(args.args) # Prevent a string from being the first constant and thus a docstring. self.add_const(self.space.w_None) lam.body.walkabout(self) Modified: pypy/branch/kill-caninline/pypy/interpreter/astcompiler/consts.py ============================================================================== --- pypy/branch/kill-caninline/pypy/interpreter/astcompiler/consts.py (original) +++ pypy/branch/kill-caninline/pypy/interpreter/astcompiler/consts.py Thu Aug 12 13:47:36 2010 @@ -17,4 +17,4 @@ PyCF_SOURCE_IS_UTF8 = 0x0100 PyCF_DONT_IMPLY_DEDENT = 0x0200 -PyCF_AST_ONLY = 0x0400 +PyCF_ONLY_AST = 0x0400 Modified: pypy/branch/kill-caninline/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/kill-caninline/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/kill-caninline/pypy/interpreter/astcompiler/symtable.py Thu Aug 12 13:47:36 2010 @@ -353,8 +353,10 @@ def visit_FunctionDef(self, func): self.note_symbol(func.name, SYM_ASSIGNED) # Function defaults and decorators happen in the outer scope. - if func.args.defaults: - self.visit_sequence(func.args.defaults) + args = func.args + assert isinstance(args, ast.arguments) + if args.defaults: + self.visit_sequence(args.defaults) if func.decorators: self.visit_sequence(func.decorators) new_scope = FunctionScope(func.name, func.lineno, func.col_offset) @@ -420,8 +422,10 @@ self.note_symbol(name, SYM_GLOBAL) def visit_Lambda(self, lamb): - if lamb.args.defaults: - self.visit_sequence(lamb.args.defaults) + args = lamb.args + assert isinstance(args, ast.arguments) + if args.defaults: + self.visit_sequence(args.defaults) new_scope = FunctionScope("lambda", lamb.lineno, lamb.col_offset) self.push_scope(new_scope, lamb) lamb.args.walkabout(self) Modified: pypy/branch/kill-caninline/pypy/interpreter/astcompiler/tools/asdl_py.py ============================================================================== --- pypy/branch/kill-caninline/pypy/interpreter/astcompiler/tools/asdl_py.py (original) +++ pypy/branch/kill-caninline/pypy/interpreter/astcompiler/tools/asdl_py.py Thu Aug 12 13:47:36 2010 @@ -100,6 +100,7 @@ self.emit("") self.make_constructor(product.fields, product) self.emit("") + self.make_mutate_over(product, name) self.emit("def walkabout(self, visitor):", 1) self.emit("visitor.visit_%s(self)" % (name,), 2) self.emit("") @@ -183,6 +184,26 @@ have_everything = self.data.required_masks[node] | \ self.data.optional_masks[node] self.emit("self.initialization_state = %i" % (have_everything,), 2) + + def make_mutate_over(self, cons, name): + self.emit("def mutate_over(self, visitor):", 1) + for field in cons.fields: + if (field.type.value not in asdl.builtin_types and + field.type.value not in self.data.simple_types): + if field.opt or field.seq: + level = 3 + self.emit("if self.%s:" % (field.name,), 2) + else: + level = 2 + if field.seq: + sub = (field.name,) + self.emit("visitor._mutate_sequence(self.%s)" % sub, level) + else: + sub = (field.name, field.name) + self.emit("self.%s = self.%s.mutate_over(visitor)" % sub, + level) + self.emit("return visitor.visit_%s(self)" % (name,), 2) + self.emit("") def visitConstructor(self, cons, base, extra_attributes): self.emit("class %s(%s):" % (cons.name, base)) @@ -199,24 +220,7 @@ self.emit("def walkabout(self, visitor):", 1) self.emit("visitor.visit_%s(self)" % (cons.name,), 2) self.emit("") - self.emit("def mutate_over(self, visitor):", 1) - for field in cons.fields: - if field.type.value not in asdl.builtin_types and \ - field.type.value not in self.data.prod_simple: - if field.opt or field.seq: - level = 3 - self.emit("if self.%s:" % (field.name,), 2) - else: - level = 2 - if field.seq: - sub = (field.name,) - self.emit("visitor._mutate_sequence(self.%s)" % sub, level) - else: - sub = (field.name, field.name) - self.emit("self.%s = self.%s.mutate_over(visitor)" % sub, - level) - self.emit("return visitor.visit_%s(self)" % (cons.name,), 2) - self.emit("") + self.make_mutate_over(cons, cons.name) self.make_var_syncer(cons.fields + self.data.cons_attributes[cons], cons, cons.name) Modified: pypy/branch/kill-caninline/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/kill-caninline/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/kill-caninline/pypy/interpreter/baseobjspace.py Thu Aug 12 13:47:36 2010 @@ -1102,17 +1102,6 @@ self.wrap('argument must be a unicode')) return self.unicode_w(w_obj) - def path_w(self, w_obj): - """ Like str_w, but if the object is unicode, encode it using - filesystemencoding - """ - filesystemencoding = self.sys.filesystemencoding - if (filesystemencoding and - self.is_true(self.isinstance(w_obj, self.w_unicode))): - w_obj = self.call_method(w_obj, "encode", - self.wrap(filesystemencoding)) - return self.str_w(w_obj) - def bool_w(self, w_obj): # Unwraps a bool, also accepting an int for compatibility. # This is here mostly just for gateway.int_unwrapping_space_method(). Modified: pypy/branch/kill-caninline/pypy/interpreter/error.py ============================================================================== --- pypy/branch/kill-caninline/pypy/interpreter/error.py (original) +++ pypy/branch/kill-caninline/pypy/interpreter/error.py Thu Aug 12 13:47:36 2010 @@ -344,7 +344,7 @@ else: _WINDOWS = True - def wrap_windowserror(space, e, filename=None): + def wrap_windowserror(space, e, w_filename=None): from pypy.rlib import rwin32 winerror = e.winerror @@ -353,19 +353,19 @@ except ValueError: msg = 'Windows Error %d' % winerror exc = space.w_WindowsError - if filename is not None: + if w_filename is not None: w_error = space.call_function(exc, space.wrap(winerror), - space.wrap(msg), space.wrap(filename)) + space.wrap(msg), w_filename) else: w_error = space.call_function(exc, space.wrap(winerror), space.wrap(msg)) return OperationError(exc, w_error) -def wrap_oserror(space, e, filename=None, exception_name='w_OSError'): +def wrap_oserror2(space, e, w_filename=None, exception_name='w_OSError'): assert isinstance(e, OSError) if _WINDOWS and isinstance(e, WindowsError): - return wrap_windowserror(space, e, filename) + return wrap_windowserror(space, e, w_filename) errno = e.errno try: @@ -373,10 +373,21 @@ except ValueError: msg = 'error %d' % errno exc = getattr(space, exception_name) - if filename is not None: + if w_filename is not None: w_error = space.call_function(exc, space.wrap(errno), - space.wrap(msg), space.wrap(filename)) + space.wrap(msg), w_filename) else: - w_error = space.call_function(exc, space.wrap(errno), space.wrap(msg)) + w_error = space.call_function(exc, space.wrap(errno), + space.wrap(msg)) return OperationError(exc, w_error) +wrap_oserror2._annspecialcase_ = 'specialize:arg(3)' + +def wrap_oserror(space, e, filename=None, exception_name='w_OSError'): + if filename is not None: + return wrap_oserror2(space, e, space.wrap(filename), + exception_name=exception_name) + else: + return wrap_oserror2(space, e, None, + exception_name=exception_name) wrap_oserror._annspecialcase_ = 'specialize:arg(3)' + Modified: pypy/branch/kill-caninline/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/kill-caninline/pypy/interpreter/gateway.py (original) +++ pypy/branch/kill-caninline/pypy/interpreter/gateway.py Thu Aug 12 13:47:36 2010 @@ -137,9 +137,6 @@ def visit_c_nonnegint(self, el, app_sig): self.checked_space_method(el, app_sig) - def visit_path(self, el, app_sig): - self.checked_space_method(el, app_sig) - def visit__Wrappable(self, el, app_sig): name = el.__name__ argname = self.orig_arg() @@ -241,9 +238,6 @@ def visit_bufferstr(self, typ): self.run_args.append("space.bufferstr_w(%s)" % (self.scopenext(),)) - def visit_path(self, typ): - self.run_args.append("space.path_w(%s)" % (self.scopenext(),)) - def visit_nonnegint(self, typ): self.run_args.append("space.nonnegint_w(%s)" % (self.scopenext(),)) @@ -371,9 +365,6 @@ def visit_bufferstr(self, typ): self.unwrap.append("space.bufferstr_w(%s)" % (self.nextarg(),)) - def visit_path(self, typ): - self.unwrap.append("space.path_w(%s)" % (self.nextarg(),)) - def visit_nonnegint(self, typ): self.unwrap.append("space.nonnegint_w(%s)" % (self.nextarg(),)) Modified: pypy/branch/kill-caninline/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/kill-caninline/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/kill-caninline/pypy/interpreter/pyopcode.py Thu Aug 12 13:47:36 2010 @@ -211,10 +211,6 @@ next_instr = block.handle(self, unroller) return next_instr # now inside a 'finally' block - if opcode == self.opcodedesc.YIELD_VALUE.index: - #self.last_instr = intmask(next_instr - 1) XXX clean up! - raise Yield - if opcode == self.opcodedesc.END_FINALLY.index: unroller = self.end_finally() if isinstance(unroller, SuspendedUnroller): @@ -239,7 +235,7 @@ if not opdesc.is_enabled(space): continue if opdesc.methodname in ( - 'EXTENDED_ARG', 'RETURN_VALUE', 'YIELD_VALUE', + 'EXTENDED_ARG', 'RETURN_VALUE', 'END_FINALLY', 'JUMP_ABSOLUTE'): continue # opcodes implemented above @@ -814,6 +810,9 @@ self.space.str_w(w_name)) self.pushvalue(w_obj) + def YIELD_VALUE(self, oparg, next_instr): + raise Yield + def jump_absolute(self, jumpto, next_instr, ec): return jumpto @@ -989,23 +988,9 @@ self.dropvalues(nargs) self.pushvalue(w_result) - def LOOKUP_METHOD(self, nameindex, next_instr): - # overridden by faster version in the standard object space. - space = self.space - w_obj = self.popvalue() - w_name = self.getname_w(nameindex) - w_value = space.getattr(w_obj, w_name) - self.pushvalue(w_value) - - def CALL_METHOD(self, nargs, next_instr): - # overridden by faster version in the standard object space. - # 'nargs' is the argument count excluding the implicit 'self' - w_callable = self.peekvalue(nargs) - try: - w_result = self.space.call_valuestack(w_callable, nargs, self) - finally: - self.dropvalues(nargs + 1) - self.pushvalue(w_result) + # overridden by faster version in the standard object space. + LOOKUP_METHOD = LOAD_ATTR + CALL_METHOD = CALL_FUNCTION def MISSING_OPCODE(self, oparg, next_instr): ofs = self.last_instr Modified: pypy/branch/kill-caninline/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/branch/kill-caninline/pypy/interpreter/test/test_compiler.py (original) +++ pypy/branch/kill-caninline/pypy/interpreter/test/test_compiler.py Thu Aug 12 13:47:36 2010 @@ -4,6 +4,7 @@ from pypy.interpreter.pycode import PyCode from pypy.interpreter.error import OperationError from pypy.interpreter.argument import Arguments +from pypy.conftest import gettestobjspace class BaseTestCompiler: def setup_method(self, method): @@ -838,6 +839,47 @@ sys.stdout = save_stdout output = s.getvalue() assert "STOP_CODE" not in output + + def test_optimize_list_comp(self): + source = """def _f(a): + return [x for x in a if None] + """ + exec source + code = _f.func_code + + import StringIO, sys, dis + s = StringIO.StringIO() + out = sys.stdout + sys.stdout = s + try: + dis.dis(code) + finally: + sys.stdout = out + output = s.getvalue() + assert "LOAD_GLOBAL" not in output + +class AppTestCallMethod(object): + def setup_class(cls): + cls.space = gettestobjspace(**{'objspace.opcodes.CALL_METHOD': True}) + + def test_call_method_kwargs(self): + source = """def _f(a): + return a.f(a=a) + """ + exec source + code = _f.func_code + + import StringIO, sys, dis + s = StringIO.StringIO() + out = sys.stdout + sys.stdout = s + try: + dis.dis(code) + finally: + sys.stdout = out + output = s.getvalue() + assert "CALL_METHOD" in output + class AppTestExceptions: def test_indentation_error(self): Modified: pypy/branch/kill-caninline/pypy/interpreter/test/test_gateway.py ============================================================================== --- pypy/branch/kill-caninline/pypy/interpreter/test/test_gateway.py (original) +++ pypy/branch/kill-caninline/pypy/interpreter/test/test_gateway.py Thu Aug 12 13:47:36 2010 @@ -454,16 +454,6 @@ assert len(l) == 1 assert space.eq_w(l[0], w("foo")) - def test_interp2app_unwrap_spec_path(self, monkeypatch): - space = self.space - def g(space, p): - return p - - app_g = gateway.interp2app(g, unwrap_spec=[gateway.ObjSpace, 'path']) - w_app_g = space.wrap(app_g) - monkeypatch.setattr(space.sys, "filesystemencoding", "utf-8") - w_res = space.call_function(w_app_g, space.wrap(u"?")) - def test_interp2app_classmethod(self): space = self.space w = space.wrap Modified: pypy/branch/kill-caninline/pypy/jit/backend/detect_cpu.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/backend/detect_cpu.py (original) +++ pypy/branch/kill-caninline/pypy/jit/backend/detect_cpu.py Thu Aug 12 13:47:36 2010 @@ -56,6 +56,8 @@ return "pypy.jit.backend.x86.runner", "CPU" elif backend_name == 'x86-without-sse2': return "pypy.jit.backend.x86.runner", "CPU386_NO_SSE2" + elif backend_name == 'x86_64': + return "pypy.jit.backend.x86.runner", "CPU_X86_64" elif backend_name == 'cli': return "pypy.jit.backend.cli.runner", "CliCPU" elif backend_name == 'llvm': Modified: pypy/branch/kill-caninline/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/branch/kill-caninline/pypy/jit/backend/llsupport/descr.py Thu Aug 12 13:47:36 2010 @@ -67,9 +67,11 @@ class BaseFieldDescr(AbstractDescr): offset = 0 # help translation + name = '' _clsname = '' - def __init__(self, offset): + def __init__(self, name, offset): + self.name = name self.offset = offset def sort_key(self): @@ -88,7 +90,7 @@ return self._is_float_field def repr_of_descr(self): - return '<%s %s>' % (self._clsname, self.offset) + return '<%s %s %s>' % (self._clsname, self.name, self.offset) class NonGcPtrFieldDescr(BaseFieldDescr): @@ -113,7 +115,8 @@ offset, _ = symbolic.get_field_token(STRUCT, fieldname, gccache.translate_support_code) FIELDTYPE = getattr(STRUCT, fieldname) - fielddescr = getFieldDescrClass(FIELDTYPE)(offset) + name = '%s.%s' % (STRUCT._name, fieldname) + fielddescr = getFieldDescrClass(FIELDTYPE)(name, offset) cachedict = cache.setdefault(STRUCT, {}) cachedict[fieldname] = fielddescr return fielddescr Modified: pypy/branch/kill-caninline/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/branch/kill-caninline/pypy/jit/backend/llsupport/gc.py Thu Aug 12 13:47:36 2010 @@ -351,7 +351,7 @@ gcrootmap = cls() self.gcrootmap = gcrootmap self.gcrefs = GcRefList() - self.single_gcref_descr = GcPtrFieldDescr(0) + self.single_gcref_descr = GcPtrFieldDescr('', 0) # make a TransformerLayoutBuilder and save it on the translator # where it can be fished and reused by the FrameworkGCTransformer Modified: pypy/branch/kill-caninline/pypy/jit/backend/llsupport/regalloc.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/backend/llsupport/regalloc.py (original) +++ pypy/branch/kill-caninline/pypy/jit/backend/llsupport/regalloc.py Thu Aug 12 13:47:36 2010 @@ -22,18 +22,19 @@ def get(self, box): return self.frame_bindings.get(box, None) - def loc(self, box, size): + def loc(self, box): res = self.get(box) if res is not None: return res - newloc = self.frame_pos(self.frame_depth, size) + newloc = self.frame_pos(self.frame_depth, box.type) self.frame_bindings[box] = newloc - self.frame_depth += size + # Objects returned by frame_pos must support frame_size() + self.frame_depth += newloc.frame_size() return newloc # abstract methods that need to be overwritten for specific assemblers @staticmethod - def frame_pos(loc, size): + def frame_pos(loc, type): raise NotImplementedError("Purely abstract") class RegisterManager(object): @@ -43,7 +44,6 @@ all_regs = [] no_lower_byte_regs = [] save_around_call_regs = [] - reg_width = 1 # in terms of stack space eaten def __init__(self, longevity, frame_manager=None, assembler=None): self.free_regs = self.all_regs[:] @@ -148,7 +148,7 @@ loc = self.reg_bindings[v_to_spill] del self.reg_bindings[v_to_spill] if self.frame_manager.get(v_to_spill) is None: - newloc = self.frame_manager.loc(v_to_spill, self.reg_width) + newloc = self.frame_manager.loc(v_to_spill) self.assembler.regalloc_mov(loc, newloc) return loc @@ -204,7 +204,7 @@ try: return self.reg_bindings[box] except KeyError: - return self.frame_manager.loc(box, self.reg_width) + return self.frame_manager.loc(box) def return_constant(self, v, forbidden_vars=[], selected_reg=None, imm_fine=True): @@ -260,7 +260,7 @@ self.reg_bindings[v] = loc self.assembler.regalloc_mov(prev_loc, loc) else: - loc = self.frame_manager.loc(v, self.reg_width) + loc = self.frame_manager.loc(v) self.assembler.regalloc_mov(prev_loc, loc) def force_result_in_reg(self, result_v, v, forbidden_vars=[]): @@ -280,7 +280,7 @@ self.free_regs = [reg for reg in self.free_regs if reg is not loc] return loc if v not in self.reg_bindings: - prev_loc = self.frame_manager.loc(v, self.reg_width) + prev_loc = self.frame_manager.loc(v) loc = self.force_allocate_reg(v, forbidden_vars) self.assembler.regalloc_mov(prev_loc, loc) assert v in self.reg_bindings @@ -300,7 +300,7 @@ def _sync_var(self, v): if not self.frame_manager.get(v): reg = self.reg_bindings[v] - to = self.frame_manager.loc(v, self.reg_width) + to = self.frame_manager.loc(v) self.assembler.regalloc_mov(reg, to) # otherwise it's clean Modified: pypy/branch/kill-caninline/pypy/jit/backend/llsupport/test/test_descr.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/backend/llsupport/test/test_descr.py (original) +++ pypy/branch/kill-caninline/pypy/jit/backend/llsupport/test/test_descr.py Thu Aug 12 13:47:36 2010 @@ -53,6 +53,10 @@ assert descr_y.__class__ is GcPtrFieldDescr assert descr_z.__class__ is NonGcPtrFieldDescr assert descr_f.__class__ is clsf + assert descr_x.name == 'S.x' + assert descr_y.name == 'S.y' + assert descr_z.name == 'S.z' + assert descr_f.name == 'S.f' if not tsc: assert descr_x.offset < descr_y.offset < descr_z.offset assert descr_x.sort_key() < descr_y.sort_key() < descr_z.sort_key() @@ -228,11 +232,11 @@ # descr2 = get_field_descr(c0, S, 'y') o, _ = symbolic.get_field_token(S, 'y', False) - assert descr2.repr_of_descr() == '' % o + assert descr2.repr_of_descr() == '' % o # descr2i = get_field_descr(c0, S, 'x') o, _ = symbolic.get_field_token(S, 'x', False) - assert descr2i.repr_of_descr() == '' % o + assert descr2i.repr_of_descr() == '' % o # descr3 = get_array_descr(c0, lltype.GcArray(lltype.Ptr(S))) assert descr3.repr_of_descr() == '' Modified: pypy/branch/kill-caninline/pypy/jit/backend/llsupport/test/test_regalloc.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/backend/llsupport/test/test_regalloc.py (original) +++ pypy/branch/kill-caninline/pypy/jit/backend/llsupport/test/test_regalloc.py Thu Aug 12 13:47:36 2010 @@ -1,5 +1,5 @@ -from pypy.jit.metainterp.history import BoxInt, ConstInt, BoxFloat +from pypy.jit.metainterp.history import BoxInt, ConstInt, BoxFloat, INT, FLOAT from pypy.jit.backend.llsupport.regalloc import FrameManager from pypy.jit.backend.llsupport.regalloc import RegisterManager as BaseRegMan @@ -26,9 +26,20 @@ def convert_to_imm(self, v): return v +class FakeFramePos(object): + def __init__(self, pos, box_type): + self.pos = pos + self.box_type = box_type + + def frame_size(self): + if self.box_type == FLOAT: + return 2 + else: + return 1 + class TFrameManager(FrameManager): - def frame_pos(self, i, size): - return i + def frame_pos(self, i, box_type): + return FakeFramePos(i, box_type) class MockAsm(object): def __init__(self): @@ -146,8 +157,8 @@ rm.next_instruction() # allocate a stack position b0, b1, b2, b3, b4 = boxes - sp = fm.loc(b0, 1) - assert sp == 0 + sp = fm.loc(b0) + assert sp.pos == 0 loc = rm.make_sure_var_in_reg(b0) assert isinstance(loc, FakeReg) rm._check_invariants() @@ -207,13 +218,13 @@ asm = MockAsm() rm = RegisterManager(longevity, frame_manager=fm, assembler=asm) rm.next_instruction() - fm.loc(b0, 1) + fm.loc(b0) rm.force_result_in_reg(b1, b0) rm._check_invariants() loc = rm.loc(b1) assert isinstance(loc, FakeReg) loc = rm.loc(b0) - assert isinstance(loc, int) + assert isinstance(loc, FakeFramePos) assert len(asm.moves) == 1 def test_return_constant(self): @@ -304,7 +315,7 @@ def test_different_frame_width(self): class XRegisterManager(RegisterManager): - reg_width = 2 + pass fm = TFrameManager() b0 = BoxInt() Modified: pypy/branch/kill-caninline/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/kill-caninline/pypy/jit/backend/test/runner_test.py Thu Aug 12 13:47:36 2010 @@ -461,6 +461,25 @@ [funcbox] + args, 'float', descr=calldescr) assert abs(res.value - 4.6) < 0.0001 + + def test_call_many_arguments(self): + # Test calling a function with a large number of arguments (more than + # 6, which will force passing some arguments on the stack on 64-bit) + + def func(*args): + assert len(args) == 16 + # Try to sum up args in a way that would probably detect a + # transposed argument + return sum(arg * (2**i) for i, arg in enumerate(args)) + + FUNC = self.FuncType([lltype.Signed]*16, lltype.Signed) + FPTR = self.Ptr(FUNC) + calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + func_ptr = llhelper(FPTR, func) + args = range(16) + funcbox = self.get_funcbox(self.cpu, func_ptr) + res = self.execute_operation(rop.CALL, [funcbox] + map(BoxInt, args), 'int', descr=calldescr) + assert res.value == func(*args) def test_call_stack_alignment(self): # test stack alignment issues, notably for Mac OS/X. @@ -638,6 +657,21 @@ assert r.value == 1 def test_array_basic(self): + a_box, A = self.alloc_array_of(rffi.SHORT, 342) + arraydescr = self.cpu.arraydescrof(A) + assert not arraydescr.is_array_of_pointers() + # + r = self.execute_operation(rop.ARRAYLEN_GC, [a_box], + 'int', descr=arraydescr) + assert r.value == 342 + r = self.execute_operation(rop.SETARRAYITEM_GC, [a_box, BoxInt(310), + BoxInt(744)], + 'void', descr=arraydescr) + assert r is None + r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(310)], + 'int', descr=arraydescr) + assert r.value == 744 + a_box, A = self.alloc_array_of(lltype.Signed, 342) arraydescr = self.cpu.arraydescrof(A) assert not arraydescr.is_array_of_pointers() @@ -978,6 +1012,8 @@ else: assert 0 operations.append(ResOperation(opnum, boxargs, boxres)) + # Unique-ify inputargs + inputargs = list(set(inputargs)) faildescr = BasicFailDescr(1) operations.append(ResOperation(rop.FINISH, [], None, descr=faildescr)) @@ -1050,9 +1086,11 @@ descr=BasicFailDescr(5))] operations[1].fail_args = [] looptoken = LoopToken() - self.cpu.compile_loop(list(testcase), operations, + # Use "set" to unique-ify inputargs + unique_testcase_list = list(set(testcase)) + self.cpu.compile_loop(unique_testcase_list, operations, looptoken) - for i, box in enumerate(testcase): + for i, box in enumerate(unique_testcase_list): self.cpu.set_future_value_float(i, box.value) fail = self.cpu.execute_token(looptoken) if fail.identifier != 5 - (expected_id^expected): @@ -1695,7 +1733,7 @@ def test_assembler_call(self): called = [] def assembler_helper(failindex, virtualizable): - assert self.cpu.get_latest_value_int(0) == 10 + assert self.cpu.get_latest_value_int(0) == 97 called.append(failindex) return 4 + 9 @@ -1708,33 +1746,41 @@ _assembler_helper_ptr) ops = ''' - [i0, i1] - i2 = int_add(i0, i1) - finish(i2)''' + [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] + i10 = int_add(i0, i1) + i11 = int_add(i10, i2) + i12 = int_add(i11, i3) + i13 = int_add(i12, i4) + i14 = int_add(i13, i5) + i15 = int_add(i14, i6) + i16 = int_add(i15, i7) + i17 = int_add(i16, i8) + i18 = int_add(i17, i9) + finish(i18)''' loop = parse(ops) looptoken = LoopToken() looptoken.outermost_jitdriver_sd = FakeJitDriverSD() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) - ARGS = [lltype.Signed, lltype.Signed] + ARGS = [lltype.Signed] * 10 RES = lltype.Signed self.cpu.portal_calldescr = self.cpu.calldescrof( lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES) - self.cpu.set_future_value_int(0, 1) - self.cpu.set_future_value_int(1, 2) + for i in range(10): + self.cpu.set_future_value_int(i, i+1) res = self.cpu.execute_token(looptoken) - assert self.cpu.get_latest_value_int(0) == 3 + assert self.cpu.get_latest_value_int(0) == 55 ops = ''' - [i4, i5] - i6 = int_add(i4, 1) - i3 = call_assembler(i6, i5, descr=looptoken) + [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] + i10 = int_add(i0, 42) + i11 = call_assembler(i10, i1, i2, i3, i4, i5, i6, i7, i8, i9, descr=looptoken) guard_not_forced()[] - finish(i3) + finish(i11) ''' loop = parse(ops, namespace=locals()) othertoken = LoopToken() self.cpu.compile_loop(loop.inputargs, loop.operations, othertoken) - self.cpu.set_future_value_int(0, 4) - self.cpu.set_future_value_int(1, 5) + for i in range(10): + self.cpu.set_future_value_int(i, i+1) res = self.cpu.execute_token(othertoken) assert self.cpu.get_latest_value_int(0) == 13 assert called Modified: pypy/branch/kill-caninline/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/kill-caninline/pypy/jit/backend/x86/assembler.py Thu Aug 12 13:47:36 2010 @@ -1,4 +1,4 @@ -import sys +import sys, os from pypy.jit.backend.llsupport import symbolic from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxFloat from pypy.jit.metainterp.history import AbstractFailDescr, INT, REF, FLOAT,\ @@ -7,23 +7,35 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.annlowlevel import llhelper from pypy.tool.uid import fixid -from pypy.jit.backend.x86.regalloc import RegAlloc, WORD,\ - X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs, FRAME_FIXED_SIZE,\ - FORCE_INDEX_OFS +from pypy.jit.backend.x86.regalloc import RegAlloc, \ + X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs + +from pypy.jit.backend.x86.arch import FRAME_FIXED_SIZE, FORCE_INDEX_OFS, WORD, IS_X86_32, IS_X86_64 + +from pypy.jit.backend.x86.regloc import (eax, ecx, edx, ebx, + esp, ebp, esi, edi, + xmm0, xmm1, xmm2, xmm3, + xmm4, xmm5, xmm6, xmm7, + r8, r9, r10, r11, + r12, r13, r14, r15, + X86_64_SCRATCH_REG, + X86_64_XMM_SCRATCH_REG, + RegLoc, StackLoc, ConstFloatLoc, + ImmedLoc, AddressLoc, imm) + from pypy.rlib.objectmodel import we_are_translated, specialize -from pypy.jit.backend.x86 import codebuf -from pypy.jit.backend.x86.ri386 import * -from pypy.jit.metainterp.resoperation import rop +from pypy.jit.backend.x86 import rx86, regloc, codebuf +from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.backend.x86.support import values_array from pypy.rlib.debug import debug_print from pypy.rlib import rgc - -# our calling convention - we pass first 6 args in registers -# and the rest stays on the stack +from pypy.jit.backend.x86.jump import remap_frame_layout +from pypy.rlib.streamio import open_file_as_stream +from pypy.jit.metainterp.history import ConstInt, BoxInt # darwin requires the stack to be 16 bytes aligned on calls. Same for gcc 4.5.0, # better safe than sorry -CALL_ALIGN = 4 +CALL_ALIGN = 16 // WORD def align_stack_words(words): return (words + CALL_ALIGN - 1) & ~(CALL_ALIGN-1) @@ -31,16 +43,36 @@ class MachineCodeBlockWrapper(object): MC_DEFAULT_SIZE = 1024*1024 - def __init__(self, bigsize, profile_agent=None): + def __init__(self, assembler, bigsize, profile_agent=None): + self.assembler = assembler self.old_mcs = [] # keepalive self.bigsize = bigsize self._mc = self._instantiate_mc() self.function_name = None self.profile_agent = profile_agent + self.reset_reserved_bytes() def _instantiate_mc(self): # hook for testing return codebuf.MachineCodeBlock(self.bigsize) + def ensure_bytes_available(self, num_bytes): + if self.bytes_free() <= (self._reserved_bytes + num_bytes): + self.make_new_mc() + + def reserve_bytes(self, num_bytes): + self.ensure_bytes_available(num_bytes) + self._reserved_bytes += num_bytes + + def reset_reserved_bytes(self): + # XXX er.... pretty random number, just to be sure + # not to write half-instruction + self._reserved_bytes = 64 + + def get_relative_pos(self): + return self._mc.get_relative_pos() + + def overwrite(self, pos, listofchars): + return self._mc.overwrite(pos, listofchars) def bytes_free(self): return self._mc._size - self._mc.get_relative_pos() @@ -62,12 +94,25 @@ def make_new_mc(self): new_mc = self._instantiate_mc() debug_print('[new machine code block at', new_mc.tell(), ']') - self._mc.JMP(rel32(new_mc.tell())) + + if IS_X86_64: + # The scratch register is sometimes used as a temporary + # register, but the JMP below might clobber it. Rather than risk + # subtle bugs, we preserve the scratch register across the jump. + self._mc.PUSH_r(X86_64_SCRATCH_REG.value) + + self._mc.JMP(imm(new_mc.tell())) + + if IS_X86_64: + # Restore scratch reg + new_mc.POP_r(X86_64_SCRATCH_REG.value) if self.function_name is not None: self.end_function(done=False) self.start_pos = new_mc.get_relative_pos() + self.assembler.write_pending_failure_recoveries() + self._mc.done() self.old_mcs.append(self._mc) self._mc = new_mc @@ -81,24 +126,39 @@ def _new_method(name): def method(self, *args): - # XXX er.... pretty random number, just to be sure - # not to write half-instruction - if self.bytes_free() < 64: + if self.bytes_free() < self._reserved_bytes: self.make_new_mc() getattr(self._mc, name)(*args) method.func_name = name return method +for _name in rx86.all_instructions + regloc.all_extra_instructions: + setattr(MachineCodeBlockWrapper, _name, _new_method(_name)) + for name in dir(codebuf.MachineCodeBlock): if name.upper() == name or name == "writechr": setattr(MachineCodeBlockWrapper, name, _new_method(name)) +class GuardToken(object): + def __init__(self, faildescr, failargs, fail_locs, exc, desc_bytes): + self.faildescr = faildescr + self.failargs = failargs + self.fail_locs = fail_locs + self.exc = exc + self.desc_bytes = desc_bytes + + def recovery_stub_size(self): + # XXX: 32 is pulled out of the air + return 32 + len(self.desc_bytes) + +DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed)) + class Assembler386(object): mc = None - mc2 = None mc_size = MachineCodeBlockWrapper.MC_DEFAULT_SIZE _float_constants = None _regalloc = None + _output_loop_log = None def __init__(self, cpu, translate_support_code=False, failargs_limit=1000): @@ -113,18 +173,26 @@ self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) self.fail_boxes_float = values_array(lltype.Float, failargs_limit) self.fail_ebp = 0 - self.loc_float_const_neg = None - self.loc_float_const_abs = None + self.loop_run_counters = [] + # if we have 10000 loops, we have some other problems I guess + self.float_const_neg_addr = 0 + self.float_const_abs_addr = 0 self.malloc_fixedsize_slowpath1 = 0 self.malloc_fixedsize_slowpath2 = 0 + self.pending_guard_tokens = None self.setup_failure_recovery() + self._debug = False + self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') def leave_jitted_hook(self): ptrs = self.fail_boxes_ptr.ar llop.gc_assume_young_pointers(lltype.Void, llmemory.cast_ptr_to_adr(ptrs)) - def make_sure_mc_exists(self): + def set_debug(self, v): + self._debug = v + + def setup(self): if self.mc is None: # the address of the function called by 'new' gc_ll_descr = self.cpu.gc_ll_descr @@ -143,11 +211,7 @@ ll_new_unicode = gc_ll_descr.get_funcptr_for_newunicode() self.malloc_unicode_func_addr = rffi.cast(lltype.Signed, ll_new_unicode) - # done - # we generate the loop body in 'mc' - # 'mc2' is for guard recovery code - self.mc = MachineCodeBlockWrapper(self.mc_size, self.cpu.profile_agent) - self.mc2 = MachineCodeBlockWrapper(self.mc_size) + self.mc = MachineCodeBlockWrapper(self, self.mc_size, self.cpu.profile_agent) self._build_failure_recovery(False) self._build_failure_recovery(True) if self.cpu.supports_floats: @@ -157,46 +221,72 @@ self._build_float_constants() if hasattr(gc_ll_descr, 'get_malloc_fixedsize_slowpath_addr'): self._build_malloc_fixedsize_slowpath() + s = os.environ.get('PYPYLOG') + if s: + if s.find(':') != -1: + s = s.split(':')[-1] + self.set_debug(True) + self._output_loop_log = s + ".count" + # Intialize here instead of __init__ to prevent + # pending_guard_tokens from being considered a prebuilt object, + # which sometimes causes memory leaks since the prebuilt list is + # still considered a GC root after we re-assign + # pending_guard_tokens in write_pending_failure_recoveries + self.pending_guard_tokens = [] + + def finish_once(self): + if self._debug: + output_log = self._output_loop_log + assert output_log is not None + f = open_file_as_stream(output_log, "w") + for i in range(len(self.loop_run_counters)): + name, struct = self.loop_run_counters[i] + f.write(name + ":" + str(struct.i) + "\n") + f.close() def _build_float_constants(self): - # 11 words: 8 words for the data, and up to 3 words for alignment - addr = lltype.malloc(rffi.CArray(lltype.Signed), 11, flavor='raw') + # 44 bytes: 32 bytes for the data, and up to 12 bytes for alignment + addr = lltype.malloc(rffi.CArray(lltype.Char), 44, flavor='raw') if not we_are_translated(): self._keepalive_malloced_float_consts = addr float_constants = rffi.cast(lltype.Signed, addr) float_constants = (float_constants + 15) & ~15 # align to 16 bytes - addr = rffi.cast(rffi.CArrayPtr(lltype.Signed), float_constants) - addr[0] = 0 # \ - addr[1] = -2147483648 # / for neg - addr[2] = 0 # - addr[3] = 0 # - addr[4] = -1 # \ - addr[5] = 2147483647 # / for abs - addr[6] = 0 # - addr[7] = 0 # - self.loc_float_const_neg = heap64(float_constants) - self.loc_float_const_abs = heap64(float_constants + 16) + addr = rffi.cast(rffi.CArrayPtr(lltype.Char), float_constants) + qword_padding = '\x00\x00\x00\x00\x00\x00\x00\x00' + # 0x8000000000000000 + neg_const = '\x00\x00\x00\x00\x00\x00\x00\x80' + # 0x7FFFFFFFFFFFFFFF + abs_const = '\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x7F' + data = neg_const + qword_padding + abs_const + qword_padding + for i in range(len(data)): + addr[i] = data[i] + self.float_const_neg_addr = float_constants + self.float_const_abs_addr = float_constants + 16 def _build_malloc_fixedsize_slowpath(self): - mc = self.mc2._mc # ---------- first helper for the slow path of malloc ---------- - self.malloc_fixedsize_slowpath1 = mc.tell() + self.malloc_fixedsize_slowpath1 = self.mc.tell() if self.cpu.supports_floats: # save the XMM registers in - for i in range(8): # the *caller* frame, from esp+8 - mc.MOVSD(mem64(esp, 8+8*i), xmm_registers[i]) - mc.SUB(edx, eax) # compute the size we want - mc.MOV(mem(esp, 4), edx) # save it as the new argument + for i in range(self.cpu.NUM_REGS):# the *caller* frame, from esp+8 + self.mc.MOVSD_sx((WORD*2)+8*i, i) + self.mc.SUB_rr(edx.value, eax.value) # compute the size we want + if IS_X86_32: + self.mc.MOV_sr(WORD, edx.value) # save it as the new argument + elif IS_X86_64: + # FIXME: We can't just clobber rdi like this, can we? + self.mc.MOV_rr(edi.value, edx.value) + addr = self.cpu.gc_ll_descr.get_malloc_fixedsize_slowpath_addr() - mc.JMP(rel32(addr)) # tail call to the real malloc + self.mc.JMP(imm(addr)) # tail call to the real malloc # ---------- second helper for the slow path of malloc ---------- - self.malloc_fixedsize_slowpath2 = mc.tell() + self.malloc_fixedsize_slowpath2 = self.mc.tell() if self.cpu.supports_floats: # restore the XMM registers - for i in range(8): # from where they were saved - mc.MOVSD(xmm_registers[i], mem64(esp, 8+8*i)) + for i in range(self.cpu.NUM_REGS):# from where they were saved + self.mc.MOVSD_xs(i, (WORD*2)+8*i) nursery_free_adr = self.cpu.gc_ll_descr.get_nursery_free_addr() - mc.MOV(edx, heap(nursery_free_adr)) # load this in EDX - mc.RET() - self.mc2.done() + self.mc.MOV(edx, heap(nursery_free_adr)) # load this in EDX + self.mc.RET() + self.mc.done() def assemble_loop(self, inputargs, operations, looptoken): """adds the following attributes to looptoken: @@ -207,15 +297,18 @@ _x86_param_depth _x86_arglocs """ + if not we_are_translated(): + # Arguments should be unique + assert len(set(inputargs)) == len(inputargs) + + self.setup() funcname = self._find_debug_merge_point(operations) - self.make_sure_mc_exists() + regalloc = RegAlloc(self, self.cpu.translate_support_code) + operations = self._inject_debugging_code(operations) arglocs = regalloc.prepare_loop(inputargs, operations, looptoken) looptoken._x86_arglocs = arglocs - needed_mem = len(arglocs[0]) * 16 + 16 - if needed_mem >= self.mc.bytes_free(): - self.mc.make_new_mc() # profile support name = "Loop # %s: %s" % (looptoken.number, funcname) @@ -230,29 +323,30 @@ self._patch_stackadjust(adr_stackadjust, frame_depth+param_depth) looptoken._x86_frame_depth = frame_depth looptoken._x86_param_depth = param_depth - # we need to make sure here that we don't overload an mc badly. - # a safe estimate is that we need at most 16 bytes per arg - needed_mem = len(arglocs[0]) * 16 + 16 - if needed_mem >= self.mc.bytes_free(): - self.mc.make_new_mc() + looptoken._x86_direct_bootstrap_code = self.mc.tell() self._assemble_bootstrap_direct_call(arglocs, curadr, frame_depth+param_depth) debug_print("Loop #", looptoken.number, "has address", looptoken._x86_loop_code, "to", self.mc.tell()) self.mc.end_function() + self.write_pending_failure_recoveries() - def assemble_bridge(self, faildescr, inputargs, operations): + if not we_are_translated(): + # Arguments should be unique + assert len(set(inputargs)) == len(inputargs) + + self.setup() funcname = self._find_debug_merge_point(operations) - self.make_sure_mc_exists() arglocs = self.rebuild_faillocs_from_descr( faildescr._x86_failure_recovery_bytecode) if not we_are_translated(): assert ([loc.assembler() for loc in arglocs] == [loc.assembler() for loc in faildescr._x86_debug_faillocs]) regalloc = RegAlloc(self, self.cpu.translate_support_code) + operations = self._inject_debugging_code(operations) fail_depths = faildescr._x86_current_depths regalloc.prepare_bridge(fail_depths, inputargs, arglocs, operations) @@ -276,25 +370,81 @@ descr_number, "has address", adr_bridge, "to", self.mc.tell()) self.mc.end_function() + self.write_pending_failure_recoveries() + + def write_pending_failure_recoveries(self): + for tok in self.pending_guard_tokens: + # Okay to write to _mc because we've already made sure that + # there's enough space by "reserving" bytes. + addr = self.generate_quick_failure(self.mc._mc, tok.faildescr, tok.failargs, tok.fail_locs, tok.exc, tok.desc_bytes) + tok.faildescr._x86_adr_recovery_stub = addr + self.patch_jump_for_descr(tok.faildescr, addr) + + self.pending_guard_tokens = [] + self.mc.reset_reserved_bytes() + self.mc.done() def _find_debug_merge_point(self, operations): + for op in operations: if op.opnum == rop.DEBUG_MERGE_POINT: - return op.args[0]._get_str() - return "" + funcname = op.args[0]._get_str() + break + else: + funcname = "" % len(self.loop_run_counters) + # invent the counter, so we don't get too confused + if self._debug: + struct = lltype.malloc(DEBUG_COUNTER, flavor='raw') + struct.i = 0 + self.loop_run_counters.append((funcname, struct)) + return funcname def patch_jump_for_descr(self, faildescr, adr_new_target): adr_jump_offset = faildescr._x86_adr_jump_offset - mc = codebuf.InMemoryCodeBuilder(adr_jump_offset, adr_jump_offset + 4) - mc.write(packimm32(adr_new_target - adr_jump_offset - 4)) + adr_recovery_stub = faildescr._x86_adr_recovery_stub + offset = adr_new_target - (adr_jump_offset + 4) + # If the new target fits within a rel32 of the jump, just patch + # that. Otherwise, leave the original rel32 to the recovery stub in + # place, but clobber the recovery stub with a jump to the real + # target. + if rx86.fits_in_32bits(offset): + mc = codebuf.InMemoryCodeBuilder(adr_jump_offset, adr_jump_offset + 4) + mc.writeimm32(offset) + else: + # "mov r11, addr; jmp r11" is 13 bytes + mc = codebuf.InMemoryCodeBuilder(adr_recovery_stub, adr_recovery_stub + 13) + mc.MOV_ri(X86_64_SCRATCH_REG.value, adr_new_target) + mc.JMP_r(X86_64_SCRATCH_REG.value) + mc.valgrind_invalidated() mc.done() + def _inject_debugging_code(self, operations): + if self._debug: + # before doing anything, let's increase a counter + c_adr = ConstInt(rffi.cast(lltype.Signed, + self.loop_run_counters[-1][1])) + box = BoxInt() + box2 = BoxInt() + ops = [ResOperation(rop.GETFIELD_RAW, [c_adr], + box, descr=self.debug_counter_descr), + ResOperation(rop.INT_ADD, [box, ConstInt(1)], box2), + ResOperation(rop.SETFIELD_RAW, [c_adr, box2], + None, descr=self.debug_counter_descr)] + operations = ops + operations + # # we need one register free (a bit of a hack, but whatever) + # self.mc.PUSH(eax) + # adr = rffi.cast(lltype.Signed, self.loop_run_counters[-1][1]) + # self.mc.MOV(eax, heap(adr)) + # self.mc.ADD(eax, imm(1)) + # self.mc.MOV(heap(adr), eax) + # self.mc.POP(eax) + return operations + def _assemble(self, regalloc, operations): self._regalloc = regalloc regalloc.walk_operations(operations) self.mc.done() - self.mc2.done() if we_are_translated() or self.cpu.dont_keepalive_stuff: self._regalloc = None # else keep it around for debugging frame_depth = regalloc.fm.frame_depth @@ -309,7 +459,7 @@ def _patchable_stackadjust(self): # stack adjustment LEA - self.mc.LEA(esp, fixedsize_ebp_ofs(0)) + self.mc.LEA32_rb(esp.value, 0) return self.mc.tell() - 4 def _patch_stackadjust(self, adr_lea, reserved_depth): @@ -318,23 +468,34 @@ # Compute the correct offset for the instruction LEA ESP, [EBP-4*words]. # Given that [EBP] is where we saved EBP, i.e. in the last word # of our fixed frame, then the 'words' value is: - words = (FRAME_FIXED_SIZE - 1) + reserved_depth + words = (self.cpu.FRAME_FIXED_SIZE - 1) + reserved_depth # align, e.g. for Mac OS X aligned_words = align_stack_words(words+2)-2 # 2 = EIP+EBP - mc.write(packimm32(-WORD * aligned_words)) + mc.writeimm32(-WORD * aligned_words) mc.done() def _call_header(self): - self.mc.PUSH(ebp) - self.mc.MOV(ebp, esp) - self.mc.PUSH(ebx) - self.mc.PUSH(esi) - self.mc.PUSH(edi) + self.mc.PUSH_r(ebp.value) + self.mc.MOV_rr(ebp.value, esp.value) + for regloc in self.cpu.CALLEE_SAVE_REGISTERS: + self.mc.PUSH_r(regloc.value) + # NB. the shape of the frame is hard-coded in get_basic_shape() too. # Also, make sure this is consistent with FRAME_FIXED_SIZE. return self._patchable_stackadjust() + def _call_footer(self): + self.mc.LEA_rb(esp.value, -len(self.cpu.CALLEE_SAVE_REGISTERS) * WORD) + + for i in range(len(self.cpu.CALLEE_SAVE_REGISTERS)-1, -1, -1): + self.mc.POP_r(self.cpu.CALLEE_SAVE_REGISTERS[i].value) + + self.mc.POP_r(ebp.value) + self.mc.RET() + def _assemble_bootstrap_direct_call(self, arglocs, jmpadr, stackdepth): + if IS_X86_64: + return self._assemble_bootstrap_direct_call_64(arglocs, jmpadr, stackdepth) # XXX pushing ebx esi and edi is a bit pointless, since we store # all regsiters anyway, for the case of guard_not_forced # XXX this can be improved greatly. Right now it'll behave like @@ -345,23 +506,81 @@ self._patch_stackadjust(adr_stackadjust, stackdepth) for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] - if isinstance(loc, REG): - self.mc.MOV(loc, mem(ebp, (2 + i) * WORD)) + if isinstance(loc, RegLoc): + assert not loc.is_xmm + self.mc.MOV_rb(loc.value, (2 + i) * WORD) loc = floatlocs[i] - if isinstance(loc, XMMREG): - self.mc.MOVSD(loc, mem64(ebp, (1 + i) * 2 * WORD)) + if isinstance(loc, RegLoc): + assert loc.is_xmm + self.mc.MOVSD_xb(loc.value, (1 + i) * 2 * WORD) tmp = eax xmmtmp = xmm0 for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] - if loc is not None and not isinstance(loc, REG): - self.mc.MOV(tmp, mem(ebp, (2 + i) * WORD)) + if loc is not None and not isinstance(loc, RegLoc): + self.mc.MOV_rb(tmp.value, (2 + i) * WORD) self.mc.MOV(loc, tmp) loc = floatlocs[i] - if loc is not None and not isinstance(loc, XMMREG): - self.mc.MOVSD(xmmtmp, mem64(ebp, (1 + i) * 2 * WORD)) - self.mc.MOVSD(loc, xmmtmp) - self.mc.JMP(rel32(jmpadr)) + if loc is not None and not isinstance(loc, RegLoc): + self.mc.MOVSD_xb(xmmtmp.value, (1 + i) * 2 * WORD) + assert isinstance(loc, StackLoc) + self.mc.MOVSD_bx(loc.value, xmmtmp.value) + self.mc.JMP_l(jmpadr) + return adr_stackadjust + + def _assemble_bootstrap_direct_call_64(self, arglocs, jmpadr, stackdepth): + # XXX: Very similar to _emit_call_64 + + src_locs = [] + dst_locs = [] + xmm_src_locs = [] + xmm_dst_locs = [] + get_from_stack = [] + + # In reverse order for use with pop() + unused_gpr = [r9, r8, ecx, edx, esi, edi] + unused_xmm = [xmm7, xmm6, xmm5, xmm4, xmm3, xmm2, xmm1, xmm0] + + nonfloatlocs, floatlocs = arglocs + adr_stackadjust = self._call_header() + self._patch_stackadjust(adr_stackadjust, stackdepth) + + # The lists are padded with Nones + assert len(nonfloatlocs) == len(floatlocs) + + for i in range(len(nonfloatlocs)): + loc = nonfloatlocs[i] + if loc is not None: + if len(unused_gpr) > 0: + src_locs.append(unused_gpr.pop()) + dst_locs.append(loc) + else: + get_from_stack.append((loc, False)) + + floc = floatlocs[i] + if floc is not None: + if len(unused_xmm) > 0: + xmm_src_locs.append(unused_xmm.pop()) + xmm_dst_locs.append(floc) + else: + get_from_stack.append((floc, True)) + + remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG) + remap_frame_layout(self, xmm_src_locs, xmm_dst_locs, X86_64_XMM_SCRATCH_REG) + + for i in range(len(get_from_stack)): + loc, is_xmm = get_from_stack[i] + if is_xmm: + self.mc.MOVSD_xb(X86_64_XMM_SCRATCH_REG.value, (2 + i) * WORD) + self.mc.MOVSD(loc, X86_64_XMM_SCRATCH_REG) + else: + self.mc.MOV_rb(X86_64_SCRATCH_REG.value, (2 + i) * WORD) + # XXX: We're assuming that "loc" won't require regloc to + # clobber the scratch register + self.mc.MOV(loc, X86_64_SCRATCH_REG) + + self.mc.JMP(imm(jmpadr)) + return adr_stackadjust def _assemble_bootstrap_code(self, inputargs, arglocs): @@ -369,11 +588,12 @@ adr_stackadjust = self._call_header() tmp = X86RegisterManager.all_regs[0] xmmtmp = X86XMMRegisterManager.all_regs[0] + self.mc._mc.begin_reuse_scratch_register() for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] if loc is None: continue - if isinstance(loc, REG): + if isinstance(loc, RegLoc): target = loc else: target = tmp @@ -387,17 +607,20 @@ adr = self.fail_boxes_int.get_addr_for_num(i) self.mc.MOV(target, heap(adr)) if target is not loc: - self.mc.MOV(loc, target) + assert isinstance(loc, StackLoc) + self.mc.MOV_br(loc.value, target.value) for i in range(len(floatlocs)): loc = floatlocs[i] if loc is None: continue adr = self.fail_boxes_float.get_addr_for_num(i) - if isinstance(loc, REG): - self.mc.MOVSD(loc, heap64(adr)) + if isinstance(loc, RegLoc): + self.mc.MOVSD(loc, heap(adr)) else: - self.mc.MOVSD(xmmtmp, heap64(adr)) - self.mc.MOVSD(loc, xmmtmp) + self.mc.MOVSD(xmmtmp, heap(adr)) + assert isinstance(loc, StackLoc) + self.mc.MOVSD_bx(loc.value, xmmtmp.value) + self.mc._mc.end_reuse_scratch_register() return adr_stackadjust def dump(self, text): @@ -410,27 +633,10 @@ finally: Box._extended_display = _prev - def _start_block(self): - # Return a 'mc' that can be used to write an "atomic" block, - # i.e. one that will not contain any JMP. - mc = self.mc._mc - if not we_are_translated(): - self._block_started_mc = (self.mc, mc.tell()) - self.mc = "block started" - return mc - - def _stop_block(self): - if not we_are_translated(): - assert self.mc == "block started" - self.mc, orgpos = self._block_started_mc - assert 0 <= self.mc._mc.tell() - orgpos <= 58, ( - "too many bytes in _start_block/_stop_block pair") - del self._block_started_mc - # ------------------------------------------------------------ def mov(self, from_loc, to_loc): - if isinstance(from_loc, XMMREG) or isinstance(to_loc, XMMREG): + if (isinstance(from_loc, RegLoc) and from_loc.is_xmm) or (isinstance(to_loc, RegLoc) and to_loc.is_xmm): self.mc.MOVSD(to_loc, from_loc) else: self.mc.MOV(to_loc, from_loc) @@ -438,24 +644,24 @@ regalloc_mov = mov # legacy interface def regalloc_push(self, loc): - if isinstance(loc, XMMREG): - self.mc.SUB(esp, imm(2*WORD)) - self.mc.MOVSD(mem64(esp, 0), loc) - elif isinstance(loc, MODRM64): + if isinstance(loc, RegLoc) and loc.is_xmm: + self.mc.SUB_ri(esp.value, 2*WORD) + self.mc.MOVSD_sx(0, loc.value) + elif WORD == 4 and isinstance(loc, StackLoc) and loc.width == 8: # XXX evil trick - self.mc.PUSH(mem(ebp, get_ebp_ofs(loc.position))) - self.mc.PUSH(mem(ebp, get_ebp_ofs(loc.position + 1))) + self.mc.PUSH_b(get_ebp_ofs(loc.position)) + self.mc.PUSH_b(get_ebp_ofs(loc.position + 1)) else: self.mc.PUSH(loc) def regalloc_pop(self, loc): - if isinstance(loc, XMMREG): - self.mc.MOVSD(loc, mem64(esp, 0)) - self.mc.ADD(esp, imm(2*WORD)) - elif isinstance(loc, MODRM64): + if isinstance(loc, RegLoc) and loc.is_xmm: + self.mc.MOVSD_xs(loc.value, 0) + self.mc.ADD_ri(esp.value, 2*WORD) + elif WORD == 4 and isinstance(loc, StackLoc) and loc.width == 8: # XXX evil trick - self.mc.POP(mem(ebp, get_ebp_ofs(loc.position + 1))) - self.mc.POP(mem(ebp, get_ebp_ofs(loc.position))) + self.mc.POP_b(get_ebp_ofs(loc.position + 1)) + self.mc.POP_b(get_ebp_ofs(loc.position)) else: self.mc.POP(loc) @@ -472,14 +678,14 @@ faildescr._x86_current_depths = current_depths failargs = guard_op.fail_args guard_opnum = guard_op.opnum - failaddr = self.implement_guard_recovery(guard_opnum, - faildescr, failargs, - faillocs) + guard_token = self.implement_guard_recovery(guard_opnum, + faildescr, failargs, + faillocs) if op is None: dispatch_opnum = guard_opnum else: dispatch_opnum = op.opnum - res = genop_guard_list[dispatch_opnum](self, op, guard_op, failaddr, + res = genop_guard_list[dispatch_opnum](self, op, guard_op, guard_token, arglocs, resloc) faildescr._x86_adr_jump_offset = res @@ -506,103 +712,161 @@ rl = result_loc.lowest8bits() if isinstance(op.args[0], Const): self.mc.CMP(arglocs[1], arglocs[0]) - getattr(self.mc, 'SET' + rev_cond)(rl) + self.mc.SET_ir(rx86.Conditions[rev_cond], rl.value) else: self.mc.CMP(arglocs[0], arglocs[1]) - getattr(self.mc, 'SET' + cond)(rl) - self.mc.MOVZX(result_loc, rl) + self.mc.SET_ir(rx86.Conditions[cond], rl.value) + self.mc.MOVZX8_rr(result_loc.value, rl.value) return genop_cmp def _cmpop_float(cond, is_ne=False): def genop_cmp(self, op, arglocs, result_loc): self.mc.UCOMISD(arglocs[0], arglocs[1]) - rl = result_loc.lowest8bits() - rh = result_loc.higher8bits() - getattr(self.mc, 'SET' + cond)(rl) + tmp1 = result_loc.lowest8bits() + if IS_X86_32: + tmp2 = result_loc.higher8bits() + elif IS_X86_64: + tmp2 = X86_64_SCRATCH_REG.lowest8bits() + + self.mc.SET_ir(rx86.Conditions[cond], tmp1.value) if is_ne: - self.mc.SETP(rh) - self.mc.OR(rl, rh) + self.mc.SET_ir(rx86.Conditions['P'], tmp2.value) + self.mc.OR8_rr(tmp1.value, tmp2.value) else: - self.mc.SETNP(rh) - self.mc.AND(rl, rh) - self.mc.MOVZX(result_loc, rl) + self.mc.SET_ir(rx86.Conditions['NP'], tmp2.value) + self.mc.AND8_rr(tmp1.value, tmp2.value) + self.mc.MOVZX8_rr(result_loc.value, tmp1.value) return genop_cmp def _cmpop_guard(cond, rev_cond, false_cond, false_rev_cond): - def genop_cmp_guard(self, op, guard_op, addr, arglocs, result_loc): + def genop_cmp_guard(self, op, guard_op, guard_token, arglocs, result_loc): guard_opnum = guard_op.opnum if isinstance(op.args[0], Const): self.mc.CMP(arglocs[1], arglocs[0]) if guard_opnum == rop.GUARD_FALSE: - name = 'J' + rev_cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(guard_token, rev_cond) else: - name = 'J' + false_rev_cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(guard_token, false_rev_cond) else: self.mc.CMP(arglocs[0], arglocs[1]) if guard_opnum == rop.GUARD_FALSE: - name = 'J' + cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(guard_token, cond) else: - name = 'J' + false_cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(guard_token, false_cond) return genop_cmp_guard def _cmpop_guard_float(cond, false_cond, need_jp): - def genop_cmp_guard_float(self, op, guard_op, addr, arglocs, + def genop_cmp_guard_float(self, op, guard_op, guard_token, arglocs, result_loc): guard_opnum = guard_op.opnum self.mc.UCOMISD(arglocs[0], arglocs[1]) + # 16 is enough space for the rel8 jumps below and the rel32 + # jump in implement_guard + self.mc.ensure_bytes_available(16 + guard_token.recovery_stub_size()) if guard_opnum == rop.GUARD_FALSE: - mc = self.mc._mc - name = 'J' + cond if need_jp: - mc.JP(rel8(6)) - getattr(mc, name)(rel32(addr)) - return mc.tell() - 4 + self.mc.J_il8(rx86.Conditions['P'], 6) + return self.implement_guard(guard_token, cond) else: if need_jp: - mc = self.mc._mc - mc.JP(rel8(2)) - getattr(mc, 'J' + cond)(rel8(5)) - return self.implement_guard(addr, mc.JMP) - name = 'J' + false_cond - return self.implement_guard(addr, getattr(self.mc, name)) + self.mc.J_il8(rx86.Conditions['P'], 2) + self.mc.J_il8(rx86.Conditions[cond], 5) + return self.implement_guard(guard_token) + return self.implement_guard(guard_token, false_cond) return genop_cmp_guard_float - @specialize.arg(5) - def _emit_call(self, x, arglocs, start=0, tmp=eax, force_mc=False, - mc=None): - if not force_mc: - mc = self.mc + def _emit_call(self, x, arglocs, start=0, tmp=eax): + if IS_X86_64: + return self._emit_call_64(x, arglocs, start) + p = 0 n = len(arglocs) for i in range(start, n): loc = arglocs[i] - if isinstance(loc, REG): - if isinstance(loc, XMMREG): - mc.MOVSD(mem64(esp, p), loc) + if isinstance(loc, RegLoc): + if loc.is_xmm: + self.mc.MOVSD_sx(p, loc.value) else: - mc.MOV(mem(esp, p), loc) + self.mc.MOV_sr(p, loc.value) p += round_up_to_4(loc.width) p = 0 for i in range(start, n): loc = arglocs[i] - if not isinstance(loc, REG): - if isinstance(loc, MODRM64): - mc.MOVSD(xmm0, loc) - mc.MOVSD(mem64(esp, p), xmm0) + if not isinstance(loc, RegLoc): + if loc.width == 8: + self.mc.MOVSD(xmm0, loc) + self.mc.MOVSD_sx(p, xmm0.value) else: - mc.MOV(tmp, loc) - mc.MOV(mem(esp, p), tmp) + self.mc.MOV(tmp, loc) + self.mc.MOV_sr(p, tmp.value) p += round_up_to_4(loc.width) self._regalloc.reserve_param(p//WORD) - mc.CALL(x) + # x is a location + self.mc.CALL(x) self.mark_gc_roots() + + def _emit_call_64(self, x, arglocs, start=0): + src_locs = [] + dst_locs = [] + xmm_src_locs = [] + xmm_dst_locs = [] + pass_on_stack = [] + + # In reverse order for use with pop() + unused_gpr = [r9, r8, ecx, edx, esi, edi] + unused_xmm = [xmm7, xmm6, xmm5, xmm4, xmm3, xmm2, xmm1, xmm0] + + for i in range(start, len(arglocs)): + loc = arglocs[i] + # XXX: Should be much simplier to tell whether a location is a + # float! It's so ugly because we have to "guard" the access to + # .type with isinstance, since not all AssemblerLocation classes + # are "typed" + if ((isinstance(loc, RegLoc) and loc.is_xmm) or + (isinstance(loc, StackLoc) and loc.type == FLOAT) or + (isinstance(loc, ConstFloatLoc))): + if len(unused_xmm) > 0: + xmm_src_locs.append(loc) + xmm_dst_locs.append(unused_xmm.pop()) + else: + pass_on_stack.append(loc) + else: + if len(unused_gpr) > 0: + src_locs.append(loc) + dst_locs.append(unused_gpr.pop()) + else: + pass_on_stack.append(loc) + # Emit instructions to pass the stack arguments + # XXX: Would be nice to let remap_frame_layout take care of this, but + # we'd need to create something like StackLoc, but relative to esp, + # and I don't know if it's worth it. + for i in range(len(pass_on_stack)): + loc = pass_on_stack[i] + if not isinstance(loc, RegLoc): + if isinstance(loc, StackLoc) and loc.type == FLOAT: + self.mc.MOVSD(X86_64_XMM_SCRATCH_REG, loc) + self.mc.MOVSD_sx(i*WORD, X86_64_XMM_SCRATCH_REG.value) + else: + self.mc.MOV(X86_64_SCRATCH_REG, loc) + self.mc.MOV_sr(i*WORD, X86_64_SCRATCH_REG.value) + else: + # It's a register + if loc.is_xmm: + self.mc.MOVSD_sx(i*WORD, loc.value) + else: + self.mc.MOV_sr(i*WORD, loc.value) + + # Handle register arguments + remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG) + remap_frame_layout(self, xmm_src_locs, xmm_dst_locs, X86_64_XMM_SCRATCH_REG) + + self._regalloc.reserve_param(len(pass_on_stack)) + self.mc.CALL(x) + self.mark_gc_roots() + def call(self, addr, args, res): - self._emit_call(rel32(addr), args) + self._emit_call(imm(addr), args) assert res is eax genop_int_neg = _unaryop("NEG") @@ -613,6 +877,9 @@ genop_int_and = _binaryop("AND", True) genop_int_or = _binaryop("OR", True) genop_int_xor = _binaryop("XOR", True) + genop_int_lshift = _binaryop("SHL") + genop_int_rshift = _binaryop("SAR") + genop_uint_rshift = _binaryop("SHR") genop_float_add = _binaryop("ADDSD", True) genop_float_sub = _binaryop('SUBSD') genop_float_mul = _binaryop('MULSD', True) @@ -659,26 +926,27 @@ genop_guard_float_gt = _cmpop_guard_float("A", "BE", False) genop_guard_float_ge = _cmpop_guard_float("AE", "B", False) - def genop_guard_float_ne(self, op, guard_op, addr, arglocs, result_loc): + def genop_guard_float_ne(self, op, guard_op, guard_token, arglocs, result_loc): guard_opnum = guard_op.opnum self.mc.UCOMISD(arglocs[0], arglocs[1]) - mc = self.mc._mc + # 16 is enough space for the rel8 jumps below and the rel32 + # jump in implement_guard + self.mc.ensure_bytes_available(16 + guard_token.recovery_stub_size()) if guard_opnum == rop.GUARD_TRUE: - mc.JP(rel8(6)) - mc.JE(rel32(addr)) - return mc.tell() - 4 - else: - mc.JP(rel8(2)) - mc.JE(rel8(5)) - return self.implement_guard(addr, mc.JMP) + self.mc.J_il8(rx86.Conditions['P'], 6) + return self.implement_guard(guard_token, 'E') + else: + self.mc.J_il8(rx86.Conditions['P'], 2) + self.mc.J_il8(rx86.Conditions['E'], 5) + return self.implement_guard(guard_token) def genop_float_neg(self, op, arglocs, resloc): # Following what gcc does: res = x ^ 0x8000000000000000 - self.mc.XORPD(arglocs[0], self.loc_float_const_neg) + self.mc.XORPD(arglocs[0], heap(self.float_const_neg_addr)) def genop_float_abs(self, op, arglocs, resloc): # Following what gcc does: res = x & 0x7FFFFFFFFFFFFFFF - self.mc.ANDPD(arglocs[0], self.loc_float_const_abs) + self.mc.ANDPD(arglocs[0], heap(self.float_const_abs_addr)) def genop_cast_float_to_int(self, op, arglocs, resloc): self.mc.CVTTSD2SI(resloc, arglocs[0]) @@ -686,70 +954,56 @@ def genop_cast_int_to_float(self, op, arglocs, resloc): self.mc.CVTSI2SD(resloc, arglocs[0]) - def genop_int_lshift(self, op, arglocs, resloc): - loc, loc2 = arglocs - if loc2 is ecx: - loc2 = cl - self.mc.SHL(loc, loc2) - - def genop_int_rshift(self, op, arglocs, resloc): - loc, loc2 = arglocs - if loc2 is ecx: - loc2 = cl - self.mc.SAR(loc, loc2) - - def genop_uint_rshift(self, op, arglocs, resloc): - loc, loc2 = arglocs - if loc2 is ecx: - loc2 = cl - self.mc.SHR(loc, loc2) - - def genop_guard_int_is_true(self, op, guard_op, addr, arglocs, resloc): + def genop_guard_int_is_true(self, op, guard_op, guard_token, arglocs, resloc): guard_opnum = guard_op.opnum - self.mc.CMP(arglocs[0], imm8(0)) + self.mc.CMP(arglocs[0], imm(0)) if guard_opnum == rop.GUARD_TRUE: - return self.implement_guard(addr, self.mc.JZ) + return self.implement_guard(guard_token, 'Z') else: - return self.implement_guard(addr, self.mc.JNZ) + return self.implement_guard(guard_token, 'NZ') def genop_int_is_true(self, op, arglocs, resloc): - self.mc.CMP(arglocs[0], imm8(0)) + self.mc.CMP(arglocs[0], imm(0)) rl = resloc.lowest8bits() - self.mc.SETNE(rl) - self.mc.MOVZX(resloc, rl) + self.mc.SET_ir(rx86.Conditions['NE'], rl.value) + self.mc.MOVZX8(resloc, rl) - def genop_guard_int_is_zero(self, op, guard_op, addr, arglocs, resloc): + def genop_guard_int_is_zero(self, op, guard_op, guard_token, arglocs, resloc): guard_opnum = guard_op.opnum - self.mc.CMP(arglocs[0], imm8(0)) + self.mc.CMP(arglocs[0], imm(0)) if guard_opnum == rop.GUARD_TRUE: - return self.implement_guard(addr, self.mc.JNZ) + return self.implement_guard(guard_token, 'NZ') else: - return self.implement_guard(addr, self.mc.JZ) + return self.implement_guard(guard_token, 'Z') def genop_int_is_zero(self, op, arglocs, resloc): - self.mc.CMP(arglocs[0], imm8(0)) + self.mc.CMP(arglocs[0], imm(0)) rl = resloc.lowest8bits() - self.mc.SETE(rl) - self.mc.MOVZX(resloc, rl) + self.mc.SET_ir(rx86.Conditions['E'], rl.value) + self.mc.MOVZX8(resloc, rl) def genop_same_as(self, op, arglocs, resloc): self.mov(arglocs[0], resloc) #genop_cast_ptr_to_int = genop_same_as def genop_int_mod(self, op, arglocs, resloc): - self.mc.CDQ() - self.mc.IDIV(ecx) + if IS_X86_32: + self.mc.CDQ() + elif IS_X86_64: + self.mc.CQO() + + self.mc.IDIV_r(ecx.value) genop_int_floordiv = genop_int_mod def genop_uint_floordiv(self, op, arglocs, resloc): - self.mc.XOR(edx, edx) - self.mc.DIV(ecx) + self.mc.XOR_rr(edx.value, edx.value) + self.mc.DIV_r(ecx.value) def genop_new_with_vtable(self, op, arglocs, result_loc): assert result_loc is eax loc_vtable = arglocs[-1] - assert isinstance(loc_vtable, IMM32) + assert isinstance(loc_vtable, ImmedLoc) arglocs = arglocs[:-1] self.call(self.malloc_func_addr, arglocs, eax) # xxx ignore NULL returns for now @@ -757,7 +1011,9 @@ def set_vtable(self, loc, loc_vtable): if self.cpu.vtable_offset is not None: - self.mc.MOV(mem(loc, self.cpu.vtable_offset), loc_vtable) + assert isinstance(loc, RegLoc) + assert isinstance(loc_vtable, ImmedLoc) + self.mc.MOV_mi((loc.value, self.cpu.vtable_offset), loc_vtable.value) # XXX genop_new is abused for all varsized mallocs with Boehm, for now # (instead of genop_new_array, genop_newstr, genop_newunicode) @@ -779,16 +1035,19 @@ def genop_getfield_gc(self, op, arglocs, resloc): base_loc, ofs_loc, size_loc = arglocs - assert isinstance(size_loc, IMM32) + assert isinstance(size_loc, ImmedLoc) + assert isinstance(resloc, RegLoc) size = size_loc.value - if size == 1: - self.mc.MOVZX(resloc, addr8_add(base_loc, ofs_loc)) + + source_addr = AddressLoc(base_loc, ofs_loc) + if resloc.is_xmm: + self.mc.MOVSD(resloc, source_addr) + elif size == 1: + self.mc.MOVZX8(resloc, source_addr) elif size == 2: - self.mc.MOVZX(resloc, addr_add(base_loc, ofs_loc)) + self.mc.MOVZX16(resloc, source_addr) elif size == WORD: - self.mc.MOV(resloc, addr_add(base_loc, ofs_loc)) - elif size == 8: - self.mc.MOVSD(resloc, addr64_add(base_loc, ofs_loc)) + self.mc.MOV(resloc, source_addr) else: raise NotImplementedError("getfield size = %d" % size) @@ -798,20 +1057,23 @@ def genop_getarrayitem_gc(self, op, arglocs, resloc): base_loc, ofs_loc, scale, ofs = arglocs - assert isinstance(ofs, IMM32) - assert isinstance(scale, IMM32) + assert isinstance(ofs, ImmedLoc) + assert isinstance(scale, ImmedLoc) if op.result.type == FLOAT: - self.mc.MOVSD(resloc, addr64_add(base_loc, ofs_loc, ofs.value, + self.mc.MOVSD(resloc, addr_add(base_loc, ofs_loc, ofs.value, scale.value)) else: if scale.value == 0: - self.mc.MOVZX(resloc, addr8_add(base_loc, ofs_loc, ofs.value, + self.mc.MOVZX8(resloc, addr_add(base_loc, ofs_loc, ofs.value, scale.value)) - elif scale.value == 2: + elif scale.value == 1: + self.mc.MOVZX16(resloc, addr_add(base_loc, ofs_loc, ofs.value, + scale.value)) + elif (1 << scale.value) == WORD: self.mc.MOV(resloc, addr_add(base_loc, ofs_loc, ofs.value, scale.value)) else: - print "[asmgen]setarrayitem unsupported size: %d" % scale.value + print "[asmgen]getarrayitem unsupported size: %d" % scale.value raise NotImplementedError() genop_getarrayitem_gc_pure = genop_getarrayitem_gc @@ -819,34 +1081,35 @@ def genop_discard_setfield_gc(self, op, arglocs): base_loc, ofs_loc, size_loc, value_loc = arglocs - assert isinstance(size_loc, IMM32) + assert isinstance(size_loc, ImmedLoc) size = size_loc.value - if size == WORD * 2: - self.mc.MOVSD(addr64_add(base_loc, ofs_loc), value_loc) + dest_addr = AddressLoc(base_loc, ofs_loc) + if isinstance(value_loc, RegLoc) and value_loc.is_xmm: + self.mc.MOVSD(dest_addr, value_loc) elif size == WORD: - self.mc.MOV(addr_add(base_loc, ofs_loc), value_loc) + self.mc.MOV(dest_addr, value_loc) elif size == 2: - self.mc.MOV16(addr_add(base_loc, ofs_loc), value_loc) + self.mc.MOV16(dest_addr, value_loc) elif size == 1: - self.mc.MOV(addr8_add(base_loc, ofs_loc), value_loc.lowest8bits()) + self.mc.MOV8(dest_addr, value_loc.lowest8bits()) else: print "[asmgen]setfield addr size %d" % size raise NotImplementedError("Addr size %d" % size) def genop_discard_setarrayitem_gc(self, op, arglocs): base_loc, ofs_loc, value_loc, scale_loc, baseofs = arglocs - assert isinstance(baseofs, IMM32) - assert isinstance(scale_loc, IMM32) + assert isinstance(baseofs, ImmedLoc) + assert isinstance(scale_loc, ImmedLoc) + dest_addr = AddressLoc(base_loc, ofs_loc, scale_loc.value, baseofs.value) if op.args[2].type == FLOAT: - self.mc.MOVSD(addr64_add(base_loc, ofs_loc, baseofs.value, - scale_loc.value), value_loc) + self.mc.MOVSD(dest_addr, value_loc) else: - if scale_loc.value == 2: - self.mc.MOV(addr_add(base_loc, ofs_loc, baseofs.value, - scale_loc.value), value_loc) + if (1 << scale_loc.value) == WORD: + self.mc.MOV(dest_addr, value_loc) + elif scale_loc.value == 1: + self.mc.MOV16(dest_addr, value_loc) elif scale_loc.value == 0: - self.mc.MOV(addr8_add(base_loc, ofs_loc, baseofs.value, - scale_loc.value), value_loc.lowest8bits()) + self.mc.MOV8(dest_addr, value_loc.lowest8bits()) else: raise NotImplementedError("scale = %d" % scale_loc.value) @@ -855,17 +1118,17 @@ basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) assert itemsize == 1 - self.mc.MOV(addr8_add(base_loc, ofs_loc, basesize), - val_loc.lowest8bits()) + dest_addr = AddressLoc(base_loc, ofs_loc, 0, basesize) + self.mc.MOV8(dest_addr, val_loc.lowest8bits()) def genop_discard_unicodesetitem(self, op, arglocs): base_loc, ofs_loc, val_loc = arglocs basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) if itemsize == 4: - self.mc.MOV(addr_add(base_loc, ofs_loc, basesize, 2), val_loc) + self.mc.MOV32(AddressLoc(base_loc, ofs_loc, 2, basesize), val_loc) elif itemsize == 2: - self.mc.MOV16(addr_add(base_loc, ofs_loc, basesize, 1), val_loc) + self.mc.MOV16(AddressLoc(base_loc, ofs_loc, 1, basesize), val_loc) else: assert 0, itemsize @@ -886,7 +1149,7 @@ def genop_arraylen_gc(self, op, arglocs, resloc): base_loc, ofs_loc = arglocs - assert isinstance(ofs_loc, IMM32) + assert isinstance(ofs_loc, ImmedLoc) self.mc.MOV(resloc, addr_add_const(base_loc, ofs_loc.value)) def genop_strgetitem(self, op, arglocs, resloc): @@ -894,83 +1157,83 @@ basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) assert itemsize == 1 - self.mc.MOVZX(resloc, addr8_add(base_loc, ofs_loc, basesize)) + self.mc.MOVZX8(resloc, AddressLoc(base_loc, ofs_loc, 0, basesize)) def genop_unicodegetitem(self, op, arglocs, resloc): base_loc, ofs_loc = arglocs basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) if itemsize == 4: - self.mc.MOV(resloc, addr_add(base_loc, ofs_loc, basesize, 2)) + self.mc.MOV32(resloc, AddressLoc(base_loc, ofs_loc, 2, basesize)) elif itemsize == 2: - self.mc.MOVZX(resloc, addr_add(base_loc, ofs_loc, basesize, 1)) + self.mc.MOVZX16(resloc, AddressLoc(base_loc, ofs_loc, 1, basesize)) else: assert 0, itemsize - def genop_guard_guard_true(self, ign_1, guard_op, addr, locs, ign_2): + def genop_guard_guard_true(self, ign_1, guard_op, guard_token, locs, ign_2): loc = locs[0] self.mc.TEST(loc, loc) - return self.implement_guard(addr, self.mc.JZ) + return self.implement_guard(guard_token, 'Z') genop_guard_guard_nonnull = genop_guard_guard_true - def genop_guard_guard_no_exception(self, ign_1, guard_op, addr, + def genop_guard_guard_no_exception(self, ign_1, guard_op, guard_token, locs, ign_2): self.mc.CMP(heap(self.cpu.pos_exception()), imm(0)) - return self.implement_guard(addr, self.mc.JNZ) + return self.implement_guard(guard_token, 'NZ') - def genop_guard_guard_exception(self, ign_1, guard_op, addr, + def genop_guard_guard_exception(self, ign_1, guard_op, guard_token, locs, resloc): loc = locs[0] loc1 = locs[1] self.mc.MOV(loc1, heap(self.cpu.pos_exception())) self.mc.CMP(loc1, loc) - addr = self.implement_guard(addr, self.mc.JNE) + addr = self.implement_guard(guard_token, 'NE') if resloc is not None: self.mc.MOV(resloc, heap(self.cpu.pos_exc_value())) self.mc.MOV(heap(self.cpu.pos_exception()), imm(0)) self.mc.MOV(heap(self.cpu.pos_exc_value()), imm(0)) return addr - def _gen_guard_overflow(self, guard_op, addr): + def _gen_guard_overflow(self, guard_op, guard_token): guard_opnum = guard_op.opnum if guard_opnum == rop.GUARD_NO_OVERFLOW: - return self.implement_guard(addr, self.mc.JO) + return self.implement_guard(guard_token, 'O') elif guard_opnum == rop.GUARD_OVERFLOW: - return self.implement_guard(addr, self.mc.JNO) + return self.implement_guard(guard_token, 'NO') else: print "int_xxx_ovf followed by", guard_op.getopname() raise AssertionError - def genop_guard_int_add_ovf(self, op, guard_op, addr, arglocs, result_loc): + def genop_guard_int_add_ovf(self, op, guard_op, guard_token, arglocs, result_loc): self.genop_int_add(op, arglocs, result_loc) - return self._gen_guard_overflow(guard_op, addr) + return self._gen_guard_overflow(guard_op, guard_token) - def genop_guard_int_sub_ovf(self, op, guard_op, addr, arglocs, result_loc): + def genop_guard_int_sub_ovf(self, op, guard_op, guard_token, arglocs, result_loc): self.genop_int_sub(op, arglocs, result_loc) - return self._gen_guard_overflow(guard_op, addr) + return self._gen_guard_overflow(guard_op, guard_token) - def genop_guard_int_mul_ovf(self, op, guard_op, addr, arglocs, result_loc): + def genop_guard_int_mul_ovf(self, op, guard_op, guard_token, arglocs, result_loc): self.genop_int_mul(op, arglocs, result_loc) - return self._gen_guard_overflow(guard_op, addr) + return self._gen_guard_overflow(guard_op, guard_token) - def genop_guard_guard_false(self, ign_1, guard_op, addr, locs, ign_2): + def genop_guard_guard_false(self, ign_1, guard_op, guard_token, locs, ign_2): loc = locs[0] self.mc.TEST(loc, loc) - return self.implement_guard(addr, self.mc.JNZ) + return self.implement_guard(guard_token, 'NZ') genop_guard_guard_isnull = genop_guard_guard_false - def genop_guard_guard_value(self, ign_1, guard_op, addr, locs, ign_2): + def genop_guard_guard_value(self, ign_1, guard_op, guard_token, locs, ign_2): if guard_op.args[0].type == FLOAT: assert guard_op.args[1].type == FLOAT self.mc.UCOMISD(locs[0], locs[1]) else: self.mc.CMP(locs[0], locs[1]) - return self.implement_guard(addr, self.mc.JNE) + return self.implement_guard(guard_token, 'NE') - def _cmp_guard_class(self, mc, locs): + def _cmp_guard_class(self, locs): offset = self.cpu.vtable_offset if offset is not None: - mc.CMP(mem(locs[0], offset), locs[1]) + self.mc.CMP(mem(locs[0], offset), locs[1]) else: # XXX hard-coded assumption: to go from an object to its class # we use the following algorithm: @@ -979,7 +1242,7 @@ # - multiply by 4 and use it as an offset in type_info_group # - add 16 bytes, to go past the TYPE_INFO structure loc = locs[1] - assert isinstance(loc, IMM32) + assert isinstance(loc, ImmedLoc) classptr = loc.value # here, we have to go back from 'classptr' to the value expected # from reading the 16 bits in the object header @@ -988,37 +1251,37 @@ type_info_group = llop.gc_get_type_info_group(llmemory.Address) type_info_group = rffi.cast(lltype.Signed, type_info_group) expected_typeid = (classptr - sizeof_ti - type_info_group) >> 2 - mc.CMP16(mem(locs[0], 0), imm32(expected_typeid)) + self.mc.CMP16(mem(locs[0], 0), ImmedLoc(expected_typeid)) - def genop_guard_guard_class(self, ign_1, guard_op, addr, locs, ign_2): - mc = self._start_block() - self._cmp_guard_class(mc, locs) - self._stop_block() - return self.implement_guard(addr, self.mc.JNE) + def genop_guard_guard_class(self, ign_1, guard_op, guard_token, locs, ign_2): + self.mc.ensure_bytes_available(256) + self._cmp_guard_class(locs) + return self.implement_guard(guard_token, 'NE') def genop_guard_guard_nonnull_class(self, ign_1, guard_op, - addr, locs, ign_2): - mc = self._start_block() - mc.CMP(locs[0], imm8(1)) - mc.JB(rel8_patched_later) - jb_location = mc.get_relative_pos() - self._cmp_guard_class(mc, locs) + guard_token, locs, ign_2): + self.mc.ensure_bytes_available(256) + self.mc.CMP(locs[0], imm(1)) + # Patched below + self.mc.J_il8(rx86.Conditions['B'], 0) + jb_location = self.mc.get_relative_pos() + self._cmp_guard_class(locs) # patch the JB above - offset = mc.get_relative_pos() - jb_location + offset = self.mc.get_relative_pos() - jb_location assert 0 < offset <= 127 - mc.overwrite(jb_location-1, [chr(offset)]) - self._stop_block() + self.mc.overwrite(jb_location-1, [chr(offset)]) # - return self.implement_guard(addr, self.mc.JNE) + return self.implement_guard(guard_token, 'NE') def implement_guard_recovery(self, guard_opnum, faildescr, failargs, fail_locs): exc = (guard_opnum == rop.GUARD_EXCEPTION or guard_opnum == rop.GUARD_NO_EXCEPTION or guard_opnum == rop.GUARD_NOT_FORCED) - return self.generate_quick_failure(faildescr, failargs, fail_locs, exc) + desc_bytes = self.failure_recovery_description(failargs, fail_locs) + return GuardToken(faildescr, failargs, fail_locs, exc, desc_bytes) - def generate_quick_failure(self, faildescr, failargs, fail_locs, exc): + def generate_quick_failure(self, mc, faildescr, failargs, fail_locs, exc, desc_bytes): """Generate the initial code for handling a failure. We try to keep it as compact as possible. The idea is that this code is executed at most once (and very often, zero times); when @@ -1026,38 +1289,43 @@ really handle recovery from this particular failure. """ fail_index = self.cpu.get_fail_descr_number(faildescr) - bytes_needed = 20 + 5 * len(failargs) # conservative estimate - if self.mc2.bytes_free() < bytes_needed: - self.mc2.make_new_mc() - mc = self.mc2._mc addr = mc.tell() withfloats = False for box in failargs: if box is not None and box.type == FLOAT: withfloats = True break - mc.CALL(rel32(self.failure_recovery_code[exc + 2 * withfloats])) + mc.CALL(imm(self.failure_recovery_code[exc + 2 * withfloats])) # write tight data that describes the failure recovery faildescr._x86_failure_recovery_bytecode = mc.tell() - self.write_failure_recovery_description(mc, failargs, fail_locs) + for byte in desc_bytes: + mc.writechr(ord(byte)) # write the fail_index too - mc.write(packimm32(fail_index)) + mc.writeimm32(fail_index) # for testing the decoding, write a final byte 0xCC if not we_are_translated(): mc.writechr(0xCC) faildescr._x86_debug_faillocs = [loc for loc in fail_locs if loc is not None] + + # Make sure the recovery stub is at least 16 bytes long (for the + # case where we overwrite the recovery stub with a 64-bit absolute + # jump) + while mc.tell() - addr < 16: + mc.writechr(0x00) return addr DESCR_REF = 0x00 DESCR_INT = 0x01 DESCR_FLOAT = 0x02 DESCR_SPECIAL = 0x03 - CODE_FROMSTACK = 4*8 + # XXX: 4*8 works on i386, should we optimize for that case? + CODE_FROMSTACK = 4*16 CODE_STOP = 0 | DESCR_SPECIAL CODE_HOLE = 4 | DESCR_SPECIAL - def write_failure_recovery_description(self, mc, failargs, locs): + def failure_recovery_description(self, failargs, locs): + desc_bytes = [] for i in range(len(failargs)): arg = failargs[i] if arg is not None: @@ -1070,24 +1338,30 @@ else: raise AssertionError("bogus kind") loc = locs[i] - if isinstance(loc, MODRM): + if isinstance(loc, StackLoc): n = self.CODE_FROMSTACK//4 + loc.position else: - assert isinstance(loc, REG) - n = loc.op + assert isinstance(loc, RegLoc) + n = loc.value n = kind + 4*n while n > 0x7F: - mc.writechr((n & 0x7F) | 0x80) + desc_bytes.append(chr((n & 0x7F) | 0x80)) n >>= 7 else: n = self.CODE_HOLE - mc.writechr(n) - mc.writechr(self.CODE_STOP) + desc_bytes.append(chr(n)) + desc_bytes.append(chr(self.CODE_STOP)) # assert that the fail_boxes lists are big enough assert len(failargs) <= self.fail_boxes_int.SIZE + return desc_bytes + + def write_failure_recovery_description(self, mc, failargs, locs): + for byte in self.failure_recovery_description(failargs, locs): + mc.writechr(ord(byte)) def rebuild_faillocs_from_descr(self, bytecode): from pypy.jit.backend.x86.regalloc import X86FrameManager + descr_to_box_type = [REF, INT, FLOAT] bytecode = rffi.cast(rffi.UCHARP, bytecode) arglocs = [] while 1: @@ -1112,7 +1386,7 @@ size = 2 else: size = 1 - loc = X86FrameManager.frame_pos(code, size) + loc = X86FrameManager.frame_pos(code, descr_to_box_type[kind]) elif code == self.CODE_STOP: break elif code == self.CODE_HOLE: @@ -1122,16 +1396,16 @@ kind = code & 3 code >>= 2 if kind == self.DESCR_FLOAT: - loc = xmm_registers[code] + loc = regloc.XMMREGLOCS[code] else: - loc = registers[code] + loc = regloc.REGLOCS[code] arglocs.append(loc) return arglocs[:] @rgc.no_collect def grab_frame_values(self, bytecode, frame_addr, allregisters): # no malloc allowed here!! - self.fail_ebp = allregisters[16 + ebp.op] + self.fail_ebp = allregisters[16 + ebp.value] num = 0 value_hi = 0 while 1: @@ -1154,7 +1428,7 @@ code = (code - self.CODE_FROMSTACK) >> 2 stackloc = frame_addr + get_ebp_ofs(code) value = rffi.cast(rffi.LONGP, stackloc)[0] - if kind == self.DESCR_FLOAT: + if kind == self.DESCR_FLOAT and WORD == 4: value_hi = value value = rffi.cast(rffi.LONGP, stackloc - 4)[0] else: @@ -1168,8 +1442,11 @@ break code >>= 2 if kind == self.DESCR_FLOAT: - value = allregisters[2*code] - value_hi = allregisters[2*code + 1] + if WORD == 4: + value = allregisters[2*code] + value_hi = allregisters[2*code + 1] + else: + value = allregisters[code] else: value = allregisters[16 + code] @@ -1180,7 +1457,8 @@ tgt = self.fail_boxes_ptr.get_addr_for_num(num) elif kind == self.DESCR_FLOAT: tgt = self.fail_boxes_float.get_addr_for_num(num) - rffi.cast(rffi.LONGP, tgt)[1] = value_hi + if WORD == 4: + rffi.cast(rffi.LONGP, tgt)[1] = value_hi else: assert 0, "bogus kind" rffi.cast(rffi.LONGP, tgt)[0] = value @@ -1189,7 +1467,8 @@ if not we_are_translated(): assert bytecode[4] == 0xCC self.fail_boxes_count = num - fail_index = rffi.cast(rffi.LONGP, bytecode)[0] + fail_index = rffi.cast(rffi.INTP, bytecode)[0] + fail_index = rffi.cast(lltype.Signed, fail_index) return fail_index def setup_failure_recovery(self): @@ -1200,8 +1479,8 @@ # original value of the registers, optionally the original # value of XMM registers, and finally a reference to the # recovery bytecode. See _build_failure_recovery() for details. - stack_at_ebp = registers[ebp.op] - bytecode = rffi.cast(rffi.UCHARP, registers[8]) + stack_at_ebp = registers[ebp.value] + bytecode = rffi.cast(rffi.UCHARP, registers[self.cpu.NUM_REGS]) allregisters = rffi.ptradd(registers, -16) return self.grab_frame_values(bytecode, stack_at_ebp, allregisters) @@ -1216,23 +1495,23 @@ self.failure_recovery_func) failure_recovery_func = rffi.cast(lltype.Signed, failure_recovery_func) - mc = self.mc2._mc + mc = self.mc._mc # Assume that we are called at the beginning, when there is no risk # that 'mc' runs out of space. Checked by asserts in mc.write(). recovery_addr = mc.tell() - mc.PUSH(edi) - mc.PUSH(esi) - mc.PUSH(ebp) - mc.PUSH(esp) # <-- not really used, but needed to take up the space - mc.PUSH(ebx) - mc.PUSH(edx) - mc.PUSH(ecx) - mc.PUSH(eax) - mc.MOV(esi, esp) + + # Push all general purpose registers + for gpr in range(self.cpu.NUM_REGS-1, -1, -1): + mc.PUSH_r(gpr) + + # ebx/rbx is callee-save in both i386 and x86-64 + mc.MOV_rr(ebx.value, esp.value) + if withfloats: - mc.SUB(esp, imm(8*8)) - for i in range(8): - mc.MOVSD(mem64(esp, 8*i), xmm_registers[i]) + # Push all float registers + mc.SUB_ri(esp.value, self.cpu.NUM_REGS*8) + for i in range(self.cpu.NUM_REGS): + mc.MOVSD_sx(8*i, i) # we call a provided function that will # - call our on_leave_jitted_hook which will mark @@ -1240,7 +1519,7 @@ # avoid unwarranted freeing # - optionally save exception depending on the flag addr = self.cpu.get_on_leave_jitted_int(save_exception=exc) - mc.CALL(rel32(addr)) + mc.CALL(imm(addr)) # the following call saves all values from the stack and from # registers to the right 'fail_boxes_' location. @@ -1250,50 +1529,58 @@ # bytecode, pushed just before by the CALL instruction written by # generate_quick_failure(). XXX misaligned stack in the call, but # it's ok because failure_recovery_func is not calling anything more - mc.PUSH(esi) - mc.CALL(rel32(failure_recovery_func)) + + # XXX + if IS_X86_32: + mc.PUSH_r(ebx.value) + elif IS_X86_64: + mc.MOV_rr(edi.value, ebx.value) + # XXX: Correct to only align the stack on 64-bit? + mc.AND_ri(esp.value, -16) + else: + raise AssertionError("Shouldn't happen") + + mc.CALL(imm(failure_recovery_func)) # returns in eax the fail_index # now we return from the complete frame, which starts from - # _assemble_bootstrap_code(). The LEA below throws away most - # of the frame, including all the PUSHes that we did just above. - mc.LEA(esp, addr_add(ebp, imm(-3 * WORD))) - mc.POP(edi) # [ebp-12] - mc.POP(esi) # [ebp-8] - mc.POP(ebx) # [ebp-4] - mc.POP(ebp) # [ebp] - mc.RET() - self.mc2.done() + # _assemble_bootstrap_code(). The LEA in _call_footer below throws + # away most of the frame, including all the PUSHes that we did just + # above. + + self._call_footer() + self.mc.done() self.failure_recovery_code[exc + 2 * withfloats] = recovery_addr def generate_failure(self, fail_index, locs, exc, locs_are_ref): - mc = self.mc + self.mc._mc.begin_reuse_scratch_register() for i in range(len(locs)): loc = locs[i] - if isinstance(loc, REG): - if loc.width == 8: + if isinstance(loc, RegLoc): + if loc.is_xmm: adr = self.fail_boxes_float.get_addr_for_num(i) - mc.MOVSD(heap64(adr), loc) + self.mc.MOVSD(heap(adr), loc) else: if locs_are_ref[i]: adr = self.fail_boxes_ptr.get_addr_for_num(i) else: adr = self.fail_boxes_int.get_addr_for_num(i) - mc.MOV(heap(adr), loc) + self.mc.MOV(heap(adr), loc) for i in range(len(locs)): loc = locs[i] - if not isinstance(loc, REG): - if loc.width == 8: - mc.MOVSD(xmm0, loc) + if not isinstance(loc, RegLoc): + if isinstance(loc, StackLoc) and loc.type == FLOAT: + self.mc.MOVSD_xb(xmm0.value, loc.value) adr = self.fail_boxes_float.get_addr_for_num(i) - mc.MOVSD(heap64(adr), xmm0) + self.mc.MOVSD(heap(adr), xmm0) else: if locs_are_ref[i]: adr = self.fail_boxes_ptr.get_addr_for_num(i) else: adr = self.fail_boxes_int.get_addr_for_num(i) - mc.MOV(eax, loc) - mc.MOV(heap(adr), eax) + self.mc.MOV(eax, loc) + self.mc.MOV(heap(adr), eax) + self.mc._mc.end_reuse_scratch_register() # we call a provided function that will # - call our on_leave_jitted_hook which will mark @@ -1301,28 +1588,31 @@ # avoid unwarranted freeing # - optionally save exception depending on the flag addr = self.cpu.get_on_leave_jitted_int(save_exception=exc) - mc.CALL(rel32(addr)) + self.mc.CALL(imm(addr)) + + self.mc.MOV_ri(eax.value, fail_index) + + # exit function + self._call_footer() - mc.LEA(esp, addr_add(ebp, imm(-3 * WORD))) - mc.MOV(eax, imm(fail_index)) - mc.POP(edi) # [ebp-12] - mc.POP(esi) # [ebp-8] - mc.POP(ebx) # [ebp-4] - mc.POP(ebp) # [ebp] - mc.RET() - - @specialize.arg(2) - def implement_guard(self, addr, emit_jump): - emit_jump(rel32(addr)) + def implement_guard(self, guard_token, condition=None): + self.mc.reserve_bytes(guard_token.recovery_stub_size()) + self.pending_guard_tokens.append(guard_token) + # XXX: These jumps are patched later, the self.mc.tell() are just + # dummy values + if condition: + self.mc.J_il(rx86.Conditions[condition], self.mc.tell()) + else: + self.mc.JMP_l(self.mc.tell()) return self.mc.tell() - 4 def genop_call(self, op, arglocs, resloc): sizeloc = arglocs[0] - assert isinstance(sizeloc, IMM32) + assert isinstance(sizeloc, ImmedLoc) size = sizeloc.value if isinstance(op.args[0], Const): - x = rel32(op.args[0].getint()) + x = imm(op.args[0].getint()) else: x = arglocs[1] if x is eax: @@ -1332,35 +1622,35 @@ self._emit_call(x, arglocs, 2, tmp=tmp) - if isinstance(resloc, MODRM64): - self.mc.FSTP(resloc) + if isinstance(resloc, StackLoc) and resloc.width == 8 and IS_X86_32: + self.mc.FSTP_b(resloc.value) elif size == 1: - self.mc.AND(eax, imm(0xff)) + self.mc.AND_ri(eax.value, 0xff) elif size == 2: - self.mc.AND(eax, imm(0xffff)) + self.mc.AND_ri(eax.value, 0xffff) - def genop_guard_call_may_force(self, op, guard_op, addr, + def genop_guard_call_may_force(self, op, guard_op, guard_token, arglocs, result_loc): faildescr = guard_op.descr fail_index = self.cpu.get_fail_descr_number(faildescr) - self.mc.MOV(mem(ebp, FORCE_INDEX_OFS), imm(fail_index)) + self.mc.MOV_bi(FORCE_INDEX_OFS, fail_index) self.genop_call(op, arglocs, result_loc) - self.mc.CMP(mem(ebp, FORCE_INDEX_OFS), imm(0)) - return self.implement_guard(addr, self.mc.JL) + self.mc.CMP_bi(FORCE_INDEX_OFS, 0) + return self.implement_guard(guard_token, 'L') - def genop_guard_call_assembler(self, op, guard_op, addr, + def genop_guard_call_assembler(self, op, guard_op, guard_token, arglocs, result_loc): faildescr = guard_op.descr fail_index = self.cpu.get_fail_descr_number(faildescr) - self.mc.MOV(mem(ebp, FORCE_INDEX_OFS), imm(fail_index)) + self.mc.MOV_bi(FORCE_INDEX_OFS, fail_index) descr = op.descr assert isinstance(descr, LoopToken) assert len(arglocs) - 2 == len(descr._x86_arglocs[0]) # # Write a call to the direct_bootstrap_code of the target assembler - self._emit_call(rel32(descr._x86_direct_bootstrap_code), arglocs, 2, + self._emit_call(imm(descr._x86_direct_bootstrap_code), arglocs, 2, tmp=eax) - mc = self._start_block() + self.mc.ensure_bytes_available(256) if op.result is None: assert result_loc is None value = self.cpu.done_with_this_frame_void_v @@ -1376,26 +1666,27 @@ value = self.cpu.done_with_this_frame_float_v else: raise AssertionError(kind) - mc.CMP(eax, imm(value)) - mc.JE(rel8_patched_later) # goto B if we get 'done_with_this_frame' - je_location = mc.get_relative_pos() + self.mc.CMP_ri(eax.value, value) + # patched later + self.mc.J_il8(rx86.Conditions['E'], 0) # goto B if we get 'done_with_this_frame' + je_location = self.mc.get_relative_pos() # # Path A: use assembler_helper_adr jd = descr.outermost_jitdriver_sd assert jd is not None asm_helper_adr = self.cpu.cast_adr_to_int(jd.assembler_helper_adr) - self._emit_call(rel32(asm_helper_adr), [eax, arglocs[1]], 0, - tmp=ecx, force_mc=True, mc=mc) - if isinstance(result_loc, MODRM64): - mc.FSTP(result_loc) + self._emit_call(imm(asm_helper_adr), [eax, arglocs[1]], 0, + tmp=ecx) + if IS_X86_32 and isinstance(result_loc, StackLoc) and result_loc.type == FLOAT: + self.mc.FSTP_b(result_loc.value) #else: result_loc is already either eax or None, checked below - mc.JMP(rel8_patched_later) # done - jmp_location = mc.get_relative_pos() + self.mc.JMP_l8(0) # jump to done, patched later + jmp_location = self.mc.get_relative_pos() # # Path B: fast path. Must load the return value, and reset the token offset = jmp_location - je_location assert 0 < offset <= 127 - mc.overwrite(je_location - 1, [chr(offset)]) + self.mc.overwrite(je_location - 1, [chr(offset)]) # # Reset the vable token --- XXX really too much special logic here:-( if jd.index_of_virtualizable >= 0: @@ -1403,8 +1694,8 @@ fielddescr = jd.vable_token_descr assert isinstance(fielddescr, BaseFieldDescr) ofs = fielddescr.offset - mc.MOV(eax, arglocs[1]) - mc.MOV(addr_add(eax, imm(ofs)), imm(0)) + self.mc.MOV(eax, arglocs[1]) + self.mc.MOV_mi((eax.value, ofs), 0) # in the line above, TOKEN_NONE = 0 # if op.result is not None: @@ -1413,27 +1704,26 @@ if kind == FLOAT: xmmtmp = X86XMMRegisterManager.all_regs[0] adr = self.fail_boxes_float.get_addr_for_num(0) - mc.MOVSD(xmmtmp, heap64(adr)) - mc.MOVSD(result_loc, xmmtmp) + self.mc.MOVSD(xmmtmp, heap(adr)) + self.mc.MOVSD(result_loc, xmmtmp) else: assert result_loc is eax if kind == INT: adr = self.fail_boxes_int.get_addr_for_num(0) - mc.MOV(eax, heap(adr)) + self.mc.MOV(eax, heap(adr)) elif kind == REF: adr = self.fail_boxes_ptr.get_addr_for_num(0) - mc.XOR(eax, eax) - mc.XCHG(eax, heap(adr)) + self.mc.XOR_rr(eax.value, eax.value) + self.mc.XCHG(eax, heap(adr)) else: raise AssertionError(kind) # # Here we join Path A and Path B again - offset = mc.get_relative_pos() - jmp_location + offset = self.mc.get_relative_pos() - jmp_location assert 0 <= offset <= 127 - mc.overwrite(jmp_location - 1, [chr(offset)]) - self._stop_block() - self.mc.CMP(mem(ebp, FORCE_INDEX_OFS), imm(0)) - return self.implement_guard(addr, self.mc.JL) + self.mc.overwrite(jmp_location - 1, [chr(offset)]) + self.mc.CMP_bi(FORCE_INDEX_OFS, 0) + return self.implement_guard(guard_token, 'L') def genop_discard_cond_call_gc_wb(self, op, arglocs): # use 'mc._mc' directly instead of 'mc', to avoid @@ -1443,31 +1733,51 @@ cls = self.cpu.gc_ll_descr.has_write_barrier_class() assert cls is not None and isinstance(descr, cls) loc_base = arglocs[0] - mc = self._start_block() - mc.TEST(mem8(loc_base, descr.jit_wb_if_flag_byteofs), - imm8(descr.jit_wb_if_flag_singlebyte)) - mc.JZ(rel8_patched_later) - jz_location = mc.get_relative_pos() + self.mc.ensure_bytes_available(256) + self.mc.TEST8_mi((loc_base.value, descr.jit_wb_if_flag_byteofs), + descr.jit_wb_if_flag_singlebyte) + self.mc.J_il8(rx86.Conditions['Z'], 0) # patched later + jz_location = self.mc.get_relative_pos() # the following is supposed to be the slow path, so whenever possible # we choose the most compact encoding over the most efficient one. for i in range(len(arglocs)-1, -1, -1): - mc.PUSH(arglocs[i]) + loc = arglocs[i] + if isinstance(loc, RegLoc): + self.mc.PUSH_r(loc.value) + else: + if IS_X86_64: + self.mc.MOV_ri(X86_64_SCRATCH_REG.value, loc.getint()) + self.mc.PUSH_r(X86_64_SCRATCH_REG.value) + else: + self.mc.PUSH_i32(loc.getint()) + + if IS_X86_64: + # We clobber these registers to pass the arguments, but that's + # okay, because consider_cond_call_gc_wb makes sure that any + # caller-save registers with values in them are present in arglocs, + # so they are saved on the stack above and restored below + self.mc.MOV_rs(edi.value, 0) + self.mc.MOV_rs(esi.value, 8) + # misaligned stack in the call, but it's ok because the write barrier # is not going to call anything more. Also, this assumes that the # write barrier does not touch the xmm registers. - mc.CALL(rel32(descr.get_write_barrier_fn(self.cpu))) + self.mc.CALL(imm(descr.get_write_barrier_fn(self.cpu))) for i in range(len(arglocs)): loc = arglocs[i] - assert isinstance(loc, REG) - mc.POP(loc) + if isinstance(loc, RegLoc): + self.mc.POP_r(loc.value) + else: + self.mc.ADD_ri(esp.value, WORD) # ignore the pushed constant # patch the JZ above - offset = mc.get_relative_pos() - jz_location + offset = self.mc.get_relative_pos() - jz_location assert 0 < offset <= 127 - mc.overwrite(jz_location-1, [chr(offset)]) - self._stop_block() + self.mc.overwrite(jz_location-1, [chr(offset)]) def genop_force_token(self, op, arglocs, resloc): - self.mc.LEA(resloc, mem(ebp, FORCE_INDEX_OFS)) + # RegAlloc.consider_force_token ensures this: + assert isinstance(resloc, RegLoc) + self.mc.LEA_rb(resloc.value, FORCE_INDEX_OFS) def not_implemented_op_discard(self, op, arglocs): msg = "not implemented operation: %s" % op.getopname() @@ -1495,17 +1805,16 @@ return loop_token._x86_arglocs def closing_jump(self, loop_token): - self.mc.JMP(rel32(loop_token._x86_loop_code)) + self.mc.JMP(imm(loop_token._x86_loop_code)) def malloc_cond_fixedsize(self, nursery_free_adr, nursery_top_adr, size, tid): - # don't use self.mc - mc = self._start_block() - mc.MOV(eax, heap(nursery_free_adr)) - mc.LEA(edx, addr_add(eax, imm(size))) - mc.CMP(edx, heap(nursery_top_adr)) - mc.JNA(rel8_patched_later) - jmp_adr = mc.get_relative_pos() + self.mc.ensure_bytes_available(256) + self.mc.MOV(eax, heap(nursery_free_adr)) + self.mc.LEA_rm(edx.value, (eax.value, size)) + self.mc.CMP(edx, heap(nursery_top_adr)) + self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later + jmp_adr = self.mc.get_relative_pos() # See comments in _build_malloc_fixedsize_slowpath for the # details of the two helper functions that we are calling below. @@ -1521,17 +1830,16 @@ # reserve room for the argument to the real malloc and the # 8 saved XMM regs self._regalloc.reserve_param(1+16) - mc.CALL(rel32(slowpath_addr1)) + self.mc.CALL(imm(slowpath_addr1)) self.mark_gc_roots() slowpath_addr2 = self.malloc_fixedsize_slowpath2 - mc.CALL(rel32(slowpath_addr2)) + self.mc.CALL(imm(slowpath_addr2)) - offset = mc.get_relative_pos() - jmp_adr + offset = self.mc.get_relative_pos() - jmp_adr assert 0 < offset <= 127 - mc.overwrite(jmp_adr-1, [chr(offset)]) - mc.MOV(addr_add(eax, imm(0)), imm(tid)) - mc.MOV(heap(nursery_free_adr), edx) - self._stop_block() + self.mc.overwrite(jmp_adr-1, [chr(offset)]) + self.mc.MOV_mi((eax.value, 0), tid) + self.mc.MOV(heap(nursery_free_adr), edx) genop_discard_list = [Assembler386.not_implemented_op_discard] * rop._LAST genop_list = [Assembler386.not_implemented_op] * rop._LAST @@ -1551,32 +1859,20 @@ num = getattr(rop, opname.upper()) genop_list[num] = value -def new_addr_add(heap, mem, memsib): - def addr_add(reg_or_imm1, reg_or_imm2, offset=0, scale=0): - if isinstance(reg_or_imm1, IMM32): - if isinstance(reg_or_imm2, IMM32): - return heap(reg_or_imm1.value + offset + - (reg_or_imm2.value << scale)) - else: - return memsib(None, reg_or_imm2, scale, reg_or_imm1.value + offset) - else: - if isinstance(reg_or_imm2, IMM32): - return mem(reg_or_imm1, offset + (reg_or_imm2.value << scale)) - else: - return memsib(reg_or_imm1, reg_or_imm2, scale, offset) - return addr_add - -addr8_add = new_addr_add(heap8, mem8, memSIB8) -addr_add = new_addr_add(heap, mem, memSIB) -addr64_add = new_addr_add(heap64, mem64, memSIB64) - -def addr_add_const(reg_or_imm1, offset): - if isinstance(reg_or_imm1, IMM32): - return heap(reg_or_imm1.value + offset) - else: - return mem(reg_or_imm1, offset) - def round_up_to_4(size): if size < 4: return 4 return size + +# XXX: ri386 migration shims: +def addr_add(reg_or_imm1, reg_or_imm2, offset=0, scale=0): + return AddressLoc(reg_or_imm1, reg_or_imm2, scale, offset) + +def addr_add_const(reg_or_imm1, offset): + return AddressLoc(reg_or_imm1, ImmedLoc(0), 0, offset) + +def mem(loc, offset): + return AddressLoc(loc, ImmedLoc(0), 0, offset) + +def heap(addr): + return AddressLoc(ImmedLoc(addr), ImmedLoc(0), 0, 0) Modified: pypy/branch/kill-caninline/pypy/jit/backend/x86/codebuf.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/backend/x86/codebuf.py (original) +++ pypy/branch/kill-caninline/pypy/jit/backend/x86/codebuf.py Thu Aug 12 13:47:36 2010 @@ -2,12 +2,20 @@ import os, sys from pypy.rpython.lltypesystem import lltype, rffi from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.jit.backend.x86.ri386 import I386CodeBuilder +from pypy.jit.backend.x86.rx86 import X86_32_CodeBuilder, X86_64_CodeBuilder +from pypy.jit.backend.x86.regloc import LocationCodeBuilder from pypy.rlib.rmmap import PTR, alloc, free from pypy.rlib.debug import make_sure_not_resized +from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 +from pypy.rlib.objectmodel import we_are_translated +# XXX: Seems nasty to change the superclass of InMemoryCodeBuilder like this +if IS_X86_32: + codebuilder_cls = X86_32_CodeBuilder +elif IS_X86_64: + codebuilder_cls = X86_64_CodeBuilder -class InMemoryCodeBuilder(I386CodeBuilder): +class InMemoryCodeBuilder(codebuilder_cls, LocationCodeBuilder): _last_dump_start = 0 def __init__(self, start, end): @@ -31,13 +39,15 @@ def write(self, listofchars): self._pos = self.overwrite(self._pos, listofchars) - def writechr(self, n): - # purely for performance: don't make the one-element list [chr(n)] + def writechar(self, char): pos = self._pos assert pos + 1 <= self._size - self._data[pos] = chr(n) + self._data[pos] = char self._pos = pos + 1 + def writechr(self, n): + self.writechar(chr(n)) + def get_relative_pos(self): return self._pos @@ -50,11 +60,6 @@ self._pos = pos self._last_dump_start = pos - def execute(self, arg1, arg2): - # XXX old testing stuff - fnptr = rffi.cast(lltype.Ptr(BINARYFN), self._data) - return fnptr(arg1, arg2) - def done(self): # normally, no special action is needed here if machine_code_dumper.enabled: @@ -77,9 +82,6 @@ valgrind.discard_translations(self._data, self._size) -BINARYFN = lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed) - - class MachineCodeDumper: enabled = True log_fd = -1 @@ -107,7 +109,10 @@ return False # log the executable name from pypy.jit.backend.hlinfo import highleveljitinfo - os.write(self.log_fd, 'BACKEND i386\n') + if IS_X86_32: + os.write(self.log_fd, 'BACKEND x86\n') + elif IS_X86_64: + os.write(self.log_fd, 'BACKEND x86_64\n') if highleveljitinfo.sys_executable: os.write(self.log_fd, 'SYS_EXECUTABLE %s\n' % ( highleveljitinfo.sys_executable,)) @@ -137,6 +142,12 @@ def __init__(self, map_size): data = alloc(map_size) + if IS_X86_64 and not we_are_translated(): + # Hack to make sure that mcs are not within 32-bits of one + # another for testing purposes + from pypy.rlib.rmmap import hint + hint.pos += 0xFFFFFFFF + self._init(data, map_size) def __del__(self): Modified: pypy/branch/kill-caninline/pypy/jit/backend/x86/jump.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/backend/x86/jump.py (original) +++ pypy/branch/kill-caninline/pypy/jit/backend/x86/jump.py Thu Aug 12 13:47:36 2010 @@ -1,23 +1,6 @@ import sys from pypy.tool.pairtype import extendabletype -from pypy.jit.backend.x86.ri386 import * - -class __extend__(OPERAND): - __metaclass__ = extendabletype - def _getregkey(self): - raise AssertionError("should only happen to registers and frame " - "positions") - -class __extend__(REG): - __metaclass__ = extendabletype - def _getregkey(self): - return ~self.op - -class __extend__(MODRM): - __metaclass__ = extendabletype - def _getregkey(self): - return self.position - +from pypy.jit.backend.x86.regloc import ImmedLoc, StackLoc def remap_frame_layout(assembler, src_locations, dst_locations, tmpreg): pending_dests = len(dst_locations) @@ -27,7 +10,7 @@ srccount[dst._getregkey()] = 0 for i in range(len(dst_locations)): src = src_locations[i] - if isinstance(src, IMM32): + if isinstance(src, ImmedLoc): continue key = src._getregkey() if key in srccount: @@ -46,7 +29,7 @@ srccount[key] = -1 # means "it's done" pending_dests -= 1 src = src_locations[i] - if not isinstance(src, IMM32): + if not isinstance(src, ImmedLoc): key = src._getregkey() if key in srccount: srccount[key] -= 1 @@ -80,7 +63,7 @@ assert pending_dests == 0 def _move(assembler, src, dst, tmpreg): - if isinstance(dst, MODRM) and isinstance(src, MODRM): + if dst.is_memory_reference() and src.is_memory_reference(): assembler.regalloc_mov(src, tmpreg) src = tmpreg assembler.regalloc_mov(src, dst) Modified: pypy/branch/kill-caninline/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/kill-caninline/pypy/jit/backend/x86/regalloc.py Thu Aug 12 13:47:36 2010 @@ -5,7 +5,7 @@ from pypy.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr, ResOperation, BoxPtr, LoopToken, INT, REF, FLOAT) -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.regloc import * from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi, rstr from pypy.rlib.objectmodel import we_are_translated from pypy.rlib import rgc @@ -17,16 +17,7 @@ from pypy.jit.backend.llsupport.descr import BaseCallDescr, BaseSizeDescr from pypy.jit.backend.llsupport.regalloc import FrameManager, RegisterManager,\ TempBox - -WORD = 4 -FRAME_FIXED_SIZE = 5 # ebp + ebx + esi + edi + force_index = 5 words -FORCE_INDEX_OFS = -4*WORD - -width_of_type = { - INT : 1, - REF : 1, - FLOAT : 2, - } +from pypy.jit.backend.x86.arch import WORD, FRAME_FIXED_SIZE, IS_X86_32, IS_X86_64 class X86RegisterManager(RegisterManager): @@ -50,12 +41,19 @@ print "convert_to_imm: got a %s" % c raise AssertionError +class X86_64_RegisterManager(X86RegisterManager): + # r11 omitted because it's used as scratch + all_regs = [eax, ecx, edx, ebx, esi, edi, r8, r9, r10, r12, r13, r14, r15] + no_lower_byte_regs = [] + save_around_call_regs = [eax, ecx, edx, esi, edi, r8, r9, r10] + class FloatConstants(object): BASE_CONSTANT_SIZE = 1000 def __init__(self): self.cur_array_free = 0 + self.const_id = 0 def _get_new_array(self): n = self.BASE_CONSTANT_SIZE @@ -71,7 +69,8 @@ n = self.cur_array_free - 1 arr[n] = floatval self.cur_array_free = n - return rffi.cast(lltype.Signed, arr) + n * 8 + self.const_id += 1 + return (self.const_id, rffi.cast(lltype.Signed, arr) + n * 8) class X86XMMRegisterManager(RegisterManager): @@ -80,7 +79,6 @@ all_regs = [xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7] # we never need lower byte I hope save_around_call_regs = all_regs - reg_width = 2 def __init__(self, longevity, frame_manager=None, assembler=None): RegisterManager.__init__(self, longevity, frame_manager=frame_manager, @@ -93,28 +91,36 @@ self.float_constants = assembler._float_constants def convert_to_imm(self, c): - adr = self.float_constants.record_float(c.getfloat()) - return heap64(adr) + const_id, adr = self.float_constants.record_float(c.getfloat()) + return ConstFloatLoc(adr, const_id) def after_call(self, v): # the result is stored in st0, but we don't have this around, # so genop_call will move it to some frame location immediately # after the call - return self.frame_manager.loc(v, 2) + return self.frame_manager.loc(v) -class X86FrameManager(FrameManager): +class X86_64_XMMRegisterManager(X86XMMRegisterManager): + # xmm15 reserved for scratch use + all_regs = [xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14] + save_around_call_regs = all_regs + + def call_result_location(self, v): + return xmm0 + + def after_call(self, v): + # We use RegisterManager's implementation, since X86XMMRegisterManager + # places the result on the stack, which we don't need to do when the + # calling convention places the result in xmm0 + return RegisterManager.after_call(self, v) +class X86FrameManager(FrameManager): @staticmethod - def frame_pos(i, size): - if size == 1: - res = mem(ebp, get_ebp_ofs(i)) - elif size == 2: - res = mem64(ebp, get_ebp_ofs(i + 1)) - else: - print "Unimplemented size %d" % i - raise NotImplementedError("unimplemented size %d" % i) - res.position = i - return res + def frame_pos(i, box_type): + if IS_X86_32 and box_type == FLOAT: + return StackLoc(i, get_ebp_ofs(i+1), 2, box_type) + else: + return StackLoc(i, get_ebp_ofs(i), 1, box_type) class RegAlloc(object): exc = False @@ -135,11 +141,21 @@ # compute longevity of variables longevity = self._compute_vars_longevity(inputargs, operations) self.longevity = longevity - self.rm = X86RegisterManager(longevity, - frame_manager = self.fm, - assembler = self.assembler) - self.xrm = X86XMMRegisterManager(longevity, frame_manager = self.fm, - assembler = self.assembler) + # XXX + if cpu.WORD == 4: + gpr_reg_mgr_cls = X86RegisterManager + xmm_reg_mgr_cls = X86XMMRegisterManager + elif cpu.WORD == 8: + gpr_reg_mgr_cls = X86_64_RegisterManager + xmm_reg_mgr_cls = X86_64_XMMRegisterManager + else: + raise AssertionError("Word size should be 4 or 8") + + self.rm = gpr_reg_mgr_cls(longevity, + frame_manager = self.fm, + assembler = self.assembler) + self.xrm = xmm_reg_mgr_cls(longevity, frame_manager = self.fm, + assembler = self.assembler) def prepare_loop(self, inputargs, operations, looptoken): self._prepare(inputargs, operations) @@ -184,7 +200,7 @@ if reg: loc = reg else: - loc = self.fm.loc(arg, width_of_type[arg.type]) + loc = self.fm.loc(arg) if arg.type == FLOAT: floatlocs[i] = loc else: @@ -252,23 +268,23 @@ arg = inputargs[i] i += 1 if arg.type == FLOAT: - if isinstance(loc, REG): + if isinstance(loc, RegLoc): self.xrm.reg_bindings[arg] = loc used[loc] = None else: self.fm.frame_bindings[arg] = loc else: - if isinstance(loc, REG): + if isinstance(loc, RegLoc): self.rm.reg_bindings[arg] = loc used[loc] = None else: self.fm.frame_bindings[arg] = loc self.rm.free_regs = [] - for reg in X86RegisterManager.all_regs: + for reg in self.rm.all_regs: if reg not in used: self.rm.free_regs.append(reg) self.xrm.free_regs = [] - for reg in X86XMMRegisterManager.all_regs: + for reg in self.xrm.all_regs: if reg not in used: self.xrm.free_regs.append(reg) # note: we need to make a copy of inputargs because possibly_free_vars @@ -646,7 +662,7 @@ vable_index = jd.index_of_virtualizable if vable_index >= 0: self.rm._sync_var(op.args[vable_index]) - vable = self.fm.loc(op.args[vable_index], 1) + vable = self.fm.loc(op.args[vable_index]) else: vable = imm(0) self._call(op, [imm(size), vable] + @@ -655,13 +671,12 @@ def consider_cond_call_gc_wb(self, op): assert op.result is None + loc_newvalue = self.rm.make_sure_var_in_reg(op.args[1], op.args) + # ^^^ we force loc_newvalue in a reg (unless it's a Const), + # because it will be needed anyway by the following setfield_gc. + # It avoids loading it twice from the memory. loc_base = self.rm.make_sure_var_in_reg(op.args[0], op.args, imm_fine=False) - loc_newvalue = self.rm.make_sure_var_in_reg(op.args[1], op.args, - imm_fine=False) - # ^^^ we also force loc_newvalue in a reg, because it will be needed - # anyway by the following setfield_gc. It avoids loading it twice - # from the memory. arglocs = [loc_base, loc_newvalue] # add eax, ecx and edx as extra "arguments" to ensure they are # saved and restored. Fish in self.rm to know which of these @@ -670,7 +685,7 @@ # function, a GC write barrier, is known not to touch them. # See remember_young_pointer() in rpython/memory/gc/generation.py. for v, reg in self.rm.reg_bindings.items(): - if ((reg is eax or reg is ecx or reg is edx) + if (reg in self.rm.save_around_call_regs and self.rm.stays_alive(v)): arglocs.append(reg) self.PerformDiscard(op, arglocs) @@ -809,7 +824,7 @@ def consider_setfield_gc(self, op): ofs_loc, size_loc, ptr = self._unpack_fielddescr(op.descr) - assert isinstance(size_loc, IMM32) + assert isinstance(size_loc, ImmedLoc) if size_loc.value == 1: need_lower_byte = True else: @@ -950,7 +965,7 @@ shape = gcrootmap.get_basic_shape() for v, val in self.fm.frame_bindings.items(): if (isinstance(v, BoxPtr) and self.rm.stays_alive(v)): - assert isinstance(val, MODRM) + assert isinstance(val, StackLoc) gcrootmap.add_ebp_offset(shape, get_ebp_ofs(val.position)) for v, reg in self.rm.reg_bindings.items(): if reg is eax: Modified: pypy/branch/kill-caninline/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/kill-caninline/pypy/jit/backend/x86/runner.py Thu Aug 12 13:47:36 2010 @@ -4,11 +4,13 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.jit.metainterp import history, compile from pypy.jit.backend.x86.assembler import Assembler386 -from pypy.jit.backend.x86.regalloc import FORCE_INDEX_OFS +from pypy.jit.backend.x86.arch import FORCE_INDEX_OFS from pypy.jit.backend.x86.profagent import ProfileAgent from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU +from pypy.jit.backend.x86 import regloc +import sys -class CPU386(AbstractLLCPU): +class AbstractX86CPU(AbstractLLCPU): debug = True supports_floats = True @@ -44,6 +46,7 @@ self.profile_agent.startup() def finish_once(self): + self.assembler.finish_once() self.profile_agent.shutdown() def compile_loop(self, inputargs, operations, looptoken): @@ -131,10 +134,28 @@ assert fail_index == fail_index_2 return faildescr +class CPU386(AbstractX86CPU): + WORD = 4 + NUM_REGS = 8 + CALLEE_SAVE_REGISTERS = [regloc.ebx, regloc.esi, regloc.edi] + FRAME_FIXED_SIZE = len(CALLEE_SAVE_REGISTERS) + 2 + + def __init__(self, *args, **kwargs): + assert sys.maxint == (2**31 - 1) + super(CPU386, self).__init__(*args, **kwargs) class CPU386_NO_SSE2(CPU386): supports_floats = False +class CPU_X86_64(AbstractX86CPU): + WORD = 8 + NUM_REGS = 16 + CALLEE_SAVE_REGISTERS = [regloc.ebx, regloc.r12, regloc.r13, regloc.r14, regloc.r15] + FRAME_FIXED_SIZE = len(CALLEE_SAVE_REGISTERS) + 2 + + def __init__(self, *args, **kwargs): + assert sys.maxint == (2**63 - 1) + super(CPU_X86_64, self).__init__(*args, **kwargs) CPU = CPU386 Modified: pypy/branch/kill-caninline/pypy/jit/backend/x86/test/conftest.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/backend/x86/test/conftest.py (original) +++ pypy/branch/kill-caninline/pypy/jit/backend/x86/test/conftest.py Thu Aug 12 13:47:36 2010 @@ -3,6 +3,5 @@ cpu = detect_cpu.autodetect() def pytest_runtest_setup(item): - if cpu != 'x86': - py.test.skip("x86 directory skipped: cpu is %r" % (cpu,)) - + if cpu not in ('x86', 'x86_64'): + py.test.skip("x86/x86_64 tests skipped: cpu is %r" % (cpu,)) Modified: pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_assembler.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_assembler.py (original) +++ pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_assembler.py Thu Aug 12 13:47:36 2010 @@ -1,14 +1,22 @@ -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.regloc import * from pypy.jit.backend.x86.assembler import Assembler386, MachineCodeBlockWrapper from pypy.jit.backend.x86.regalloc import X86FrameManager, get_ebp_ofs -from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat +from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, INT, REF, FLOAT from pypy.rlib.rarithmetic import intmask from pypy.rpython.lltypesystem import lltype, llmemory, rffi +from pypy.jit.backend.x86.arch import WORD, IS_X86_32, IS_X86_64 +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.regalloc import X86RegisterManager, X86_64_RegisterManager, X86XMMRegisterManager, X86_64_XMMRegisterManager +ACTUAL_CPU = getcpuclass() class FakeCPU: rtyper = None supports_floats = True + NUM_REGS = ACTUAL_CPU.NUM_REGS + + def fielddescrof(self, STRUCT, name): + return 42 class FakeMC: def __init__(self, base_address=0): @@ -25,7 +33,14 @@ self.content.append(("JMP", args)) def done(self): pass + def PUSH_r(self, reg): + pass + def POP_r(self, reg): + pass +class FakeAssembler: + def write_pending_failure_recoveries(self): + pass def test_write_failure_recovery_description(): assembler = Assembler386(FakeCPU()) @@ -33,12 +48,12 @@ failargs = [BoxInt(), BoxPtr(), BoxFloat()] * 3 failargs.insert(6, None) failargs.insert(7, None) - locs = [X86FrameManager.frame_pos(0, 1), - X86FrameManager.frame_pos(1, 1), - X86FrameManager.frame_pos(10, 2), - X86FrameManager.frame_pos(100, 1), - X86FrameManager.frame_pos(101, 1), - X86FrameManager.frame_pos(110, 2), + locs = [X86FrameManager.frame_pos(0, INT), + X86FrameManager.frame_pos(1, REF), + X86FrameManager.frame_pos(10, FLOAT), + X86FrameManager.frame_pos(100, INT), + X86FrameManager.frame_pos(101, REF), + X86FrameManager.frame_pos(110, FLOAT), None, None, ebx, @@ -46,17 +61,17 @@ xmm2] assert len(failargs) == len(locs) assembler.write_failure_recovery_description(mc, failargs, locs) - nums = [Assembler386.DESCR_INT + 4*(8+0), - Assembler386.DESCR_REF + 4*(8+1), - Assembler386.DESCR_FLOAT + 4*(8+10), - Assembler386.DESCR_INT + 4*(8+100), - Assembler386.DESCR_REF + 4*(8+101), - Assembler386.DESCR_FLOAT + 4*(8+110), + nums = [Assembler386.DESCR_INT + 4*(16+0), + Assembler386.DESCR_REF + 4*(16+1), + Assembler386.DESCR_FLOAT + 4*(16+10), + Assembler386.DESCR_INT + 4*(16+100), + Assembler386.DESCR_REF + 4*(16+101), + Assembler386.DESCR_FLOAT + 4*(16+110), Assembler386.CODE_HOLE, Assembler386.CODE_HOLE, - Assembler386.DESCR_INT + 4*ebx.op, - Assembler386.DESCR_REF + 4*esi.op, - Assembler386.DESCR_FLOAT + 4*xmm2.op] + Assembler386.DESCR_INT + 4*ebx.value, + Assembler386.DESCR_REF + 4*esi.value, + Assembler386.DESCR_FLOAT + 4*xmm2.value] double_byte_nums = [] for num in nums[3:6]: double_byte_nums.append((num & 0x7F) | 0x80) @@ -94,6 +109,9 @@ return lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S)) def get_random_float(): + # Returns , , + # NB: on 64-bit, will be the entire float and + # will be random garbage from malloc! assert withfloats value = random.random() - 0.5 # make sure it fits into 64 bits @@ -101,9 +119,16 @@ rffi.cast(rffi.DOUBLEP, tmp)[0] = value return rffi.cast(rffi.DOUBLEP, tmp)[0], tmp[0], tmp[1] + if IS_X86_32: + main_registers = X86RegisterManager.all_regs + xmm_registers = X86XMMRegisterManager.all_regs + elif IS_X86_64: + main_registers = X86_64_RegisterManager.all_regs + xmm_registers = X86_64_XMMRegisterManager.all_regs + # memory locations: 26 integers, 26 pointers, 26 floats # main registers: half of them as signed and the other half as ptrs - # xmm registers: all floats, from xmm0 to xmm7 + # xmm registers: all floats, from xmm0 to xmm(7|15) # holes: 8 locations = [] baseloc = 4 @@ -117,18 +142,17 @@ content = ([('int', locations.pop()) for _ in range(26)] + [('ptr', locations.pop()) for _ in range(26)] + [(['int', 'ptr'][random.randrange(0, 2)], reg) - for reg in [eax, ecx, edx, ebx, esi, edi]]) + for reg in main_registers]) if withfloats: content += ([('float', locations.pop()) for _ in range(26)] + - [('float', reg) for reg in [xmm0, xmm1, xmm2, xmm3, - xmm4, xmm5, xmm6, xmm7]]) + [('float', reg) for reg in xmm_registers]) for i in range(8): content.append(('hole', None)) random.shuffle(content) # prepare the expected target arrays, the descr_bytecode, # the 'registers' and the 'stack' arrays according to 'content' - xmmregisters = lltype.malloc(rffi.LONGP.TO, 16+9, flavor='raw') + xmmregisters = lltype.malloc(rffi.LONGP.TO, 16+ACTUAL_CPU.NUM_REGS+1, flavor='raw') registers = rffi.ptradd(xmmregisters, 16) stacklen = baseloc + 10 stack = lltype.malloc(rffi.LONGP.TO, stacklen, flavor='raw') @@ -140,8 +164,8 @@ assert loc >= 0 ofs = get_ebp_ofs(loc) assert ofs < 0 - assert (ofs % 4) == 0 - stack[stacklen + ofs//4] = value + assert (ofs % WORD) == 0 + stack[stacklen + ofs//WORD] = value descr_bytecode = [] for i, (kind, loc) in enumerate(content): @@ -152,12 +176,18 @@ value, lo, hi = get_random_float() expected_floats[i] = value kind = Assembler386.DESCR_FLOAT - if isinstance(loc, REG): - xmmregisters[2*loc.op] = lo - xmmregisters[2*loc.op+1] = hi + if isinstance(loc, RegLoc): + if WORD == 4: + xmmregisters[2*loc.value] = lo + xmmregisters[2*loc.value+1] = hi + elif WORD == 8: + xmmregisters[loc.value] = lo else: - write_in_stack(loc, hi) - write_in_stack(loc+1, lo) + if WORD == 4: + write_in_stack(loc, hi) + write_in_stack(loc+1, lo) + elif WORD == 8: + write_in_stack(loc, lo) else: if kind == 'int': value = get_random_int() @@ -170,15 +200,15 @@ value = rffi.cast(rffi.LONG, value) else: assert 0, kind - if isinstance(loc, REG): - registers[loc.op] = value + if isinstance(loc, RegLoc): + registers[loc.value] = value else: write_in_stack(loc, value) - if isinstance(loc, REG): - num = kind + 4*loc.op + if isinstance(loc, RegLoc): + num = kind + 4*loc.value else: - num = kind + 4*(8+loc) + num = kind + Assembler386.CODE_FROMSTACK + (4*loc) while num >= 0x80: descr_bytecode.append((num & 0x7F) | 0x80) num >>= 7 @@ -195,8 +225,8 @@ for i in range(len(descr_bytecode)): assert 0 <= descr_bytecode[i] <= 255 descr_bytes[i] = rffi.cast(rffi.UCHAR, descr_bytecode[i]) - registers[8] = rffi.cast(rffi.LONG, descr_bytes) - registers[ebp.op] = rffi.cast(rffi.LONG, stack) + 4*stacklen + registers[ACTUAL_CPU.NUM_REGS] = rffi.cast(rffi.LONG, descr_bytes) + registers[ebp.value] = rffi.cast(rffi.LONG, stack) + WORD*stacklen # run! assembler = Assembler386(FakeCPU()) @@ -237,7 +267,8 @@ def test_mc_wrapper_profile_agent(): agent = FakeProfileAgent() - mc = FakeMCWrapper(100, agent) + assembler = FakeAssembler() + mc = FakeMCWrapper(assembler, 100, agent) mc.start_function("abc") mc.writechr("x") mc.writechr("x") Modified: pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_basic.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_basic.py (original) +++ pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_basic.py Thu Aug 12 13:47:36 2010 @@ -1,5 +1,5 @@ import py -from pypy.jit.backend.x86.runner import CPU386 +from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.metainterp.warmspot import ll_meta_interp from pypy.jit.metainterp.test import test_basic from pypy.jit.codewriter.policy import StopAtXPolicy @@ -7,7 +7,7 @@ class Jit386Mixin(test_basic.LLJitMixin): type_system = 'lltype' - CPUClass = CPU386 + CPUClass = getcpuclass() def check_jumps(self, maxcount): pass Modified: pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_gc_integration.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_gc_integration.py (original) +++ pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_gc_integration.py Thu Aug 12 13:47:36 2010 @@ -9,13 +9,13 @@ from pypy.jit.codewriter import heaptracker from pypy.jit.backend.llsupport.descr import GcCache from pypy.jit.backend.llsupport.gc import GcLLDescription -from pypy.jit.backend.x86.runner import CPU -from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, FRAME_FIXED_SIZE +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.regalloc import RegAlloc +from pypy.jit.backend.x86.arch import WORD, FRAME_FIXED_SIZE from pypy.jit.metainterp.test.oparser import parse from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.annlowlevel import llhelper from pypy.rpython.lltypesystem import rclass, rstr -from pypy.jit.backend.x86.ri386 import * from pypy.jit.backend.llsupport.gc import GcLLDescr_framework, GcRefList, GcPtrFieldDescr from pypy.jit.backend.x86.test.test_regalloc import MockAssembler @@ -23,6 +23,8 @@ from pypy.jit.backend.x86.regalloc import X86RegisterManager, X86FrameManager,\ X86XMMRegisterManager +CPU = getcpuclass() + class MockGcRootMap(object): def get_basic_shape(self): return ['shape'] @@ -51,7 +53,7 @@ def initialize(self): self.gcrefs = GcRefList() self.gcrefs.initialize() - self.single_gcref_descr = GcPtrFieldDescr(0) + self.single_gcref_descr = GcPtrFieldDescr('', 0) rewrite_assembler = GcLLDescr_framework.rewrite_assembler.im_func @@ -84,7 +86,7 @@ mark = regalloc.get_mark_gc_roots(cpu.gc_ll_descr.gcrootmap) assert mark[0] == 'compressed' base = -WORD * FRAME_FIXED_SIZE - expected = ['ebx', 'esi', 'edi', base, base-4, base-8] + expected = ['ebx', 'esi', 'edi', base, base-WORD, base-WORD*2] assert dict.fromkeys(mark[1:]) == dict.fromkeys(expected) class TestRegallocGcIntegration(BaseTestRegalloc): @@ -175,7 +177,7 @@ self.addrs[1] = self.addrs[0] + 64 # 64 bytes def malloc_slowpath(size): - assert size == 8 + assert size == WORD*2 nadr = rffi.cast(lltype.Signed, self.nursery) self.addrs[0] = nadr + size return nadr @@ -199,7 +201,7 @@ return rffi.cast(lltype.Signed, self.addrs) def get_nursery_top_addr(self): - return rffi.cast(lltype.Signed, self.addrs) + 4 + return rffi.cast(lltype.Signed, self.addrs) + WORD def get_malloc_fixedsize_slowpath_addr(self): fptr = llhelper(lltype.Ptr(self.MALLOC_SLOWPATH), self.malloc_slowpath) @@ -213,7 +215,7 @@ def setup_method(self, method): cpu = CPU(None, None) - cpu.vtable_offset = 4 + cpu.vtable_offset = WORD cpu.gc_ll_descr = GCDescrFastpathMalloc() NODE = lltype.Struct('node', ('tid', lltype.Signed), @@ -249,7 +251,7 @@ assert gc_ll_descr.nursery[0] == self.nodedescr.tid assert gc_ll_descr.nursery[1] == 42 nurs_adr = rffi.cast(lltype.Signed, gc_ll_descr.nursery) - assert gc_ll_descr.addrs[0] == nurs_adr + 8 + assert gc_ll_descr.addrs[0] == nurs_adr + (WORD*2) def test_malloc_slowpath(self): ops = ''' @@ -269,7 +271,7 @@ # this should call slow path once gc_ll_descr = self.cpu.gc_ll_descr nadr = rffi.cast(lltype.Signed, gc_ll_descr.nursery) - assert gc_ll_descr.addrs[0] == nadr + 8 + assert gc_ll_descr.addrs[0] == nadr + (WORD*2) def test_new_with_vtable(self): ops = ''' @@ -284,4 +286,4 @@ assert gc_ll_descr.nursery[0] == self.descrsize.tid assert gc_ll_descr.nursery[1] == self.vtable_int nurs_adr = rffi.cast(lltype.Signed, gc_ll_descr.nursery) - assert gc_ll_descr.addrs[0] == nurs_adr + 12 + assert gc_ll_descr.addrs[0] == nurs_adr + (WORD*3) Modified: pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_jump.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_jump.py (original) +++ pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_jump.py Thu Aug 12 13:47:36 2010 @@ -1,6 +1,7 @@ -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.regloc import * from pypy.jit.backend.x86.regalloc import X86FrameManager from pypy.jit.backend.x86.jump import remap_frame_layout +from pypy.jit.metainterp.history import INT frame_pos = X86FrameManager.frame_pos @@ -25,7 +26,7 @@ continue assert len(op1) == len(op2) for x, y in zip(op1, op2): - if isinstance(x, MODRM) and isinstance(y, MODRM): + if isinstance(x, StackLoc) and isinstance(y, MODRM): assert x.byte == y.byte assert x.extradata == y.extradata else: @@ -41,9 +42,9 @@ remap_frame_layout(assembler, [eax, ebx, ecx, edx, esi, edi], [eax, ebx, ecx, edx, esi, edi], '?') assert assembler.ops == [] - s8 = frame_pos(1, 1) - s12 = frame_pos(31, 1) - s20 = frame_pos(6, 1) + s8 = frame_pos(1, INT) + s12 = frame_pos(31, INT) + s20 = frame_pos(6, INT) remap_frame_layout(assembler, [eax, ebx, ecx, s20, s8, edx, s12, esi, edi], [eax, ebx, ecx, s20, s8, edx, s12, esi, edi], '?') @@ -58,10 +59,10 @@ def test_simple_framelocs(): assembler = MockAssembler() - s8 = frame_pos(0, 1) - s12 = frame_pos(13, 1) - s20 = frame_pos(20, 1) - s24 = frame_pos(221, 1) + s8 = frame_pos(0, INT) + s12 = frame_pos(13, INT) + s20 = frame_pos(20, INT) + s24 = frame_pos(221, INT) remap_frame_layout(assembler, [s8, eax, s12], [s20, s24, edi], edx) assert assembler.ops == [('mov', s8, edx), ('mov', edx, s20), @@ -70,10 +71,10 @@ def test_reordering(): assembler = MockAssembler() - s8 = frame_pos(8, 1) - s12 = frame_pos(12, 1) - s20 = frame_pos(19, 1) - s24 = frame_pos(1, 1) + s8 = frame_pos(8, INT) + s12 = frame_pos(12, INT) + s20 = frame_pos(19, INT) + s24 = frame_pos(1, INT) remap_frame_layout(assembler, [eax, s8, s20, ebx], [s8, ebx, eax, edi], '?') assert assembler.got([('mov', ebx, edi), @@ -83,10 +84,10 @@ def test_cycle(): assembler = MockAssembler() - s8 = frame_pos(8, 1) - s12 = frame_pos(12, 1) - s20 = frame_pos(19, 1) - s24 = frame_pos(1, 1) + s8 = frame_pos(8, INT) + s12 = frame_pos(12, INT) + s20 = frame_pos(19, INT) + s24 = frame_pos(1, INT) remap_frame_layout(assembler, [eax, s8, s20, ebx], [s8, ebx, eax, s20], '?') assert assembler.got([('push', s8), @@ -97,12 +98,12 @@ def test_cycle_2(): assembler = MockAssembler() - s8 = frame_pos(8, 1) - s12 = frame_pos(12, 1) - s20 = frame_pos(19, 1) - s24 = frame_pos(1, 1) - s2 = frame_pos(2, 1) - s3 = frame_pos(3, 1) + s8 = frame_pos(8, INT) + s12 = frame_pos(12, INT) + s20 = frame_pos(19, INT) + s24 = frame_pos(1, INT) + s2 = frame_pos(2, INT) + s3 = frame_pos(3, INT) remap_frame_layout(assembler, [eax, s8, edi, s20, eax, s20, s24, esi, s2, s3], [s8, s20, edi, eax, edx, s24, ebx, s12, s3, s2], @@ -127,14 +128,14 @@ remap_frame_layout(assembler, [c3], [eax], '?') assert assembler.ops == [('mov', c3, eax)] assembler = MockAssembler() - s12 = frame_pos(12, 1) + s12 = frame_pos(12, INT) remap_frame_layout(assembler, [c3], [s12], '?') assert assembler.ops == [('mov', c3, s12)] def test_constants_and_cycle(): assembler = MockAssembler() c3 = imm(3) - s12 = frame_pos(13, 1) + s12 = frame_pos(13, INT) remap_frame_layout(assembler, [ebx, c3, s12], [s12, eax, ebx], edi) assert assembler.ops == [('mov', c3, eax), Modified: pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_recompilation.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_recompilation.py (original) +++ pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_recompilation.py Thu Aug 12 13:47:36 2010 @@ -1,6 +1,5 @@ - -from pypy.jit.backend.x86.runner import CPU from pypy.jit.backend.x86.test.test_regalloc import BaseTestRegalloc +from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 class TestRecompilation(BaseTestRegalloc): def test_compile_bridge_not_deeper(self): @@ -51,7 +50,9 @@ descr = loop.operations[2].descr new = descr._x86_bridge_frame_depth assert descr._x86_bridge_param_depth == 0 - assert new > previous + # XXX: Maybe add enough ops to force stack on 64-bit as well? + if IS_X86_32: + assert new > previous self.cpu.set_future_value_int(0, 0) fail = self.run(loop) assert fail.identifier == 2 @@ -111,7 +112,9 @@ guard_op = loop.operations[5] loop_frame_depth = loop.token._x86_frame_depth assert loop.token._x86_param_depth == 0 - assert guard_op.descr._x86_bridge_frame_depth > loop_frame_depth + # XXX: Maybe add enough ops to force stack on 64-bit as well? + if IS_X86_32: + assert guard_op.descr._x86_bridge_frame_depth > loop_frame_depth assert guard_op.descr._x86_bridge_param_depth == 0 self.cpu.set_future_value_int(0, 0) self.cpu.set_future_value_int(1, 0) Modified: pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_regalloc.py Thu Aug 12 13:47:36 2010 @@ -7,15 +7,17 @@ BoxPtr, ConstPtr, LoopToken, BasicFailDescr from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.backend.llsupport.descr import GcCache -from pypy.jit.backend.x86.runner import CPU -from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, X86RegisterManager,\ +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.regalloc import RegAlloc, X86RegisterManager,\ FloatConstants +from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 from pypy.jit.metainterp.test.oparser import parse from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.annlowlevel import llhelper from pypy.rpython.lltypesystem import rclass, rstr -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.rx86 import * +CPU = getcpuclass() class MockGcDescr(GcCache): def get_funcptr_for_new(self): return 123 @@ -92,13 +94,20 @@ def f2(x, y): return x*y + def f10(*args): + assert len(args) == 10 + return sum(args) + F1PTR = lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Signed)) F2PTR = lltype.Ptr(lltype.FuncType([lltype.Signed]*2, lltype.Signed)) + F10PTR = lltype.Ptr(lltype.FuncType([lltype.Signed]*10, lltype.Signed)) f1ptr = llhelper(F1PTR, f1) f2ptr = llhelper(F2PTR, f2) + f10ptr = llhelper(F10PTR, f10) f1_calldescr = cpu.calldescrof(F1PTR.TO, F1PTR.TO.ARGS, F1PTR.TO.RESULT) f2_calldescr = cpu.calldescrof(F2PTR.TO, F2PTR.TO.ARGS, F2PTR.TO.RESULT) + f10_calldescr = cpu.calldescrof(F10PTR.TO, F10PTR.TO.ARGS, F10PTR.TO.RESULT) namespace = locals().copy() type_system = 'lltype' @@ -541,6 +550,12 @@ assert self.getints(9) == [0, 1, 1, 1, 1, 1, 1, 1, 1] class TestRegAllocCallAndStackDepth(BaseTestRegalloc): + def expected_param_depth(self, num_args): + # Assumes the arguments are all non-float + if IS_X86_32: + return num_args + elif IS_X86_64: + return max(num_args - 6, 0) def test_one_call(self): ops = ''' @@ -550,7 +565,7 @@ ''' loop = self.interpret(ops, [4, 7, 9, 9 ,9, 9, 9, 9, 9, 9, 9]) assert self.getints(11) == [5, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9] - assert loop.token._x86_param_depth == 1 + assert loop.token._x86_param_depth == self.expected_param_depth(1) def test_two_calls(self): ops = ''' @@ -561,8 +576,21 @@ ''' loop = self.interpret(ops, [4, 7, 9, 9 ,9, 9, 9, 9, 9, 9, 9]) assert self.getints(11) == [5*7, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9] - assert loop.token._x86_param_depth == 2 - + assert loop.token._x86_param_depth == self.expected_param_depth(2) + + def test_call_many_arguments(self): + # NB: The first and last arguments in the call are constants. This + # is primarily for x86-64, to ensure that loading a constant to an + # argument register or to the stack works correctly + ops = ''' + [i0, i1, i2, i3, i4, i5, i6, i7] + i8 = call(ConstClass(f10ptr), 1, i0, i1, i2, i3, i4, i5, i6, i7, 10, descr=f10_calldescr) + finish(i8) + ''' + loop = self.interpret(ops, [2, 3, 4, 5, 6, 7, 8, 9]) + assert self.getint(0) == 55 + assert loop.token._x86_param_depth == self.expected_param_depth(10) + def test_bridge_calls_1(self): ops = ''' [i0, i1] @@ -579,7 +607,7 @@ ''' bridge = self.attach_bridge(ops, loop, -2) - assert loop.operations[-2].descr._x86_bridge_param_depth == 2 + assert loop.operations[-2].descr._x86_bridge_param_depth == self.expected_param_depth(2) self.cpu.set_future_value_int(0, 4) self.cpu.set_future_value_int(1, 7) @@ -602,7 +630,7 @@ ''' bridge = self.attach_bridge(ops, loop, -2) - assert loop.operations[-2].descr._x86_bridge_param_depth == 2 + assert loop.operations[-2].descr._x86_bridge_param_depth == self.expected_param_depth(2) self.cpu.set_future_value_int(0, 4) self.cpu.set_future_value_int(1, 7) Modified: pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_regalloc2.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_regalloc2.py (original) +++ pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_regalloc2.py Thu Aug 12 13:47:36 2010 @@ -2,7 +2,9 @@ from pypy.jit.metainterp.history import ResOperation, BoxInt, ConstInt,\ BoxPtr, ConstPtr, BasicFailDescr, LoopToken from pypy.jit.metainterp.resoperation import rop -from pypy.jit.backend.x86.runner import CPU +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.arch import WORD +CPU = getcpuclass() def test_bug_rshift(): v1 = BoxInt() @@ -281,5 +283,8 @@ assert cpu.get_latest_value_int(16) == -57344 assert cpu.get_latest_value_int(17) == 1 assert cpu.get_latest_value_int(18) == -1 - assert cpu.get_latest_value_int(19) == -2147483648 + if WORD == 4: + assert cpu.get_latest_value_int(19) == -2147483648 + elif WORD == 8: + assert cpu.get_latest_value_int(19) == 19327352832 assert cpu.get_latest_value_int(20) == -49 Modified: pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_runner.py Thu Aug 12 13:47:36 2010 @@ -1,16 +1,22 @@ import py from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rstr, rclass +from pypy.rpython.annlowlevel import llhelper from pypy.jit.metainterp.history import ResOperation, LoopToken -from pypy.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstPtr, - Box, BasicFailDescr) -from pypy.jit.backend.x86.runner import CPU -from pypy.jit.backend.x86.regalloc import WORD +from pypy.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstFloat, + ConstPtr, Box, BoxFloat, BasicFailDescr) +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.arch import WORD from pypy.jit.backend.llsupport import symbolic from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.executor import execute from pypy.jit.backend.test.runner_test import LLtypeBackendTest +from pypy.jit.metainterp.test.oparser import parse +from pypy.tool.udir import udir import ctypes import sys +import os + +CPU = getcpuclass() class FakeStats(object): pass @@ -56,7 +62,7 @@ assert u.chars[3] == u'd' @staticmethod - def _resbuf(res, item_tp=ctypes.c_int): + def _resbuf(res, item_tp=ctypes.c_long): return ctypes.cast(res.value._obj.intval, ctypes.POINTER(item_tp)) def test_allocations(self): @@ -71,8 +77,11 @@ return ctypes.cast(buf, ctypes.c_void_p).value func = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int)(f) addr = ctypes.cast(func, ctypes.c_void_p).value + # ctypes produces an unsigned value. We need it to be signed for, eg, + # relative addressing to work properly. + addr = rffi.cast(lltype.Signed, addr) - self.cpu.assembler.make_sure_mc_exists() + self.cpu.assembler.setup() self.cpu.assembler.malloc_func_addr = addr ofs = symbolic.get_field_token(rstr.STR, 'chars', False)[0] @@ -357,7 +366,9 @@ self.cpu.compile_bridge(faildescr1, [i1b], bridge) name, address, size = agent.functions[1] assert name == "Bridge # 0: bye" - assert address == loopaddress + loopsize + # Would be exactly ==, but there are some guard failure recovery + # stubs in-between + assert address >= loopaddress + loopsize assert size >= 10 # randomish number self.cpu.set_future_value_int(0, 2) @@ -366,6 +377,19 @@ res = self.cpu.get_latest_value_int(0) assert res == 20 + def test_call_with_const_floats(self): + def func(f1, f2): + return f1 + f2 + + FUNC = self.FuncType([lltype.Float, lltype.Float], lltype.Float) + FPTR = self.Ptr(FUNC) + calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + func_ptr = llhelper(FPTR, func) + funcbox = self.get_funcbox(self.cpu, func_ptr) + res = self.execute_operation(rop.CALL, [funcbox, ConstFloat(1.5), ConstFloat(2.5)], 'float', descr=calldescr) + assert res.value == 4.0 + + class TestX86OverflowMC(TestX86): def setup_method(self, meth): @@ -383,10 +407,100 @@ ops.append(ResOperation(rop.FINISH, [v], None, descr=BasicFailDescr())) looptoken = LoopToken() - self.cpu.assembler.make_sure_mc_exists() + self.cpu.assembler.setup() old_mc_mc = self.cpu.assembler.mc._mc self.cpu.compile_loop([base_v], ops, looptoken) assert self.cpu.assembler.mc._mc != old_mc_mc # overflowed self.cpu.set_future_value_int(0, base_v.value) self.cpu.execute_token(looptoken) assert self.cpu.get_latest_value_int(0) == 1024 + + def test_overflow_guard_float_cmp(self): + # The float comparisons on x86 tend to use small relative jumps, + # which may run into trouble if they fall on the edge of a + # MachineCodeBlock change. + a = BoxFloat(1.0) + b = BoxFloat(2.0) + failed = BoxInt(41) + finished = BoxInt(42) + + # We select guards that will always succeed, so that execution will + # continue through the entire set of comparisions + ops_to_test = ( + (rop.FLOAT_LT, [a, b], rop.GUARD_TRUE), + (rop.FLOAT_LT, [b, a], rop.GUARD_FALSE), + + (rop.FLOAT_LE, [a, a], rop.GUARD_TRUE), + (rop.FLOAT_LE, [a, b], rop.GUARD_TRUE), + (rop.FLOAT_LE, [b, a], rop.GUARD_FALSE), + + (rop.FLOAT_EQ, [a, a], rop.GUARD_TRUE), + (rop.FLOAT_EQ, [a, b], rop.GUARD_FALSE), + + (rop.FLOAT_NE, [a, b], rop.GUARD_TRUE), + (rop.FLOAT_NE, [a, a], rop.GUARD_FALSE), + + (rop.FLOAT_GT, [b, a], rop.GUARD_TRUE), + (rop.FLOAT_GT, [a, b], rop.GUARD_FALSE), + + (rop.FLOAT_GE, [a, a], rop.GUARD_TRUE), + (rop.FLOAT_GE, [b, a], rop.GUARD_TRUE), + (rop.FLOAT_GE, [a, b], rop.GUARD_FALSE), + ) + + for float_op, args, guard_op in ops_to_test: + ops = [] + + for i in range(200): + cmp_result = BoxInt() + ops.append(ResOperation(float_op, args, cmp_result)) + ops.append(ResOperation(guard_op, [cmp_result], None, descr=BasicFailDescr())) + ops[-1].fail_args = [failed] + + ops.append(ResOperation(rop.FINISH, [finished], None, descr=BasicFailDescr())) + + looptoken = LoopToken() + self.cpu.compile_loop([a, b, failed, finished], ops, looptoken) + self.cpu.set_future_value_float(0, a.value) + self.cpu.set_future_value_float(1, b.value) + self.cpu.set_future_value_int(2, failed.value) + self.cpu.set_future_value_int(3, finished.value) + self.cpu.execute_token(looptoken) + + # Really just a sanity check. We're actually interested in + # whether the test segfaults. + assert self.cpu.get_latest_value_int(0) == finished.value + + +class TestDebuggingAssembler(object): + def setup_method(self, meth): + self.pypylog = os.environ.get('PYPYLOG', None) + self.logfile = str(udir.join('x86_runner.log')) + os.environ['PYPYLOG'] = "mumble:" + self.logfile + self.cpu = CPU(rtyper=None, stats=FakeStats()) + + def teardown_method(self, meth): + if self.pypylog is not None: + os.environ['PYPYLOG'] = self.pypylog + + def test_debugger_on(self): + loop = """ + [i0] + debug_merge_point('xyz') + i1 = int_add(i0, 1) + i2 = int_ge(i1, 10) + guard_false(i2) [] + jump(i1) + """ + ops = parse(loop) + self.cpu.assembler.set_debug(True) + self.cpu.compile_loop(ops.inputargs, ops.operations, ops.token) + self.cpu.set_future_value_int(0, 0) + self.cpu.execute_token(ops.token) + # check debugging info + name, struct = self.cpu.assembler.loop_run_counters[0] + assert name == 'xyz' + assert struct.i == 10 + self.cpu.finish_once() + lines = py.path.local(self.logfile + ".count").readlines() + assert lines[0] == 'xyz:10\n' Modified: pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_symbolic_x86.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_symbolic_x86.py (original) +++ pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_symbolic_x86.py Thu Aug 12 13:47:36 2010 @@ -1,6 +1,7 @@ import py from pypy.jit.backend.llsupport.symbolic import * from pypy.rpython.lltypesystem import lltype, rffi +from pypy.jit.backend.x86.arch import WORD # This test file is here and not in llsupport/test/ because it checks # that we get correct numbers for a 32-bit machine. @@ -19,32 +20,32 @@ ofs_z, size_z = get_field_token(S, 'z', False) # ofs_x might be 0 or not, depending on how we count the headers # but the rest should be as expected for a 386 machine - assert size_x == size_y == size_z == 4 + assert size_x == size_y == size_z == WORD assert ofs_x >= 0 - assert ofs_y == ofs_x + 4 - assert ofs_z == ofs_x + 8 + assert ofs_y == ofs_x + WORD + assert ofs_z == ofs_x + (WORD*2) def test_struct_size(): ofs_z, size_z = get_field_token(S, 'z', False) totalsize = get_size(S, False) - assert totalsize == ofs_z + 4 + assert totalsize == ofs_z + WORD def test_primitive_size(): - assert get_size(lltype.Signed, False) == 4 + assert get_size(lltype.Signed, False) == WORD assert get_size(lltype.Char, False) == 1 - assert get_size(lltype.Ptr(S), False) == 4 + assert get_size(lltype.Ptr(S), False) == WORD def test_array_token(): A = lltype.GcArray(lltype.Char) basesize, itemsize, ofs_length = get_array_token(A, False) - assert basesize >= 4 # at least the 'length', maybe some gc headers + assert basesize >= WORD # at least the 'length', maybe some gc headers assert itemsize == 1 - assert ofs_length == basesize - 4 + assert ofs_length == basesize - WORD A = lltype.GcArray(lltype.Signed) basesize, itemsize, ofs_length = get_array_token(A, False) - assert basesize >= 4 # at least the 'length', maybe some gc headers - assert itemsize == 4 - assert ofs_length == basesize - 4 + assert basesize >= WORD # at least the 'length', maybe some gc headers + assert itemsize == WORD + assert ofs_length == basesize - WORD def test_varsized_struct_size(): S1 = lltype.GcStruct('S1', ('parent', S), @@ -54,9 +55,9 @@ ofs_extra, size_extra = get_field_token(S1, 'extra', False) basesize, itemsize, ofs_length = get_array_token(S1, False) assert size_parent == ofs_extra - assert size_extra == 4 - assert ofs_length == ofs_extra + 4 - assert basesize == ofs_length + 4 + assert size_extra == WORD + assert ofs_length == ofs_extra + WORD + assert basesize == ofs_length + WORD assert itemsize == 1 def test_string(): Modified: pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_zll_random.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_zll_random.py (original) +++ pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_zll_random.py Thu Aug 12 13:47:36 2010 @@ -1,9 +1,11 @@ from pypy.jit.backend.test.test_random import check_random_function, Random from pypy.jit.backend.test.test_ll_random import LLtypeOperationBuilder -from pypy.jit.backend.x86.runner import CPU386 +from pypy.jit.backend.detect_cpu import getcpuclass + +CPU = getcpuclass() def test_stress(): - cpu = CPU386(None, None) + cpu = CPU(None, None) r = Random() for i in range(1000): check_random_function(cpu, LLtypeOperationBuilder, r, i, 1000) Modified: pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_zrpy_gc.py Thu Aug 12 13:47:36 2010 @@ -17,6 +17,8 @@ from pypy.jit.backend.llsupport.gc import GcRefList, GcRootMap_asmgcc from pypy.jit.backend.llsupport.gc import GcLLDescr_framework from pypy.tool.udir import udir +from pypy.jit.backend.x86.arch import IS_X86_64 +import py.test class X(object): def __init__(self, x=0): @@ -126,6 +128,10 @@ class TestCompileHybrid(object): def setup_class(cls): + if IS_X86_64: + # No hybrid GC on 64-bit for the time being + py.test.skip() + funcs = [] name_to_func = {} for fullname in dir(cls): Modified: pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_ztranslation.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_ztranslation.py (original) +++ pypy/branch/kill-caninline/pypy/jit/backend/x86/test/test_ztranslation.py Thu Aug 12 13:47:36 2010 @@ -3,13 +3,14 @@ from pypy.rlib.jit import JitDriver, OPTIMIZER_FULL, unroll_parameters from pypy.rlib.jit import PARAMETERS, dont_look_inside from pypy.jit.metainterp.jitprof import Profiler -from pypy.jit.backend.x86.runner import CPU386 +from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.backend.test.support import CCompiledMixin from pypy.jit.codewriter.policy import StopAtXPolicy from pypy.translator.translator import TranslationContext +from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 class TestTranslationX86(CCompiledMixin): - CPUClass = CPU386 + CPUClass = getcpuclass() def _check_cbuilder(self, cbuilder): # We assume here that we have sse2. If not, the CPUClass @@ -113,7 +114,7 @@ class TestTranslationRemoveTypePtrX86(CCompiledMixin): - CPUClass = CPU386 + CPUClass = getcpuclass() def _get_TranslationContext(self): t = TranslationContext() @@ -124,6 +125,10 @@ return t def test_external_exception_handling_translates(self): + # FIXME + if IS_X86_64: + import py.test; py.test.skip() + jitdriver = JitDriver(greens = [], reds = ['n', 'total']) class ImDone(Exception): Modified: pypy/branch/kill-caninline/pypy/jit/backend/x86/tool/viewcode.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/backend/x86/tool/viewcode.py (original) +++ pypy/branch/kill-caninline/pypy/jit/backend/x86/tool/viewcode.py Thu Aug 12 13:47:36 2010 @@ -31,16 +31,23 @@ if sys.platform == "win32": XXX # lots more in Psyco -def machine_code_dump(data, originaddr): - # the disassembler to use. 'objdump' writes GNU-style instructions. - # 'ndisasm' would use Intel syntax, but you need to fix the output parsing. - objdump = ('objdump -M intel -b binary -m i386 ' +def machine_code_dump(data, originaddr, backend_name): + objdump_backend_option = { + 'x86': 'i386', + 'x86_64': 'x86-64', + 'i386': 'i386', + } + objdump = ('objdump -M intel,%(backend)s -b binary -m i386 ' '--adjust-vma=%(origin)d -D %(file)s') # f = open(tmpfile, 'wb') f.write(data) f.close() - g = os.popen(objdump % {'file': tmpfile, 'origin': originaddr}, 'r') + g = os.popen(objdump % { + 'file': tmpfile, + 'origin': originaddr, + 'backend': objdump_backend_option[backend_name], + }, 'r') result = g.readlines() g.close() return result[6:] # drop some objdump cruft @@ -126,7 +133,7 @@ def disassemble(self): if not hasattr(self, 'text'): - lines = machine_code_dump(self.data, self.addr) + lines = machine_code_dump(self.data, self.addr, self.world.backend_name) # instead of adding symbol names in the dumps we could # also make the 0xNNNNNNNN addresses be red and show the # symbol name when the mouse is over them @@ -171,10 +178,13 @@ self.jumps = {} self.symbols = {} self.logentries = {} + self.backend_name = None def parse(self, f, textonly=True): for line in f: - if line.startswith('CODE_DUMP '): + if line.startswith('BACKEND '): + self.backend_name = line.split(' ')[1].strip() + elif line.startswith('CODE_DUMP '): pieces = line.split() assert pieces[1].startswith('@') assert pieces[2].startswith('+') Modified: pypy/branch/kill-caninline/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/kill-caninline/pypy/jit/codewriter/codewriter.py Thu Aug 12 13:47:36 2010 @@ -14,11 +14,12 @@ class CodeWriter(object): callcontrol = None # for tests - def __init__(self, cpu=None, jitdrivers_sd=[]): + def __init__(self, cpu=None, jitdrivers_sd=[], debug=False): self.cpu = cpu self.assembler = Assembler() self.callcontrol = CallControl(cpu, jitdrivers_sd) self._seen_files = set() + self.debug = debug def transform_func_to_jitcode(self, func, values, type_system='lltype'): """For testing.""" @@ -60,7 +61,8 @@ self.assembler.assemble(ssarepr, jitcode) # # print the resulting assembler - self.print_ssa_repr(ssarepr, portal_jd, verbose) + if self.debug: + self.print_ssa_repr(ssarepr, portal_jd, verbose) def make_jitcodes(self, verbose=False): log.info("making JitCodes...") Modified: pypy/branch/kill-caninline/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/kill-caninline/pypy/jit/codewriter/jtransform.py Thu Aug 12 13:47:36 2010 @@ -702,15 +702,40 @@ raise NotImplementedError("cast_ptr_to_int") def rewrite_op_force_cast(self, op): - from pypy.rpython.lltypesystem.rffi import size_and_sign + from pypy.rpython.lltypesystem.rffi import size_and_sign, sizeof from pypy.rlib.rarithmetic import intmask assert not self._is_gc(op.args[0]) size1, unsigned1 = size_and_sign(op.args[0].concretetype) size2, unsigned2 = size_and_sign(op.result.concretetype) - if size1 == size2 and unsigned1 == unsigned2: - return - raise NotImplementedError("cast not supported yet: %s" % (op, )) - + if size2 >= sizeof(lltype.Signed): + return # the target type is LONG or ULONG + # + def bounds(size, unsigned): + if unsigned: + return 0, 1<<(8*size) + else: + return -(1<<(8*size-1)), 1<<(8*size-1) + min1, max1 = bounds(size1, unsigned1) + min2, max2 = bounds(size2, unsigned2) + if min2 <= min1 <= max1 <= max2: + return # the target type includes the source range + # + result = [] + v1 = op.args[0] + if min2: + c_min2 = Constant(min2, lltype.Signed) + v2 = Variable(); v2.concretetype = lltype.Signed + result.append(SpaceOperation('int_sub', [v1, c_min2], v2)) + else: + v2 = v1 + c_mask = Constant(int((1<<(8*size2))-1), lltype.Signed) + v3 = Variable(); v3.concretetype = lltype.Signed + result.append(SpaceOperation('int_and', [v2, c_mask], v3)) + if min2: + result.append(SpaceOperation('int_add', [v3, c_min2], op.result)) + else: + result[-1].result = op.result + return result # ---------- # Renames, from the _old opname to the _new one. Modified: pypy/branch/kill-caninline/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/kill-caninline/pypy/jit/codewriter/test/test_flatten.py Thu Aug 12 13:47:36 2010 @@ -732,25 +732,77 @@ """, transform=True) def test_force_cast(self): - py.test.skip("later") from pypy.rpython.lltypesystem import rffi - def f(n): - c = chr(n) - return rffi.cast(rffi.INT, c) - self.encoding_test(f, [42], """ - int_return %i0 - """, transform=True) - def g(n): - return rffi.cast(rffi.UCHAR, n) - self.encoding_test(g, [42], """ - int_and %i0, $255 -> %i1 - int_return %i1 - """, transform=True) - def h(n): - return rffi.cast(rffi.SCHAR, n) - self.encoding_test(h, [42], """ - ... - """, transform=True) + + for FROM, TO, expected in [ + (rffi.SIGNEDCHAR, rffi.SIGNEDCHAR, ""), + (rffi.SIGNEDCHAR, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.SIGNEDCHAR, rffi.SHORT, ""), + (rffi.SIGNEDCHAR, rffi.USHORT, "int_and %i0, $65535 -> %i1"), + (rffi.SIGNEDCHAR, rffi.LONG, ""), + (rffi.SIGNEDCHAR, rffi.ULONG, ""), + + (rffi.UCHAR, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.UCHAR, rffi.UCHAR, ""), + (rffi.UCHAR, rffi.SHORT, ""), + (rffi.UCHAR, rffi.USHORT, ""), + (rffi.UCHAR, rffi.LONG, ""), + (rffi.UCHAR, rffi.ULONG, ""), + + (rffi.SHORT, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.SHORT, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.SHORT, rffi.SHORT, ""), + (rffi.SHORT, rffi.USHORT, "int_and %i0, $65535 -> %i1"), + (rffi.SHORT, rffi.LONG, ""), + (rffi.SHORT, rffi.ULONG, ""), + + (rffi.USHORT, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.USHORT, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.USHORT, rffi.SHORT, """int_sub %i0, $-32768 -> %i1 + int_and %i1, $65535 -> %i2 + int_add %i2, $-32768 -> %i3"""), + (rffi.USHORT, rffi.USHORT, ""), + (rffi.USHORT, rffi.LONG, ""), + (rffi.USHORT, rffi.ULONG, ""), + + (rffi.LONG, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.LONG, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.LONG, rffi.SHORT, """int_sub %i0, $-32768 -> %i1 + int_and %i1, $65535 -> %i2 + int_add %i2, $-32768 -> %i3"""), + (rffi.LONG, rffi.USHORT, "int_and %i0, $65535 -> %i1"), + (rffi.LONG, rffi.LONG, ""), + (rffi.LONG, rffi.ULONG, ""), + + (rffi.ULONG, rffi.SIGNEDCHAR, """int_sub %i0, $-128 -> %i1 + int_and %i1, $255 -> %i2 + int_add %i2, $-128 -> %i3"""), + (rffi.ULONG, rffi.UCHAR, "int_and %i0, $255 -> %i1"), + (rffi.ULONG, rffi.SHORT, """int_sub %i0, $-32768 -> %i1 + int_and %i1, $65535 -> %i2 + int_add %i2, $-32768 -> %i3"""), + (rffi.ULONG, rffi.USHORT, "int_and %i0, $65535 -> %i1"), + (rffi.ULONG, rffi.LONG, ""), + (rffi.ULONG, rffi.ULONG, ""), + ]: + expected = [s.strip() for s in expected.splitlines()] + check_force_cast(FROM, TO, expected, 42) + check_force_cast(FROM, TO, expected, -42) + expected.append('int_return %i' + str(len(expected))) + expected = '\n'.join(expected) + # + def f(n): + return rffi.cast(TO, n) + self.encoding_test(f, [rffi.cast(FROM, 42)], expected, + transform=True) def test_force_cast_pointer(self): from pypy.rpython.lltypesystem import rffi @@ -759,3 +811,27 @@ self.encoding_test(h, [lltype.nullptr(rffi.CCHARP.TO)], """ int_return %i0 """, transform=True) + + +def check_force_cast(FROM, TO, operations, value): + """Check that the test is correctly written...""" + from pypy.rpython.lltypesystem import rffi + import re + r = re.compile('(\w+) \%i\d, \$(-?\d+)') + # + value = rffi.cast(FROM, value) + value = rffi.cast(lltype.Signed, value) + # + expected_value = rffi.cast(TO, value) + expected_value = rffi.cast(lltype.Signed, expected_value) + # + for op in operations: + match = r.match(op) + assert match, "line %r does not match regexp" % (op,) + opname = match.group(1) + if opname == 'int_add': value += int(match.group(2)) + elif opname == 'int_sub': value -= int(match.group(2)) + elif opname == 'int_and': value &= int(match.group(2)) + else: assert 0, opname + # + assert rffi.cast(lltype.Signed, value) == expected_value Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/optimizeopt.py Thu Aug 12 13:47:36 2010 @@ -1,7 +1,7 @@ from pypy.jit.metainterp.history import Box, BoxInt, LoopToken, BoxFloat,\ ConstFloat from pypy.jit.metainterp.history import Const, ConstInt, ConstPtr, ConstObj, REF -from pypy.jit.metainterp.resoperation import rop, ResOperation +from pypy.jit.metainterp.resoperation import rop, ResOperation, opboolinvers, opboolreflex from pypy.jit.metainterp import jitprof from pypy.jit.metainterp.executor import execute_nonspec from pypy.jit.metainterp.specnode import SpecNode, NotSpecNode, ConstantSpecNode @@ -18,6 +18,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.jit.metainterp.history import AbstractDescr, make_hashable_int + def optimize_loop_1(metainterp_sd, loop): """Optimize loop.operations to make it match the input of loop.specnodes and to remove internal overheadish operations. Note that loop.specnodes @@ -611,12 +612,59 @@ assert oldop.opnum == op.opnum self.make_equal_to(op.result, self.getvalue(oldop.result)) return + elif self.find_rewritable_bool(op, args): + return else: self.pure_operations[args] = op # otherwise, the operation remains self.emit_operation(op) + + def try_boolinvers(self, op, targs): + oldop = self.pure_operations.get(targs, None) + if oldop is not None and oldop.descr is op.descr: + value = self.getvalue(oldop.result) + if value.is_constant(): + if value.box is CONST_1: + self.make_constant(op.result, CONST_0) + return True + elif value.box is CONST_0: + self.make_constant(op.result, CONST_1) + return True + return False + + + def find_rewritable_bool(self, op, args): + try: + oldopnum = opboolinvers[op.opnum] + targs = [args[0], args[1], ConstInt(oldopnum)] + if self.try_boolinvers(op, targs): + return True + except KeyError: + pass + + try: + oldopnum = opboolreflex[op.opnum] + targs = [args[1], args[0], ConstInt(oldopnum)] + oldop = self.pure_operations.get(targs, None) + if oldop is not None and oldop.descr is op.descr: + self.make_equal_to(op.result, self.getvalue(oldop.result)) + return True + except KeyError: + pass + + try: + oldopnum = opboolinvers[opboolreflex[op.opnum]] + targs = [args[1], args[0], ConstInt(oldopnum)] + if self.try_boolinvers(op, targs): + return True + except KeyError: + pass + + return False + + def optimize_JUMP(self, op): orgop = self.loop.operations[-1] exitargs = [] @@ -992,6 +1040,25 @@ self.make_equal_to(op.result, v1) else: self.optimize_default(op) + + def optimize_INT_SUB(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + if v2.is_constant() and v2.box.getint() == 0: + self.make_equal_to(op.result, v1) + else: + return self.optimize_default(op) + + def optimize_INT_ADD(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + # If one side of the op is 0 the result is the other side. + if v1.is_constant() and v1.box.getint() == 0: + self.make_equal_to(op.result, v2) + elif v2.is_constant() and v2.box.getint() == 0: + self.make_equal_to(op.result, v1) + else: + self.optimize_default(op) optimize_ops = _findall(Optimizer, 'optimize_') Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/resoperation.py Thu Aug 12 13:47:36 2010 @@ -274,3 +274,51 @@ setup(__name__ == '__main__') # print out the table when run directly del _oplist + +opboolinvers = { + rop.INT_EQ: rop.INT_NE, + rop.INT_NE: rop.INT_EQ, + rop.INT_LT: rop.INT_GE, + rop.INT_GE: rop.INT_LT, + rop.INT_GT: rop.INT_LE, + rop.INT_LE: rop.INT_GT, + + rop.UINT_LT: rop.UINT_GE, + rop.UINT_GE: rop.UINT_LT, + rop.UINT_GT: rop.UINT_LE, + rop.UINT_LE: rop.UINT_GT, + + rop.FLOAT_EQ: rop.FLOAT_NE, + rop.FLOAT_NE: rop.FLOAT_EQ, + rop.FLOAT_LT: rop.FLOAT_GE, + rop.FLOAT_GE: rop.FLOAT_LT, + rop.FLOAT_GT: rop.FLOAT_LE, + rop.FLOAT_LE: rop.FLOAT_GT, + + rop.PTR_EQ: rop.PTR_NE, + rop.PTR_NE: rop.PTR_EQ, + } + +opboolreflex = { + rop.INT_EQ: rop.INT_EQ, + rop.INT_NE: rop.INT_NE, + rop.INT_LT: rop.INT_GT, + rop.INT_GE: rop.INT_LE, + rop.INT_GT: rop.INT_LT, + rop.INT_LE: rop.INT_GE, + + rop.UINT_LT: rop.UINT_GT, + rop.UINT_GE: rop.UINT_LE, + rop.UINT_GT: rop.UINT_LT, + rop.UINT_LE: rop.UINT_GE, + + rop.FLOAT_EQ: rop.FLOAT_EQ, + rop.FLOAT_NE: rop.FLOAT_NE, + rop.FLOAT_LT: rop.FLOAT_GT, + rop.FLOAT_GE: rop.FLOAT_LE, + rop.FLOAT_GT: rop.FLOAT_LT, + rop.FLOAT_LE: rop.FLOAT_GE, + + rop.PTR_EQ: rop.PTR_EQ, + rop.PTR_NE: rop.PTR_NE, + } Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_executor.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_executor.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_executor.py Thu Aug 12 13:47:36 2010 @@ -4,14 +4,14 @@ from pypy.rpython.lltypesystem import lltype, llmemory from pypy.jit.metainterp.executor import execute from pypy.jit.metainterp.executor import execute_varargs, execute_nonspec -from pypy.jit.metainterp.resoperation import rop +from pypy.jit.metainterp.resoperation import rop, opboolinvers, opboolreflex, opname from pypy.jit.metainterp.history import BoxInt, ConstInt from pypy.jit.metainterp.history import BoxPtr, ConstPtr from pypy.jit.metainterp.history import BoxFloat, ConstFloat from pypy.jit.metainterp.history import AbstractDescr, Box from pypy.jit.metainterp import history from pypy.jit.backend.model import AbstractCPU - +from pypy.rpython.lltypesystem import llmemory, rffi class FakeDescr(AbstractDescr): pass @@ -312,3 +312,40 @@ assert box.getint() == retvalue else: assert 0, "rettype is %r" % (rettype,) + +def make_args_for_op(op, a, b): + n=opname[op] + if n[0:3] == 'INT' or n[0:4] == 'UINT': + arg1 = ConstInt(a) + arg2 = ConstInt(b) + elif n[0:5] == 'FLOAT': + arg1 = ConstFloat(float(a)) + arg2 = ConstFloat(float(b)) + elif n[0:3] == 'PTR': + arg1 = ConstPtr(rffi.cast(llmemory.GCREF, a)) + arg2 = ConstPtr(rffi.cast(llmemory.GCREF, b)) + else: + raise NotImplementedError( + "Don't know how to make args for " + n) + return arg1, arg2 + + +def test_opboolinvers(): + cpu = FakeCPU() + for op1, op2 in opboolinvers.items(): + for a in (1,2,3): + for b in (1,2,3): + arg1, arg2 = make_args_for_op(op1, a, b) + box1 = execute(cpu, None, op1, None, arg1, arg2) + box2 = execute(cpu, None, op2, None, arg1, arg2) + assert box1.value == (not box2.value) + +def test_opboolreflex(): + cpu = FakeCPU() + for op1, op2 in opboolreflex.items(): + for a in (1,2,3): + for b in (1,2,3): + arg1, arg2 = make_args_for_op(op1, a, b) + box1 = execute(cpu, None, op1, None, arg1, arg2) + box2 = execute(cpu, None, op2, None, arg2, arg1) + assert box1.value == box2.value Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_loop.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_loop.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_loop.py Thu Aug 12 13:47:36 2010 @@ -9,6 +9,10 @@ class LoopTest(object): optimizer = OPTIMIZER_SIMPLE + automatic_promotion_result = { + 'int_add' : 6, 'int_gt' : 1, 'guard_false' : 1, 'jump' : 1, + 'guard_value' : 3 + } def meta_interp(self, f, args, policy=None): return ll_meta_interp(f, args, optimizer=self.optimizer, @@ -477,9 +481,9 @@ res = self.meta_interp(main_interpreter_loop, [1]) assert res == main_interpreter_loop(1) self.check_loop_count(1) - # XXX maybe later optimize guard_value away - self.check_loops({'int_add' : 6, 'int_gt' : 1, - 'guard_false' : 1, 'jump' : 1, 'guard_value' : 3}) + # These loops do different numbers of ops based on which optimizer we + # are testing with. + self.check_loops(self.automatic_promotion_result) def test_can_enter_jit_outside_main_loop(self): myjitdriver = JitDriver(greens=[], reds=['i', 'j', 'a']) Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_loop_spec.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_loop_spec.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_loop_spec.py Thu Aug 12 13:47:36 2010 @@ -5,6 +5,10 @@ class LoopSpecTest(test_loop.LoopTest): optimizer = OPTIMIZER_FULL + automatic_promotion_result = { + 'int_add' : 3, 'int_gt' : 1, 'guard_false' : 1, 'jump' : 1, + 'guard_value' : 1 + } # ====> test_loop.py Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_optimizeopt.py Thu Aug 12 13:47:36 2010 @@ -364,6 +364,74 @@ """ self.optimize_loop(ops, 'Not', expected) + def test_constant_boolrewrite_lt(self): + ops = """ + [i0] + i1 = int_lt(i0, 0) + guard_true(i1) [] + i2 = int_ge(i0, 0) + guard_false(i2) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_lt(i0, 0) + guard_true(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_constant_boolrewrite_gt(self): + ops = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + i2 = int_le(i0, 0) + guard_false(i2) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_constant_boolrewrite_reflex(self): + ops = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + i2 = int_lt(0, i0) + guard_true(i2) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_constant_boolrewrite_reflex_invers(self): + ops = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + i2 = int_ge(0, i0) + guard_false(i2) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + def test_remove_consecutive_guard_value_constfold(self): ops = """ [] @@ -411,7 +479,6 @@ self.optimize_loop(ops, 'Not', expected) def test_int_is_true_1(self): - py.test.skip("XXX implement me") ops = """ [i0] i1 = int_is_true(i0) @@ -806,16 +873,10 @@ guard_nonnull(p0) [] i7 = ptr_ne(p0, p1) guard_true(i7) [] - i8 = ptr_eq(p0, p1) - guard_false(i8) [] i9 = ptr_ne(p0, p2) guard_true(i9) [] - i10 = ptr_eq(p0, p2) - guard_false(i10) [] i11 = ptr_ne(p2, p1) guard_true(i11) [] - i12 = ptr_eq(p2, p1) - guard_false(i12) [] jump(p0, p1, p2) """ self.optimize_loop(ops, 'Not, Not, Not', expected2) @@ -2051,8 +2112,41 @@ jump(i1, i0) """ self.optimize_loop(ops, 'Not, Not', expected) - - + + def test_fold_partially_constant_ops(self): + ops = """ + [i0] + i1 = int_sub(i0, 0) + jump(i1) + """ + expected = """ + [i0] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + ops = """ + [i0] + i1 = int_add(i0, 0) + jump(i1) + """ + expected = """ + [i0] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + ops = """ + [i0] + i1 = int_add(0, i0) + jump(i1) + """ + expected = """ + [i0] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + # ---------- def make_fail_descr(self): Modified: pypy/branch/kill-caninline/pypy/jit/tl/pypyjit.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/tl/pypyjit.py (original) +++ pypy/branch/kill-caninline/pypy/jit/tl/pypyjit.py Thu Aug 12 13:47:36 2010 @@ -37,6 +37,7 @@ set_opt_level(config, level='jit') config.objspace.allworkingmodules = False config.objspace.usemodules.pypyjit = True +config.objspace.usemodules.array = True config.objspace.usemodules._weakref = False config.objspace.usemodules._sre = False set_pypy_opt_level(config, level='jit') Modified: pypy/branch/kill-caninline/pypy/jit/tl/pypyjit_demo.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/tl/pypyjit_demo.py (original) +++ pypy/branch/kill-caninline/pypy/jit/tl/pypyjit_demo.py Thu Aug 12 13:47:36 2010 @@ -1,38 +1,64 @@ -base = object +## base = object -class Number(base): - __slots__ = ('val', ) - def __init__(self, val=0): - self.val = val - - def __add__(self, other): - if not isinstance(other, int): - other = other.val - return Number(val=self.val + other) +## class Number(base): +## __slots__ = ('val', ) +## def __init__(self, val=0): +## self.val = val + +## def __add__(self, other): +## if not isinstance(other, int): +## other = other.val +## return Number(val=self.val + other) - def __cmp__(self, other): - val = self.val - if not isinstance(other, int): - other = other.val - return cmp(val, other) - - def __nonzero__(self): - return bool(self.val) - -def g(x, inc=2): - return x + inc - -def f(n, x, inc): - while x < n: - x = g(x, inc=1) - return x - -import time -#t1 = time.time() -#f(10000000, Number(), 1) -#t2 = time.time() -#print t2 - t1 -t1 = time.time() -f(10000000, 0, 1) -t2 = time.time() -print t2 - t1 +## def __cmp__(self, other): +## val = self.val +## if not isinstance(other, int): +## other = other.val +## return cmp(val, other) + +## def __nonzero__(self): +## return bool(self.val) + +## def g(x, inc=2): +## return x + inc + +## def f(n, x, inc): +## while x < n: +## x = g(x, inc=1) +## return x + +## import time +## #t1 = time.time() +## #f(10000000, Number(), 1) +## #t2 = time.time() +## #print t2 - t1 +## t1 = time.time() +## f(10000000, 0, 1) +## t2 = time.time() +## print t2 - t1 + +try: + from array import array + def f(img): + i=0 + sa=0 + while i < img.__len__(): + sa+=img[i] + i+=1 + return sa + + img=array('h',(1,2,3,4)) + print f(img) +except Exception, e: + print "Exception: ", type(e) + print e + +## def f(): +## a=7 +## i=0 +## while i<4: +## if i<0: break +## if i<0: break +## i+=1 + +## f() Modified: pypy/branch/kill-caninline/pypy/module/__builtin__/compiling.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/__builtin__/compiling.py (original) +++ pypy/branch/kill-caninline/pypy/module/__builtin__/compiling.py Thu Aug 12 13:47:36 2010 @@ -38,7 +38,7 @@ str_ = space.str_w(w_source) ec = space.getexecutioncontext() - if flags & ~(ec.compiler.compiler_flags | consts.PyCF_AST_ONLY | + if flags & ~(ec.compiler.compiler_flags | consts.PyCF_ONLY_AST | consts.PyCF_DONT_IMPLY_DEDENT | consts.PyCF_SOURCE_IS_UTF8): raise OperationError(space.w_ValueError, space.wrap("compile() unrecognized flags")) @@ -53,7 +53,7 @@ "or 'eval' or 'single'")) if ast_node is None: - if flags & consts.PyCF_AST_ONLY: + if flags & consts.PyCF_ONLY_AST: mod = ec.compiler.compile_to_ast(str_, filename, mode, flags) return space.wrap(mod) else: Modified: pypy/branch/kill-caninline/pypy/module/__builtin__/test/test_buffer.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/__builtin__/test/test_buffer.py (original) +++ pypy/branch/kill-caninline/pypy/module/__builtin__/test/test_buffer.py Thu Aug 12 13:47:36 2010 @@ -1,8 +1,11 @@ """Tests some behaviour of the buffer type that is not tested in lib-python/2.5.2/test/test_types.py where the stdlib buffer tests live.""" import autopath +from pypy.conftest import gettestobjspace class AppTestBuffer: + def setup_class(cls): + cls.space = gettestobjspace(usemodules=('array',)) def test_unicode_buffer(self): import sys Modified: pypy/branch/kill-caninline/pypy/module/_ast/__init__.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/_ast/__init__.py (original) +++ pypy/branch/kill-caninline/pypy/module/_ast/__init__.py Thu Aug 12 13:47:36 2010 @@ -5,7 +5,7 @@ class Module(MixedModule): interpleveldefs = { - "PyCF_AST_ONLY" : "space.wrap(%s)" % consts.PyCF_AST_ONLY + "PyCF_ONLY_AST" : "space.wrap(%s)" % consts.PyCF_ONLY_AST } appleveldefs = {} Modified: pypy/branch/kill-caninline/pypy/module/_ast/test/test_ast.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/_ast/test/test_ast.py (original) +++ pypy/branch/kill-caninline/pypy/module/_ast/test/test_ast.py Thu Aug 12 13:47:36 2010 @@ -10,7 +10,7 @@ cls.w_get_ast = cls.space.appexec([], """(): def get_ast(source, mode="exec"): import _ast as ast - mod = compile(source, "", mode, ast.PyCF_AST_ONLY) + mod = compile(source, "", mode, ast.PyCF_ONLY_AST) assert isinstance(mod, ast.mod) return mod return get_ast""") Modified: pypy/branch/kill-caninline/pypy/module/_codecs/test/test_codecs.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/_codecs/test/test_codecs.py (original) +++ pypy/branch/kill-caninline/pypy/module/_codecs/test/test_codecs.py Thu Aug 12 13:47:36 2010 @@ -123,6 +123,10 @@ class AppTestPartialEvaluation: + def setup_class(cls): + space = gettestobjspace(usemodules=('array',)) + cls.space = space + def test_partial_utf8(self): import _codecs encoding = 'utf-8' Modified: pypy/branch/kill-caninline/pypy/module/_demo/demo.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/_demo/demo.py (original) +++ pypy/branch/kill-caninline/pypy/module/_demo/demo.py Thu Aug 12 13:47:36 2010 @@ -4,11 +4,14 @@ from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.tool import rffi_platform +from pypy.translator.tool.cbuild import ExternalCompilationInfo import sys, math time_t = rffi_platform.getsimpletype('time_t', '#include ', rffi.LONG) -time = rffi.llexternal('time', [rffi.VOIDP], time_t, includes=['time.h']) +eci = ExternalCompilationInfo(includes=['time.h']) +time = rffi.llexternal('time', [int], time_t, + compilation_info=eci) def get(space, name): w_module = space.getbuiltinmodule('_demo') @@ -20,10 +23,10 @@ w_DemoError = get(space, 'DemoError') msg = "repetition count must be > 0" raise OperationError(w_DemoError, space.wrap(msg)) - starttime = time(None) + starttime = time(0) for i in range(repetitions): space.call_function(w_callable) - endtime = time(None) + endtime = time(0) return space.wrap(endtime - starttime) measuretime.unwrap_spec = [ObjSpace, int, W_Root] @@ -62,11 +65,16 @@ self.x = space.int_w(w_value) def mytype_new(space, w_subtype, x): + if x == 3: + return space.wrap(MySubType(space, x)) return space.wrap(W_MyType(space, x)) mytype_new.unwrap_spec = [ObjSpace, W_Root, int] getset_x = GetSetProperty(W_MyType.fget_x, W_MyType.fset_x, cls=W_MyType) +class MySubType(W_MyType): + pass + W_MyType.typedef = TypeDef('MyType', __new__ = interp2app(mytype_new), x = getset_x, Modified: pypy/branch/kill-caninline/pypy/module/_file/interp_file.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/_file/interp_file.py (original) +++ pypy/branch/kill-caninline/pypy/module/_file/interp_file.py Thu Aug 12 13:47:36 2010 @@ -4,6 +4,7 @@ from pypy.rlib.rarithmetic import r_longlong from pypy.module._file.interp_stream import W_AbstractStream from pypy.module._file.interp_stream import StreamErrors, wrap_streamerror +from pypy.module.posix.interp_posix import dispatch_filename from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import ObjSpace, W_Root, Arguments from pypy.interpreter.typedef import TypeDef, GetSetProperty @@ -81,11 +82,11 @@ # file lock. They don't convert StreamErrors to OperationErrors, too. def direct___init__(self, w_name, mode='r', buffering=-1): - name = self.space.str_w(w_name) self.direct_close() self.w_name = w_name self.check_mode_ok(mode) - stream = streamio.open_file_as_stream(name, mode, buffering) + stream = dispatch_filename(streamio.open_file_as_stream)( + self.space, w_name, mode, buffering) fd = stream.try_to_find_file_descriptor() self.fdopenstream(stream, fd, mode) Modified: pypy/branch/kill-caninline/pypy/module/_file/test/test_file.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/_file/test/test_file.py (original) +++ pypy/branch/kill-caninline/pypy/module/_file/test/test_file.py Thu Aug 12 13:47:36 2010 @@ -125,6 +125,15 @@ assert type(res) is str f.close() + def test_unicode_filename(self): + import sys + try: + u'\xe9'.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + skip("encoding not good enough") + f = self.file(self.temppath + u'\xe9', "w") + f.close() + def test_oserror_has_filename(self): try: f = self.file("file that is clearly not there") Modified: pypy/branch/kill-caninline/pypy/module/_file/test/test_file_extra.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/_file/test/test_file_extra.py (original) +++ pypy/branch/kill-caninline/pypy/module/_file/test/test_file_extra.py Thu Aug 12 13:47:36 2010 @@ -353,6 +353,10 @@ class AppTestAFewExtra: + def setup_class(cls): + space = gettestobjspace(usemodules=('array',)) + cls.space = space + def setup_method(self, method): fn = str(udir.join('temptestfile')) self.w_temptestfile = self.space.wrap(fn) Modified: pypy/branch/kill-caninline/pypy/module/_rawffi/test/test__rawffi.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/_rawffi/test/test__rawffi.py (original) +++ pypy/branch/kill-caninline/pypy/module/_rawffi/test/test__rawffi.py Thu Aug 12 13:47:36 2010 @@ -677,7 +677,12 @@ a = A(1) a[0] = -1234 a.free() - + + def test_long_with_fromaddress(self): + import _rawffi + addr = -1 + raises(ValueError, _rawffi.Array('u').fromaddress, addr, 100) + def test_passing_raw_pointers(self): import _rawffi lib = _rawffi.CDLL(self.lib_name) Modified: pypy/branch/kill-caninline/pypy/module/_socket/test/test_sock_app.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/_socket/test/test_sock_app.py (original) +++ pypy/branch/kill-caninline/pypy/module/_socket/test/test_sock_app.py Thu Aug 12 13:47:36 2010 @@ -4,7 +4,7 @@ from pypy.tool.udir import udir def setup_module(mod): - mod.space = gettestobjspace(usemodules=['_socket']) + mod.space = gettestobjspace(usemodules=['_socket', 'array']) global socket import socket mod.w_socket = space.appexec([], "(): import _socket as m; return m") Modified: pypy/branch/kill-caninline/pypy/module/_sre/test/test_app_sre.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/_sre/test/test_app_sre.py (original) +++ pypy/branch/kill-caninline/pypy/module/_sre/test/test_app_sre.py Thu Aug 12 13:47:36 2010 @@ -87,7 +87,9 @@ class AppTestSreMatch: - + def setup_class(cls): + cls.space = gettestobjspace(usemodules=('array', )) + def test_copy(self): import re # copy support is disabled by default in _sre.c Modified: pypy/branch/kill-caninline/pypy/module/_stackless/interp_coroutine.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/_stackless/interp_coroutine.py (original) +++ pypy/branch/kill-caninline/pypy/module/_stackless/interp_coroutine.py Thu Aug 12 13:47:36 2010 @@ -265,10 +265,14 @@ instr += 1 oparg = ord(code[instr]) | ord(code[instr + 1]) << 8 nargs = oparg & 0xff + nkwds = (oparg >> 8) & 0xff if space.config.objspace.opcodes.CALL_METHOD and opcode == map['CALL_METHOD']: - chain = resume_state_create(chain, 'CALL_METHOD', frame, - nargs) - elif opcode == map['CALL_FUNCTION'] and (oparg >> 8) & 0xff == 0: + if nkwds == 0: # only positional arguments + chain = resume_state_create(chain, 'CALL_METHOD', frame, + nargs) + else: # includes keyword arguments + chain = resume_state_create(chain, 'CALL_METHOD_KW', frame) + elif opcode == map['CALL_FUNCTION'] and nkwds == 0: # Only positional arguments # case1: ("CALL_FUNCTION", f, nargs, returns=w_result) chain = resume_state_create(chain, 'CALL_FUNCTION', frame, Modified: pypy/branch/kill-caninline/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/cpyext/api.py (original) +++ pypy/branch/kill-caninline/pypy/module/cpyext/api.py Thu Aug 12 13:47:36 2010 @@ -1,5 +1,5 @@ import ctypes -import sys +import sys, os import atexit import py @@ -896,6 +896,8 @@ initfunctype = lltype.Ptr(lltype.FuncType([], lltype.Void)) @unwrap_spec(ObjSpace, str, str) def load_extension_module(space, path, name): + if os.sep not in path: + path = os.curdir + os.sep + path # force a '/' in the path state = space.fromcache(State) state.package_context = name try: Modified: pypy/branch/kill-caninline/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/cpyext/methodobject.py (original) +++ pypy/branch/kill-caninline/pypy/module/cpyext/methodobject.py Thu Aug 12 13:47:36 2010 @@ -100,7 +100,11 @@ return generic_cpy_call(space, self.ml.c_ml_meth, w_self, w_arg) def get_doc(space, self): - return space.wrap(rffi.charp2str(self.ml.c_ml_doc)) + doc = self.ml.c_ml_doc + if doc: + return space.wrap(rffi.charp2str(doc)) + else: + return space.w_None class W_PyCMethodObject(W_PyCFunctionObject): Modified: pypy/branch/kill-caninline/pypy/module/fcntl/test/test_fcntl.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/fcntl/test/test_fcntl.py (original) +++ pypy/branch/kill-caninline/pypy/module/fcntl/test/test_fcntl.py Thu Aug 12 13:47:36 2010 @@ -13,7 +13,7 @@ class AppTestFcntl: def setup_class(cls): - space = gettestobjspace(usemodules=('fcntl',)) + space = gettestobjspace(usemodules=('fcntl', 'array')) cls.space = space tmpprefix = str(udir.ensure('test_fcntl', dir=1).join('tmp_')) cls.w_tmp = space.wrap(tmpprefix) Modified: pypy/branch/kill-caninline/pypy/module/marshal/test/test_marshalimpl.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/marshal/test/test_marshalimpl.py (original) +++ pypy/branch/kill-caninline/pypy/module/marshal/test/test_marshalimpl.py Thu Aug 12 13:47:36 2010 @@ -1,9 +1,13 @@ from pypy.module.marshal import interp_marshal from pypy.interpreter.error import OperationError +from pypy.conftest import gettestobjspace import sys class AppTestMarshalMore: + def setup_class(cls): + space = gettestobjspace(usemodules=('array',)) + cls.space = space def test_long_0(self): import marshal Modified: pypy/branch/kill-caninline/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/kill-caninline/pypy/module/posix/interp_posix.py Thu Aug 12 13:47:36 2010 @@ -1,8 +1,9 @@ from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped from pypy.rlib import rposix +from pypy.rlib.objectmodel import specialize from pypy.rlib.rarithmetic import r_longlong from pypy.rlib.unroll import unrolling_iterable -from pypy.interpreter.error import OperationError, wrap_oserror +from pypy.interpreter.error import OperationError, wrap_oserror, wrap_oserror2 from pypy.rpython.module.ll_os import RegisterOs from pypy.rpython.module import ll_os_stat from pypy.rpython.lltypesystem import rffi, lltype @@ -12,15 +13,78 @@ import os, sys _WIN = sys.platform == 'win32' -def open(space, fname, flag, mode=0777): +class FileEncoder: + def __init__(self, space, w_obj): + self.space = space + self.w_obj = w_obj + + def as_bytes(self): + from pypy.module.sys.interp_encoding import getfilesystemencoding + space = self.space + w_bytes = space.call_method(self.w_obj, 'encode', + getfilesystemencoding(space)) + return space.str_w(w_bytes) + + def as_unicode(self): + return self.space.unicode_w(self.w_obj) + +class FileDecoder: + def __init__(self, space, w_obj): + self.space = space + self.w_obj = w_obj + + def as_bytes(self): + return self.space.str_w(self.w_obj) + + def as_unicode(self): + from pypy.module.sys.interp_encoding import getfilesystemencoding + space = self.space + w_unicode = space.call_method(self.w_obj, 'decode', + getfilesystemencoding(space)) + return space.unicode_w(w_unicode) + + at specialize.memo() +def dispatch_filename(func, tag=0): + def dispatch(space, w_fname, *args): + if space.isinstance_w(w_fname, space.w_unicode): + fname = FileEncoder(space, w_fname) + return func(fname, *args) + else: + fname = space.str_w(w_fname) + return func(fname, *args) + return dispatch + + at specialize.memo() +def dispatch_filename_2(func): + def dispatch(space, w_fname1, w_fname2, *args): + if space.isinstance_w(w_fname1, space.w_unicode): + fname1 = FileEncoder(space, w_fname1) + if space.isinstance_w(w_fname2, space.w_unicode): + fname2 = FileEncoder(space, w_fname2) + return func(fname1, fname2, *args) + else: + fname2 = FileDecoder(space, w_fname2) + return func(fname1, fname2, *args) + else: + fname1 = FileDecoder(space, w_fname1) + if space.isinstance_w(w_fname2, space.w_unicode): + fname2 = FileEncoder(space, w_fname2) + return func(fname1, fname2, *args) + else: + fname2 = FileDecoder(space, w_fname2) + return func(fname1, fname2, *args) + return dispatch + +def open(space, w_fname, flag, mode=0777): """Open a file (for low level IO). Return a file descriptor (a small integer).""" - try: - fd = os.open(fname, flag, mode) + try: + fd = dispatch_filename(rposix.open)( + space, w_fname, flag, mode) except OSError, e: - raise wrap_oserror(space, e, fname) + raise wrap_oserror2(space, e, w_fname) return space.wrap(fd) -open.unwrap_spec = [ObjSpace, 'path', "c_int", "c_int"] +open.unwrap_spec = [ObjSpace, W_Root, "c_int", "c_int"] def lseek(space, fd, pos, how): """Set the current position of a file descriptor. Return the new position. @@ -159,7 +223,7 @@ return build_stat_result(space, st) fstat.unwrap_spec = [ObjSpace, "c_int"] -def stat(space, path): +def stat(space, w_path): """Perform a stat system call on the given path. Return an object with (at least) the following attributes: st_mode @@ -175,22 +239,22 @@ """ try: - st = os.stat(path) + st = dispatch_filename(rposix.stat)(space, w_path) except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) else: return build_stat_result(space, st) -stat.unwrap_spec = [ObjSpace, 'path'] +stat.unwrap_spec = [ObjSpace, W_Root] -def lstat(space, path): +def lstat(space, w_path): "Like stat(path), but do no follow symbolic links." try: - st = os.lstat(path) + st = dispatch_filename(rposix.lstat)(space, w_path) except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) else: return build_stat_result(space, st) -lstat.unwrap_spec = [ObjSpace, 'path'] +lstat.unwrap_spec = [ObjSpace, W_Root] class StatState(object): def __init__(self, space): @@ -231,7 +295,7 @@ raise wrap_oserror(space, e) dup2.unwrap_spec = [ObjSpace, "c_int", "c_int"] -def access(space, path, mode): +def access(space, w_path, mode): """ access(path, mode) -> 1 if granted, 0 otherwise @@ -242,12 +306,12 @@ existence, or the inclusive-OR of R_OK, W_OK, and X_OK. """ try: - ok = os.access(path, mode) - except OSError, e: - raise wrap_oserror(space, e, path) + ok = dispatch_filename(rposix.access)(space, w_path, mode) + except OSError, e: + raise wrap_oserror2(space, e, w_path) else: return space.wrap(ok) -access.unwrap_spec = [ObjSpace, str, "c_int"] +access.unwrap_spec = [ObjSpace, W_Root, "c_int"] def times(space): @@ -278,32 +342,38 @@ return space.wrap(rc) system.unwrap_spec = [ObjSpace, str] -def unlink(space, path): +def unlink(space, w_path): """Remove a file (same as remove(path)).""" try: - os.unlink(path) - except OSError, e: - raise wrap_oserror(space, e, path) -unlink.unwrap_spec = [ObjSpace, 'path'] + dispatch_filename(rposix.unlink)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +unlink.unwrap_spec = [ObjSpace, W_Root] -def remove(space, path): +def remove(space, w_path): """Remove a file (same as unlink(path)).""" try: - os.unlink(path) - except OSError, e: - raise wrap_oserror(space, e, path) -remove.unwrap_spec = [ObjSpace, 'path'] + dispatch_filename(rposix.unlink)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +remove.unwrap_spec = [ObjSpace, W_Root] -def _getfullpathname(space, path): +def _getfullpathname(space, w_path): """helper for ntpath.abspath """ - posix = __import__(os.name) # nt specific try: - fullpath = posix._getfullpathname(path) + if space.isinstance_w(w_path, space.w_unicode): + path = FileEncoder(space, w_path) + fullpath = rposix._getfullpathname(path) + w_fullpath = space.wrap(fullpath) + else: + path = space.str_w(w_path) + fullpath = rposix._getfullpathname(path) + w_fullpath = space.wrap(fullpath) except OSError, e: - raise wrap_oserror(space, e, path) - else: - return space.wrap(fullpath) -_getfullpathname.unwrap_spec = [ObjSpace, str] + raise wrap_oserror2(space, e, w_path) + else: + return w_fullpath +_getfullpathname.unwrap_spec = [ObjSpace, W_Root] def getcwd(space): """Return the current working directory.""" @@ -315,35 +385,46 @@ return space.wrap(cur) getcwd.unwrap_spec = [ObjSpace] -def getcwdu(space): - """Return the current working directory as a unicode string.""" - # XXX ascii encoding for now - return space.call_method(getcwd(space), 'decode') +if sys.platform == 'win32': + def getcwdu(space): + """Return the current working directory as a unicode string.""" + try: + cur = os.getcwdu() + except OSError, e: + raise wrap_oserror(space, e) + else: + return space.wrap(cur) +else: + def getcwdu(space): + """Return the current working directory as a unicode string.""" + filesystemencoding = space.sys.filesystemencoding + return space.call_method(getcwd(space), 'decode', + space.wrap(filesystemencoding)) getcwdu.unwrap_spec = [ObjSpace] -def chdir(space, path): +def chdir(space, w_path): """Change the current working directory to the specified path.""" try: - os.chdir(path) - except OSError, e: - raise wrap_oserror(space, e, path) -chdir.unwrap_spec = [ObjSpace, str] + dispatch_filename(rposix.chdir)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +chdir.unwrap_spec = [ObjSpace, W_Root] -def mkdir(space, path, mode=0777): +def mkdir(space, w_path, mode=0777): """Create a directory.""" try: - os.mkdir(path, mode) - except OSError, e: - raise wrap_oserror(space, e, path) -mkdir.unwrap_spec = [ObjSpace, str, "c_int"] + dispatch_filename(rposix.mkdir)(space, w_path, mode) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +mkdir.unwrap_spec = [ObjSpace, W_Root, "c_int"] -def rmdir(space, path): +def rmdir(space, w_path): """Remove a directory.""" try: - os.rmdir(path) - except OSError, e: - raise wrap_oserror(space, e, path) -rmdir.unwrap_spec = [ObjSpace, str] + dispatch_filename(rposix.rmdir)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +rmdir.unwrap_spec = [ObjSpace, W_Root] def strerror(space, errno): """Translate an error code to a message string.""" @@ -410,7 +491,7 @@ unsetenv.unwrap_spec = [ObjSpace, str] -def listdir(space, dirname): +def listdir(space, w_dirname): """Return a list containing the names of the entries in the directory. \tpath: path of directory to list @@ -418,12 +499,18 @@ The list is in arbitrary order. It does not include the special entries '.' and '..' even if they are present in the directory.""" try: - result = os.listdir(dirname) + if space.isinstance_w(w_dirname, space.w_unicode): + dirname = FileEncoder(space, w_dirname) + result = rposix.listdir(dirname) + result_w = [space.wrap(s) for s in result] + else: + dirname = space.str_w(w_dirname) + result = rposix.listdir(dirname) + result_w = [space.wrap(s) for s in result] except OSError, e: - raise wrap_oserror(space, e, dirname) - result_w = [space.wrap(s) for s in result] + raise wrap_oserror2(space, e, w_dirname) return space.newlist(result_w) -listdir.unwrap_spec = [ObjSpace, str] +listdir.unwrap_spec = [ObjSpace, W_Root] def pipe(space): "Create a pipe. Returns (read_end, write_end)." @@ -434,21 +521,21 @@ return space.newtuple([space.wrap(fd1), space.wrap(fd2)]) pipe.unwrap_spec = [ObjSpace] -def chmod(space, path, mode): +def chmod(space, w_path, mode): "Change the access permissions of a file." - try: - os.chmod(path, mode) - except OSError, e: - raise wrap_oserror(space, e, path) -chmod.unwrap_spec = [ObjSpace, str, "c_int"] + try: + dispatch_filename(rposix.chmod)(space, w_path, mode) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +chmod.unwrap_spec = [ObjSpace, W_Root, "c_int"] -def rename(space, old, new): +def rename(space, w_old, w_new): "Rename a file or directory." - try: - os.rename(old, new) - except OSError, e: + try: + dispatch_filename_2(rposix.rename)(space, w_old, w_new) + except OSError, e: raise wrap_oserror(space, e) -rename.unwrap_spec = [ObjSpace, str, str] +rename.unwrap_spec = [ObjSpace, W_Root, W_Root] def umask(space, mask): "Set the current numeric umask and return the previous umask." @@ -576,7 +663,7 @@ raise wrap_oserror(space, e) execve.unwrap_spec = [ObjSpace, str, W_Root, W_Root] -def utime(space, path, w_tuple): +def utime(space, w_path, w_tuple): """ utime(path, (atime, mtime)) utime(path, None) @@ -585,10 +672,10 @@ """ if space.is_w(w_tuple, space.w_None): try: - os.utime(path, None) + dispatch_filename(rposix.utime, 1)(space, w_path, None) return except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) try: msg = "utime() arg 2 must be a tuple (atime, mtime) or None" args_w = space.fixedview(w_tuple) @@ -596,14 +683,14 @@ raise OperationError(space.w_TypeError, space.wrap(msg)) actime = space.float_w(args_w[0]) modtime = space.float_w(args_w[1]) - os.utime(path, (actime, modtime)) + dispatch_filename(rposix.utime, 2)(space, w_path, (actime, modtime)) except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) except OperationError, e: if not e.match(space, space.w_TypeError): raise raise OperationError(space.w_TypeError, space.wrap(msg)) -utime.unwrap_spec = [ObjSpace, str, W_Root] +utime.unwrap_spec = [ObjSpace, W_Root, W_Root] def setsid(space): """setsid() -> pid Modified: pypy/branch/kill-caninline/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/posix/test/test_posix2.py (original) +++ pypy/branch/kill-caninline/pypy/module/posix/test/test_posix2.py Thu Aug 12 13:47:36 2010 @@ -32,6 +32,9 @@ # even when running on top of CPython 2.4. os.stat_float_times(True) + # Initialize sys.filesystemencoding + space.call_method(space.getbuiltinmodule('sys'), 'getfilesystemencoding') + def need_sparse_files(): if sys.platform == 'darwin': py.test.skip("no sparse files on default Mac OS X file system") @@ -706,6 +709,28 @@ except OSError: pass +class AppTestUnicodeFilename: + def setup_class(cls): + ufilename = (unicode(udir.join('test_unicode_filename_')) + + u'\u65e5\u672c.txt') # "Japan" + try: + f = file(ufilename, 'w') + except UnicodeEncodeError: + py.test.skip("encoding not good enough") + f.write("test") + f.close() + cls.space = space + cls.w_filename = space.wrap(ufilename) + cls.w_posix = space.appexec([], GET_POSIX) + + def test_open(self): + fd = self.posix.open(self.filename, self.posix.O_RDONLY) + try: + content = self.posix.read(fd, 50) + finally: + self.posix.close(fd) + assert content == "test" + class TestPexpect(object): # XXX replace with AppExpectTest class as soon as possible Modified: pypy/branch/kill-caninline/pypy/module/pypyjit/policy.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/pypyjit/policy.py (original) +++ pypy/branch/kill-caninline/pypy/module/pypyjit/policy.py Thu Aug 12 13:47:36 2010 @@ -11,7 +11,7 @@ if '.' in modname: modname, _ = modname.split('.', 1) if modname in ['pypyjit', 'signal', 'micronumpy', 'math', 'exceptions', - 'imp', 'sys']: + 'imp', 'sys', 'array']: return True return False Modified: pypy/branch/kill-caninline/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/kill-caninline/pypy/module/pypyjit/test/test_pypy_c.py Thu Aug 12 13:47:36 2010 @@ -615,6 +615,237 @@ return total ''', 170, ([], 4999450000L)) + def test_boolrewrite_invers(self): + for a, b, res, ops in (('2000', '2000', 20001000, 51), + ( '500', '500', 15001500, 81), + ( '300', '600', 16001700, 83), + ( 'a', 'b', 16001700, 89), + ( 'a', 'a', 13001700, 85)): + + self.run_source(''' + def main(): + sa = 0 + a = 300 + b = 600 + for i in range(1000): + if i < %s: sa += 1 + else: sa += 2 + if i >= %s: sa += 10000 + else: sa += 20000 + return sa + '''%(a, b), ops, ([], res)) + + def test_boolrewrite_reflex(self): + for a, b, res, ops in (('2000', '2000', 10001000, 51), + ( '500', '500', 15001500, 81), + ( '300', '600', 14001700, 83), + ( 'a', 'b', 14001700, 89), + ( 'a', 'a', 17001700, 85)): + + self.run_source(''' + def main(): + sa = 0 + a = 300 + b = 600 + for i in range(1000): + if i < %s: sa += 1 + else: sa += 2 + if %s > i: sa += 10000 + else: sa += 20000 + return sa + '''%(a, b), ops, ([], res)) + + + def test_boolrewrite_correct_invers(self): + def opval(i, op, a): + if eval('%d %s %d' % (i, op, a)): return 1 + return 2 + + ops = ('<', '>', '<=', '>=', '==', '!=') + for op1 in ops: + for op2 in ops: + for a,b in ((500, 500), (300, 600)): + res = 0 + res += opval(a-1, op1, a) * (a) + res += opval( a, op1, a) + res += opval(a+1, op1, a) * (1000 - a - 1) + res += opval(b-1, op2, b) * 10000 * (b) + res += opval( b, op2, b) * 10000 + res += opval(b+1, op2, b) * 10000 * (1000 - b - 1) + + self.run_source(''' + def main(): + sa = 0 + for i in range(1000): + if i %s %d: sa += 1 + else: sa += 2 + if i %s %d: sa += 10000 + else: sa += 20000 + return sa + '''%(op1, a, op2, b), 83, ([], res)) + + self.run_source(''' + def main(): + sa = 0 + i = 0.0 + while i < 250.0: + if i %s %f: sa += 1 + else: sa += 2 + if i %s %f: sa += 10000 + else: sa += 20000 + i += 0.25 + return sa + '''%(op1, float(a)/4.0, op2, float(b)/4.0), 109, ([], res)) + + + def test_boolrewrite_correct_reflex(self): + def opval(i, op, a): + if eval('%d %s %d' % (i, op, a)): return 1 + return 2 + + ops = ('<', '>', '<=', '>=', '==', '!=') + for op1 in ops: + for op2 in ops: + for a,b in ((500, 500), (300, 600)): + res = 0 + res += opval(a-1, op1, a) * (a) + res += opval( a, op1, a) + res += opval(a+1, op1, a) * (1000 - a - 1) + res += opval(b, op2, b-1) * 10000 * (b) + res += opval(b, op2, b) * 10000 + res += opval(b, op2, b+1) * 10000 * (1000 - b - 1) + + self.run_source(''' + def main(): + sa = 0 + for i in range(1000): + if i %s %d: sa += 1 + else: sa += 2 + if %d %s i: sa += 10000 + else: sa += 20000 + return sa + '''%(op1, a, b, op2), 83, ([], res)) + + self.run_source(''' + def main(): + sa = 0 + i = 0.0 + while i < 250.0: + if i %s %f: sa += 1 + else: sa += 2 + if %f %s i: sa += 10000 + else: sa += 20000 + i += 0.25 + return sa + '''%(op1, float(a)/4.0, float(b)/4.0, op2), 109, ([], res)) + + def test_boolrewrite_ptr(self): + compares = ('a == b', 'b == a', 'a != b', 'b != a', 'a == c', 'c != b') + for e1 in compares: + for e2 in compares: + a, b, c = 1, 2, 3 + if eval(e1): res = 752 * 1 + else: res = 752 * 2 + if eval(e2): res += 752 * 10000 + else: res += 752 * 20000 + a = b + if eval(e1): res += 248 * 1 + else: res += 248 * 2 + if eval(e2): res += 248 * 10000 + else: res += 248 * 20000 + + + if 'c' in e1 or 'c' in e2: + n = 337 + else: + n = 215 + + self.run_source(''' + class tst: + pass + def main(): + a = tst() + b = tst() + c = tst() + sa = 0 + for i in range(1000): + if %s: sa += 1 + else: sa += 2 + if %s: sa += 10000 + else: sa += 20000 + if i > 750: a = b + return sa + '''%(e1, e2), n, ([], res)) + + def test_array_sum(self): + for tc, maxops in zip('bhilBHILfd', (38,) * 6 + (40, 40, 41, 38)): + res = 19352859 + if tc in 'IL': + res = long(res) + elif tc in 'fd': + res = float(res) + + self.run_source(''' + from array import array + + def main(): + img = array("%s", range(127) * 5) * 484 + l, i = 0, 0 + while i < 640 * 480: + l += img[i] + i += 1 + return l + ''' % tc, maxops, ([], res)) + + def test_array_sum_char(self): + self.run_source(''' + from array import array + + def main(): + img = array("c", "Hello") * 130 * 480 + l, i = 0, 0 + while i < 640 * 480: + l += ord(img[i]) + i += 1 + return l + ''', 60, ([], 30720000)) + + def test_array_sum_unicode(self): + self.run_source(''' + from array import array + + def main(): + img = array("u", u"Hello") * 130 * 480 + l, i = 0, 0 + while i < 640 * 480: + if img[i] == u"l": + l += 1 + i += 1 + return l + ''', 65, ([], 122880)) + + def test_array_intimg(self): + for tc, maxops in zip('ilILd', (67, 67, 69, 69, 61)): + res = 73574560 + if tc in 'IL': + res = long(res) + elif tc in 'fd': + res = float(res) + + self.run_source(''' + from array import array + + def main(tc): + img = array(tc, range(3)) * (350 * 480) + intimg = array(tc, (0,)) * (640 * 480) + l, i = 0, 640 + while i < 640 * 480: + l = l + img[i] + intimg[i] = (intimg[i-640] + l) + i += 1 + return intimg[i - 1] + ''', maxops, ([tc], res)) + class AppTestJIT(PyPyCJITTests): def setup_class(cls): if not option.runappdirect: @@ -637,6 +868,7 @@ cls.counter = 0 cls.pypy_c = option.pypy_c + def has_info(pypy_c, option): g = os.popen('"%s" --info' % pypy_c, 'r') lines = g.readlines() Modified: pypy/branch/kill-caninline/pypy/module/signal/interp_signal.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/signal/interp_signal.py (original) +++ pypy/branch/kill-caninline/pypy/module/signal/interp_signal.py Thu Aug 12 13:47:36 2010 @@ -7,6 +7,7 @@ from pypy.translator.tool.cbuild import ExternalCompilationInfo import py from pypy.tool import autopath +from pypy.rlib import jit def setup(): for key, value in cpy_signal.__dict__.items(): @@ -159,10 +160,12 @@ return space.wrap(SIG_DFL) getsignal.unwrap_spec = [ObjSpace, int] + at jit.dont_look_inside def alarm(space, timeout): return space.wrap(c_alarm(timeout)) alarm.unwrap_spec = [ObjSpace, int] + at jit.dont_look_inside def pause(space): c_pause() return space.w_None @@ -173,6 +176,7 @@ raise OperationError(space.w_ValueError, space.wrap("signal number out of range")) + at jit.dont_look_inside def signal(space, signum, w_handler): """ signal(sig, action) -> action Modified: pypy/branch/kill-caninline/pypy/module/thread/test/test_gil.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/thread/test/test_gil.py (original) +++ pypy/branch/kill-caninline/pypy/module/thread/test/test_gil.py Thu Aug 12 13:47:36 2010 @@ -1,7 +1,6 @@ import time from pypy.module.thread import gil from pypy.module.thread.test import test_ll_thread -from pypy.rpython.lltypesystem import rffi from pypy.module.thread import ll_thread as thread from pypy.rlib.objectmodel import we_are_translated Modified: pypy/branch/kill-caninline/pypy/objspace/std/callmethod.py ============================================================================== --- pypy/branch/kill-caninline/pypy/objspace/std/callmethod.py (original) +++ pypy/branch/kill-caninline/pypy/objspace/std/callmethod.py Thu Aug 12 13:47:36 2010 @@ -12,7 +12,7 @@ from pypy.interpreter import function from pypy.objspace.descroperation import object_getattribute -from pypy.rlib import rstack # for resume points +from pypy.rlib import jit, rstack # for resume points # This module exports two extra methods for StdObjSpaceFrame implementing # the LOOKUP_METHOD and CALL_METHOD opcodes in an efficient way, as well @@ -56,16 +56,41 @@ f.pushvalue(w_value) f.pushvalue(None) -def CALL_METHOD(f, nargs, *ignored): - # 'nargs' is the argument count excluding the implicit 'self' - w_self = f.peekvalue(nargs) - w_callable = f.peekvalue(nargs + 1) - n = nargs + (w_self is not None) - try: - w_result = f.space.call_valuestack(w_callable, n, f) - rstack.resume_point("CALL_METHOD", f, nargs, returns=w_result) - finally: - f.dropvalues(nargs + 2) + at jit.unroll_safe +def CALL_METHOD(f, oparg, *ignored): + # opargs contains the arg, and kwarg count, excluding the implicit 'self' + n_args = oparg & 0xff + n_kwargs = (oparg >> 8) & 0xff + w_self = f.peekvalue(n_args + (2 * n_kwargs)) + n = n_args + (w_self is not None) + + if not n_kwargs: + w_callable = f.peekvalue(n_args + (2 * n_kwargs) + 1) + try: + w_result = f.space.call_valuestack(w_callable, n, f) + rstack.resume_point("CALL_METHOD", f, n_args, returns=w_result) + finally: + f.dropvalues(n_args + 2) + else: + keywords = [None] * n_kwargs + keywords_w = [None] * n_kwargs + while True: + n_kwargs -= 1 + if n_kwargs < 0: + break + w_value = f.popvalue() + w_key = f.popvalue() + key = f.space.str_w(w_key) + keywords[n_kwargs] = key + keywords_w[n_kwargs] = w_value + + arguments = f.popvalues(n) # includes w_self if it is not None + args = f.argument_factory(arguments, keywords, keywords_w, None, None) + if w_self is None: + f.popvalue() # removes w_self, which is None + w_callable = f.popvalue() + w_result = f.space.call_args(w_callable, args) + rstack.resume_point("CALL_METHOD_KW", f, returns=w_result) f.pushvalue(w_result) Modified: pypy/branch/kill-caninline/pypy/objspace/std/itertype.py ============================================================================== --- pypy/branch/kill-caninline/pypy/objspace/std/itertype.py (original) +++ pypy/branch/kill-caninline/pypy/objspace/std/itertype.py Thu Aug 12 13:47:36 2010 @@ -1,5 +1,6 @@ from pypy.interpreter import gateway from pypy.objspace.std.stdtypedef import StdTypeDef +from pypy.interpreter.error import OperationError # ____________________________________________________________ @@ -8,6 +9,11 @@ XXX to do: remove this __reduce__ method and do a registration with copy_reg, instead. """ + + # cpython does not support pickling iterators but stackless python do + #msg = 'Pickling for iterators dissabled as cpython does not support it' + #raise OperationError(space.w_TypeError, space.wrap(msg)) + from pypy.objspace.std.iterobject import W_AbstractSeqIterObject assert isinstance(w_self, W_AbstractSeqIterObject) from pypy.interpreter.mixedmodule import MixedModule Modified: pypy/branch/kill-caninline/pypy/objspace/std/model.py ============================================================================== --- pypy/branch/kill-caninline/pypy/objspace/std/model.py (original) +++ pypy/branch/kill-caninline/pypy/objspace/std/model.py Thu Aug 12 13:47:36 2010 @@ -88,6 +88,7 @@ import pypy.objspace.std.default # register a few catch-all multimethods import pypy.objspace.std.marshal_impl # install marshal multimethods + import pypy.module.array # the set of implementation types self.typeorder = { @@ -140,6 +141,8 @@ # check if we missed implementations for implcls in _registered_implementations: + if hasattr(implcls, 'register'): + implcls.register(self.typeorder) assert (implcls in self.typeorder or implcls in self.imported_but_not_registered), ( "please add %r in StdTypeModel.typeorder" % (implcls,)) Modified: pypy/branch/kill-caninline/pypy/objspace/std/test/test_callmethod.py ============================================================================== --- pypy/branch/kill-caninline/pypy/objspace/std/test/test_callmethod.py (original) +++ pypy/branch/kill-caninline/pypy/objspace/std/test/test_callmethod.py Thu Aug 12 13:47:36 2010 @@ -106,6 +106,15 @@ else: raise Exception("did not raise?") """ + + def test_kwargs(self): + exec """if 1: + class C(object): + def f(self, a): + return a + 2 + + assert C().f(a=3) == 5 + """ class AppTestCallMethodWithGetattributeShortcut(AppTestCallMethod): Modified: pypy/branch/kill-caninline/pypy/rlib/objectmodel.py ============================================================================== --- pypy/branch/kill-caninline/pypy/rlib/objectmodel.py (original) +++ pypy/branch/kill-caninline/pypy/rlib/objectmodel.py Thu Aug 12 13:47:36 2010 @@ -19,32 +19,80 @@ # def f(... # -class _AttachSpecialization(object): +class _Specialize(object): + def memo(self): + """ Specialize functions based on argument values. All arguments has + to be constant at the compile time. The whole function call is replaced + by a call result then. + """ + def decorated_func(func): + func._annspecialcase_ = 'specialize:memo' + return func + return decorated_func - def __init__(self, tag): - self.tag = tag + def arg(self, *args): + """ Specialize function based on values of given positions of arguments. + They must be compile-time constants in order to work. + + There will be a copy of provided function for each combination + of given arguments on positions in args (that can lead to + exponential behavior!). + """ + def decorated_func(func): + func._annspecialcase_ = 'specialize:arg' + self._wrap(args) + return func - def __call__(self, *args): - if not args: - args = "" - else: - args = "("+','.join([repr(arg) for arg in args]) +")" - specialcase = "specialize:%s%s" % (self.tag, args) - - def specialize_decorator(func): - "NOT_RPYTHON" - func._annspecialcase_ = specialcase + return decorated_func + + def argtype(self, *args): + """ Specialize function based on types of arguments on given positions. + + There will be a copy of provided function for each combination + of given arguments on positions in args (that can lead to + exponential behavior!). + """ + def decorated_func(func): + func._annspecialcase_ = 'specialize:argtype' + self._wrap(args) return func - return specialize_decorator - -class _Specialize(object): + return decorated_func - def __getattr__(self, name): - return _AttachSpecialization(name) + def ll(self): + """ This is version of argtypes that cares about low-level types + (so it'll get additional copies for two different types of pointers + for example). Same warnings about exponential behavior apply. + """ + def decorated_func(func): + func._annspecialcase_ = 'specialize:ll' + return func + + return decorated_func + + def ll_and_arg(self, arg): + """ XXX what does that do? + """ + def decorated_func(func): + func._annspecialcase_ = 'specialize:ll_and_arg(%d)' % arg + return func + + return decorated_func + + def _wrap(self, args): + return "("+','.join([repr(arg) for arg in args]) +")" specialize = _Specialize() +def enforceargs(*args): + """ Decorate a function with forcing of RPython-level types on arguments. + None means no enforcing. + + XXX shouldn't we also add asserts in function body? + """ + def decorator(f): + f._annenforceargs_ = args + return f + return decorator + # ____________________________________________________________ class Symbolic(object): Modified: pypy/branch/kill-caninline/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/branch/kill-caninline/pypy/rlib/rarithmetic.py (original) +++ pypy/branch/kill-caninline/pypy/rlib/rarithmetic.py Thu Aug 12 13:47:36 2010 @@ -33,7 +33,7 @@ """ -import math +import sys, math from pypy.rpython import extregistry from pypy.rlib import objectmodel @@ -113,14 +113,23 @@ "NOT_RPYTHON" return _local_ovfcheck(int(long(a) << b)) -FL_MAXINT = float(LONG_TEST-1) -FL_MININT = float(-LONG_TEST) - -def ovfcheck_float_to_int(x): - _, intp = math.modf(x) - if FL_MININT < intp < FL_MAXINT: - return int(intp) - raise OverflowError +# Strange things happening for float to int on 64 bit: +# int(float(i)) != i because of rounding issues. +# These are the minimum and maximum float value that can +# successfully be casted to an int. +if sys.maxint == 2147483647: + def ovfcheck_float_to_int(x): + if -2147483649.0 < x < 2147483648.0: + return int(x) + raise OverflowError +else: + # The following values are not quite +/-sys.maxint. + # Note the "<= x <" here, as opposed to "< x <" above. + # This is justified by test_typed in translator/c/test. + def ovfcheck_float_to_int(x): + if -9223372036854776832.0 <= x < 9223372036854775296.0: + return int(x) + raise OverflowError def compute_restype(self_type, other_type): if self_type is other_type: Modified: pypy/branch/kill-caninline/pypy/rlib/rdynload.py ============================================================================== --- pypy/branch/kill-caninline/pypy/rlib/rdynload.py (original) +++ pypy/branch/kill-caninline/pypy/rlib/rdynload.py Thu Aug 12 13:47:36 2010 @@ -18,8 +18,6 @@ if _WIN32: from pypy.rlib import rwin32 - -if _WIN32: includes = ['windows.h'] else: includes = ['dlfcn.h'] Modified: pypy/branch/kill-caninline/pypy/rlib/rmmap.py ============================================================================== --- pypy/branch/kill-caninline/pypy/rlib/rmmap.py (original) +++ pypy/branch/kill-caninline/pypy/rlib/rmmap.py Thu Aug 12 13:47:36 2010 @@ -646,8 +646,14 @@ hintp = rffi.cast(PTR, hint.pos) res = c_mmap_safe(hintp, map_size, prot, flags, -1, 0) if res == rffi.cast(PTR, -1): - raise MemoryError - hint.pos += map_size + # some systems (some versions of OS/X?) complain if they + # are passed a non-zero address. Try again. + hintp = rffi.cast(PTR, 0) + res = c_mmap_safe(hintp, map_size, prot, flags, -1, 0) + if res == rffi.cast(PTR, -1): + raise MemoryError + else: + hint.pos += map_size return res alloc._annenforceargs_ = (int,) Modified: pypy/branch/kill-caninline/pypy/rlib/rposix.py ============================================================================== --- pypy/branch/kill-caninline/pypy/rlib/rposix.py (original) +++ pypy/branch/kill-caninline/pypy/rlib/rposix.py Thu Aug 12 13:47:36 2010 @@ -3,6 +3,7 @@ from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rlib.rarithmetic import intmask +from pypy.rlib.objectmodel import specialize class CConstantErrno(CConstant): # these accessors are used when calling get_errno() or set_errno() @@ -42,3 +43,102 @@ os.close(fd) except OSError: pass + +#___________________________________________________________________ +# Wrappers around posix functions, that accept either strings, or +# instances with a "as_bytes()" method. +# - pypy.modules.posix.interp_posix passes an object containing a unicode path +# which can encode itself with sys.filesystemencoding. +# - but pypy.rpython.module.ll_os.py on Windows will replace these functions +# with other wrappers that directly handle unicode strings. + at specialize.argtype(0) +def open(path, flags, mode): + if isinstance(path, str): + return os.open(path, flags, mode) + else: + return os.open(path.as_bytes(), flags, mode) + + at specialize.argtype(0) +def stat(path): + if isinstance(path, str): + return os.stat(path) + else: + return os.stat(path.as_bytes()) + + at specialize.argtype(0) +def lstat(path): + if isinstance(path, str): + return os.lstat(path) + else: + return os.lstat(path.as_bytes()) + + at specialize.argtype(0) +def unlink(path): + if isinstance(path, str): + return os.unlink(path) + else: + return os.unlink(path.as_bytes()) + + at specialize.argtype(0, 1) +def rename(path1, path2): + if isinstance(path1, str): + return os.rename(path1, path2) + else: + return os.rename(path1.as_bytes(), path2.as_bytes()) + + at specialize.argtype(0) +def listdir(dirname): + if isinstance(dirname, str): + return os.listdir(dirname) + else: + return os.listdir(dirname.as_bytes()) + + at specialize.argtype(0) +def access(path, mode): + if isinstance(path, str): + return os.access(path, mode) + else: + return os.access(path.as_bytes(), mode) + + at specialize.argtype(0) +def chmod(path, mode): + if isinstance(path, str): + return os.chmod(path, mode) + else: + return os.chmod(path.as_bytes(), mode) + + at specialize.argtype(0, 1) +def utime(path, times): + if isinstance(path, str): + return os.utime(path, times) + else: + return os.utime(path.as_bytes(), times) + + at specialize.argtype(0) +def chdir(path): + if isinstance(path, str): + return os.chdir(path) + else: + return os.chdir(path.as_bytes()) + + at specialize.argtype(0) +def mkdir(path, mode=0777): + if isinstance(path, str): + return os.mkdir(path, mode) + else: + return os.mkdir(path.as_bytes(), mode) + + at specialize.argtype(0) +def rmdir(path): + if isinstance(path, str): + return os.rmdir(path) + else: + return os.rmdir(path.as_bytes()) + +if os.name == 'nt': + import nt + def _getfullpathname(path): + if isinstance(path, str): + return nt._getfullpathname(path) + else: + return nt._getfullpathname(path.as_bytes()) Modified: pypy/branch/kill-caninline/pypy/rlib/rwin32.py ============================================================================== --- pypy/branch/kill-caninline/pypy/rlib/rwin32.py (original) +++ pypy/branch/kill-caninline/pypy/rlib/rwin32.py Thu Aug 12 13:47:36 2010 @@ -31,6 +31,7 @@ DWORD = rffi_platform.SimpleType("DWORD", rffi.UINT) BOOL = rffi_platform.SimpleType("BOOL", rffi.LONG) BYTE = rffi_platform.SimpleType("BYTE", rffi.UCHAR) + WCHAR = rffi_platform.SimpleType("WCHAR", rffi.UCHAR) INT = rffi_platform.SimpleType("INT", rffi.INT) LONG = rffi_platform.SimpleType("LONG", rffi.LONG) PLONG = rffi_platform.SimpleType("PLONG", rffi.LONGP) @@ -38,6 +39,8 @@ LPCVOID = rffi_platform.SimpleType("LPCVOID", rffi.VOIDP) LPSTR = rffi_platform.SimpleType("LPSTR", rffi.CCHARP) LPCSTR = rffi_platform.SimpleType("LPCSTR", rffi.CCHARP) + LPWSTR = rffi_platform.SimpleType("LPWSTR", rffi.CWCHARP) + LPCWSTR = rffi_platform.SimpleType("LPCWSTR", rffi.CWCHARP) LPDWORD = rffi_platform.SimpleType("LPDWORD", rffi.INTP) SIZE_T = rffi_platform.SimpleType("SIZE_T", rffi.SIZE_T) ULONG_PTR = rffi_platform.SimpleType("ULONG_PTR", rffi.ULONG) @@ -87,6 +90,10 @@ GetLastError = winexternal('GetLastError', [], DWORD) SetLastError = winexternal('SetLastError', [DWORD], lltype.Void) + # In tests, the first call to GetLastError is always wrong, because error + # is hidden by operations in ll2ctypes. Call it now. + GetLastError() + LoadLibrary = winexternal('LoadLibraryA', [rffi.CCHARP], rffi.VOIDP) GetProcAddress = winexternal('GetProcAddress', [rffi.VOIDP, rffi.CCHARP], @@ -129,13 +136,29 @@ } return 0; }''') - exename = static_platform.compile( - [cfile], ExternalCompilationInfo(), - outputfilename = "dosmaperr", - standalone=True) - output = os.popen(str(exename)) - errors = dict(map(int, line.split()) - for line in output) + try: + exename = static_platform.compile( + [cfile], ExternalCompilationInfo(), + outputfilename = "dosmaperr", + standalone=True) + except WindowsError: + # Fallback for the mingw32 compiler + errors = { + 2: 2, 3: 2, 4: 24, 5: 13, 6: 9, 7: 12, 8: 12, 9: 12, 10: 7, + 11: 8, 15: 2, 16: 13, 17: 18, 18: 2, 19: 13, 20: 13, 21: 13, + 22: 13, 23: 13, 24: 13, 25: 13, 26: 13, 27: 13, 28: 13, + 29: 13, 30: 13, 31: 13, 32: 13, 33: 13, 34: 13, 35: 13, + 36: 13, 53: 2, 65: 13, 67: 2, 80: 17, 82: 13, 83: 13, 89: 11, + 108: 13, 109: 32, 112: 28, 114: 9, 128: 10, 129: 10, 130: 9, + 132: 13, 145: 41, 158: 13, 161: 2, 164: 11, 167: 13, 183: 17, + 188: 8, 189: 8, 190: 8, 191: 8, 192: 8, 193: 8, 194: 8, + 195: 8, 196: 8, 197: 8, 198: 8, 199: 8, 200: 8, 201: 8, + 202: 8, 206: 2, 215: 11, 1816: 12, + } + else: + output = os.popen(str(exename)) + errors = dict(map(int, line.split()) + for line in output) return errors, errno.EINVAL # A bit like strerror... Modified: pypy/branch/kill-caninline/pypy/rlib/streamio.py ============================================================================== --- pypy/branch/kill-caninline/pypy/rlib/streamio.py (original) +++ pypy/branch/kill-caninline/pypy/rlib/streamio.py Thu Aug 12 13:47:36 2010 @@ -38,7 +38,9 @@ # import os, sys +from pypy.rlib.objectmodel import specialize from pypy.rlib.rarithmetic import r_longlong, intmask +from pypy.rlib import rposix from os import O_RDONLY, O_WRONLY, O_RDWR, O_CREAT, O_TRUNC O_BINARY = getattr(os, "O_BINARY", 0) @@ -71,6 +73,7 @@ return s.join(string.split(c)) + at specialize.argtype(0) def open_file_as_stream(path, mode="r", buffering=-1): os_flags, universal, reading, writing, basemode, binary = decode_mode(mode) stream = open_path_helper(path, os_flags, basemode == "a") @@ -89,9 +92,10 @@ return construct_stream_tower(stream, buffering, universal, reading, writing, binary) + at specialize.argtype(0) def open_path_helper(path, os_flags, append): # XXX for now always return DiskFile - fd = os.open(path, os_flags, 0666) + fd = rposix.open(path, os_flags, 0666) if append: try: os.lseek(fd, 0, 2) Modified: pypy/branch/kill-caninline/pypy/rlib/test/test_objectmodel.py ============================================================================== --- pypy/branch/kill-caninline/pypy/rlib/test/test_objectmodel.py (original) +++ pypy/branch/kill-caninline/pypy/rlib/test/test_objectmodel.py Thu Aug 12 13:47:36 2010 @@ -404,6 +404,13 @@ assert f._annspecialcase_ == 'specialize:arg(1)' +def test_enforceargs_decorator(): + @enforceargs(int, str, None) + def f(a, b, c): + pass + + assert f._annenforceargs_ == (int, str, None) + def getgraph(f, argtypes): from pypy.translator.translator import TranslationContext, graphof from pypy.translator.backendopt.all import backend_optimizations Modified: pypy/branch/kill-caninline/pypy/rlib/test/test_rarithmetic.py ============================================================================== --- pypy/branch/kill-caninline/pypy/rlib/test/test_rarithmetic.py (original) +++ pypy/branch/kill-caninline/pypy/rlib/test/test_rarithmetic.py Thu Aug 12 13:47:36 2010 @@ -271,26 +271,29 @@ assert ovfcheck_float_to_int(13.0) == 13 assert ovfcheck_float_to_int(-1.0) == -1 assert ovfcheck_float_to_int(-13.0) == -13 - # strange things happening for float to int on 64 bit - maxint32 = 2 ** 31 - 1 - assert ovfcheck_float_to_int(float(maxint32-1)) == maxint32-1 - #assert ovfcheck_float_to_int(float(maxint32)) == maxint32 - assert ovfcheck_float_to_int(float(-maxint32)) == -maxint32 - #assert ovfcheck_float_to_int(float(-maxint32-1)) == -maxint32-1 - - try: - ovfcheck_float_to_int(float(-sys.maxint-1)-1) - except OverflowError: - pass - else: - assert False - - try: - ovfcheck_float_to_int(float(sys.maxint)+1) - except OverflowError: - pass - else: - assert False + + # strange things happening for float to int on 64 bit: + # int(float(i)) != i because of rounding issues + x = sys.maxint + while int(float(x)) > sys.maxint: + x -= 1 + assert ovfcheck_float_to_int(float(x)) == int(float(x)) + + x = sys.maxint + 1 + while int(float(x)) <= sys.maxint: + x += 1 + py.test.raises(OverflowError, ovfcheck_float_to_int, x) + + x = -sys.maxint-1 + while int(float(x)) < -sys.maxint-1: + x += 1 + assert ovfcheck_float_to_int(float(x)) == int(float(x)) + + x = -sys.maxint-1 + while int(float(x)) >= -sys.maxint-1: + x -= 1 + py.test.raises(OverflowError, ovfcheck_float_to_int, x) + def test_abs(): assert type(abs(r_longlong(1))) is r_longlong Modified: pypy/branch/kill-caninline/pypy/rpython/extfunc.py ============================================================================== --- pypy/branch/kill-caninline/pypy/rpython/extfunc.py (original) +++ pypy/branch/kill-caninline/pypy/rpython/extfunc.py Thu Aug 12 13:47:36 2010 @@ -52,7 +52,10 @@ return super(ExtRegistryEntry, self).__getattr__(attr) raise exc, exc_inst, tb -def registering(func): +def registering(func, condition=True): + if not condition: + return lambda method: None + def decorator(method): method._registering_func = func return method @@ -63,11 +66,9 @@ func = getattr(ns, name) except AttributeError: condition = False + func = None - if condition: - return registering(func) - else: - return lambda method: None + return registering(func, condition=condition) class LazyRegisteringMeta(type): def __new__(self, _name, _type, _vars): @@ -167,8 +168,6 @@ return signature_args def compute_result_annotation(self, *args_s): - if hasattr(self, 'ann_hook'): - self.ann_hook() self.normalize_args(*args_s) # check arguments return self.signature_result @@ -235,7 +234,6 @@ def register_external(function, args, result=None, export_name=None, llimpl=None, ooimpl=None, llfakeimpl=None, oofakeimpl=None, - annotation_hook=None, sandboxsafe=False): """ function: the RPython function that will be rendered as an external function (e.g.: math.floor) @@ -244,7 +242,6 @@ export_name: the name of the function as it will be seen by the backends llimpl, ooimpl: optional; if provided, these RPython functions are called instead of the target function llfakeimpl, oofakeimpl: optional; if provided, they are called by the llinterpreter - annotationhook: optional; a callable that is called during annotation, useful for genc hacks sandboxsafe: use True if the function performs no I/O (safe for --sandbox) """ @@ -271,8 +268,6 @@ lltypefakeimpl = staticmethod(llfakeimpl) if oofakeimpl: ootypefakeimpl = staticmethod(oofakeimpl) - if annotation_hook: - ann_hook = staticmethod(annotation_hook) if export_name: FunEntry.__name__ = export_name Modified: pypy/branch/kill-caninline/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/kill-caninline/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/kill-caninline/pypy/rpython/lltypesystem/ll2ctypes.py Thu Aug 12 13:47:36 2010 @@ -24,6 +24,7 @@ from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE from pypy.rpython import raddress from pypy.translator.platform import platform +from array import array def uaddressof(obj): return fixid(ctypes.addressof(obj)) @@ -756,7 +757,15 @@ elif T is lltype.Char: llobj = chr(cobj) elif T is lltype.UniChar: - llobj = unichr(cobj) + try: + llobj = unichr(cobj) + except (ValueError, OverflowError): + for tc in 'HIL': + if array(tc).itemsize == array('u').itemsize: + llobj = array('u', array(tc, (cobj,)).tostring())[0] + break + else: + raise elif T is lltype.Signed: llobj = cobj elif T is lltype.Bool: Modified: pypy/branch/kill-caninline/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/kill-caninline/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/kill-caninline/pypy/rpython/lltypesystem/rffi.py Thu Aug 12 13:47:36 2010 @@ -176,6 +176,15 @@ # XXX leaks if a str2charp() fails with MemoryError # and was not the first in this function freeme = arg + elif TARGET == CWCHARP: + if arg is None: + arg = lltype.nullptr(CWCHARP.TO) # None => (wchar_t*)NULL + freeme = arg + elif isinstance(arg, unicode): + arg = unicode2wcharp(arg) + # XXX leaks if a unicode2wcharp() fails with MemoryError + # and was not the first in this function + freeme = arg elif _isfunctype(TARGET) and not _isllptr(arg): # XXX pass additional arguments if invoke_around_handlers: Modified: pypy/branch/kill-caninline/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/kill-caninline/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/kill-caninline/pypy/rpython/lltypesystem/rstr.py Thu Aug 12 13:47:36 2010 @@ -2,7 +2,7 @@ from pypy.tool.pairtype import pairtype from pypy.rpython.error import TyperError from pypy.rlib.objectmodel import malloc_zero_filled, we_are_translated -from pypy.rlib.objectmodel import _hash_string +from pypy.rlib.objectmodel import _hash_string, enforceargs from pypy.rlib.debug import ll_assert from pypy.rlib.jit import purefunction from pypy.rpython.robject import PyObjRepr, pyobj_repr @@ -56,6 +56,7 @@ llmemory.itemoffsetof(TP.chars, 0) + llmemory.sizeof(CHAR_TP) * item) + @enforceargs(None, None, int, int, int) def copy_string_contents(src, dst, srcstart, dststart, length): assert srcstart >= 0 assert dststart >= 0 @@ -674,6 +675,7 @@ res_index += item_len i += 1 return result + ll_join_strs._annenforceargs_ = [int, None] def ll_join_chars(length, chars): # no need to optimize this, will be replaced by string builder Modified: pypy/branch/kill-caninline/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/kill-caninline/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/kill-caninline/pypy/rpython/module/ll_os.py Thu Aug 12 13:47:36 2010 @@ -6,12 +6,15 @@ # might be found in doc/rffi.txt import os, sys, errno +import py from pypy.rpython.module.support import ll_strcpy, OOSupport -from pypy.tool.sourcetools import func_with_new_name +from pypy.tool.sourcetools import func_with_new_name, func_renamer from pypy.rlib.rarithmetic import r_longlong -from pypy.rpython.extfunc import BaseLazyRegistering +from pypy.rpython.extfunc import ( + BaseLazyRegistering, lazy_register, register_external) from pypy.rpython.extfunc import registering, registering_if, extdef -from pypy.annotation.model import SomeInteger, SomeString, SomeTuple, SomeFloat +from pypy.annotation.model import ( + SomeInteger, SomeString, SomeTuple, SomeFloat, SomeUnicodeString) from pypy.annotation.model import s_ImpossibleValue, s_None, s_Bool from pypy.rpython.lltypesystem import rffi from pypy.rpython.lltypesystem import lltype @@ -26,7 +29,99 @@ from pypy.rpython.lltypesystem.rstr import STR from pypy.rpython.annlowlevel import llstr from pypy.rlib import rgc -from pypy.rlib.objectmodel import keepalive_until_here +from pypy.rlib.objectmodel import keepalive_until_here, specialize + +def monkeypatch_rposix(posixfunc, unicodefunc, signature): + func_name = posixfunc.__name__ + + if hasattr(signature, '_default_signature_'): + signature = signature._default_signature_ + arglist = ['arg%d' % (i,) for i in range(len(signature))] + transformed_arglist = arglist[:] + for i, arg in enumerate(signature): + if arg is unicode: + transformed_arglist[i] = transformed_arglist[i] + '.as_unicode()' + + args = ', '.join(arglist) + transformed_args = ', '.join(transformed_arglist) + main_arg = 'arg%d' % (signature.index(unicode),) + + source = py.code.Source(""" + def %(func_name)s(%(args)s): + if isinstance(%(main_arg)s, str): + return posixfunc(%(args)s) + else: + return unicodefunc(%(transformed_args)s) + """ % locals()) + miniglobals = {'posixfunc' : posixfunc, + 'unicodefunc': unicodefunc, + '__name__': __name__, # for module name propagation + } + exec source.compile() in miniglobals + new_func = miniglobals[func_name] + specialized_args = [i for i in range(len(signature)) + if signature[i] in (unicode, None)] + new_func = specialize.argtype(*specialized_args)(new_func) + + # Monkeypatch the function in pypy.rlib.rposix + setattr(rposix, func_name, new_func) + +class StringTraits: + str = str + CHAR = rffi.CHAR + CCHARP = rffi.CCHARP + charp2str = staticmethod(rffi.charp2str) + str2charp = staticmethod(rffi.str2charp) + free_charp = staticmethod(rffi.free_charp) + + @staticmethod + def posix_function_name(name): + return underscore_on_windows + name + + @staticmethod + def ll_os_name(name): + return 'll_os.ll_os_' + name + +class UnicodeTraits: + str = unicode + CHAR = rffi.WCHAR_T + CCHARP = rffi.CWCHARP + charp2str = staticmethod(rffi.wcharp2unicode) + str2charp = staticmethod(rffi.unicode2wcharp) + free_charp = staticmethod(rffi.free_wcharp) + + @staticmethod + def posix_function_name(name): + return underscore_on_windows + 'w' + name + + @staticmethod + def ll_os_name(name): + return 'll_os.ll_os_w' + name + +def registering_str_unicode(posixfunc, condition=True): + if not condition: + return registering(None, condition=False) + + func_name = posixfunc.__name__ + + def register_posixfunc(self, method): + val = method(self, StringTraits()) + register_external(posixfunc, *val.def_args, **val.def_kwds) + + if sys.platform == 'win32': + val = method(self, UnicodeTraits()) + @func_renamer(func_name + "_unicode") + def unicodefunc(*args): + return posixfunc(*args) + register_external(unicodefunc, *val.def_args, **val.def_kwds) + signature = val.def_args[0] + monkeypatch_rposix(posixfunc, unicodefunc, signature) + + def decorator(method): + decorated = lambda self: register_posixfunc(self, method) + decorated._registering_func = posixfunc + return decorated + return decorator posix = __import__(os.name) @@ -282,8 +377,8 @@ return extdef([int, int], s_None, llimpl=dup2_llimpl, export_name="ll_os.ll_os_dup2") - @registering(os.utime) - def register_os_utime(self): + @registering_str_unicode(os.utime) + def register_os_utime(self, traits): UTIMBUFP = lltype.Ptr(self.UTIMBUF) os_utime = self.llexternal('utime', [rffi.CCHARP, UTIMBUFP], rffi.INT) @@ -336,6 +431,9 @@ # tp is known to be None, and one version where it is known # to be a tuple of 2 floats. if not _WIN32: + assert traits.str is str + + @specialize.argtype(1) def os_utime_llimpl(path, tp): if tp is None: error = os_utime(path, lltype.nullptr(UTIMBUFP.TO)) @@ -346,85 +444,13 @@ if error == -1: raise OSError(rposix.get_errno(), "os_utime failed") else: - from pypy.rlib import rwin32 - from pypy.rpython.module.ll_os_stat import time_t_to_FILE_TIME - - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h'], - ) - - FILE_WRITE_ATTRIBUTES = platform.ConstantInteger( - 'FILE_WRITE_ATTRIBUTES') - OPEN_EXISTING = platform.ConstantInteger( - 'OPEN_EXISTING') - FILE_FLAG_BACKUP_SEMANTICS = platform.ConstantInteger( - 'FILE_FLAG_BACKUP_SEMANTICS') - globals().update(platform.configure(CConfig)) - - CreateFile = rffi.llexternal( - 'CreateFileA', - [rwin32.LPCSTR, rwin32.DWORD, rwin32.DWORD, - rwin32.LPSECURITY_ATTRIBUTES, rwin32.DWORD, rwin32.DWORD, - rwin32.HANDLE], - rwin32.HANDLE, - calling_conv='win') - - GetSystemTime = rffi.llexternal( - 'GetSystemTime', - [lltype.Ptr(rwin32.SYSTEMTIME)], - lltype.Void, - calling_conv='win') - - SystemTimeToFileTime = rffi.llexternal( - 'SystemTimeToFileTime', - [lltype.Ptr(rwin32.SYSTEMTIME), - lltype.Ptr(rwin32.FILETIME)], - rwin32.BOOL, - calling_conv='win') - - SetFileTime = rffi.llexternal( - 'SetFileTime', - [rwin32.HANDLE, - lltype.Ptr(rwin32.FILETIME), - lltype.Ptr(rwin32.FILETIME), - lltype.Ptr(rwin32.FILETIME)], - rwin32.BOOL, - calling_conv = 'win') + from pypy.rpython.module.ll_win32file import make_utime_impl + os_utime_llimpl = make_utime_impl(traits) - def os_utime_llimpl(path, tp): - hFile = CreateFile(path, - FILE_WRITE_ATTRIBUTES, 0, - None, OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS, 0) - if hFile == rwin32.INVALID_HANDLE_VALUE: - raise rwin32.lastWindowsError() - ctime = lltype.nullptr(rwin32.FILETIME) - atime = lltype.malloc(rwin32.FILETIME, flavor='raw') - mtime = lltype.malloc(rwin32.FILETIME, flavor='raw') - try: - if tp is None: - now = lltype.malloc(rwin32.SYSTEMTIME, flavor='raw') - try: - GetSystemTime(now) - if (not SystemTimeToFileTime(now, atime) or - not SystemTimeToFileTime(now, mtime)): - raise rwin32.lastWindowsError() - finally: - lltype.free(now, flavor='raw') - else: - actime, modtime = tp - time_t_to_FILE_TIME(actime, atime) - time_t_to_FILE_TIME(modtime, mtime) - if not SetFileTime(hFile, ctime, atime, mtime): - raise rwin32.lastWindowsError() - finally: - rwin32.CloseHandle(hFile) - lltype.free(atime, flavor='raw') - lltype.free(mtime, flavor='raw') - os_utime_llimpl._annspecialcase_ = 'specialize:argtype(1)' - - s_string = SomeString() + if traits.str is str: + s_string = SomeString() + else: + s_string = SomeUnicodeString() s_tuple_of_2_floats = SomeTuple([SomeFloat(), SomeFloat()]) def os_utime_normalize_args(s_path, s_times): @@ -445,12 +471,12 @@ else: raise Exception("os.utime() arg 2 must be None or a tuple of " "2 floats, got %s" % (s_times,)) + os_utime_normalize_args._default_signature_ = [traits.str, None] return extdef(os_utime_normalize_args, s_None, "ll_os.ll_os_utime", llimpl=os_utime_llimpl) - @registering(os.times) def register_os_times(self): if sys.platform.startswith('win'): @@ -687,22 +713,21 @@ def register_os_setsid(self): return self.extdef_for_os_function_returning_int('setsid') - @registering(os.open) - def register_os_open(self): - os_open = self.llexternal(underscore_on_windows+'open', - [rffi.CCHARP, rffi.INT, rffi.MODE_T], + @registering_str_unicode(os.open) + def register_os_open(self, traits): + os_open = self.llexternal(traits.posix_function_name('open'), + [traits.CCHARP, rffi.INT, rffi.MODE_T], rffi.INT) - def os_open_llimpl(path, flags, mode): result = rffi.cast(rffi.LONG, os_open(path, flags, mode)) if result == -1: raise OSError(rposix.get_errno(), "os_open failed") return result - def os_open_oofakeimpl(o_path, flags, mode): - return os.open(o_path._str, flags, mode) + def os_open_oofakeimpl(path, flags, mode): + return os.open(OOSupport.from_rstr(path), flags, mode) - return extdef([str, int, int], int, "ll_os.ll_os_open", + return extdef([traits.str, int, int], int, traits.ll_os_name('open'), llimpl=os_open_llimpl, oofakeimpl=os_open_oofakeimpl) # ------------------------------- os.read ------------------------------- @@ -862,10 +887,10 @@ llimpl=fdatasync_llimpl, export_name="ll_os.ll_os_fdatasync") - @registering(os.access) - def register_os_access(self): - os_access = self.llexternal(underscore_on_windows + 'access', - [rffi.CCHARP, rffi.INT], + @registering_str_unicode(os.access) + def register_os_access(self, traits): + os_access = self.llexternal(traits.posix_function_name('access'), + [traits.CCHARP, rffi.INT], rffi.INT) if sys.platform.startswith('win'): @@ -882,44 +907,22 @@ def os_access_oofakeimpl(path, mode): return os.access(OOSupport.from_rstr(path), mode) - return extdef([str, int], s_Bool, llimpl=access_llimpl, - export_name="ll_os.ll_os_access", + return extdef([traits.str, int], s_Bool, llimpl=access_llimpl, + export_name=traits.ll_os_name("access"), oofakeimpl=os_access_oofakeimpl) - @registering_if(posix, '_getfullpathname') - def register_posix__getfullpathname(self): - from pypy.rlib import rwin32 + @registering_str_unicode(getattr(posix, '_getfullpathname', None), + condition=sys.platform=='win32') + def register_posix__getfullpathname(self, traits): # this nt function is not exposed via os, but needed # to get a correct implementation of os.abspath - # XXX why do we ignore WINAPI conventions everywhere? - LPSTRP = rffi.CArrayPtr(rwin32.LPSTR) - # XXX unicode? - GetFullPathName = self.llexternal( - 'GetFullPathNameA', - [rwin32.LPCSTR, - rwin32.DWORD, - rwin32.LPSTR, - rffi.CArrayPtr(rwin32.LPSTR)], - rwin32.DWORD) - - def _getfullpathname_llimpl(lpFileName): - nBufferLength = rwin32.MAX_PATH + 1 - lpBuffer = lltype.malloc(rwin32.LPSTR.TO, nBufferLength, flavor='raw') - try: - res = GetFullPathName( - lpFileName, rffi.cast(rwin32.DWORD, nBufferLength), - lpBuffer, lltype.nullptr(LPSTRP.TO)) - if res == 0: - raise rwin32.lastWindowsError("_getfullpathname failed") - result = rffi.charp2str(lpBuffer) - return result - finally: - lltype.free(lpBuffer, flavor='raw') + from pypy.rpython.module.ll_win32file import make_getfullpathname_impl + getfullpathname_llimpl = make_getfullpathname_impl(traits) - return extdef([str], # a single argument which is a str - str, # returns a string - "ll_os.posix__getfullpathname", - llimpl=_getfullpathname_llimpl) + return extdef([traits.str], # a single argument which is a str + traits.str, # returns a string + traits.ll_os_name('_getfullpathname'), + llimpl=getfullpathname_llimpl) @registering(os.getcwd) def register_os_getcwd(self): @@ -953,71 +956,42 @@ "ll_os.ll_os_getcwd", llimpl=os_getcwd_llimpl, oofakeimpl=os_getcwd_oofakeimpl) - @registering(os.listdir) - def register_os_listdir(self): - # we need a different approach on Windows and on Posix - if sys.platform.startswith('win'): - from pypy.rlib import rwin32 - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h'] - ) - WIN32_FIND_DATA = platform.Struct('struct _WIN32_FIND_DATAA', - [('cFileName', lltype.FixedSizeArray(rffi.CHAR, 1))]) - ERROR_FILE_NOT_FOUND = platform.ConstantInteger( - 'ERROR_FILE_NOT_FOUND') - ERROR_NO_MORE_FILES = platform.ConstantInteger( - 'ERROR_NO_MORE_FILES') + @registering(os.getcwdu, condition=sys.platform=='win32') + def register_os_getcwdu(self): + os_wgetcwd = self.llexternal(underscore_on_windows + 'wgetcwd', + [rffi.CWCHARP, rffi.SIZE_T], + rffi.CWCHARP) - config = platform.configure(CConfig) - WIN32_FIND_DATA = config['WIN32_FIND_DATA'] - ERROR_FILE_NOT_FOUND = config['ERROR_FILE_NOT_FOUND'] - ERROR_NO_MORE_FILES = config['ERROR_NO_MORE_FILES'] - LPWIN32_FIND_DATA = lltype.Ptr(WIN32_FIND_DATA) - - FindFirstFile = self.llexternal('FindFirstFile', - [rwin32.LPCSTR, LPWIN32_FIND_DATA], - rwin32.HANDLE) - FindNextFile = self.llexternal('FindNextFile', - [rwin32.HANDLE, LPWIN32_FIND_DATA], - rwin32.BOOL) - FindClose = self.llexternal('FindClose', - [rwin32.HANDLE], - rwin32.BOOL) + def os_getcwd_llimpl(): + bufsize = 256 + while True: + buf = lltype.malloc(rffi.CWCHARP.TO, bufsize, flavor='raw') + res = os_wgetcwd(buf, rffi.cast(rffi.SIZE_T, bufsize)) + if res: + break # ok + error = rposix.get_errno() + lltype.free(buf, flavor='raw') + if error != errno.ERANGE: + raise OSError(error, "getcwd failed") + # else try again with a larger buffer, up to some sane limit + bufsize *= 4 + if bufsize > 1024*1024: # xxx hard-coded upper limit + raise OSError(error, "getcwd result too large") + result = rffi.wcharp2unicode(res) + lltype.free(buf, flavor='raw') + return result - def os_listdir_llimpl(path): - if path and path[-1] not in ('/', '\\', ':'): - path += '/' - path += '*.*' - filedata = lltype.malloc(WIN32_FIND_DATA, flavor='raw') - try: - result = [] - hFindFile = FindFirstFile(path, filedata) - if hFindFile == rwin32.INVALID_HANDLE_VALUE: - error = rwin32.GetLastError() - if error == ERROR_FILE_NOT_FOUND: - return result - else: - raise WindowsError(error, "FindFirstFile failed") - while True: - name = rffi.charp2str(rffi.cast(rffi.CCHARP, - filedata.c_cFileName)) - if name != "." and name != "..": # skip these - result.append(name) - if not FindNextFile(hFindFile, filedata): - break - # FindNextFile sets error to ERROR_NO_MORE_FILES if - # it got to the end of the directory - error = rwin32.GetLastError() - FindClose(hFindFile) - if error == ERROR_NO_MORE_FILES: - return result - else: - raise WindowsError(error, "FindNextFile failed") - finally: - lltype.free(filedata, flavor='raw') + return extdef([], unicode, + "ll_os.ll_os_wgetcwd", llimpl=os_getcwd_llimpl) + @registering_str_unicode(os.listdir) + def register_os_listdir(self, traits): + # we need a different approach on Windows and on Posix + if sys.platform.startswith('win'): + from pypy.rpython.module.ll_win32file import make_listdir_impl + os_listdir_llimpl = make_listdir_impl(traits) else: + assert traits.str is str compilation_info = ExternalCompilationInfo( includes = ['sys/types.h', 'dirent.h'] ) @@ -1057,9 +1031,9 @@ raise OSError(error, "os_readdir failed") return result - return extdef([str], # a single argument which is a str - [str], # returns a list of strings - "ll_os.ll_os_listdir", + return extdef([traits.str], # a single argument which is a str + [traits.str], # returns a list of strings + traits.ll_os_name('listdir'), llimpl=os_listdir_llimpl) @registering(os.pipe) @@ -1234,38 +1208,40 @@ return extdef([str], int, llimpl=system_llimpl, export_name="ll_os.ll_os_system") - @registering(os.unlink) - def register_os_unlink(self): - os_unlink = self.llexternal(underscore_on_windows+'unlink', [rffi.CCHARP], rffi.INT) + @registering_str_unicode(os.unlink) + def register_os_unlink(self, traits): + os_unlink = self.llexternal(traits.posix_function_name('unlink'), + [traits.CCHARP], rffi.INT) def unlink_llimpl(pathname): res = rffi.cast(lltype.Signed, os_unlink(pathname)) if res < 0: raise OSError(rposix.get_errno(), "os_unlink failed") - return extdef([str], s_None, llimpl=unlink_llimpl, - export_name="ll_os.ll_os_unlink") + return extdef([traits.str], s_None, llimpl=unlink_llimpl, + export_name=traits.ll_os_name('unlink')) - @registering(os.chdir) - def register_os_chdir(self): - os_chdir = self.llexternal(underscore_on_windows+'chdir', [rffi.CCHARP], rffi.INT) + @registering_str_unicode(os.chdir) + def register_os_chdir(self, traits): + os_chdir = self.llexternal(traits.posix_function_name('chdir'), + [traits.CCHARP], rffi.INT) def chdir_llimpl(path): res = rffi.cast(lltype.Signed, os_chdir(path)) if res < 0: raise OSError(rposix.get_errno(), "os_chdir failed") - return extdef([str], s_None, llimpl=chdir_llimpl, - export_name="ll_os.ll_os_chdir") + return extdef([traits.str], s_None, llimpl=chdir_llimpl, + export_name=traits.ll_os_name('chdir')) - @registering(os.mkdir) - def register_os_mkdir(self): + @registering_str_unicode(os.mkdir) + def register_os_mkdir(self, traits): if os.name == 'nt': ARG2 = [] # no 'mode' argument on Windows - just ignored else: ARG2 = [rffi.MODE_T] - os_mkdir = self.llexternal(underscore_on_windows+'mkdir', - [rffi.CCHARP]+ARG2, rffi.INT) + os_mkdir = self.llexternal(traits.posix_function_name('mkdir'), + [traits.CCHARP] + ARG2, rffi.INT) IGNORE_MODE = len(ARG2) == 0 def mkdir_llimpl(pathname, mode): @@ -1277,46 +1253,47 @@ if res < 0: raise OSError(rposix.get_errno(), "os_mkdir failed") - return extdef([str, int], s_None, llimpl=mkdir_llimpl, - export_name="ll_os.ll_os_mkdir") + return extdef([traits.str, int], s_None, llimpl=mkdir_llimpl, + export_name=traits.ll_os_name('mkdir')) - @registering(os.rmdir) - def register_os_rmdir(self): - os_rmdir = self.llexternal(underscore_on_windows+'rmdir', [rffi.CCHARP], rffi.INT) + @registering_str_unicode(os.rmdir) + def register_os_rmdir(self, traits): + os_rmdir = self.llexternal(traits.posix_function_name('rmdir'), + [traits.CCHARP], rffi.INT) def rmdir_llimpl(pathname): res = rffi.cast(lltype.Signed, os_rmdir(pathname)) if res < 0: raise OSError(rposix.get_errno(), "os_rmdir failed") - return extdef([str], s_None, llimpl=rmdir_llimpl, - export_name="ll_os.ll_os_rmdir") + return extdef([traits.str], s_None, llimpl=rmdir_llimpl, + export_name=traits.ll_os_name('rmdir')) - @registering(os.chmod) - def register_os_chmod(self): - os_chmod = self.llexternal(underscore_on_windows+'chmod', [rffi.CCHARP, rffi.MODE_T], - rffi.INT) + @registering_str_unicode(os.chmod) + def register_os_chmod(self, traits): + os_chmod = self.llexternal(traits.posix_function_name('chmod'), + [traits.CCHARP, rffi.MODE_T], rffi.INT) def chmod_llimpl(path, mode): res = rffi.cast(lltype.Signed, os_chmod(path, rffi.cast(rffi.MODE_T, mode))) if res < 0: raise OSError(rposix.get_errno(), "os_chmod failed") - return extdef([str, int], s_None, llimpl=chmod_llimpl, - export_name="ll_os.ll_os_chmod") + return extdef([traits.str, int], s_None, llimpl=chmod_llimpl, + export_name=traits.ll_os_name('chmod')) - @registering(os.rename) - def register_os_rename(self): - os_rename = self.llexternal('rename', [rffi.CCHARP, rffi.CCHARP], - rffi.INT) + @registering_str_unicode(os.rename) + def register_os_rename(self, traits): + os_rename = self.llexternal(traits.posix_function_name('rename'), + [traits.CCHARP, traits.CCHARP], rffi.INT) def rename_llimpl(oldpath, newpath): res = rffi.cast(lltype.Signed, os_rename(oldpath, newpath)) if res < 0: raise OSError(rposix.get_errno(), "os_rename failed") - return extdef([str, str], s_None, llimpl=rename_llimpl, - export_name="ll_os.ll_os_rename") + return extdef([traits.str, traits.str], s_None, llimpl=rename_llimpl, + export_name=traits.ll_os_name('rename')) @registering(os.umask) def register_os_umask(self): @@ -1425,17 +1402,17 @@ @registering(os.fstat) def register_os_fstat(self): from pypy.rpython.module import ll_os_stat - ll_os_stat.register_stat_variant('fstat') + return ll_os_stat.register_stat_variant('fstat', StringTraits()) - @registering(os.stat) - def register_os_stat(self): + @registering_str_unicode(os.stat) + def register_os_stat(self, traits): from pypy.rpython.module import ll_os_stat - ll_os_stat.register_stat_variant('stat') + return ll_os_stat.register_stat_variant('stat', traits) - @registering(os.lstat) - def register_os_lstat(self): + @registering_str_unicode(os.lstat) + def register_os_lstat(self, traits): from pypy.rpython.module import ll_os_stat - ll_os_stat.register_stat_variant('lstat') + return ll_os_stat.register_stat_variant('lstat', traits) # ------------------------------- os.W* --------------------------------- Modified: pypy/branch/kill-caninline/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/branch/kill-caninline/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/branch/kill-caninline/pypy/rpython/module/ll_os_stat.py Thu Aug 12 13:47:36 2010 @@ -5,13 +5,14 @@ import os, sys from pypy.annotation import model as annmodel from pypy.tool.pairtype import pairtype -from pypy.tool.sourcetools import func_with_new_name +from pypy.tool.sourcetools import func_with_new_name, func_renamer from pypy.rpython import extregistry -from pypy.rpython.extfunc import register_external +from pypy.rpython.extfunc import register_external, extdef from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.tool import rffi_platform as platform from pypy.rpython.lltypesystem.rtupletype import TUPLE_TYPE from pypy.rlib import rposix +from pypy.rlib.objectmodel import specialize from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rpython.annlowlevel import hlstr @@ -211,13 +212,27 @@ return make_stat_result(result) -def register_stat_variant(name): - if sys.platform.startswith('win'): - _functions = {'stat': '_stati64', - 'fstat': '_fstati64', - 'lstat': '_stati64'} # no lstat on Windows - c_func_name = _functions[name] - elif sys.platform.startswith('linux'): +def register_stat_variant(name, traits): + if name != 'fstat': + arg_is_path = True + s_arg = traits.str + ARG1 = traits.CCHARP + else: + arg_is_path = False + s_arg = int + ARG1 = rffi.INT + + if sys.platform == 'win32': + # See Win32 implementation below + posix_stat_llimpl = make_win32_stat_impl(name, traits) + + return extdef( + [s_arg], s_StatResult, traits.ll_os_name(name), + llimpl=posix_stat_llimpl) + + assert traits.str is str + + if sys.platform.startswith('linux'): # because we always use _FILE_OFFSET_BITS 64 - this helps things work that are not a c compiler _functions = {'stat': 'stat64', 'fstat': 'fstat64', @@ -226,22 +241,26 @@ else: c_func_name = name - arg_is_path = (name != 'fstat') + posix_mystat = rffi.llexternal(c_func_name, + [ARG1, STAT_STRUCT], rffi.INT, + compilation_info=compilation_info) + @func_renamer('os_%s_llimpl' % (name,)) def posix_stat_llimpl(arg): stresult = lltype.malloc(STAT_STRUCT.TO, flavor='raw') try: if arg_is_path: - arg = rffi.str2charp(arg) + arg = traits.str2charp(arg) error = rffi.cast(rffi.LONG, posix_mystat(arg, stresult)) if arg_is_path: - rffi.free_charp(arg) + traits.free_charp(arg) if error != 0: raise OSError(rposix.get_errno(), "os_?stat failed") return build_stat_result(stresult) finally: lltype.free(stresult, flavor='raw') + @func_renamer('os_%s_fake' % (name,)) def posix_fakeimpl(arg): if s_arg == str: arg = hlstr(arg) @@ -259,40 +278,17 @@ setattr(ll_tup, 'item%d' % i, val) return ll_tup - if arg_is_path: - s_arg = str - ARG1 = rffi.CCHARP - else: - s_arg = int - ARG1 = rffi.INT + return extdef( + [s_arg], s_StatResult, "ll_os.ll_os_%s" % (name,), + llimpl=posix_stat_llimpl, llfakeimpl=posix_fakeimpl) - if sys.platform != 'win32': - posix_mystat = rffi.llexternal(c_func_name, - [ARG1, STAT_STRUCT], rffi.INT, - compilation_info=compilation_info) - - register_external( - getattr(os, name), [s_arg], s_StatResult, - "ll_os.ll_os_%s" % (name,), - llimpl=func_with_new_name(posix_stat_llimpl, - 'os_%s_llimpl' % (name,)), - llfakeimpl=func_with_new_name(posix_fakeimpl, - 'os_%s_fake' % (name,)), - ) - else: - # See Win32 implementation below - register_external( - getattr(os, name), [s_arg], s_StatResult, - "ll_os.ll_os_%s" % (name,), - llimpl=func_with_new_name(globals()['win32_%s_llimpl' % (name,)], - 'os_%s_llimpl' % (name,)), - ) +def make_win32_stat_impl(name, traits): + from pypy.rlib import rwin32 + from pypy.rpython.module.ll_win32file import make_win32_traits + win32traits = make_win32_traits(traits) -# ____________________________________________________________ -if sys.platform == 'win32': # The CRT of Windows has a number of flaws wrt. its stat() implementation: - # - for when we implement subsecond resolution in RPython, time stamps - # would be restricted to second resolution + # - time stamps are restricted to second resolution # - file modification times suffer from forth-and-back conversions between # UTC and local time # Therefore, we implement our own stat, based on the Win32 API directly. @@ -302,122 +298,18 @@ assert len(STAT_FIELDS) == 10 # no extra fields on Windows - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h', 'winbase.h', 'sys/stat.h'], - ) - - GetFileExInfoStandard = platform.ConstantInteger( - 'GetFileExInfoStandard') - FILE_ATTRIBUTE_DIRECTORY = platform.ConstantInteger( - 'FILE_ATTRIBUTE_DIRECTORY') - FILE_ATTRIBUTE_READONLY = platform.ConstantInteger( - 'FILE_ATTRIBUTE_READONLY') - ERROR_SHARING_VIOLATION = platform.ConstantInteger( - 'ERROR_SHARING_VIOLATION') - _S_IFDIR = platform.ConstantInteger('_S_IFDIR') - _S_IFREG = platform.ConstantInteger('_S_IFREG') - _S_IFCHR = platform.ConstantInteger('_S_IFCHR') - _S_IFIFO = platform.ConstantInteger('_S_IFIFO') - FILE_TYPE_UNKNOWN = platform.ConstantInteger('FILE_TYPE_UNKNOWN') - FILE_TYPE_CHAR = platform.ConstantInteger('FILE_TYPE_CHAR') - FILE_TYPE_PIPE = platform.ConstantInteger('FILE_TYPE_PIPE') - - WIN32_FILE_ATTRIBUTE_DATA = platform.Struct( - 'WIN32_FILE_ATTRIBUTE_DATA', - [('dwFileAttributes', rwin32.DWORD), - ('nFileSizeHigh', rwin32.DWORD), - ('nFileSizeLow', rwin32.DWORD), - ('ftCreationTime', rwin32.FILETIME), - ('ftLastAccessTime', rwin32.FILETIME), - ('ftLastWriteTime', rwin32.FILETIME)]) - - BY_HANDLE_FILE_INFORMATION = platform.Struct( - 'BY_HANDLE_FILE_INFORMATION', - [('dwFileAttributes', rwin32.DWORD), - ('nFileSizeHigh', rwin32.DWORD), - ('nFileSizeLow', rwin32.DWORD), - ('nNumberOfLinks', rwin32.DWORD), - ('nFileIndexHigh', rwin32.DWORD), - ('nFileIndexLow', rwin32.DWORD), - ('ftCreationTime', rwin32.FILETIME), - ('ftLastAccessTime', rwin32.FILETIME), - ('ftLastWriteTime', rwin32.FILETIME)]) - - WIN32_FIND_DATA = platform.Struct( - 'WIN32_FIND_DATAA', - # Only interesting fields - [('dwFileAttributes', rwin32.DWORD), - ('nFileSizeHigh', rwin32.DWORD), - ('nFileSizeLow', rwin32.DWORD), - ('ftCreationTime', rwin32.FILETIME), - ('ftLastAccessTime', rwin32.FILETIME), - ('ftLastWriteTime', rwin32.FILETIME)]) - - globals().update(platform.configure(CConfig)) - GET_FILEEX_INFO_LEVELS = rffi.ULONG # an enumeration - - GetFileAttributesEx = rffi.llexternal( - 'GetFileAttributesExA', - [rffi.CCHARP, GET_FILEEX_INFO_LEVELS, - lltype.Ptr(WIN32_FILE_ATTRIBUTE_DATA)], - rwin32.BOOL, - calling_conv='win') - - GetFileInformationByHandle = rffi.llexternal( - 'GetFileInformationByHandle', - [rwin32.HANDLE, lltype.Ptr(BY_HANDLE_FILE_INFORMATION)], - rwin32.BOOL, - calling_conv='win') - - GetFileType = rffi.llexternal( - 'GetFileType', - [rwin32.HANDLE], - rwin32.DWORD, - calling_conv='win') - - FindFirstFile = rffi.llexternal( - 'FindFirstFileA', - [rffi.CCHARP, lltype.Ptr(WIN32_FIND_DATA)], - rwin32.HANDLE, - calling_conv='win') - - FindClose = rffi.llexternal( - 'FindClose', - [rwin32.HANDLE], - rwin32.BOOL, - calling_conv='win') - def attributes_to_mode(attributes): m = 0 - if attributes & FILE_ATTRIBUTE_DIRECTORY: - m |= _S_IFDIR | 0111 # IFEXEC for user,group,other + if attributes & win32traits.FILE_ATTRIBUTE_DIRECTORY: + m |= win32traits._S_IFDIR | 0111 # IFEXEC for user,group,other else: - m |= _S_IFREG - if attributes & FILE_ATTRIBUTE_READONLY: + m |= win32traits._S_IFREG + if attributes & win32traits.FILE_ATTRIBUTE_READONLY: m |= 0444 else: m |= 0666 return m - def make_longlong(high, low): - return (lltype.r_longlong(high) << 32) + lltype.r_longlong(low) - - # Seconds between 1.1.1601 and 1.1.1970 - secs_between_epochs = lltype.r_longlong(11644473600) - - def FILE_TIME_to_time_t_nsec(filetime): - ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime) - # FILETIME is in units of 100 nsec - nsec = (ft % 10000000) * 100 - time = (ft / 10000000) - secs_between_epochs - return time, nsec - - def time_t_to_FILE_TIME(time, filetime): - ft = lltype.r_longlong((time + secs_between_epochs) * 10000000) - filetime.c_dwHighDateTime = lltype.r_uint(ft >> 32) - filetime.c_dwLowDateTime = lltype.r_uint(ft & ((1 << 32) - 1)) - def attribute_data_to_stat(info): st_mode = attributes_to_mode(info.c_dwFileAttributes) st_size = make_longlong(info.c_nFileSizeHigh, info.c_nFileSizeLow) @@ -456,65 +348,94 @@ return make_stat_result(result) def attributes_from_dir(l_path, data): - filedata = lltype.malloc(WIN32_FIND_DATA, flavor='raw') - hFindFile = FindFirstFile(l_path, filedata) - if hFindFile == rwin32.INVALID_HANDLE_VALUE: - return 0 - FindClose(hFindFile) - data.c_dwFileAttributes = filedata.c_dwFileAttributes - rffi.structcopy(data.c_ftCreationTime, filedata.c_ftCreationTime) - rffi.structcopy(data.c_ftLastAccessTime, filedata.c_ftLastAccessTime) - rffi.structcopy(data.c_ftLastWriteTime, filedata.c_ftLastWriteTime) - data.c_nFileSizeHigh = filedata.c_nFileSizeHigh - data.c_nFileSizeLow = filedata.c_nFileSizeLow - return 1 + filedata = lltype.malloc(win32traits.WIN32_FIND_DATA, flavor='raw') + try: + hFindFile = win32traits.FindFirstFile(l_path, filedata) + if hFindFile == rwin32.INVALID_HANDLE_VALUE: + return 0 + win32traits.FindClose(hFindFile) + data.c_dwFileAttributes = filedata.c_dwFileAttributes + rffi.structcopy(data.c_ftCreationTime, filedata.c_ftCreationTime) + rffi.structcopy(data.c_ftLastAccessTime, filedata.c_ftLastAccessTime) + rffi.structcopy(data.c_ftLastWriteTime, filedata.c_ftLastWriteTime) + data.c_nFileSizeHigh = filedata.c_nFileSizeHigh + data.c_nFileSizeLow = filedata.c_nFileSizeLow + return 1 + finally: + lltype.free(filedata, flavor='raw') def win32_stat_llimpl(path): - data = lltype.malloc(WIN32_FILE_ATTRIBUTE_DATA, flavor='raw') + data = lltype.malloc(win32traits.WIN32_FILE_ATTRIBUTE_DATA, flavor='raw') try: - l_path = rffi.str2charp(path) - res = GetFileAttributesEx(l_path, GetFileExInfoStandard, data) + l_path = traits.str2charp(path) + res = win32traits.GetFileAttributesEx(l_path, win32traits.GetFileExInfoStandard, data) errcode = rwin32.GetLastError() if res == 0: - if errcode == ERROR_SHARING_VIOLATION: + if errcode == win32traits.ERROR_SHARING_VIOLATION: res = attributes_from_dir(l_path, data) errcode = rwin32.GetLastError() - rffi.free_charp(l_path) + traits.free_charp(l_path) if res == 0: raise WindowsError(errcode, "os_stat failed") return attribute_data_to_stat(data) finally: lltype.free(data, flavor='raw') - win32_lstat_llimpl = win32_stat_llimpl def win32_fstat_llimpl(fd): handle = rwin32._get_osfhandle(fd) - filetype = GetFileType(handle) - if filetype == FILE_TYPE_CHAR: + filetype = win32traits.GetFileType(handle) + if filetype == win32traits.FILE_TYPE_CHAR: # console or LPT device - return make_stat_result((_S_IFCHR, + return make_stat_result((win32traits._S_IFCHR, 0, 0, 0, 0, 0, 0, 0, 0, 0)) - elif filetype == FILE_TYPE_PIPE: + elif filetype == win32traits.FILE_TYPE_PIPE: # socket or named pipe - return make_stat_result((_S_IFIFO, + return make_stat_result((win32traits._S_IFIFO, 0, 0, 0, 0, 0, 0, 0, 0, 0)) - elif filetype == FILE_TYPE_UNKNOWN: + elif filetype == win32traits.FILE_TYPE_UNKNOWN: error = rwin32.GetLastError() if error != 0: raise WindowsError(error, "os_fstat failed") # else: unknown but valid file # normal disk file (FILE_TYPE_DISK) - info = lltype.malloc(BY_HANDLE_FILE_INFORMATION, flavor='raw', - zero=True) + info = lltype.malloc(win32traits.BY_HANDLE_FILE_INFORMATION, + flavor='raw', zero=True) try: - res = GetFileInformationByHandle(handle, info) + res = win32traits.GetFileInformationByHandle(handle, info) if res == 0: raise WindowsError(rwin32.GetLastError(), "os_fstat failed") return by_handle_info_to_stat(info) finally: lltype.free(info, flavor='raw') + if name == 'fstat': + return win32_fstat_llimpl + else: + return win32_stat_llimpl + + +#__________________________________________________ +# Helper functions for win32 + +def make_longlong(high, low): + return (lltype.r_longlong(high) << 32) + lltype.r_longlong(low) + +# Seconds between 1.1.1601 and 1.1.1970 +secs_between_epochs = lltype.r_longlong(11644473600) + +def FILE_TIME_to_time_t_nsec(filetime): + ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime) + # FILETIME is in units of 100 nsec + nsec = (ft % 10000000) * 100 + time = (ft / 10000000) - secs_between_epochs + return time, nsec + +def time_t_to_FILE_TIME(time, filetime): + ft = lltype.r_longlong((time + secs_between_epochs) * 10000000) + filetime.c_dwHighDateTime = lltype.r_uint(ft >> 32) + filetime.c_dwLowDateTime = lltype.r_uint(ft & ((1 << 32) - 1)) + Modified: pypy/branch/kill-caninline/pypy/rpython/module/test/test_ll_os_stat.py ============================================================================== --- pypy/branch/kill-caninline/pypy/rpython/module/test/test_ll_os_stat.py (original) +++ pypy/branch/kill-caninline/pypy/rpython/module/test/test_ll_os_stat.py Thu Aug 12 13:47:36 2010 @@ -1,4 +1,4 @@ -from pypy.rpython.module import ll_os_stat +from pypy.rpython.module import ll_os_stat, ll_os import sys, os import py @@ -8,14 +8,18 @@ py.test.skip("win32 specific tests") def test_stat(self): - stat = ll_os_stat.win32_stat_llimpl + stat = ll_os_stat.make_win32_stat_impl('stat', ll_os.StringTraits()) + wstat = ll_os_stat.make_win32_stat_impl('stat', ll_os.UnicodeTraits()) def check(f): - assert stat(f).st_mtime == os.stat(f).st_mtime + expected = os.stat(f).st_mtime + assert stat(f).st_mtime == expected + assert wstat(unicode(f)).st_mtime == expected check('c:/') check('c:/temp') check('c:/pagefile.sys') def test_fstat(self): - stat = ll_os_stat.win32_fstat_llimpl(0) # stdout + fstat = ll_os_stat.make_win32_stat_impl('fstat', ll_os.StringTraits()) + stat = fstat(0) # stdout assert stat.st_mode != 0 Modified: pypy/branch/kill-caninline/pypy/rpython/rstr.py ============================================================================== --- pypy/branch/kill-caninline/pypy/rpython/rstr.py (original) +++ pypy/branch/kill-caninline/pypy/rpython/rstr.py Thu Aug 12 13:47:36 2010 @@ -288,7 +288,11 @@ if not hop.args_s[1].is_constant(): raise TyperError("encoding must be constant") encoding = hop.args_s[1].const - v_self = hop.inputarg(self.repr, 0) + if encoding == "ascii" and self.lowleveltype == UniChar: + expect = UniChar # only for unichar.encode('ascii') + else: + expect = self.repr # must be a regular unicode string + v_self = hop.inputarg(expect, 0) hop.exception_is_here() if encoding == "ascii": return hop.gendirectcall(self.ll_str, v_self) @@ -415,7 +419,17 @@ sourcevars.append((v_item, r_arg)) return r_str.ll.do_stringformat(hop, sourcevars) - + + +class __extend__(AbstractCharRepr): + def ll_str(self, ch): + return self.ll.ll_chr2str(ch) + +class __extend__(AbstractUniCharRepr): + def ll_str(self, ch): + # xxx suboptimal, maybe + return str(unicode(ch)) + class __extend__(AbstractCharRepr, AbstractUniCharRepr): @@ -433,9 +447,6 @@ get_ll_fasthash_function = get_ll_hash_function - def ll_str(self, ch): - return self.ll.ll_chr2str(ch) - def rtype_len(_, hop): return hop.inputconst(Signed, 1) Modified: pypy/branch/kill-caninline/pypy/rpython/test/test_extfunc.py ============================================================================== --- pypy/branch/kill-caninline/pypy/rpython/test/test_extfunc.py (original) +++ pypy/branch/kill-caninline/pypy/rpython/test/test_extfunc.py Thu Aug 12 13:47:36 2010 @@ -6,150 +6,164 @@ from pypy.annotation.policy import AnnotatorPolicy from pypy.rpython.test.test_llinterp import interpret -def b(x): - return eval("x+40") +class TestExtFuncEntry: -class BTestFuncEntry(ExtFuncEntry): - _about_ = b - name = 'b' - signature_args = [annmodel.SomeInteger()] - signature_result = annmodel.SomeInteger() - -def test_annotation_b(): - def f(): - return b(1) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeInteger) - -def test_rtyping_b(): - def f(): - return b(2) - - res = interpret(f, []) - assert res == 42 - -def c(y, x): - yyy - -class CTestFuncEntry(ExtFuncEntry): - _about_ = c - name = 'ccc' - signature_args = [annmodel.SomeInteger()] * 2 - signature_result = annmodel.SomeInteger() - - def lltypeimpl(y, x): - return y + x - lltypeimpl = staticmethod(lltypeimpl) - -def test_interp_c(): - def f(): - return c(3, 4) - - res = interpret(f, []) - assert res == 7 - -def d(y): - return eval("y()") - -class DTestFuncEntry(ExtFuncEntry): - _about_ = d - name = 'd' - signature_args = [annmodel.SomeGenericCallable(args=[], result= - annmodel.SomeFloat())] - signature_result = annmodel.SomeFloat() - -def test_callback(): - def callback(): - return 2.5 - - def f(): - return d(callback) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeFloat) - assert a.translator._graphof(callback) - -def dd(): - pass - -register_external(dd, [int], int) - -def test_register_external_signature(): - def f(): - return dd(3) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeInteger) - - -def function_with_tuple_arg(): - """ - Dummy function which is declared via register_external to take a tuple as - an argument so that register_external's behavior for tuple-taking functions - can be verified. - """ -register_external(function_with_tuple_arg, [(int,)], int) - -def test_register_external_tuple_args(): - """ - Verify the annotation of a registered external function which takes a tuple - argument. - """ - def f(): - return function_with_tuple_arg((1,)) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - - # Not a very good assertion, but at least it means _something_ happened. - assert isinstance(s, annmodel.SomeInteger) - -def function_with_list(): - pass -register_external(function_with_list, [[int]], int) - -def function_returning_list(): - pass -register_external(function_returning_list, [], [int]) - -def test_register_external_return_goes_back(): - """ - Check whether it works to pass the same list from one external - fun to another - [bookkeeper and list joining issues] - """ - def f(): - return function_with_list(function_returning_list()) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeInteger) - -def function_withspecialcase(arg): - return repr(arg) -register_external(function_withspecialcase, args=None, result=str) - -def test_register_external_specialcase(): - def f(): - x = function_withspecialcase - return x(33) + x("aaa") + x([]) + "\n" - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeString) + def test_basic(self): + """ + A ExtFuncEntry provides an annotation for a function, no need to flow + its graph. + """ + def b(x): + "NOT_RPYTHON" + return eval("x+40") + + class BTestFuncEntry(ExtFuncEntry): + _about_ = b + name = 'b' + signature_args = [annmodel.SomeInteger()] + signature_result = annmodel.SomeInteger() + + def f(): + return b(2) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeInteger) + + res = interpret(f, []) + assert res == 42 + + def test_lltypeimpl(self): + """ + interpret() calls lltypeimpl instead of of the function/ + """ + def c(y, x): + yyy + + class CTestFuncEntry(ExtFuncEntry): + _about_ = c + name = 'ccc' + signature_args = [annmodel.SomeInteger()] * 2 + signature_result = annmodel.SomeInteger() + + def lltypeimpl(y, x): + return y + x + lltypeimpl = staticmethod(lltypeimpl) + + def f(): + return c(3, 4) + + res = interpret(f, []) + assert res == 7 + + def test_callback(self): + """ + Verify annotation when a callback function is in the arguments list. + """ + def d(y): + return eval("y()") + + class DTestFuncEntry(ExtFuncEntry): + _about_ = d + name = 'd' + signature_args = [annmodel.SomeGenericCallable(args=[], result= + annmodel.SomeFloat())] + signature_result = annmodel.SomeFloat() + + def callback(): + return 2.5 + + def f(): + return d(callback) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeFloat) + assert a.translator._graphof(callback) + + def test_register_external_signature(self): + """ + Test the standard interface for external functions. + """ + def dd(): + pass + register_external(dd, [int], int) + + def f(): + return dd(3) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeInteger) + + def test_register_external_tuple_args(self): + """ + Verify the annotation of a registered external function which takes a + tuple argument. + """ + + def function_with_tuple_arg(): + """ + Dummy function which is declared via register_external to take a + tuple as an argument so that register_external's behavior for + tuple-taking functions can be verified. + """ + register_external(function_with_tuple_arg, [(int,)], int) + + def f(): + return function_with_tuple_arg((1,)) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + + # Not a very good assertion, but at least it means _something_ happened. + assert isinstance(s, annmodel.SomeInteger) + + def test_register_external_return_goes_back(self): + """ + Check whether it works to pass the same list from one external + fun to another + [bookkeeper and list joining issues] + """ + def function_with_list(): + pass + register_external(function_with_list, [[int]], int) + + def function_returning_list(): + pass + register_external(function_returning_list, [], [int]) + + def f(): + return function_with_list(function_returning_list()) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeInteger) + + def test_register_external_specialcase(self): + """ + When args=None, the external function accepts any arguments unmodified. + """ + def function_withspecialcase(arg): + return repr(arg) + register_external(function_withspecialcase, args=None, result=str) + + def f(): + x = function_withspecialcase + return x(33) + x("aaa") + x([]) + "\n" + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeString) Modified: pypy/branch/kill-caninline/pypy/rpython/test/test_rstr.py ============================================================================== --- pypy/branch/kill-caninline/pypy/rpython/test/test_rstr.py (original) +++ pypy/branch/kill-caninline/pypy/rpython/test/test_rstr.py Thu Aug 12 13:47:36 2010 @@ -863,6 +863,24 @@ res = self.interpret(f, [1]) assert self.ll_to_string(res) == "hello" + def test_str_unichar(self): + def f(i): + c = u"abc" + return str(c[i])[0] + assert self.interpret(f, [1]) == "b" + + def test_encode_char(self): + def f(i): + c = u"abc" + return c[i].encode("ascii") + assert self.ll_to_string(self.interpret(f, [0])) == "a" + + def test_encode_char_latin1(self): + def f(i): + c = u"abc" + return c[i].encode("latin-1") + assert self.ll_to_string(self.interpret(f, [0])) == "a" + def FIXME_test_str_to_pystringobj(): def f(n): if n >= 0: Modified: pypy/branch/kill-caninline/pypy/rpython/tool/rfficache.py ============================================================================== --- pypy/branch/kill-caninline/pypy/rpython/tool/rfficache.py (original) +++ pypy/branch/kill-caninline/pypy/rpython/tool/rfficache.py Thu Aug 12 13:47:36 2010 @@ -29,7 +29,7 @@ } ''' % (include_string, add_source, str(question))) c_file = udir.join("gcctest.c") - c_file.write(c_source) + c_file.write(str(c_source) + '\n') eci = ExternalCompilationInfo() return build_executable_cache([c_file], eci) Modified: pypy/branch/kill-caninline/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/kill-caninline/pypy/translator/c/genc.py (original) +++ pypy/branch/kill-caninline/pypy/translator/c/genc.py Thu Aug 12 13:47:36 2010 @@ -1,7 +1,6 @@ import autopath import py import sys, os -from pypy.translator.c.node import PyObjectNode, FuncNode from pypy.translator.c.database import LowLevelDatabase from pypy.translator.c.extfunc import pre_include_code_lines from pypy.translator.llsupport.wrapper import new_wrapper @@ -196,7 +195,7 @@ all = [] for node in self.db.globalcontainers(): - eci = getattr(node, 'compilation_info', None) + eci = node.compilation_info() if eci: all.append(eci) self.merge_eci(*all) @@ -222,7 +221,7 @@ graphs = db.all_graphs() db.gctransformer.prepare_inline_helpers(graphs) for node in db.containerlist: - if isinstance(node, FuncNode): + if hasattr(node, 'funcgens'): for funcgen in node.funcgens: funcgen.patch_graph(copy_graph=False) return db Modified: pypy/branch/kill-caninline/pypy/translator/c/node.py ============================================================================== --- pypy/branch/kill-caninline/pypy/translator/c/node.py (original) +++ pypy/branch/kill-caninline/pypy/translator/c/node.py Thu Aug 12 13:47:36 2010 @@ -466,15 +466,15 @@ class ContainerNode(object): - if USESLOTS: - __slots__ = """db T obj + if USESLOTS: # keep the number of slots down! + __slots__ = """db obj typename implementationtypename - name ptrname compilation_info + name ptrname globalcontainer""".split() + eci_name = '_compilation_info' def __init__(self, db, T, obj): self.db = db - self.T = T self.obj = obj #self.dependencies = {} self.typename = db.gettype(T) #, who_asks=self) @@ -489,16 +489,22 @@ else: self.globalcontainer = False parentnode = db.getcontainernode(parent) - defnode = db.gettypedefnode(parentnode.T) + defnode = db.gettypedefnode(parentnode.getTYPE()) self.name = defnode.access_expr(parentnode.name, parentindex) if self.typename != self.implementationtypename: if db.gettypedefnode(T).extra_union_for_varlength: self.name += '.b' - self.compilation_info = getattr(obj, '_compilation_info', None) self.ptrname = '(&%s)' % self.name + def getTYPE(self): + return typeOf(self.obj) + def is_thread_local(self): - return hasattr(self.T, "_hints") and self.T._hints.get('thread_local') + T = self.getTYPE() + return hasattr(T, "_hints") and T._hints.get('thread_local') + + def compilation_info(self): + return getattr(self.obj, self.eci_name, None) def get_declaration(self): if self.name[-2:] == '.b': @@ -546,27 +552,31 @@ __slots__ = () def basename(self): - return self.T._name + T = self.getTYPE() + return T._name def enum_dependencies(self): - for name in self.T._names: + T = self.getTYPE() + for name in T._names: yield getattr(self.obj, name) def getlength(self): - if self.T._arrayfld is None: + T = self.getTYPE() + if T._arrayfld is None: return 1 else: - array = getattr(self.obj, self.T._arrayfld) + array = getattr(self.obj, T._arrayfld) return len(array.items) def initializationexpr(self, decoration=''): + T = self.getTYPE() is_empty = True yield '{' - defnode = self.db.gettypedefnode(self.T) + defnode = self.db.gettypedefnode(T) data = [] - if needs_gcheader(self.T): + if needs_gcheader(T): gc_init = self.db.gcpolicy.struct_gcheader_initdata(self) data.append(('gcheader', gc_init)) @@ -578,16 +588,16 @@ # '.fieldname = value'. But here we don't know which of the # fields need initialization, so XXX we pick the first one # arbitrarily. - if hasattr(self.T, "_hints") and self.T._hints.get('union'): + if hasattr(T, "_hints") and T._hints.get('union'): data = data[0:1] - if 'get_padding_drop' in self.T._hints: + if 'get_padding_drop' in T._hints: d = {} for name, _ in data: - T = defnode.c_struct_field_type(name) - typename = self.db.gettype(T) + T1 = defnode.c_struct_field_type(name) + typename = self.db.gettype(T1) d[name] = cdecl(typename, '') - padding_drop = self.T._hints['get_padding_drop'](d) + padding_drop = T._hints['get_padding_drop'](d) else: padding_drop = [] @@ -617,9 +627,10 @@ return 'struct _hashT_%s @' % self.name def forward_declaration(self): + T = self.getTYPE() assert self.typename == self.implementationtypename # no array part hash_typename = self.get_hash_typename() - hash_offset = self.db.gctransformer.get_hash_offset(self.T) + hash_offset = self.db.gctransformer.get_hash_offset(T) yield '%s {' % cdecl(hash_typename, '') yield '\tunion {' yield '\t\t%s;' % cdecl(self.implementationtypename, 'head') @@ -671,22 +682,23 @@ return len(self.obj.items) def initializationexpr(self, decoration=''): - defnode = self.db.gettypedefnode(self.T) + T = self.getTYPE() + defnode = self.db.gettypedefnode(T) yield '{' - if needs_gcheader(self.T): + if needs_gcheader(T): gc_init = self.db.gcpolicy.array_gcheader_initdata(self) lines = generic_initializationexpr(self.db, gc_init, 'gcheader', '%sgcheader' % (decoration,)) for line in lines: yield line - if self.T._hints.get('nolength', False): + if T._hints.get('nolength', False): length = '' else: length = '%d, ' % len(self.obj.items) - if self.T.OF is Void or len(self.obj.items) == 0: + if T.OF is Void or len(self.obj.items) == 0: yield '\t%s' % length.rstrip(', ') yield '}' - elif self.T.OF == Char: + elif T.OF == Char: if len(self.obj.items) and self.obj.items[0] is None: s = ''.join([self.obj.getitem(i) for i in range(len(self.obj.items))]) else: @@ -694,7 +706,7 @@ yield '\t%s%s' % (length, c_char_array_constant(s)) yield '}' else: - barebone = barebonearray(self.T) + barebone = barebonearray(T) if not barebone: yield '\t%s{' % length for j in range(len(self.obj.items)): @@ -722,7 +734,8 @@ self.ptrname = self.name def basename(self): - return self.T._name + T = self.getTYPE() + return T._name def enum_dependencies(self): for i in range(self.obj.getlength()): @@ -732,11 +745,12 @@ return 1 # not variable-sized! def initializationexpr(self, decoration=''): + T = self.getTYPE() assert self.typename == self.implementationtypename # not var-sized is_empty = True yield '{' # _names == ['item0', 'item1', ...] - for j, name in enumerate(self.T._names): + for j, name in enumerate(T._names): value = getattr(self.obj, name) lines = generic_initializationexpr(self.db, value, '%s[%d]' % (self.name, j), @@ -777,6 +791,7 @@ class FuncNode(ContainerNode): nodekind = 'func' + eci_name = 'compilation_info' # there not so many node of this kind, slots should not # be necessary @@ -794,7 +809,6 @@ else: self.name = (forcename or db.namespace.uniquename('g_' + self.basename())) - self.compilation_info = getattr(obj, 'compilation_info', None) self.make_funcgens() #self.dependencies = {} self.typename = db.gettype(T) #, who_asks=self) @@ -939,18 +953,20 @@ return [] def initializationexpr(self, decoration=''): - yield 'RPyOpaque_INITEXPR_%s' % (self.T.tag,) + T = self.getTYPE() + yield 'RPyOpaque_INITEXPR_%s' % (T.tag,) def startupcode(self): + T = self.getTYPE() args = [self.ptrname] # XXX how to make this code more generic? - if self.T.tag == 'ThreadLock': + if T.tag == 'ThreadLock': lock = self.obj.externalobj if lock.locked(): args.append('1') else: args.append('0') - yield 'RPyOpaque_SETUP_%s(%s);' % (self.T.tag, ', '.join(args)) + yield 'RPyOpaque_SETUP_%s(%s);' % (T.tag, ', '.join(args)) def opaquenode_factory(db, T, obj): Modified: pypy/branch/kill-caninline/pypy/translator/c/test/test_typed.py ============================================================================== --- pypy/branch/kill-caninline/pypy/translator/c/test/test_typed.py (original) +++ pypy/branch/kill-caninline/pypy/translator/c/test/test_typed.py Thu Aug 12 13:47:36 2010 @@ -789,3 +789,37 @@ return u'hello' + unichr(i) f = self.getcompiled(func, [int]) assert f(0x1234) == u'hello\u1234' + + def test_ovfcheck_float_to_int(self): + from pypy.rlib.rarithmetic import ovfcheck_float_to_int + + def func(fl): + try: + return ovfcheck_float_to_int(fl) + except OverflowError: + return -666 + f = self.getcompiled(func, [float]) + assert f(-123.0) == -123 + + for frac in [0.0, 0.01, 0.99]: + # strange things happening for float to int on 64 bit: + # int(float(i)) != i because of rounding issues + x = sys.maxint + while int(x + frac) > sys.maxint: + x -= 1 + assert f(x + frac) == int(x + frac) + + x = sys.maxint + while int(x - frac) <= sys.maxint: + x += 1 + assert f(x - frac) == -666 + + x = -sys.maxint-1 + while int(x - frac) < -sys.maxint-1: + x += 1 + assert f(x - frac) == int(x - frac) + + x = -sys.maxint-1 + while int(x + frac) >= -sys.maxint-1: + x -= 1 + assert f(x + frac) == -666 Modified: pypy/branch/kill-caninline/pypy/translator/goal/app_main.py ============================================================================== --- pypy/branch/kill-caninline/pypy/translator/goal/app_main.py (original) +++ pypy/branch/kill-caninline/pypy/translator/goal/app_main.py Thu Aug 12 13:47:36 2010 @@ -223,7 +223,6 @@ path = os.getenv('PYTHONPATH') if path: newpath = path.split(os.pathsep) + newpath - newpath.insert(0, '') # remove duplicates _seen = {} del sys.path[:] @@ -327,6 +326,10 @@ except: print >> sys.stderr, "'import site' failed" + # update sys.path *after* loading site.py, in case there is a + # "site.py" file in the script's directory. + sys.path.insert(0, '') + if warnoptions: sys.warnoptions.append(warnoptions) from warnings import _processoptions Modified: pypy/branch/kill-caninline/pypy/translator/goal/test2/test_app_main.py ============================================================================== --- pypy/branch/kill-caninline/pypy/translator/goal/test2/test_app_main.py (original) +++ pypy/branch/kill-caninline/pypy/translator/goal/test2/test_app_main.py Thu Aug 12 13:47:36 2010 @@ -5,6 +5,7 @@ import sys, os, re import autopath from pypy.tool.udir import udir +from contextlib import contextmanager banner = sys.version.splitlines()[0] @@ -326,8 +327,9 @@ class TestNonInteractive: def run(self, cmdline, senddata='', expect_prompt=False, - expect_banner=False): - cmdline = '%s "%s" %s' % (sys.executable, app_main, cmdline) + expect_banner=False, python_flags=''): + cmdline = '%s %s "%s" %s' % (sys.executable, python_flags, + app_main, cmdline) print 'POPEN:', cmdline child_in, child_out_err = os.popen4(cmdline) child_in.write(senddata) @@ -449,6 +451,43 @@ assert data == '\x00(STDOUT)\n\x00' # from stdout child_out_err.close() + def test_proper_sys_path(self, tmpdir): + + @contextmanager + def chdir_and_unset_pythonpath(new_cwd): + old_cwd = new_cwd.chdir() + old_pythonpath = os.getenv('PYTHONPATH') + os.unsetenv('PYTHONPATH') + try: + yield + finally: + old_cwd.chdir() + os.putenv('PYTHONPATH', old_pythonpath) + + tmpdir.join('site.py').write('print "SHOULD NOT RUN"') + runme_py = tmpdir.join('runme.py') + runme_py.write('print "some text"') + + cmdline = str(runme_py) + + with chdir_and_unset_pythonpath(tmpdir): + data = self.run(cmdline, python_flags='-S') + + assert data == "some text\n" + + runme2_py = tmpdir.mkdir('otherpath').join('runme2.py') + runme2_py.write('print "some new text"\n' + 'import sys\n' + 'print sys.path\n') + + cmdline2 = str(runme2_py) + + with chdir_and_unset_pythonpath(tmpdir): + data = self.run(cmdline2, python_flags='-S') + + assert data.startswith("some new text\n") + assert repr(str(tmpdir.join('otherpath'))) in data + class AppTestAppMain: Modified: pypy/branch/kill-caninline/pypy/translator/platform/__init__.py ============================================================================== --- pypy/branch/kill-caninline/pypy/translator/platform/__init__.py (original) +++ pypy/branch/kill-caninline/pypy/translator/platform/__init__.py Thu Aug 12 13:47:36 2010 @@ -161,7 +161,7 @@ return (library_dirs + self.link_flags + export_flags + link_files + list(eci.link_extra) + libraries) - def _exportsymbols_link_flags(self, eci): + def _exportsymbols_link_flags(self, eci, relto=None): if eci.export_symbols: raise ValueError("This platform does not support export symbols") return [] Modified: pypy/branch/kill-caninline/pypy/translator/platform/darwin.py ============================================================================== --- pypy/branch/kill-caninline/pypy/translator/platform/darwin.py (original) +++ pypy/branch/kill-caninline/pypy/translator/platform/darwin.py Thu Aug 12 13:47:36 2010 @@ -56,7 +56,7 @@ include_dirs = self._includedirs(eci.include_dirs) return (args + frameworks + include_dirs) - def _exportsymbols_link_flags(self, eci): + def _exportsymbols_link_flags(self, eci, relto=None): if not eci.export_symbols: return [] @@ -65,6 +65,9 @@ for sym in eci.export_symbols: f.write("_%s\n" % (sym,)) f.close() + + if relto: + response_file = relto.bestrelpath(response_file) return ["-Wl,-exported_symbols_list,%s" % (response_file,)] class Darwin_i386(Darwin): Modified: pypy/branch/kill-caninline/pypy/translator/platform/posix.py ============================================================================== --- pypy/branch/kill-caninline/pypy/translator/platform/posix.py (original) +++ pypy/branch/kill-caninline/pypy/translator/platform/posix.py Thu Aug 12 13:47:36 2010 @@ -14,7 +14,10 @@ def __init__(self, cc=None): if cc is None: - cc = 'gcc' + try: + cc = os.environ['CC'] + except KeyError: + cc = 'gcc' self.cc = cc def _libs(self, libraries): @@ -39,7 +42,7 @@ def _link_args_from_eci(self, eci, standalone): return Platform._link_args_from_eci(self, eci, standalone) - def _exportsymbols_link_flags(self, eci): + def _exportsymbols_link_flags(self, eci, relto=None): if not eci.export_symbols: return [] @@ -50,6 +53,9 @@ f.write("%s;\n" % (sym,)) f.write("};") f.close() + + if relto: + response_file = relto.bestrelpath(response_file) return ["-Wl,--export-dynamic,--version-script=%s" % (response_file,)] def _link(self, cc, ofiles, link_args, standalone, exe_name): @@ -90,7 +96,7 @@ if shared: linkflags = self._args_for_shared(linkflags) - linkflags += self._exportsymbols_link_flags(eci) + linkflags += self._exportsymbols_link_flags(eci, relto=path) if shared: libname = exe_name.new(ext='').basename Modified: pypy/branch/kill-caninline/pypy/translator/platform/windows.py ============================================================================== --- pypy/branch/kill-caninline/pypy/translator/platform/windows.py (original) +++ pypy/branch/kill-caninline/pypy/translator/platform/windows.py Thu Aug 12 13:47:36 2010 @@ -141,7 +141,7 @@ # Windows needs to resolve all symbols even for DLLs return super(MsvcPlatform, self)._link_args_from_eci(eci, standalone=True) - def _exportsymbols_link_flags(self, eci): + def _exportsymbols_link_flags(self, eci, relto=None): if not eci.export_symbols: return [] @@ -150,6 +150,9 @@ for sym in eci.export_symbols: f.write("/EXPORT:%s\n" % (sym,)) f.close() + + if relto: + response_file = relto.bestrelpath(response_file) return ["@%s" % (response_file,)] def _compile_c_file(self, cc, cfile, compile_args): @@ -219,7 +222,7 @@ if shared: linkflags = self._args_for_shared(linkflags) + [ '/EXPORT:$(PYPY_MAIN_FUNCTION)'] - linkflags += self._exportsymbols_link_flags(eci) + linkflags += self._exportsymbols_link_flags(eci, relto=path) if shared: so_name = exe_name.new(purebasename='lib' + exe_name.purebasename, From arigo at codespeak.net Thu Aug 12 13:49:25 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 12 Aug 2010 13:49:25 +0200 (CEST) Subject: [pypy-svn] r76604 - in pypy/trunk/pypy/module/array: . benchmark test Message-ID: <20100812114925.2A913282B9E@codespeak.net> Author: arigo Date: Thu Aug 12 13:49:23 2010 New Revision: 76604 Modified: pypy/trunk/pypy/module/array/ (props changed) pypy/trunk/pypy/module/array/__init__.py (props changed) pypy/trunk/pypy/module/array/benchmark/ (props changed) pypy/trunk/pypy/module/array/benchmark/intimg.c (props changed) pypy/trunk/pypy/module/array/benchmark/intimgtst.c (props changed) pypy/trunk/pypy/module/array/benchmark/intimgtst.py (props changed) pypy/trunk/pypy/module/array/benchmark/loop.c (props changed) pypy/trunk/pypy/module/array/benchmark/loop.py (props changed) pypy/trunk/pypy/module/array/benchmark/sum.c (props changed) pypy/trunk/pypy/module/array/benchmark/sumtst.c (props changed) pypy/trunk/pypy/module/array/benchmark/sumtst.py (props changed) pypy/trunk/pypy/module/array/interp_array.py (props changed) pypy/trunk/pypy/module/array/test/ (props changed) pypy/trunk/pypy/module/array/test/__init__.py (props changed) pypy/trunk/pypy/module/array/test/test_array.py (props changed) Log: fixeol From arigo at codespeak.net Thu Aug 12 13:54:14 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 12 Aug 2010 13:54:14 +0200 (CEST) Subject: [pypy-svn] r76605 - pypy/extradoc/planning Message-ID: <20100812115414.3FD86282B9E@codespeak.net> Author: arigo Date: Thu Aug 12 13:54:12 2010 New Revision: 76605 Modified: pypy/extradoc/planning/jit.txt Log: Add a task. Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Thu Aug 12 13:54:12 2010 @@ -31,6 +31,9 @@ E.g. http://paste.pocoo.org/show/181319/ with B being old-style and C being new-style, or vice-versa. +- maybe refactor a bit the x86 backend, particularly the register + allocation + OPTIMIZATIONS ------------- From arigo at codespeak.net Thu Aug 12 15:26:10 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 12 Aug 2010 15:26:10 +0200 (CEST) Subject: [pypy-svn] r76606 - pypy/branch/asmgcc-64/pypy/jit/backend/x86 Message-ID: <20100812132610.8EBBE282B9E@codespeak.net> Author: arigo Date: Thu Aug 12 15:26:07 2010 New Revision: 76606 Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/assembler.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/regalloc.py Log: Fix the comments. Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/assembler.py Thu Aug 12 15:26:07 2010 @@ -273,7 +273,8 @@ if IS_X86_32: self.mc.MOV_sr(WORD, edx.value) # save it as the new argument elif IS_X86_64: - # FIXME: We can't just clobber rdi like this, can we? + # rdi can be clobbered: its content was forced to the stack + # by _fastpath_malloc(), like all other save_around_call_regs. self.mc.MOV_rr(edi.value, edx.value) addr = self.cpu.gc_ll_descr.get_malloc_fixedsize_slowpath_addr() Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/regalloc.py Thu Aug 12 15:26:07 2010 @@ -707,12 +707,14 @@ def _fastpath_malloc(self, op, descr): assert isinstance(descr, BaseSizeDescr) gc_ll_descr = self.assembler.cpu.gc_ll_descr - tmp0 = TempBox() self.rm.force_allocate_reg(op.result, selected_reg=eax) + # We need to force-allocate each of save_around_call_regs now. + # The alternative would be to save and restore them around the + # actual call to malloc(), in the rare case where we need to do + # it; however, mark_gc_roots() would need to be adapted to know + # where the variables end up being saved. Messy. for reg in self.rm.save_around_call_regs: if reg is not eax: - # FIXME: Is this the right way to spill a register? And do we - # need to do this for all non-callee-save regs? tmp_box = TempBox() self.rm.force_allocate_reg(tmp_box, selected_reg=reg) self.rm.possibly_free_var(tmp_box) From hakanardo at codespeak.net Thu Aug 12 16:16:58 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Thu, 12 Aug 2010 16:16:58 +0200 (CEST) Subject: [pypy-svn] r76607 - pypy/branch/jit-bounds Message-ID: <20100812141658.EC563282BEC@codespeak.net> Author: hakanardo Date: Thu Aug 12 16:16:51 2010 New Revision: 76607 Added: pypy/branch/jit-bounds/ (props changed) - copied from r76606, pypy/trunk/ Log: Introducing upperbound and lowerbound attributes in OptValue in optimizeopt to track what has been assumed about the value by the guards? To make it possible to remove a guard(i<4) following a guard(i<5) From hakanardo at codespeak.net Thu Aug 12 19:02:14 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Thu, 12 Aug 2010 19:02:14 +0200 (CEST) Subject: [pypy-svn] r76608 - in pypy/branch/jit-bounds/pypy/jit/metainterp: . test Message-ID: <20100812170214.45AC5282B9E@codespeak.net> Author: hakanardo Date: Thu Aug 12 19:02:10 2010 New Revision: 76608 Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py Log: Basic int_lt case. Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py Thu Aug 12 19:02:10 2010 @@ -50,13 +50,44 @@ level = LEVEL_UNKNOWN known_class = None + maxint = None + minint = None - def __init__(self, box): + def __init__(self, box, producer=None): self.box = box + self.producer = producer if isinstance(box, Const): self.level = LEVEL_CONSTANT # invariant: box is a Const if and only if level == LEVEL_CONSTANT + def get_maxint(self): + if self.level == LEVEL_CONSTANT: + return self.box.getint() + else: + return self.maxint + + def get_minint(self): + if self.level == LEVEL_CONSTANT: + return self.box.getint() + else: + return self.minint + + def boundint_lt(self, val): + if val is None: return + self.maxint = val - 1 + + def bountint_le(self, val): + if val is None: return + self.maxint = val + + def boundint_gt(self, val): + if val is None: return + self.minint = val + 1 + + def bountint_ge(self, val): + if val is None: return + self.minint = val + def force_box(self): return self.box @@ -412,6 +443,7 @@ self.bool_boxes = {} self.loop_invariant_results = {} self.pure_operations = args_dict() + self.producer = {} def forget_numberings(self, virtualbox): self.metainterp_sd.profiler.count(jitprof.OPT_FORCINGS) @@ -527,6 +559,7 @@ self.exception_might_have_happened = False self.newoperations = [] for op in self.loop.operations: + self.producer[op.result] = op opnum = op.opnum for value, func in optimize_ops: if opnum == value: @@ -538,6 +571,20 @@ # accumulate counters self.resumedata_memo.update_counters(self.metainterp_sd.profiler) + def propagate_bounds_backward(self, box): + print box + try: + op = self.producer[box] + except KeyError: + return + print op + + opnum = op.opnum + for value, func in propagate_bounds_ops: + if opnum == value: + func(self, op) + break + def emit_operation(self, op): self.heap_op_optimizer.emitting_operation(op) self._emit_operation(op) @@ -689,6 +736,7 @@ if emit_operation: self.emit_operation(op) value.make_constant(constbox) + self.propagate_bounds_backward(op.args[0]) def optimize_GUARD_ISNULL(self, op): value = self.getvalue(op.args[0]) @@ -1060,8 +1108,40 @@ else: self.optimize_default(op) + def optimize_INT_LT(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + min1 = v1.get_minint() + max1 = v1.get_maxint() + min2 = v2.get_minint() + max2 = v1.get_maxint() + if max1 is not None and min2 is not None and max1 < min2: + self.make_constant_int(op.result, 1) + elif min1 is not None and max2 is not None and max2 <= min1: + self.make_constant_int(op.result, 0) + else: + self.optimize_default(op) + + + def propagate_bounds_INT_LT(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + r = self.getvalue(op.result) + if r.is_constant(): + if r.box.same_constant(CONST_1): + v1.boundint_lt(v2.get_maxint()) + v2.boundint_gt(v1.get_minint()) + elif r.box.same_constant(CONST_0): + v1.boundint_ge(v2.get_minint()) + v2.boundint_le(v1.get_maxint()) + else: + assert False, "Boolean neither True nor False" + self.propagate_bounds_backward(op.args[0]) + self.propagate_bounds_backward(op.args[1]) + optimize_ops = _findall(Optimizer, 'optimize_') +propagate_bounds_ops = _findall(Optimizer, 'propagate_bounds_') class CachedArrayItems(object): Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py Thu Aug 12 19:02:10 2010 @@ -3044,6 +3044,26 @@ ''' self.optimize_loop(ops, 'Not', expected) + def test_upperbound(self): + ops = """ + [i0] + i1 = int_lt(i0, 4) + guard_true(i1) [] + i2 = int_lt(i0, 5) + guard_true(i2) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_lt(i0, 4) + guard_true(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + + + ##class TestOOtype(BaseTestOptimizeOpt, OOtypeMixin): ## def test_instanceof(self): From getxsick at codespeak.net Thu Aug 12 19:09:01 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Thu, 12 Aug 2010 19:09:01 +0200 (CEST) Subject: [pypy-svn] r76609 - in pypy/branch/fast-ctypes/pypy/module/jitffi: . test Message-ID: <20100812170901.7F2FC282B9E@codespeak.net> Author: getxsick Date: Thu Aug 12 19:08:59 2010 New Revision: 76609 Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py Log: fix for commented test and translastion Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Thu Aug 12 19:08:59 2010 @@ -160,8 +160,9 @@ charp = rffi.cast(rffi.CCHARP, addr) return space.wrap(rffi.charp2str(charp)) # XXX free it? - def adr_to_intp_w(self, space, addr): - return space.wrap(rffi.cast(rffi.INTP, addr)) + def get_int_from_addr_w(self, space, addr): + intp = rffi.cast(rffi.INTP, addr) + return space.wrap(intp[0]) # return the first element def W_Test___new__(space, w_x): return space.wrap(W_Test(space)) @@ -175,6 +176,6 @@ unwrap_spec=['self', ObjSpace, str]), get_str = interp2app(W_Test.get_str_w, unwrap_spec=['self', ObjSpace, int]), - adr_to_intp = interp2app(W_Test.adr_to_intp_w, + get_int_from_addr = interp2app(W_Test.get_int_from_addr_w, unwrap_spec=['self', ObjSpace, int]) ) Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py Thu Aug 12 19:08:59 2010 @@ -151,15 +151,15 @@ func.call([charp]) assert t.get_str(charp) == 'xbxbxb' - #def test_get_ptr(self): - # import jitffi - # t = jitffi.Test() - # lib = jitffi.CDLL(self.lib_name) + def test_get_ptr(self): + import jitffi + t = jitffi.Test() + lib = jitffi.CDLL(self.lib_name) - # func = lib.get('return_intptr', ['i'], 'p') - # addr = func.call([22]) - # ret = t.adr_to_intp(addr) - # assert ret[0] == 22 + func = lib.get('return_intptr', ['i'], 'p') + addr = func.call([22]) + ret = t.get_int_from_addr(addr) + assert ret == 22 def test_undefined_func(self): import jitffi From hakanardo at codespeak.net Thu Aug 12 19:24:00 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Thu, 12 Aug 2010 19:24:00 +0200 (CEST) Subject: [pypy-svn] r76610 - pypy/branch/jit-bounds/pypy/jit/metainterp Message-ID: <20100812172400.E3D83282B9E@codespeak.net> Author: hakanardo Date: Thu Aug 12 19:23:59 2010 New Revision: 76610 Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py Log: Basic int_lt case. Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py Thu Aug 12 19:23:59 2010 @@ -88,6 +88,27 @@ if val is None: return self.minint = val + def known_lt(self, other): + max1 = self.get_maxint() + min2 = other.get_minint() + if max1 is not None and min2 is not None and max1 < min2: + return True + return False + + def known_le(self, other): + max1 = self.get_maxint() + min2 = other.get_minint() + if max1 is not None and min2 is not None and max1 <= min2: + return True + return False + + def known_gt(self, other): + return other.known_lt(self) + + def known_ge(self, other): + return other.known_le(self) + + def force_box(self): return self.box @@ -1111,11 +1132,21 @@ def optimize_INT_LT(self, op): v1 = self.getvalue(op.args[0]) v2 = self.getvalue(op.args[1]) + if v1.known_lt(v2): + self.make_constant_int(op.result, 1) + elif v1.known_ge(v2): + self.make_constant_int(op.result, 0) + else: + self.optimize_default(op) + + def optimize_INT_GT(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) min1 = v1.get_minint() max1 = v1.get_maxint() min2 = v2.get_minint() max2 = v1.get_maxint() - if max1 is not None and min2 is not None and max1 < min2: + if min1 is not None and min2 is not None and max2 < min1 self.make_constant_int(op.result, 1) elif min1 is not None and max2 is not None and max2 <= min1: self.make_constant_int(op.result, 0) From hakanardo at codespeak.net Thu Aug 12 19:32:50 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Thu, 12 Aug 2010 19:32:50 +0200 (CEST) Subject: [pypy-svn] r76611 - in pypy/branch/jit-bounds/pypy/jit/metainterp: . test Message-ID: <20100812173250.E9467282B9E@codespeak.net> Author: hakanardo Date: Thu Aug 12 19:32:48 2010 New Revision: 76611 Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py Log: more tests, int_gt Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py Thu Aug 12 19:32:48 2010 @@ -76,7 +76,7 @@ if val is None: return self.maxint = val - 1 - def bountint_le(self, val): + def boundint_le(self, val): if val is None: return self.maxint = val @@ -84,7 +84,7 @@ if val is None: return self.minint = val + 1 - def bountint_ge(self, val): + def boundint_ge(self, val): if val is None: return self.minint = val @@ -1142,18 +1142,13 @@ def optimize_INT_GT(self, op): v1 = self.getvalue(op.args[0]) v2 = self.getvalue(op.args[1]) - min1 = v1.get_minint() - max1 = v1.get_maxint() - min2 = v2.get_minint() - max2 = v1.get_maxint() - if min1 is not None and min2 is not None and max2 < min1 + if v1.known_gt(v2): self.make_constant_int(op.result, 1) - elif min1 is not None and max2 is not None and max2 <= min1: + elif v1.known_le(v2): self.make_constant_int(op.result, 0) else: self.optimize_default(op) - - + def propagate_bounds_INT_LT(self, op): v1 = self.getvalue(op.args[0]) v2 = self.getvalue(op.args[1]) @@ -1170,6 +1165,22 @@ self.propagate_bounds_backward(op.args[0]) self.propagate_bounds_backward(op.args[1]) + def propagate_bounds_INT_GT(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + r = self.getvalue(op.result) + if r.is_constant(): + if r.box.same_constant(CONST_1): + v2.boundint_lt(v1.get_maxint()) + v1.boundint_gt(v2.get_minint()) + elif r.box.same_constant(CONST_0): + v2.boundint_ge(v1.get_minint()) + v1.boundint_le(v2.get_maxint()) + else: + assert False, "Boolean neither True nor False" + self.propagate_bounds_backward(op.args[0]) + self.propagate_bounds_backward(op.args[1]) + optimize_ops = _findall(Optimizer, 'optimize_') propagate_bounds_ops = _findall(Optimizer, 'propagate_bounds_') Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py Thu Aug 12 19:32:48 2010 @@ -3044,7 +3044,7 @@ ''' self.optimize_loop(ops, 'Not', expected) - def test_upperbound(self): + def test_bound_lt(self): ops = """ [i0] i1 = int_lt(i0, 4) @@ -3061,6 +3061,74 @@ """ self.optimize_loop(ops, 'Not', expected) + def test_bound_lt_noguard(self): + ops = """ + [i0] + i1 = int_lt(i0, 4) + i2 = int_lt(i0, 5) + jump(i2) + """ + expected = """ + [i0] + i1 = int_lt(i0, 4) + i2 = int_lt(i0, 5) + jump(i2) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_bound_lt_noopt(self): + ops = """ + [i0] + i1 = int_lt(i0, 4) + guard_false(i1) [] + i2 = int_lt(i0, 5) + guard_true(i2) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_lt(i0, 4) + guard_false(i1) [] + i2 = int_lt(i0, 5) + guard_true(i2) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_bound_lt_rev(self): + ops = """ + [i0] + i1 = int_lt(i0, 4) + guard_false(i1) [] + i2 = int_gt(i0, 3) + guard_true(i2) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_lt(i0, 4) + guard_false(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_bound_gt(self): + ops = """ + [i0] + i1 = int_gt(i0, 5) + guard_true(i1) [] + i2 = int_gt(i0, 4) + guard_true(i2) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_gt(i0, 5) + guard_true(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + From hakanardo at codespeak.net Thu Aug 12 23:00:27 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Thu, 12 Aug 2010 23:00:27 +0200 (CEST) Subject: [pypy-svn] r76612 - in pypy/branch/jit-bounds/pypy/jit/metainterp: . test Message-ID: <20100812210027.46D2D282B9E@codespeak.net> Author: hakanardo Date: Thu Aug 12 23:00:24 2010 New Revision: 76612 Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py Log: makeing sure never to lessen a bound and not to overflow Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py Thu Aug 12 23:00:24 2010 @@ -18,6 +18,9 @@ from pypy.rpython.lltypesystem import lltype from pypy.jit.metainterp.history import AbstractDescr, make_hashable_int +import sys +SYSMAXINT = sys.maxint +SYSMININT = -sys.maxint - 1 def optimize_loop_1(metainterp_sd, loop): """Optimize loop.operations to make it match the input of loop.specnodes @@ -74,19 +77,27 @@ def boundint_lt(self, val): if val is None: return - self.maxint = val - 1 + if self.maxint is None or val <= self.maxint: + if val <= SYSMININT: + self.maxint = None + self.maxint = val - 1 def boundint_le(self, val): if val is None: return - self.maxint = val + if self.maxint is None or val < self.maxint: + self.maxint = val def boundint_gt(self, val): if val is None: return - self.minint = val + 1 + if self.minint is None or val >= self.minint: + if val >= SYSMAXINT: + self.minint = None + self.minint = val + 1 def boundint_ge(self, val): if val is None: return - self.minint = val + if self.minint is None or val > self.minint: + self.minint = val def known_lt(self, other): max1 = self.get_maxint() Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py Thu Aug 12 23:00:24 2010 @@ -3129,6 +3129,25 @@ """ self.optimize_loop(ops, 'Not', expected) + def test_bound_lt_tripple(self): + ops = """ + [i0] + i1 = int_lt(i0, 0) + guard_true(i1) [] + i2 = int_lt(i0, 7) + guard_true(i2) [] + i3 = int_lt(i0, 5) + guard_true(i3) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_lt(i0, 0) + guard_true(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + From jcreigh at codespeak.net Fri Aug 13 15:07:30 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Fri, 13 Aug 2010 15:07:30 +0200 (CEST) Subject: [pypy-svn] r76613 - in pypy/branch/asmgcc-64: . lib-python lib-python/modified-2.5.2/test lib_pypy pypy/config pypy/jit/metainterp/doc pypy/module/array pypy/module/array/benchmark pypy/module/array/test pypy/module/cpyext pypy/rpython/lltypesystem Message-ID: <20100813130730.281E3282B9E@codespeak.net> Author: jcreigh Date: Fri Aug 13 15:07:27 2010 New Revision: 76613 Removed: pypy/branch/asmgcc-64/pypy/jit/metainterp/doc/ Modified: pypy/branch/asmgcc-64/ (props changed) pypy/branch/asmgcc-64/lib-python/conftest.py pypy/branch/asmgcc-64/lib-python/modified-2.5.2/test/test_array.py pypy/branch/asmgcc-64/lib_pypy/array.py pypy/branch/asmgcc-64/pypy/config/pypyoption.py pypy/branch/asmgcc-64/pypy/module/array/ (props changed) pypy/branch/asmgcc-64/pypy/module/array/__init__.py (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/ (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/Makefile (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/intimg.c (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/intimgtst.c (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/intimgtst.py (contents, props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/loop.c (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/loop.py (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/sum.c (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/sumtst.c (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/sumtst.py (contents, props changed) pypy/branch/asmgcc-64/pypy/module/array/interp_array.py (contents, props changed) pypy/branch/asmgcc-64/pypy/module/array/test/ (props changed) pypy/branch/asmgcc-64/pypy/module/array/test/__init__.py (props changed) pypy/branch/asmgcc-64/pypy/module/array/test/test_array.py (contents, props changed) pypy/branch/asmgcc-64/pypy/module/array/test/test_array_old.py (props changed) pypy/branch/asmgcc-64/pypy/module/cpyext/api.py pypy/branch/asmgcc-64/pypy/rpython/lltypesystem/ll2ctypes.py Log: merged changes to trunk through r76612 Modified: pypy/branch/asmgcc-64/lib-python/conftest.py ============================================================================== --- pypy/branch/asmgcc-64/lib-python/conftest.py (original) +++ pypy/branch/asmgcc-64/lib-python/conftest.py Fri Aug 13 15:07:27 2010 @@ -132,7 +132,7 @@ RegrTest('test_ast.py', core=True), RegrTest('test_anydbm.py'), RegrTest('test_applesingle.py', skip=True), - RegrTest('test_array.py', core=True, usemodules='struct'), + RegrTest('test_array.py', core=True, usemodules='struct array'), RegrTest('test_asynchat.py', usemodules='thread'), RegrTest('test_atexit.py', core=True), RegrTest('test_audioop.py', skip=True), Modified: pypy/branch/asmgcc-64/lib-python/modified-2.5.2/test/test_array.py ============================================================================== --- pypy/branch/asmgcc-64/lib-python/modified-2.5.2/test/test_array.py (original) +++ pypy/branch/asmgcc-64/lib-python/modified-2.5.2/test/test_array.py Fri Aug 13 15:07:27 2010 @@ -269,9 +269,11 @@ ) b = array.array(self.badtypecode()) - self.assertRaises(TypeError, a.__add__, b) + #self.assertRaises(TypeError, a.__add__, b) + #self.assertRaises(TypeError, a.__add__, "bad") + self.assertRaises(TypeError, lambda i, j: i + j, a, b) + self.assertRaises(TypeError, lambda i, j: i + j, a, "bad") - self.assertRaises(TypeError, a.__add__, "bad") def test_iadd(self): a = array.array(self.typecode, self.example[::-1]) @@ -284,9 +286,12 @@ ) b = array.array(self.badtypecode()) - self.assertRaises(TypeError, a.__add__, b) - - self.assertRaises(TypeError, a.__iadd__, "bad") + #self.assertRaises(TypeError, a.__add__, b) + #self.assertRaises(TypeError, a.__iadd__, "bad") + def f(i, j): + i += j + self.assertRaises(TypeError, f, a, b) + self.assertRaises(TypeError, f, a, "bad") def test_mul(self): a = 5*array.array(self.typecode, self.example) @@ -313,7 +318,8 @@ array.array(self.typecode) ) - self.assertRaises(TypeError, a.__mul__, "bad") + #self.assertRaises(TypeError, a.__mul__, "bad") + self.assertRaises(TypeError, lambda i, j: i * j, a, "bad") def test_imul(self): a = array.array(self.typecode, self.example) @@ -342,7 +348,10 @@ a *= -1 self.assertEqual(a, array.array(self.typecode)) - self.assertRaises(TypeError, a.__imul__, "bad") + #self.assertRaises(TypeError, a.__imul__, "bad") + def f(i, j): + i *= j + self.assertRaises(TypeError, f, a, "bad") def test_getitem(self): a = array.array(self.typecode, self.example) Modified: pypy/branch/asmgcc-64/lib_pypy/array.py ============================================================================== --- pypy/branch/asmgcc-64/lib_pypy/array.py (original) +++ pypy/branch/asmgcc-64/lib_pypy/array.py Fri Aug 13 15:07:27 2010 @@ -1,24 +1,531 @@ -from array import array as _array +"""This module defines an object type which can efficiently represent +an array of basic values: characters, integers, floating point +numbers. Arrays are sequence types and behave very much like lists, +except that the type of objects stored in them is constrained. The +type is specified at object creation time by using a type code, which +is a single character. The following type codes are defined: + + Type code C Type Minimum size in bytes + 'c' character 1 + 'b' signed integer 1 + 'B' unsigned integer 1 + 'u' Unicode character 2 + 'h' signed integer 2 + 'H' unsigned integer 2 + 'i' signed integer 2 + 'I' unsigned integer 2 + 'l' signed integer 4 + 'L' unsigned integer 4 + 'f' floating point 4 + 'd' floating point 8 + +The constructor is: + +array(typecode [, initializer]) -- create a new array +""" + +from struct import calcsize, pack, pack_into, unpack_from +import operator + +# the buffer-like object to use internally: trying from +# various places in order... +try: + import _rawffi # a reasonable implementation based + _RAWARRAY = _rawffi.Array('c') # on raw_malloc, and providing a + def bytebuffer(size): # real address + return _RAWARRAY(size, autofree=True) + def getbufaddress(buf): + return buf.buffer +except ImportError: + try: + from __pypy__ import bytebuffer # a reasonable implementation + def getbufaddress(buf): # compatible with oo backends, + return 0 # but no address + except ImportError: + # not running on PyPy. Fall back to ctypes... + import ctypes + bytebuffer = ctypes.create_string_buffer + def getbufaddress(buf): + voidp = ctypes.cast(ctypes.pointer(buf), ctypes.c_void_p) + return voidp.value + +# ____________________________________________________________ + +TYPECODES = "cbBuhHiIlLfd" class array(object): - def __init__(self, typecode, initializer=None): - self._array = _array(typecode) - if initializer is not None: + """array(typecode [, initializer]) -> array + + Return a new array whose items are restricted by typecode, and + initialized from the optional initializer value, which must be a list, + string. or iterable over elements of the appropriate type. + + Arrays represent basic values and behave very much like lists, except + the type of objects stored in them is constrained. + + Methods: + + append() -- append a new item to the end of the array + buffer_info() -- return information giving the current memory info + byteswap() -- byteswap all the items of the array + count() -- return number of occurences of an object + extend() -- extend array by appending multiple elements from an iterable + fromfile() -- read items from a file object + fromlist() -- append items from the list + fromstring() -- append items from the string + index() -- return index of first occurence of an object + insert() -- insert a new item into the array at a provided position + pop() -- remove and return item (default last) + read() -- DEPRECATED, use fromfile() + remove() -- remove first occurence of an object + reverse() -- reverse the order of the items in the array + tofile() -- write all items to a file object + tolist() -- return the array converted to an ordinary list + tostring() -- return the array converted to a string + write() -- DEPRECATED, use tofile() + + Attributes: + + typecode -- the typecode character used to create the array + itemsize -- the length in bytes of one array item + """ + __slots__ = ["typecode", "itemsize", "_data", "_descriptor", "__weakref__"] + + def __new__(cls, typecode, initializer=[], **extrakwds): + self = object.__new__(cls) + if cls is array and extrakwds: + raise TypeError("array() does not take keyword arguments") + if not isinstance(typecode, str) or len(typecode) != 1: + raise TypeError( + "array() argument 1 must be char, not %s" % type(typecode)) + if typecode not in TYPECODES: + raise ValueError( + "bad typecode (must be one of %s)" % ', '.join(TYPECODES)) + self._data = bytebuffer(0) + self.typecode = typecode + self.itemsize = calcsize(typecode) + if isinstance(initializer, list): + self.fromlist(initializer) + elif isinstance(initializer, str): + self.fromstring(initializer) + elif isinstance(initializer, unicode) and self.typecode == "u": + self.fromunicode(initializer) + else: self.extend(initializer) + return self + def _clear(self): + self._data = bytebuffer(0) - def append(self ,x): - self._array.append(x) - def __getitem__(self, idx): - return self._array[idx] - def __setitem__(self, idx, val): - self._array[idx]=val - def __len__(self): - return len(self._array) + ##### array-specific operations + def fromfile(self, f, n): + """Read n objects from the file object f and append them to the end of + the array. Also called as read.""" + if not isinstance(f, file): + raise TypeError("arg1 must be open file") + size = self.itemsize * n + item = f.read(size) + if len(item) < size: + raise EOFError("not enough items in file") + self.fromstring(item) + + def fromlist(self, l): + """Append items to array from list.""" + if not isinstance(l, list): + raise TypeError("arg must be list") + self._fromiterable(l) + + def fromstring(self, s): + """Appends items from the string, interpreting it as an array of machine + values, as if it had been read from a file using the fromfile() + method.""" + if isinstance(s, unicode): + s = str(s) + self._frombuffer(s) + + def _frombuffer(self, s): + length = len(s) + if length % self.itemsize != 0: + raise ValueError("string length not a multiple of item size") + boundary = len(self._data) + newdata = bytebuffer(boundary + length) + newdata[:boundary] = self._data + newdata[boundary:] = s + self._data = newdata + + def fromunicode(self, ustr): + """Extends this array with data from the unicode string ustr. The array + must be a type 'u' array; otherwise a ValueError is raised. Use + array.fromstring(ustr.encode(...)) to append Unicode data to an array of + some other type.""" + if not self.typecode == "u": + raise ValueError( + "fromunicode() may only be called on type 'u' arrays") + # XXX the following probable bug is not emulated: + # CPython accepts a non-unicode string or a buffer, and then + # behaves just like fromstring(), except that it strangely truncates + # string arguments at multiples of the unicode byte size. + # Let's only accept unicode arguments for now. + if not isinstance(ustr, unicode): + raise TypeError("fromunicode() argument should probably be " + "a unicode string") + # _frombuffer() does the currect thing using + # the buffer behavior of unicode objects + self._frombuffer(buffer(ustr)) + + def tofile(self, f): + """Write all items (as machine values) to the file object f. Also + called as write.""" + if not isinstance(f, file): + raise TypeError("arg must be open file") + f.write(self.tostring()) + + def tolist(self): + """Convert array to an ordinary list with the same items.""" + count = len(self._data) // self.itemsize + return list(unpack_from('%d%s' % (count, self.typecode), self._data)) + + def tostring(self): + return self._data[:] + + def __buffer__(self): + return buffer(self._data) + + def tounicode(self): + """Convert the array to a unicode string. The array must be a type 'u' + array; otherwise a ValueError is raised. Use array.tostring().decode() + to obtain a unicode string from an array of some other type.""" + if self.typecode != "u": + raise ValueError("tounicode() may only be called on type 'u' arrays") + # XXX performance is not too good + return u"".join(self.tolist()) + + def byteswap(self): + """Byteswap all items of the array. If the items in the array are not + 1, 2, 4, or 8 bytes in size, RuntimeError is raised.""" + if self.itemsize not in [1, 2, 4, 8]: + raise RuntimeError("byteswap not supported for this array") + # XXX slowish + itemsize = self.itemsize + bytes = self._data + for start in range(0, len(bytes), itemsize): + stop = start + itemsize + bytes[start:stop] = bytes[start:stop][::-1] + + def buffer_info(self): + """Return a tuple (address, length) giving the current memory address + and the length in items of the buffer used to hold array's contents. The + length should be multiplied by the itemsize attribute to calculate the + buffer length in bytes. On PyPy the address might be meaningless + (returned as 0), depending on the available modules.""" + return (getbufaddress(self._data), len(self)) + read = fromfile + + write = tofile + + ##### general object protocol + + def __repr__(self): + if len(self._data) == 0: + return "array('%s')" % self.typecode + elif self.typecode == "c": + return "array('%s', %s)" % (self.typecode, repr(self.tostring())) + elif self.typecode == "u": + return "array('%s', %s)" % (self.typecode, repr(self.tounicode())) + else: + return "array('%s', %s)" % (self.typecode, repr(self.tolist())) + + def __copy__(self): + a = array(self.typecode) + a._data = bytebuffer(len(self._data)) + a._data[:] = self._data + return a + + def __eq__(self, other): + if not isinstance(other, array): + return NotImplemented + if self.typecode == 'c': + return buffer(self._data) == buffer(other._data) + else: + return self.tolist() == other.tolist() + + def __ne__(self, other): + if not isinstance(other, array): + return NotImplemented + if self.typecode == 'c': + return buffer(self._data) != buffer(other._data) + else: + return self.tolist() != other.tolist() + + def __lt__(self, other): + if not isinstance(other, array): + return NotImplemented + if self.typecode == 'c': + return buffer(self._data) < buffer(other._data) + else: + return self.tolist() < other.tolist() + + def __gt__(self, other): + if not isinstance(other, array): + return NotImplemented + if self.typecode == 'c': + return buffer(self._data) > buffer(other._data) + else: + return self.tolist() > other.tolist() + + def __le__(self, other): + if not isinstance(other, array): + return NotImplemented + if self.typecode == 'c': + return buffer(self._data) <= buffer(other._data) + else: + return self.tolist() <= other.tolist() + + def __ge__(self, other): + if not isinstance(other, array): + return NotImplemented + if self.typecode == 'c': + return buffer(self._data) >= buffer(other._data) + else: + return self.tolist() >= other.tolist() + + def __reduce__(self): + dict = getattr(self, '__dict__', None) + data = self.tostring() + if data: + initargs = (self.typecode, data) + else: + initargs = (self.typecode,) + return (type(self), initargs, dict) + + ##### list methods + + def append(self, x): + """Append new value x to the end of the array.""" + self._frombuffer(pack(self.typecode, x)) + + def count(self, x): + """Return number of occurences of x in the array.""" + return operator.countOf(self, x) + def extend(self, iterable): - for i in iterable: self.append(i) + """Append items to the end of the array.""" + if isinstance(iterable, array) \ + and not self.typecode == iterable.typecode: + raise TypeError("can only extend with array of same kind") + self._fromiterable(iterable) + + def index(self, x): + """Return index of first occurence of x in the array.""" + return operator.indexOf(self, x) + def insert(self, i, x): + """Insert a new item x into the array before position i.""" + seqlength = len(self) + if i < 0: + i += seqlength + if i < 0: + i = 0 + elif i > seqlength: + i = seqlength + boundary = i * self.itemsize + data = pack(self.typecode, x) + newdata = bytebuffer(len(self._data) + len(data)) + newdata[:boundary] = self._data[:boundary] + newdata[boundary:boundary+self.itemsize] = data + newdata[boundary+self.itemsize:] = self._data[boundary:] + self._data = newdata + + def pop(self, i=-1): + """Return the i-th element and delete it from the array. i defaults to + -1.""" + seqlength = len(self) + if i < 0: + i += seqlength + if not (0 <= i < seqlength): + raise IndexError(i) + boundary = i * self.itemsize + result = unpack_from(self.typecode, self._data, boundary)[0] + newdata = bytebuffer(len(self._data) - self.itemsize) + newdata[:boundary] = self._data[:boundary] + newdata[boundary:] = self._data[boundary+self.itemsize:] + self._data = newdata + return result + + def remove(self, x): + """Remove the first occurence of x in the array.""" + self.pop(self.index(x)) + + def reverse(self): + """Reverse the order of the items in the array.""" + lst = self.tolist() + lst.reverse() + self._clear() + self.fromlist(lst) + ##### list protocol + def __len__(self): + return len(self._data) // self.itemsize + + def __add__(self, other): + if not isinstance(other, array): + raise TypeError("can only append array to array") + if self.typecode != other.typecode: + raise TypeError("bad argument type for built-in operation") + return array(self.typecode, buffer(self._data) + buffer(other._data)) + + def __mul__(self, repeat): + return array(self.typecode, buffer(self._data) * repeat) + + __rmul__ = __mul__ + + def __getitem__(self, i): + seqlength = len(self) + if isinstance(i, slice): + start, stop, step = i.indices(seqlength) + if step != 1: + sublist = self.tolist()[i] # fall-back + return array(self.typecode, sublist) + if start < 0: + start = 0 + if stop < start: + stop = start + assert stop <= seqlength + return array(self.typecode, self._data[start * self.itemsize : + stop * self.itemsize]) + else: + if i < 0: + i += seqlength + if self.typecode == 'c': # speed trick + return self._data[i] + if not (0 <= i < seqlength): + raise IndexError(i) + boundary = i * self.itemsize + return unpack_from(self.typecode, self._data, boundary)[0] + + def __getslice__(self, i, j): + return self.__getitem__(slice(i, j)) + + def __setitem__(self, i, x): + if isinstance(i, slice): + if (not isinstance(x, array) + or self.typecode != x.typecode): + raise TypeError("can only assign array of same kind" + " to array slice") + seqlength = len(self) + start, stop, step = i.indices(seqlength) + if step != 1: + sublist = self.tolist() # fall-back + sublist[i] = x.tolist() + self._clear() + self.fromlist(sublist) + return + if start < 0: + start = 0 + if stop < start: + stop = start + assert stop <= seqlength + boundary1 = start * self.itemsize + boundary2 = stop * self.itemsize + boundary2new = boundary1 + len(x._data) + if boundary2 == boundary2new: + self._data[boundary1:boundary2] = x._data + else: + newdata = bytebuffer(len(self._data) + boundary2new-boundary2) + newdata[:boundary1] = self._data[:boundary1] + newdata[boundary1:boundary2new] = x._data + newdata[boundary2new:] = self._data[boundary2:] + self._data = newdata + else: + seqlength = len(self) + if i < 0: + i += seqlength + if self.typecode == 'c': # speed trick + self._data[i] = x + return + if not (0 <= i < seqlength): + raise IndexError(i) + boundary = i * self.itemsize + pack_into(self.typecode, self._data, boundary, x) + + def __setslice__(self, i, j, x): + self.__setitem__(slice(i, j), x) + + def __delitem__(self, i): + if isinstance(i, slice): + seqlength = len(self) + start, stop, step = i.indices(seqlength) + if start < 0: + start = 0 + if stop < start: + stop = start + assert stop <= seqlength + if step != 1: + sublist = self.tolist() # fall-back + del sublist[i] + self._clear() + self.fromlist(sublist) + return + dellength = stop - start + boundary1 = start * self.itemsize + boundary2 = stop * self.itemsize + newdata = bytebuffer(len(self._data) - (boundary2-boundary1)) + newdata[:boundary1] = self._data[:boundary1] + newdata[boundary1:] = self._data[boundary2:] + self._data = newdata + else: + seqlength = len(self) + if i < 0: + i += seqlength + if not (0 <= i < seqlength): + raise IndexError(i) + boundary = i * self.itemsize + newdata = bytebuffer(len(self._data) - self.itemsize) + newdata[:boundary] = self._data[:boundary] + newdata[boundary:] = self._data[boundary+self.itemsize:] + self._data = newdata + + def __delslice__(self, i, j): + self.__delitem__(slice(i, j)) + + def __contains__(self, item): + for x in self: + if x == item: + return True + return False + + def __iadd__(self, other): + if not isinstance(other, array): + raise TypeError("can only extend array with array") + self.extend(other) + return self + + def __imul__(self, repeat): + newdata = buffer(self._data) * repeat + self._data = bytebuffer(len(newdata)) + self._data[:] = newdata + return self + + def __iter__(self): + p = 0 + typecode = self.typecode + itemsize = self.itemsize + while p < len(self._data): + yield unpack_from(typecode, self._data, p)[0] + p += itemsize + + ##### internal methods + + def _fromiterable(self, iterable): + iterable = tuple(iterable) + n = len(iterable) + boundary = len(self._data) + newdata = bytebuffer(boundary + n * self.itemsize) + newdata[:boundary] = self._data + pack_into('%d%s' % (n, self.typecode), newdata, boundary, *iterable) + self._data = newdata + +ArrayType = array Modified: pypy/branch/asmgcc-64/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/config/pypyoption.py (original) +++ pypy/branch/asmgcc-64/pypy/config/pypyoption.py Fri Aug 13 15:07:27 2010 @@ -30,7 +30,7 @@ "rctime" , "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios", "zlib", "struct", "md5", "sha", "bz2", "_minimal_curses", "cStringIO", - "thread", "itertools", "pyexpat", "_ssl", "cpyext"] + "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array"] )) working_oo_modules = default_modules.copy() Modified: pypy/branch/asmgcc-64/pypy/module/array/benchmark/intimgtst.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/array/benchmark/intimgtst.py (original) +++ pypy/branch/asmgcc-64/pypy/module/array/benchmark/intimgtst.py Fri Aug 13 15:07:27 2010 @@ -1,7 +1,5 @@ #!/usr/bin/python -from time import time - -from array import array, simple_array +from array import array def f(img, intimg): l=0 @@ -14,13 +12,7 @@ -if True: - img=array('d','\x00'*640*480*8) - intimg=array('d','\x00'*640*480*8) -else: - img=simple_array(640*480) - intimg=simple_array(640*480) +img=array('d','\x00'*640*480*8) +intimg=array('d','\x00'*640*480*8) -start=time() for l in range(500): f(img, intimg) -print time()-start Modified: pypy/branch/asmgcc-64/pypy/module/array/benchmark/sumtst.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/array/benchmark/sumtst.py (original) +++ pypy/branch/asmgcc-64/pypy/module/array/benchmark/sumtst.py Fri Aug 13 15:07:27 2010 @@ -1,7 +1,6 @@ #!/usr/bin/python from array import array -#img=array('d',(0,)*640*480); def f(img): l=0 i=0; @@ -11,12 +10,9 @@ return l img=array('d', (0,)) * (640*480) -#img=array('d', [0]*640*480) -#img=array('d', (0,))*(640*480) for l in range(500): f(img) -#print f(img) # C pypy-simple pypy cpython -# sumtst: 0m0.630s 0m0.659s 0m0.762s 0m33.447s -# intimg: 0m0.646s 0m1.078s 0m1.357s 1m0.279s +# sumtst: 0m0.630s 0m0.659s 0m0.851s 0m33.447s +# intimg: 0m0.646s 0m1.078s 0m1.446s 1m0.279s Modified: pypy/branch/asmgcc-64/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/array/interp_array.py (original) +++ pypy/branch/asmgcc-64/pypy/module/array/interp_array.py Fri Aug 13 15:07:27 2010 @@ -16,8 +16,8 @@ from pypy.module._file.interp_file import W_File from pypy.interpreter.buffer import RWBuffer -def w_array(space, w_cls, typecode, w_initializer=None, w_args=None): - if len(w_args.arguments_w) > 0: +def w_array(space, w_cls, typecode, w_args=None): + if len(w_args.arguments_w) > 1: msg = 'array() takes at most 2 arguments' raise OperationError(space.w_TypeError, space.wrap(msg)) if len(typecode) != 1: @@ -30,23 +30,23 @@ a = space.allocate_instance(types[tc].w_class, w_cls) a.__init__(space) - if w_initializer is not None: - if not space.is_w(w_initializer, space.w_None): - if space.type(w_initializer) is space.w_str: - a.fromstring(w_initializer) - elif space.type(w_initializer) is space.w_unicode: - a.fromsequence(w_initializer) - elif space.type(w_initializer) is space.w_list: - a.fromlist(w_initializer) - else: - a.extend(w_initializer) + if len(w_args.arguments_w) > 0: + w_initializer = w_args.arguments_w[0] + if space.type(w_initializer) is space.w_str: + a.fromstring(w_initializer) + elif space.type(w_initializer) is space.w_unicode: + a.fromsequence(w_initializer) + elif space.type(w_initializer) is space.w_list: + a.fromlist(w_initializer) + else: + a.extend(w_initializer) break else: msg = 'bad typecode (must be c, b, B, u, h, H, i, I, l, L, f or d)' raise OperationError(space.w_ValueError, space.wrap(msg)) return a -w_array.unwrap_spec = (ObjSpace, W_Root, str, W_Root, Arguments) +w_array.unwrap_spec = (ObjSpace, W_Root, str, Arguments) array_append = SMM('append', 2) @@ -475,7 +475,10 @@ return self def mul__Array_ANY(space, self, w_repeat): - repeat = space.int_w(w_repeat) + try: + repeat = space.int_w(w_repeat) + except OperationError: + return space.w_NotImplemented a = mytype.w_class(space) repeat = max(repeat, 0) a.setlen(self.len * repeat) @@ -488,7 +491,10 @@ return mul__Array_ANY(space, self, w_repeat) def inplace_mul__Array_ANY(space, self, w_repeat): - repeat = space.int_w(w_repeat) + try: + repeat = space.int_w(w_repeat) + except OperationError: + return space.w_NotImplemented oldlen = self.len repeat = max(repeat, 0) self.setlen(self.len * repeat) @@ -579,7 +585,7 @@ w_lst2 = space.call_method(other, 'tolist') return space.cmp(w_lst1, w_lst2) else: - raise OperationError(space.w_NotImplementedError, space.wrap('')) + return space.w_NotImplemented # Misc methods @@ -598,7 +604,11 @@ args = [space.wrap(mytype.typecode), w_s] else: args = [space.wrap(mytype.typecode)] - return space.newtuple([space.type(self), space.newtuple(args)]) + try: + dct = space.getattr(self, space.wrap('__dict__')) + except OperationError: + dct = space.w_None + return space.newtuple([space.type(self), space.newtuple(args), dct]) def array_byteswap__Array(space, self): if mytype.bytes not in [1, 2, 4, 8]: @@ -656,3 +666,5 @@ for mytype in types.values(): make_array(mytype) + +register_all(locals(), globals()) Modified: pypy/branch/asmgcc-64/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/array/test/test_array.py (original) +++ pypy/branch/asmgcc-64/pypy/module/array/test/test_array.py Fri Aug 13 15:07:27 2010 @@ -61,6 +61,7 @@ for tc in 'bhilBHILfd': assert self.array(tc).typecode == tc + raises(TypeError, self.array, tc, None) def test_value_range(self): values = (-129, 128, -128, 127, 0, 255, -1, 256, @@ -468,6 +469,12 @@ assert repr(a) == "array('i', [20, 8, 2, 9, 7, 10])" def test_compare(self): + class comparable(object): + def __cmp__(self, other): + return 0 + class incomparable(object): + pass + for v1, v2, tt in (([1, 2, 3], [1, 3, 2], 'bhilBHIL'), ('abc', 'acb', 'c'), (unicode('abc'), unicode('acb'), 'u')): @@ -476,6 +483,13 @@ b = self.array(t, v1) c = self.array(t, v2) + print (a==7) + assert (a == 7) is False + assert (comparable() == a) is True + assert (a == comparable()) is True + assert (a == incomparable()) is False + assert (incomparable() == a) is False + assert (a == a) is True assert (a == b) is True assert (b == a) is True @@ -579,13 +593,16 @@ raises(TypeError, "a = self.array('i') + 2") raises(TypeError, "self.array('i') + self.array('b')") + a = self.array('i') + raises(TypeError, "a += 7") # Calling __add__ directly raises TypeError in cpython but # returns NotImplemented in pypy if placed within a # try: except TypeError: construction. # - # raises(TypeError, self.array('i').__add__, (2,)) - # raises(TypeError, self.array('i').__add__, self.array('b')) + #raises(TypeError, self.array('i').__add__, (2,)) + #raises(TypeError, self.array('i').__iadd__, (2,)) + #raises(TypeError, self.array('i').__add__, self.array('b')) class addable(object): def __add__(self, other): @@ -597,6 +614,10 @@ assert addable() + self.array('i') == 'add' assert self.array('i') + addable() == 'radd' + a = self.array('i') + a += addable() + assert a == 'radd' + a = self.array('i', [1, 2]) assert a * -1 == self.array('i') b = a @@ -604,6 +625,24 @@ assert a == self.array('i') assert b == self.array('i') + a = self.array('i') + raises(TypeError, "a * 'hi'") + raises(TypeError, "'hi' * a") + + class mulable(object): + def __mul__(self, other): + return "mul" + + def __rmul__(self, other): + return "rmul" + + assert mulable() * self.array('i') == 'mul' + assert self.array('i') * mulable() == 'rmul' + + a = self.array('i') + a *= mulable() + assert a == 'rmul' + def test_delitem(self): a = self.array('i', [1, 2, 3]) del a[1] @@ -731,6 +770,12 @@ assert repr(mya('i', [1, 2, 3])) == "array('i', [1, 2, 3])" assert repr(mya('i', (1, 2, 3))) == "array('i', [1, 2, 3])" + def test_unicode_outofrange(self): + a = self.array('u', unicode(r'\x01\u263a\x00\ufeff', 'unicode-escape')) + b = self.array('u', unicode(r'\x01\u263a\x00\ufeff', 'unicode-escape')) + b.byteswap() + assert a != b + class TestCPythonsOwnArray(BaseArrayTests): Modified: pypy/branch/asmgcc-64/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/cpyext/api.py (original) +++ pypy/branch/asmgcc-64/pypy/module/cpyext/api.py Fri Aug 13 15:07:27 2010 @@ -1,5 +1,5 @@ import ctypes -import sys +import sys, os import atexit import py @@ -896,6 +896,8 @@ initfunctype = lltype.Ptr(lltype.FuncType([], lltype.Void)) @unwrap_spec(ObjSpace, str, str) def load_extension_module(space, path, name): + if os.sep not in path: + path = os.curdir + os.sep + path # force a '/' in the path state = space.fromcache(State) state.package_context = name try: Modified: pypy/branch/asmgcc-64/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/asmgcc-64/pypy/rpython/lltypesystem/ll2ctypes.py Fri Aug 13 15:07:27 2010 @@ -24,6 +24,7 @@ from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE from pypy.rpython import raddress from pypy.translator.platform import platform +from array import array def uaddressof(obj): return fixid(ctypes.addressof(obj)) @@ -756,7 +757,15 @@ elif T is lltype.Char: llobj = chr(cobj) elif T is lltype.UniChar: - llobj = unichr(cobj) + try: + llobj = unichr(cobj) + except (ValueError, OverflowError): + for tc in 'HIL': + if array(tc).itemsize == array('u').itemsize: + llobj = array('u', array(tc, (cobj,)).tostring())[0] + break + else: + raise elif T is lltype.Signed: llobj = cobj elif T is lltype.Bool: From arigo at codespeak.net Fri Aug 13 17:18:16 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 13 Aug 2010 17:18:16 +0200 (CEST) Subject: [pypy-svn] r76614 - pypy/branch/kill-caninline/pypy/jit/metainterp Message-ID: <20100813151816.E58D5282C0A@codespeak.net> Author: arigo Date: Fri Aug 13 17:18:14 2010 New Revision: 76614 Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/blackhole.py Log: Document a new case in which we can fall (I think this is only possible on the kill-caninline branch). Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/blackhole.py Fri Aug 13 17:18:14 2010 @@ -1168,7 +1168,9 @@ # we now proceed to interpret the bytecode in this frame self.run() # - except JitException: + except JitException, e: + if self.nextblackholeinterp and self.jitcode.is_portal: + self._handle_jitexception_in_portal(e) raise # go through except Exception, e: # if we get an exception, return it to the caller frame @@ -1270,6 +1272,26 @@ e = lltype.cast_opaque_ptr(llmemory.GCREF, e) raise sd.ExitFrameWithExceptionRef(self.cpu, e) + def _handle_jitexception_in_portal(self, e): + # This case is really rare, but can occur if + # convert_and_run_from_pyjitpl() gets called in this situation: + # + # [function 1] <---- top BlackholeInterpreter() + # [recursive main jit code] + # ... + # [bottom main jit code] <---- bottom BlackholeInterpreter() + # + # and then "function 1" contains a call to "function 2", which + # calls "can_enter_jit". The latter can terminate by raising a + # JitException. In that case, the JitException is not supposed + # to fall through the whole chain of BlackholeInterpreters, but + # be caught and handled just below the level "recursive main jit + # code". The present function is called in this case, with self + # being the blackhole interpreter of "recursive main jit code". + print e + print "XXX must handle this JitException here!" + assert False, "XXX must handle this JitException here!" + def _copy_data_from_miframe(self, miframe): self.setposition(miframe.jitcode, miframe.pc) for i in range(self.jitcode.num_regs_i()): From arigo at codespeak.net Fri Aug 13 18:12:07 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 13 Aug 2010 18:12:07 +0200 (CEST) Subject: [pypy-svn] r76615 - pypy/branch/kill-caninline/pypy/jit/metainterp/test Message-ID: <20100813161207.5F77C282B9E@codespeak.net> Author: arigo Date: Fri Aug 13 18:12:04 2010 New Revision: 76615 Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_recursive.py Log: A test. Without the changes to blackhole.py in the previous revision, this fails as observed in a pypy-c: it ignores the tail of one iteration of one loop, and just returns the string '...fghiJ' instead of '...fghijJ'. Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_recursive.py Fri Aug 13 18:12:04 2010 @@ -1027,6 +1027,41 @@ assert res == 2095 self.check_loops(call_assembler=6, everywhere=True) + def test_handle_jitexception_in_portal(self): + # a test for _handle_jitexception_in_portal in blackhole.py + driver = JitDriver(greens = ['codeno'], reds = ['i', 'str'], + get_printable_location = lambda codeno: str(codeno)) + def do_can_enter_jit(codeno, i, str): + driver.can_enter_jit(codeno=codeno, i=i, str=str) + def intermediate(codeno, i, str): + if i == 9: + do_can_enter_jit(codeno, i, str) + def portal(codeno, str): + i = value.initial + while i < 10: + intermediate(codeno, i, str) + driver.jit_merge_point(codeno=codeno, i=i, str=str) + i += 1 + if codeno == 64 and i == 10: + str = portal(96, str) + str += chr(codeno+i) + return str + class Value: + initial = -1 + value = Value() + def main(): + value.initial = 0 + return (portal(64, '') + + portal(64, '') + + portal(64, '') + + portal(64, '') + + portal(64, '')) + assert main() == 'ABCDEFGHIabcdefghijJ' * 5 + for tlimit in [90, 110, 150]: + print 'tlimit =', tlimit + res = self.meta_interp(main, [], inline=True, trace_limit=tlimit) + assert ''.join(res.chars) == 'ABCDEFGHIabcdefghijJ' * 5 + # There is a test which I fail to write. # * what happens if we call recursive_call while blackholing # this seems to be completely corner case and not really happening From arigo at codespeak.net Fri Aug 13 18:15:31 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 13 Aug 2010 18:15:31 +0200 (CEST) Subject: [pypy-svn] r76616 - pypy/branch/kill-caninline/pypy/jit/metainterp/test Message-ID: <20100813161531.BE1CC282B9E@codespeak.net> Author: arigo Date: Fri Aug 13 18:15:30 2010 New Revision: 76616 Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_recursive.py Log: Remove this note from r70777, which I am guessing is exactly what my test covers. If I'm right, then no wonder you failed to write the test so far, because I think it could not happen before the branch/kill-caninline. Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_recursive.py Fri Aug 13 18:15:30 2010 @@ -1062,10 +1062,6 @@ res = self.meta_interp(main, [], inline=True, trace_limit=tlimit) assert ''.join(res.chars) == 'ABCDEFGHIabcdefghijJ' * 5 - # There is a test which I fail to write. - # * what happens if we call recursive_call while blackholing - # this seems to be completely corner case and not really happening - # in the wild class TestLLtype(RecursiveTests, LLJitMixin): pass From arigo at codespeak.net Fri Aug 13 19:07:35 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 13 Aug 2010 19:07:35 +0200 (CEST) Subject: [pypy-svn] r76617 - pypy/branch/kill-caninline/pypy/jit/metainterp/test Message-ID: <20100813170735.CF0C1282B9E@codespeak.net> Author: arigo Date: Fri Aug 13 19:07:33 2010 New Revision: 76617 Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_recursive.py Log: The previous problem-detection code is not enough. Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/test/test_recursive.py Fri Aug 13 19:07:33 2010 @@ -1032,6 +1032,7 @@ driver = JitDriver(greens = ['codeno'], reds = ['i', 'str'], get_printable_location = lambda codeno: str(codeno)) def do_can_enter_jit(codeno, i, str): + i = (i+1)-1 # some operations driver.can_enter_jit(codeno=codeno, i=i, str=str) def intermediate(codeno, i, str): if i == 9: @@ -1057,7 +1058,7 @@ portal(64, '') + portal(64, '')) assert main() == 'ABCDEFGHIabcdefghijJ' * 5 - for tlimit in [90, 110, 150]: + for tlimit in [95, 90, 102]: print 'tlimit =', tlimit res = self.meta_interp(main, [], inline=True, trace_limit=tlimit) assert ''.join(res.chars) == 'ABCDEFGHIabcdefghijJ' * 5 From arigo at codespeak.net Fri Aug 13 19:14:55 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 13 Aug 2010 19:14:55 +0200 (CEST) Subject: [pypy-svn] r76618 - pypy/branch/kill-caninline/pypy/jit/metainterp Message-ID: <20100813171455.DDF16282B9E@codespeak.net> Author: arigo Date: Fri Aug 13 19:14:53 2010 New Revision: 76618 Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/blackhole.py Log: "Fix" for the previous test (or more accurately, make it fail as expected). Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/blackhole.py Fri Aug 13 19:14:53 2010 @@ -1169,8 +1169,7 @@ self.run() # except JitException, e: - if self.nextblackholeinterp and self.jitcode.is_portal: - self._handle_jitexception_in_portal(e) + self._handle_jitexception(e) raise # go through except Exception, e: # if we get an exception, return it to the caller frame @@ -1272,7 +1271,7 @@ e = lltype.cast_opaque_ptr(llmemory.GCREF, e) raise sd.ExitFrameWithExceptionRef(self.cpu, e) - def _handle_jitexception_in_portal(self, e): + def _handle_jitexception(self, e): # This case is really rare, but can occur if # convert_and_run_from_pyjitpl() gets called in this situation: # @@ -1286,8 +1285,21 @@ # JitException. In that case, the JitException is not supposed # to fall through the whole chain of BlackholeInterpreters, but # be caught and handled just below the level "recursive main jit - # code". The present function is called in this case, with self - # being the blackhole interpreter of "recursive main jit code". + # code". + blackholeinterp = self + while True: + # If we reach "bottom main jit code", just re-raise. + # This case is handled by the fact that the call chain + # leading to convert_and_run_from_pyjitpl() contains it + # own ll_portal_runner(). + if not blackholeinterp.nextblackholeinterp: + raise e + # If we reach an intermediate portal, special code needed. + if blackholeinterp.jitcode.is_portal: + break + # Else, continue searching in the parent. + blackholeinterp = blackholeinterp.nextblackholeinterp + # print e print "XXX must handle this JitException here!" assert False, "XXX must handle this JitException here!" From getxsick at codespeak.net Fri Aug 13 22:07:47 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Fri, 13 Aug 2010 22:07:47 +0200 (CEST) Subject: [pypy-svn] r76619 - in pypy/branch/fast-ctypes/pypy/module/_ctypes: . test Message-ID: <20100813200747.56F0C282B9E@codespeak.net> Author: getxsick Date: Fri Aug 13 22:07:44 2010 New Revision: 76619 Added: pypy/branch/fast-ctypes/pypy/module/_ctypes/ pypy/branch/fast-ctypes/pypy/module/_ctypes/__init__.py pypy/branch/fast-ctypes/pypy/module/_ctypes/test/ pypy/branch/fast-ctypes/pypy/module/_ctypes/test/__init__.py Log: initial checkin for RPython implementation of ctypes Added: pypy/branch/fast-ctypes/pypy/module/_ctypes/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/fast-ctypes/pypy/module/_ctypes/__init__.py Fri Aug 13 22:07:44 2010 @@ -0,0 +1,5 @@ +from pypy.interpreter.mixedmodule import MixedModule + +class Module(MixedModule): + interpleveldefs = {} + appleveldefs = {} Added: pypy/branch/fast-ctypes/pypy/module/_ctypes/test/__init__.py ============================================================================== From arigo at codespeak.net Sat Aug 14 09:42:53 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 14 Aug 2010 09:42:53 +0200 (CEST) Subject: [pypy-svn] r76620 - pypy/branch/kill-caninline/pypy/jit/metainterp Message-ID: <20100814074253.03F37282C1B@codespeak.net> Author: arigo Date: Sat Aug 14 09:42:51 2010 New Revision: 76620 Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/blackhole.py pypy/branch/kill-caninline/pypy/jit/metainterp/jitdriver.py pypy/branch/kill-caninline/pypy/jit/metainterp/warmspot.py pypy/branch/kill-caninline/pypy/jit/metainterp/warmstate.py Log: Fix the bug shown in r76617. Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/blackhole.py Sat Aug 14 09:42:51 2010 @@ -1169,7 +1169,6 @@ self.run() # except JitException, e: - self._handle_jitexception(e) raise # go through except Exception, e: # if we get an exception, return it to the caller frame @@ -1271,38 +1270,32 @@ e = lltype.cast_opaque_ptr(llmemory.GCREF, e) raise sd.ExitFrameWithExceptionRef(self.cpu, e) - def _handle_jitexception(self, e): + def _handle_jitexception_in_portal(self, e): # This case is really rare, but can occur if # convert_and_run_from_pyjitpl() gets called in this situation: # # [function 1] <---- top BlackholeInterpreter() - # [recursive main jit code] + # [recursive portal jit code] # ... - # [bottom main jit code] <---- bottom BlackholeInterpreter() + # [bottom portal jit code] <---- bottom BlackholeInterpreter() # # and then "function 1" contains a call to "function 2", which # calls "can_enter_jit". The latter can terminate by raising a # JitException. In that case, the JitException is not supposed # to fall through the whole chain of BlackholeInterpreters, but - # be caught and handled just below the level "recursive main jit - # code". - blackholeinterp = self - while True: - # If we reach "bottom main jit code", just re-raise. - # This case is handled by the fact that the call chain - # leading to convert_and_run_from_pyjitpl() contains it - # own ll_portal_runner(). - if not blackholeinterp.nextblackholeinterp: - raise e - # If we reach an intermediate portal, special code needed. - if blackholeinterp.jitcode.is_portal: + # be caught and handled just below the level "recursive portal + # jit code". The present function is called to handle the case + # of recursive portal jit codes. + for jd in self.builder.metainterp_sd.jitdrivers_sd: + if jd.mainjitcode is self.jitcode: break - # Else, continue searching in the parent. - blackholeinterp = blackholeinterp.nextblackholeinterp - # - print e - print "XXX must handle this JitException here!" - assert False, "XXX must handle this JitException here!" + else: + assert 0, "portal jitcode not found??" + # call the helper in warmspot.py. It might either raise a + # regular exception (which should then be propagated outside + # of 'self', not caught inside), or return (the return value + # gets stored in nextblackholeinterp). + jd.handle_jitexc_from_bh(self.nextblackholeinterp, e) def _copy_data_from_miframe(self, miframe): self.setposition(miframe.jitcode, miframe.pc) @@ -1325,10 +1318,32 @@ while True: try: current_exc = blackholeinterp._resume_mainloop(current_exc) - finally: - blackholeinterp.builder.release_interp(blackholeinterp) + except JitException, e: + blackholeinterp, current_exc = _handle_jitexception( + blackholeinterp, e) + blackholeinterp.builder.release_interp(blackholeinterp) blackholeinterp = blackholeinterp.nextblackholeinterp +def _handle_jitexception(blackholeinterp, jitexc): + # See comments in _handle_jitexception_in_portal(). + while not blackholeinterp.jitcode.is_portal: + blackholeinterp.builder.release_interp(blackholeinterp) + blackholeinterp = blackholeinterp.nextblackholeinterp + if blackholeinterp.nextblackholeinterp is None: + blackholeinterp.builder.release_interp(blackholeinterp) + raise jitexc # bottommost entry: go through + # We have reached a recursive portal level. + try: + blackholeinterp._handle_jitexception_in_portal(jitexc) + except Exception, e: + # It raised a general exception (it should not be a JitException here). + lle = get_llexception(blackholeinterp.cpu, e) + else: + # It set up the nextblackholeinterp to contain the return value. + lle = lltype.nullptr(rclass.OBJECTPTR.TO) + # We will continue to loop in _run_forever() from the parent level. + return blackholeinterp, lle + def resume_in_blackhole(metainterp_sd, jitdriver_sd, resumedescr, all_virtuals=None): from pypy.jit.metainterp.resume import blackhole_from_resumedata Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/jitdriver.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/jitdriver.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/jitdriver.py Sat Aug 14 09:42:51 2010 @@ -12,6 +12,7 @@ # self.result_type ... pypy.jit.metainterp.warmspot # self.virtualizable_info... pypy.jit.metainterp.warmspot # self.warmstate ... pypy.jit.metainterp.warmspot + # self.handle_jitexc_from_bh pypy.jit.metainterp.warmspot # self.index ... pypy.jit.codewriter.call # self.mainjitcode ... pypy.jit.codewriter.call Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/warmspot.py Sat Aug 14 09:42:51 2010 @@ -558,6 +558,7 @@ # Prepare the portal_runner() helper # from pypy.jit.metainterp.warmstate import specialize_value + from pypy.jit.metainterp.warmstate import unspecialize_value portal_ptr = self.cpu.ts.functionptr(PORTALFUNC, 'portal', graph = portalgraph) jd._portal_ptr = portal_ptr @@ -612,6 +613,37 @@ value = cast_base_ptr_to_instance(Exception, value) raise Exception, value + def handle_jitexception(e): + # XXX the bulk of this function is a copy-paste from above :-( + try: + raise e + except self.ContinueRunningNormally, e: + args = () + for ARGTYPE, attrname, count in portalfunc_ARGS: + x = getattr(e, attrname)[count] + x = specialize_value(ARGTYPE, x) + args = args + (x,) + return ll_portal_runner(*args) + except self.DoneWithThisFrameVoid: + assert result_kind == 'void' + return + except self.DoneWithThisFrameInt, e: + assert result_kind == 'int' + return specialize_value(RESULT, e.result) + except self.DoneWithThisFrameRef, e: + assert result_kind == 'ref' + return specialize_value(RESULT, e.result) + except self.DoneWithThisFrameFloat, e: + assert result_kind == 'float' + return specialize_value(RESULT, e.result) + except self.ExitFrameWithExceptionRef, e: + value = ts.cast_to_baseclass(e.value) + if not we_are_translated(): + raise LLException(ts.get_typeptr(value), value) + else: + value = cast_base_ptr_to_instance(Exception, value) + raise Exception, value + jd._ll_portal_runner = ll_portal_runner # for debugging jd.portal_runner_ptr = self.helper_func(jd._PTR_PORTAL_FUNCTYPE, ll_portal_runner) @@ -632,32 +664,8 @@ vinfo.reset_vable_token(virtualizable) try: loop_token = fail_descr.handle_fail(self.metainterp_sd, jd) - except self.ContinueRunningNormally, e: - args = () - for ARGTYPE, attrname, count in portalfunc_ARGS: - x = getattr(e, attrname)[count] - x = specialize_value(ARGTYPE, x) - args = args + (x,) - return ll_portal_runner(*args) - except self.DoneWithThisFrameVoid: - assert result_kind == 'void' - return - except self.DoneWithThisFrameInt, e: - assert result_kind == 'int' - return specialize_value(RESULT, e.result) - except self.DoneWithThisFrameRef, e: - assert result_kind == 'ref' - return specialize_value(RESULT, e.result) - except self.DoneWithThisFrameFloat, e: - assert result_kind == 'float' - return specialize_value(RESULT, e.result) - except self.ExitFrameWithExceptionRef, e: - value = ts.cast_to_baseclass(e.value) - if not we_are_translated(): - raise LLException(ts.get_typeptr(value), value) - else: - value = cast_base_ptr_to_instance(Exception, value) - raise Exception, value + except JitException, e: + return handle_jitexception(e) fail_descr = self.cpu.execute_token(loop_token) jd._assembler_call_helper = assembler_call_helper # for debugging @@ -669,6 +677,21 @@ if vinfo is not None: jd.vable_token_descr = vinfo.vable_token_descr + def handle_jitexception_from_blackhole(bhcaller, e): + result = handle_jitexception(e) + # + if result_kind != 'void': + result = unspecialize_value(result) + if result_kind == 'int': + bhcaller._setup_return_value_i(result) + elif result_kind == 'ref': + bhcaller._setup_return_value_r(result) + elif result_kind == 'float': + bhcaller._setup_return_value_f(result) + else: + assert False + jd.handle_jitexc_from_bh = handle_jitexception_from_blackhole + # ____________________________________________________________ # Now mutate origportalgraph to end with a call to portal_runner_ptr # Modified: pypy/branch/kill-caninline/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/kill-caninline/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/kill-caninline/pypy/jit/metainterp/warmstate.py Sat Aug 14 09:42:51 2010 @@ -30,6 +30,22 @@ else: return lltype.cast_opaque_ptr(TYPE, x) + at specialize.ll() +def unspecialize_value(value): + """Casts 'value' to a Signed, a GCREF or a Float.""" + if isinstance(lltype.typeOf(value), lltype.Ptr): + if lltype.typeOf(value).TO._gckind == 'gc': + return lltype.cast_opaque_ptr(llmemory.GCREF, value) + else: + adr = llmemory.cast_ptr_to_adr(value) + return heaptracker.adr2int(adr) + elif isinstance(lltype.typeOf(value), ootype.OOType): + return ootype.cast_to_object(value) + elif isinstance(value, float): + return value + else: + return intmask(value) + @specialize.arg(0) def unwrap(TYPE, box): if TYPE is lltype.Void: From arigo at codespeak.net Sat Aug 14 09:46:34 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 14 Aug 2010 09:46:34 +0200 (CEST) Subject: [pypy-svn] r76621 - pypy/branch/kill-caninline/pypy/module/pypyjit Message-ID: <20100814074634.19E05282C1B@codespeak.net> Author: arigo Date: Sat Aug 14 09:46:33 2010 New Revision: 76621 Modified: pypy/branch/kill-caninline/pypy/module/pypyjit/interp_jit.py Log: Improve the handling of generators, possibly. Modified: pypy/branch/kill-caninline/pypy/module/pypyjit/interp_jit.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/pypyjit/interp_jit.py (original) +++ pypy/branch/kill-caninline/pypy/module/pypyjit/interp_jit.py Sat Aug 14 09:46:33 2010 @@ -20,6 +20,7 @@ 'fastlocals_w[*]', 'last_exception', 'lastblock', + 'last_yield', ] JUMP_ABSOLUTE = opmap['JUMP_ABSOLUTE'] @@ -62,10 +63,10 @@ PyFrame_execute_generator_frame = PyFrame.execute_generator_frame class __extend__(PyFrame): - just_resuming_generator = False + last_yield = -1 def execute_generator_frame(self, w_inputvalue, ex=False): - self.just_resuming_generator = True + self.last_yield = self.last_instr return PyFrame_execute_generator_frame(self, w_inputvalue, ex) def dispatch(self, pycode, next_instr, ec): @@ -81,14 +82,22 @@ except ExitFrame: return self.popvalue() - def jump_absolute(self, jumpto, _, ec=None): - if self.just_resuming_generator: - self.just_resuming_generator = False + def jump_absolute(self, jumpto, _, ec): + # Custom logic. A JUMP_ABSOLUTE closes a loop, except when it is + # a jump over the most recent YIELD_VALUE. + if jumpto <= self.last_yield: + self.last_yield = -1 return jumpto + # In jitted mode, we call bytecode_trace() only once per loop, + # instead of every N instructions. if we_are_jitted(): self.last_instr = intmask(jumpto) ec.bytecode_trace(self) jumpto = r_uint(self.last_instr) + # Call the main hook on pypyjitdriver. In the non-JITted version + # of this code, this becomes a call to a profiling helper that may + # jump to the JITted version. In the JITted version, it marks the + # start and the end of all loops. pypyjitdriver.can_enter_jit(frame=self, ec=ec, next_instr=jumpto, pycode=self.getcode()) return jumpto From arigo at codespeak.net Sat Aug 14 10:46:40 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 14 Aug 2010 10:46:40 +0200 (CEST) Subject: [pypy-svn] r76622 - in pypy/trunk/pypy/tool/release: . test Message-ID: <20100814084640.483B1282B90@codespeak.net> Author: arigo Date: Sat Aug 14 10:46:38 2010 New Revision: 76622 Modified: pypy/trunk/pypy/tool/release/package.py pypy/trunk/pypy/tool/release/test/test_package.py Log: On Windows, don't rely on the presence of the 'tar' tool. Add a test (that also runs on linux) checking compression with the tarfile module. Modified: pypy/trunk/pypy/tool/release/package.py ============================================================================== --- pypy/trunk/pypy/tool/release/package.py (original) +++ pypy/trunk/pypy/tool/release/package.py Sat Aug 14 10:46:38 2010 @@ -11,11 +11,12 @@ import py import os import fnmatch -import tarfile from pypy.tool.udir import udir if sys.version_info < (2,6): py.test.skip("requires 2.6 so far") +USE_TARFILE_MODULE = sys.platform == 'win32' + def ignore_patterns(*patterns): """Function that can be used as copytree() ignore parameter. @@ -69,9 +70,17 @@ old_dir = os.getcwd() try: os.chdir(str(builddir)) - os.system("strip " + str(archive_pypy_c)) - os.system('tar cvjf ' + str(builddir.join(name + '.tar.bz2')) + - " " + name) + os.system("strip " + str(archive_pypy_c)) # ignore errors + if USE_TARFILE_MODULE: + import tarfile + tf = tarfile.open(str(builddir.join(name + '.tar.bz2')), 'w:bz2') + tf.add(name) + tf.close() + else: + e = os.system('tar cvjf ' + str(builddir.join(name + '.tar.bz2')) + + " " + name) + if e: + raise OSError('"tar" returned exit status %r' % e) finally: os.chdir(old_dir) if copy_to_dir is not None: Modified: pypy/trunk/pypy/tool/release/test/test_package.py ============================================================================== --- pypy/trunk/pypy/tool/release/test/test_package.py (original) +++ pypy/trunk/pypy/tool/release/test/test_package.py Sat Aug 14 10:46:38 2010 @@ -5,7 +5,7 @@ from pypy.module.sys.version import CPYTHON_VERSION import tarfile, os -def test_dir_structure(): +def test_dir_structure(test='test'): # make sure we have sort of pypy-c pypy_c = py.path.local(pypydir).join('translator', 'goal', 'pypy-c') if not pypy_c.check(): @@ -14,8 +14,8 @@ else: fake_pypy_c = False try: - builddir = package(py.path.local(pypydir).dirpath(), 'test') - prefix = builddir.join('test') + builddir = package(py.path.local(pypydir).dirpath(), test) + prefix = builddir.join(test) cpyver = '%d.%d.%d' % CPYTHON_VERSION[:3] assert prefix.join('lib-python', cpyver, 'test').check() assert prefix.join('bin', 'pypy-c').check() @@ -24,18 +24,27 @@ assert not prefix.join('lib_pypy', 'ctypes_configure').check() assert prefix.join('LICENSE').check() assert prefix.join('README').check() - th = tarfile.open(str(builddir.join('test.tar.bz2'))) - assert th.getmember('test/lib_pypy/syslog.py') + th = tarfile.open(str(builddir.join('%s.tar.bz2' % test))) + assert th.getmember('%s/lib_pypy/syslog.py' % test) # the headers file could be not there, because they are copied into # trunk/include only during translation includedir = py.path.local(pypydir).dirpath().join('include') def check_include(name): if includedir.join(name).check(file=True): - assert th.getmember('test/include/%s' % name) + assert th.getmember('%s/include/%s' % (test, name)) check_include('Python.h') check_include('modsupport.inl') check_include('pypy_decl.h') finally: if fake_pypy_c: pypy_c.remove() + +def test_with_tarfile_module(): + from pypy.tool.release import package + prev = package.USE_TARFILE_MODULE + try: + package.USE_TARFILE_MODULE = True + test_dir_structure(test='testtarfile') + finally: + package.USE_TARFILE_MODULE = prev From hakanardo at codespeak.net Sat Aug 14 12:13:53 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Sat, 14 Aug 2010 12:13:53 +0200 (CEST) Subject: [pypy-svn] r76623 - in pypy/branch/jit-bounds/pypy/jit/metainterp: . test Message-ID: <20100814101353.6428A282B90@codespeak.net> Author: hakanardo Date: Sat Aug 14 12:13:51 2010 New Revision: 76623 Added: pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_intbound.py Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py Log: Moved bound calculations to separate class, support for lt, gt, le, ge, add, sub Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py Sat Aug 14 12:13:51 2010 @@ -17,10 +17,8 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.rpython.lltypesystem import lltype from pypy.jit.metainterp.history import AbstractDescr, make_hashable_int - -import sys -SYSMAXINT = sys.maxint -SYSMININT = -sys.maxint - 1 +from pypy.rlib.rarithmetic import ovfcheck +from copy import copy def optimize_loop_1(metainterp_sd, loop): """Optimize loop.operations to make it match the input of loop.specnodes @@ -41,75 +39,43 @@ # ____________________________________________________________ -LEVEL_UNKNOWN = '\x00' -LEVEL_NONNULL = '\x01' -LEVEL_KNOWNCLASS = '\x02' # might also mean KNOWNARRAYDESCR, for arrays -LEVEL_CONSTANT = '\x03' - - -class OptValue(object): - _attrs_ = ('box', 'known_class', 'last_guard_index', 'level') - last_guard_index = -1 +class IntBound(object): + def __init__(self, lower, upper): + self.has_upper = True + self.has_lower = True + self.upper = upper + self.lower = lower + + # Returns True if the bound was updated + def make_le(self, other): + if other.has_upper: + if not self.has_upper or other.upper < self.upper: + self.has_upper = True + self.upper = other.upper + return True + return False - level = LEVEL_UNKNOWN - known_class = None - maxint = None - minint = None + def make_lt(self, other): + return self.make_le(other.add(-1)) - def __init__(self, box, producer=None): - self.box = box - self.producer = producer - if isinstance(box, Const): - self.level = LEVEL_CONSTANT - # invariant: box is a Const if and only if level == LEVEL_CONSTANT + def make_ge(self, other): + if other.has_lower: + if not self.has_lower or other.lower > self.lower: + self.has_lower = True + self.lower = other.lower + return True + return False - def get_maxint(self): - if self.level == LEVEL_CONSTANT: - return self.box.getint() - else: - return self.maxint - - def get_minint(self): - if self.level == LEVEL_CONSTANT: - return self.box.getint() - else: - return self.minint - - def boundint_lt(self, val): - if val is None: return - if self.maxint is None or val <= self.maxint: - if val <= SYSMININT: - self.maxint = None - self.maxint = val - 1 - - def boundint_le(self, val): - if val is None: return - if self.maxint is None or val < self.maxint: - self.maxint = val - - def boundint_gt(self, val): - if val is None: return - if self.minint is None or val >= self.minint: - if val >= SYSMAXINT: - self.minint = None - self.minint = val + 1 - - def boundint_ge(self, val): - if val is None: return - if self.minint is None or val > self.minint: - self.minint = val + def make_gt(self, other): + return self.make_ge(other.add(1)) def known_lt(self, other): - max1 = self.get_maxint() - min2 = other.get_minint() - if max1 is not None and min2 is not None and max1 < min2: + if self.has_upper and other.has_lower and self.upper < other.lower: return True return False def known_le(self, other): - max1 = self.get_maxint() - min2 = other.get_minint() - if max1 is not None and min2 is not None and max1 <= min2: + if self.has_upper and other.has_lower and self.upper <= other.lower: return True return False @@ -119,7 +85,121 @@ def known_ge(self, other): return other.known_le(self) + def intersect(self, other): + r = False + + if other.has_lower: + if other.lower > self.lower or not self.has_lower: + self.lower = other.lower + self.has_lower = True + r = True + + if other.has_upper: + if other.upper < self.upper or not self.has_upper: + self.upper = other.upper + self.has_upper = True + r = True + + return r + + def add(self, offset): + res = copy(self) + try: + res.lower = ovfcheck(res.lower + offset) + except OverflowError: + res.has_lower = False + try: + res.upper = ovfcheck(res.upper + offset) + except OverflowError: + res.has_upper = False + return res + + def add_bound(self, other): + res = copy(self) + if other.has_upper: + res.upper += other.upper + else: + res.has_upper = False + if other.has_lower: + res.lower += other.lower + else: + res.has_lower = False + return res + + def sub_bound(self, other): + res = copy(self) + if other.has_upper: + res.lower -= other.upper + else: + res.has_lower = False + if other.has_lower: + res.upper -= other.lower + else: + res.has_upper = False + return res + + def contains(self, val): + if self.has_lower and val < self.lower: + return False + if self.has_upper and val > self.upper: + return False + return True + + def __repr__(self): + if self.has_lower: + l = '%4d' % self.lower + else: + l = '-Inf' + if self.has_upper: + u = '%3d' % self.upper + else: + u = 'Inf' + return '%s <= x <= %s' % (l, u) + + + +class IntUpperBound(IntBound): + def __init__(self, upper): + self.has_upper = True + self.has_lower = False + self.upper = upper + self.lower = 0 + +class IntLowerBound(IntBound): + def __init__(self, lower): + self.has_upper = False + self.has_lower = True + self.upper = 0 + self.lower = lower + +class IntUnbounded(IntBound): + def __init__(self): + self.has_upper = False + self.has_lower = False + self.upper = 0 + self.lower = 0 + +LEVEL_UNKNOWN = '\x00' +LEVEL_NONNULL = '\x01' +LEVEL_KNOWNCLASS = '\x02' # might also mean KNOWNARRAYDESCR, for arrays +LEVEL_CONSTANT = '\x03' + +class OptValue(object): + _attrs_ = ('box', 'known_class', 'last_guard_index', 'level') + last_guard_index = -1 + + level = LEVEL_UNKNOWN + known_class = None + + def __init__(self, box, producer=None): + self.box = box + self.producer = producer + self.intbound = IntUnbounded() + if isinstance(box, Const): + self.make_constant(box) + # invariant: box is a Const if and only if level == LEVEL_CONSTANT + def force_box(self): return self.box @@ -147,6 +227,10 @@ assert isinstance(constbox, Const) self.box = constbox self.level = LEVEL_CONSTANT + try: + self.intbound = IntBound(self.box.getint(),self. box.getint()) + except NotImplementedError: + self.intbound = IntUnbounded() def get_constant_class(self, cpu): level = self.level @@ -205,10 +289,8 @@ raise NotImplementedError class ConstantValue(OptValue): - level = LEVEL_CONSTANT - def __init__(self, box): - self.box = box + self.make_constant(box) CONST_0 = ConstInt(0) CONST_1 = ConstInt(1) @@ -604,13 +686,10 @@ self.resumedata_memo.update_counters(self.metainterp_sd.profiler) def propagate_bounds_backward(self, box): - print box try: op = self.producer[box] except KeyError: return - print op - opnum = op.opnum for value, func in propagate_bounds_ops: if opnum == value: @@ -1127,7 +1206,10 @@ if v2.is_constant() and v2.box.getint() == 0: self.make_equal_to(op.result, v1) else: - return self.optimize_default(op) + self.optimize_default(op) + r = self.getvalue(op.result) + print v1.intbound.sub_bound(v2.intbound) + r.intbound.intersect(v1.intbound.sub_bound(v2.intbound)) def optimize_INT_ADD(self, op): v1 = self.getvalue(op.args[0]) @@ -1139,13 +1221,15 @@ self.make_equal_to(op.result, v1) else: self.optimize_default(op) + r = self.getvalue(op.result) + r.intbound.intersect(v1.intbound.add_bound(v2.intbound)) def optimize_INT_LT(self, op): v1 = self.getvalue(op.args[0]) v2 = self.getvalue(op.args[1]) - if v1.known_lt(v2): + if v1.intbound.known_lt(v2.intbound): self.make_constant_int(op.result, 1) - elif v1.known_ge(v2): + elif v1.intbound.known_ge(v2.intbound): self.make_constant_int(op.result, 0) else: self.optimize_default(op) @@ -1153,45 +1237,109 @@ def optimize_INT_GT(self, op): v1 = self.getvalue(op.args[0]) v2 = self.getvalue(op.args[1]) - if v1.known_gt(v2): + if v1.intbound.known_gt(v2.intbound): self.make_constant_int(op.result, 1) - elif v1.known_le(v2): + elif v1.intbound.known_le(v2.intbound): self.make_constant_int(op.result, 0) else: self.optimize_default(op) - def propagate_bounds_INT_LT(self, op): + def optimize_INT_LE(self, op): v1 = self.getvalue(op.args[0]) v2 = self.getvalue(op.args[1]) + if v1.intbound.known_le(v2.intbound): + self.make_constant_int(op.result, 1) + elif v1.intbound.known_gt(v2.intbound): + self.make_constant_int(op.result, 0) + else: + self.optimize_default(op) + + def optimize_INT_GE(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + if v1.intbound.known_ge(v2.intbound): + self.make_constant_int(op.result, 1) + elif v1.intbound.known_lt(v2.intbound): + self.make_constant_int(op.result, 0) + else: + self.optimize_default(op) + + def make_int_lt(self, args): + v1 = self.getvalue(args[0]) + v2 = self.getvalue(args[1]) + if v1.intbound.make_lt(v2.intbound): + self.propagate_bounds_backward(args[0]) + if v2.intbound.make_gt(v1.intbound): + self.propagate_bounds_backward(args[1]) + + + def make_int_le(self, args): + v1 = self.getvalue(args[0]) + v2 = self.getvalue(args[1]) + if v1.intbound.make_le(v2.intbound): + self.propagate_bounds_backward(args[0]) + if v2.intbound.make_ge(v1.intbound): + self.propagate_bounds_backward(args[1]) + + def make_int_gt(self, args): + self.make_int_lt([args[1], args[0]]) + + def make_int_ge(self, args): + self.make_int_le([args[1], args[0]]) + + def propagate_bounds_INT_LT(self, op): r = self.getvalue(op.result) if r.is_constant(): if r.box.same_constant(CONST_1): - v1.boundint_lt(v2.get_maxint()) - v2.boundint_gt(v1.get_minint()) - elif r.box.same_constant(CONST_0): - v1.boundint_ge(v2.get_minint()) - v2.boundint_le(v1.get_maxint()) + self.make_int_lt(op.args) else: - assert False, "Boolean neither True nor False" - self.propagate_bounds_backward(op.args[0]) - self.propagate_bounds_backward(op.args[1]) + self.make_int_ge(op.args) def propagate_bounds_INT_GT(self, op): - v1 = self.getvalue(op.args[0]) - v2 = self.getvalue(op.args[1]) r = self.getvalue(op.result) if r.is_constant(): if r.box.same_constant(CONST_1): - v2.boundint_lt(v1.get_maxint()) - v1.boundint_gt(v2.get_minint()) - elif r.box.same_constant(CONST_0): - v2.boundint_ge(v1.get_minint()) - v1.boundint_le(v2.get_maxint()) + self.make_int_gt(op.args) + else: + self.make_int_le(op.args) + + def propagate_bounds_INT_LE(self, op): + r = self.getvalue(op.result) + if r.is_constant(): + if r.box.same_constant(CONST_1): + self.make_int_le(op.args) else: - assert False, "Boolean neither True nor False" - self.propagate_bounds_backward(op.args[0]) + self.make_int_gt(op.args) + + def propagate_bounds_INT_GE(self, op): + r = self.getvalue(op.result) + if r.is_constant(): + if r.box.same_constant(CONST_1): + self.make_int_ge(op.args) + else: + self.make_int_lt(op.args) + + def propagate_bounds_INT_ADD(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + r = self.getvalue(op.result) + b = r.intbound.sub_bound(v2.intbound) + if v1.intbound.intersect(b): + self.propagate_bounds_backward(op.args[0]) + b = r.intbound.sub_bound(v1.intbound) + if v2.intbound.intersect(b): self.propagate_bounds_backward(op.args[1]) + def propagate_bounds_INT_SUB(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + r = self.getvalue(op.result) + b = r.intbound.add_bound(v2.intbound) + if v1.intbound.intersect(b): + self.propagate_bounds_backward(op.args[0]) + # FIXME: + # b = r.intbound.sub_bound(v1.intbound).mul(-1) + # if v2.intbound.intersect(b): optimize_ops = _findall(Optimizer, 'optimize_') propagate_bounds_ops = _findall(Optimizer, 'propagate_bounds_') Added: pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_intbound.py ============================================================================== --- (empty file) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_intbound.py Sat Aug 14 12:13:51 2010 @@ -0,0 +1,199 @@ +from pypy.jit.metainterp.optimizeopt import IntBound, IntUpperBound, \ + IntLowerBound, IntUnbounded +from copy import copy + +def bound(a,b): + if a is None and b is None: + return IntUnbounded() + elif a is None: + return IntUpperBound(b) + elif b is None: + return IntLowerBound(a) + else: + return IntBound(a,b) + +def const(a): + return bound(a,a) + +def some_bounds(): + brd = [None] + range(-2, 3) + for lower in brd: + for upper in brd: + if lower is not None and upper is not None and lower > upper: + continue + yield (lower, upper, bound(lower, upper)) + +nbr = range(-5, 6) + +def test_known(): + for lower, upper, b in some_bounds(): + inside = [] + border = [] + for n in nbr: + if (lower is None or n >= lower) and \ + (upper is None or n <= upper): + if n == lower or n ==upper: + border.append(n) + else: + inside.append(n) + + for n in nbr: + c = const(n) + if n in inside: + assert b.contains(n) + assert not b.known_lt(c) + assert not b.known_gt(c) + assert not b.known_le(c) + assert not b.known_ge(c) + elif n in border: + assert b.contains(n) + if n == upper: + assert b.known_le(const(upper)) + else: + assert b.known_ge(const(lower)) + else: + assert not b.contains(n) + some = (border + inside)[0] + if n < some: + assert b.known_gt(c) + else: + assert b.known_lt(c) + + +def test_make(): + for _, _, b1 in some_bounds(): + for _, _, b2 in some_bounds(): + lt = IntUnbounded() + lt.make_lt(b1) + lt.make_lt(b2) + for n in nbr: + c = const(n) + if b1.known_le(c) or b2.known_le(c): + assert lt.known_lt(c) + else: + assert not lt.known_lt(c) + assert not lt.known_gt(c) + assert not lt.known_ge(c) + + gt = IntUnbounded() + gt.make_gt(b1) + gt.make_gt(b2) + for n in nbr: + c = const(n) + if b1.known_ge(c) or b2.known_ge(c): + assert gt.known_gt(c) + else: + assert not gt.known_gt(c) + assert not gt.known_lt(c) + assert not gt.known_le(c) + + le = IntUnbounded() + le.make_le(b1) + le.make_le(b2) + for n in nbr: + c = const(n) + if b1.known_le(c) or b2.known_le(c): + assert le.known_le(c) + else: + assert not le.known_le(c) + assert not le.known_gt(c) + assert not le.known_ge(c) + + + ge = IntUnbounded() + ge.make_ge(b1) + ge.make_ge(b2) + for n in nbr: + c = const(n) + if b1.known_ge(c) or b2.known_ge(c): + assert ge.known_ge(c) + else: + assert not ge.known_ge(c) + assert not ge.known_lt(c) + assert not ge.known_le(c) + + gl = IntUnbounded() + gl.make_ge(b1) + gl.make_le(b2) + for n in nbr: + c = const(n) + if b1.known_ge(c): + assert gl.known_ge(c) + else: + assert not gl.known_ge(c) + assert not gl.known_gt(c) + if b2.known_le(c): + assert gl.known_le(c) + else: + assert not gl.known_le(c) + assert not gl.known_lt(c) + +def test_intersect(): + for _, _, b1 in some_bounds(): + for _, _, b2 in some_bounds(): + b = copy(b1) + b.intersect(b2) + for n in nbr: + if b1.contains(n) and b2.contains(n): + assert b.contains(n) + else: + assert not b.contains(n) + +def test_add(): + for _, _, b1 in some_bounds(): + for n1 in nbr: + b2 = b1.add(n1) + for n2 in nbr: + c1 = const(n2) + c2 = const(n2 + n1) + + if b1.known_le(c1): + assert b2.known_le(c2) + else: + assert not b2.known_le(c2) + + if b1.known_ge(c1): + assert b2.known_ge(c2) + else: + assert not b2.known_ge(c2) + + if b1.known_le(c1): + assert b2.known_le(c2) + else: + assert not b2.known_lt(c2) + + if b1.known_lt(c1): + assert b2.known_lt(c2) + else: + assert not b2.known_lt(c2) + + if b1.known_gt(c1): + assert b2.known_gt(c2) + else: + assert not b2.known_gt(c2) + +def test_add_bound(): + for _, _, b1 in some_bounds(): + for _, _, b2 in some_bounds(): + b3 = b1.add_bound(b2) + for n1 in nbr: + for n2 in nbr: + if b1.contains(n1) and b2.contains(n2): + assert b3.contains(n1 + n2) + + a=bound(2, 4).add_bound(bound(1, 2)) + assert not a.contains(2) + assert not a.contains(7) + +def test_sub_bound(): + for _, _, b1 in some_bounds(): + for _, _, b2 in some_bounds(): + b3 = b1.sub_bound(b2) + for n1 in nbr: + for n2 in nbr: + if b1.contains(n1) and b2.contains(n2): + assert b3.contains(n1 - n2) + + a=bound(2, 4).sub_bound(bound(1, 2)) + assert not a.contains(-1) + assert not a.contains(4) Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py Sat Aug 14 12:13:51 2010 @@ -3112,6 +3112,135 @@ """ self.optimize_loop(ops, 'Not', expected) + def test_bound_lt_tripple(self): + ops = """ + [i0] + i1 = int_lt(i0, 0) + guard_true(i1) [] + i2 = int_lt(i0, 7) + guard_true(i2) [] + i3 = int_lt(i0, 5) + guard_true(i3) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_lt(i0, 0) + guard_true(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_bound_lt_add(self): + ops = """ + [i0] + i1 = int_lt(i0, 4) + guard_true(i1) [] + i2 = int_add(i0, 10) + i3 = int_lt(i2, 15) + guard_true(i3) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_lt(i0, 4) + guard_true(i1) [] + i2 = int_add(i0, 10) + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_bound_lt_add_before(self): + ops = """ + [i0] + i2 = int_add(i0, 10) + i3 = int_lt(i2, 15) + guard_true(i3) [] + i1 = int_lt(i0, 6) + guard_true(i1) [] + jump(i0) + """ + expected = """ + [i0] + i2 = int_add(i0, 10) + i3 = int_lt(i2, 15) + guard_true(i3) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_bound_lt_sub(self): + ops = """ + [i0] + i1 = int_lt(i0, 4) + guard_true(i1) [] + i2 = int_sub(i0, 10) + i3 = int_lt(i2, -5) + guard_true(i3) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_lt(i0, 4) + guard_true(i1) [] + i2 = int_sub(i0, 10) + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_bound_lt_sub_before(self): + ops = """ + [i0] + i2 = int_sub(i0, 10) + i3 = int_lt(i2, -5) + guard_true(i3) [] + i1 = int_lt(i0, 5) + guard_true(i1) [] + jump(i0) + """ + expected = """ + [i0] + i2 = int_sub(i0, 10) + i3 = int_lt(i2, -5) + guard_true(i3) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_bound_ltle(self): + ops = """ + [i0] + i1 = int_lt(i0, 4) + guard_true(i1) [] + i2 = int_le(i0, 3) + guard_true(i2) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_lt(i0, 4) + guard_true(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_bound_lelt(self): + ops = """ + [i0] + i1 = int_le(i0, 4) + guard_true(i1) [] + i2 = int_lt(i0, 5) + guard_true(i2) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_le(i0, 4) + guard_true(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + def test_bound_gt(self): ops = """ [i0] @@ -3129,27 +3258,40 @@ """ self.optimize_loop(ops, 'Not', expected) - def test_bound_lt_tripple(self): + def test_bound_gtge(self): ops = """ [i0] - i1 = int_lt(i0, 0) + i1 = int_gt(i0, 5) guard_true(i1) [] - i2 = int_lt(i0, 7) + i2 = int_ge(i0, 6) guard_true(i2) [] - i3 = int_lt(i0, 5) - guard_true(i3) [] jump(i0) """ expected = """ [i0] - i1 = int_lt(i0, 0) + i1 = int_gt(i0, 5) guard_true(i1) [] jump(i0) """ self.optimize_loop(ops, 'Not', expected) - - + def test_bound_gegt(self): + ops = """ + [i0] + i1 = int_ge(i0, 5) + guard_true(i1) [] + i2 = int_gt(i0, 4) + guard_true(i2) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_ge(i0, 5) + guard_true(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + ##class TestOOtype(BaseTestOptimizeOpt, OOtypeMixin): From dan at codespeak.net Sat Aug 14 12:17:12 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Sat, 14 Aug 2010 12:17:12 +0200 (CEST) Subject: [pypy-svn] r76624 - in pypy/branch/micronumpy/pypy: module/micronumpy module/micronumpy/test tool Message-ID: <20100814101712.4729A282B90@codespeak.net> Author: dan Date: Sat Aug 14 12:17:10 2010 New Revision: 76624 Modified: pypy/branch/micronumpy/pypy/module/micronumpy/array.py pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py pypy/branch/micronumpy/pypy/tool/convolve.py pypy/branch/micronumpy/pypy/tool/numpybench.py Log: Passing way more tests, most slicing working. Translated with JIT does not. Preparing for sync with trunk. Modified: pypy/branch/micronumpy/pypy/module/micronumpy/array.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/array.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/array.py Sat Aug 14 12:17:10 2010 @@ -26,14 +26,17 @@ size *= dimension return size -def normalize_slice_starts(starts, shape): - result = starts[:] - for i in range(len(result)): - if result[i] < 0: - result[i] += shape[i] - elif result[i] >= shape[i]: +def normalize_slice_starts(slice_starts, shape): + for i in range(len(slice_starts)): + if slice_starts[i] < 0: + slice_starts[i] += shape[i] + elif slice_starts[i] >= shape[i]: raise IndexError("invalid index") - return result + return slice_starts + +def squeeze_shape(shape): + "Simple squeeze." + return [x for x in shape if x != 1] def squeeze_slice(current_shape, starts, shape, step): current_shape = current_shape[:] Modified: pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py Sat Aug 14 12:17:10 2010 @@ -15,6 +15,8 @@ from pypy.module.micronumpy.array import infer_shape from pypy.module.micronumpy.array import stride_row, stride_column, size_from_shape from pypy.module.micronumpy.array import normalize_slice_starts +from pypy.module.micronumpy.array import squeeze_slice, squeeze_shape +from pypy.module.micronumpy.array import shape_prefix def index_w(space, w_index): return space.int_w(space.index(w_index)) @@ -22,11 +24,15 @@ class MicroIter(Wrappable): _immutable_fields_ = ['array', 'step', 'stop', 'ndim'] def __init__(self, array): + #print "MicroIter.__init__(array=%s)" % array # XXX: DEBUG self.array = array - self.i = array.slice_starts[0] - self.step = array.slice_steps[0] - self.stop = self.i + array.shape[0] - self.ndim = len(self.array.shape) + self.i = 0 + #self.i = array.slice_starts[array.offset] # XXX: DEBUG + self.index = array.slice_starts[:] + #print "slice_starts:", self.index # XXX: DEBUG + self.step = array.slice_steps[array.offset] + self.stop = array.shape[array.offset] + self.ndim = len(array.shape[array.offset:]) def descr_iter(self, space): return space.wrap(self) @@ -35,17 +41,22 @@ def descr_next(self, space): if self.i < self.stop: if self.ndim > 1: - ar = MicroArray(self.array.shape[1:], + ar = MicroArray(self.array.shape, self.array.dtype, parent=self.array, - strides=self.array.strides[1:], - slice_starts=self.array.offsets + [self.i], - slice_steps=self.array.slice_steps[1:]) + offset=self.array.offset + 1, + strides=self.array.strides, + slice_starts=self.index, + slice_steps=self.array.slice_steps) self.i += 1 + self.index[self.array.offset] += self.step return space.wrap(ar) elif self.ndim == 1: - next = self.array.getitem(space, self.array.offset + self.array.flatten_index([self.i])) + #print "getting at index:", self.index # XXX: DEBUG + next = self.array.getitem(space, self.array.flatten_index(self.index)) + #print "done (got '%s')" % next # XXX: DEBUG self.i += 1 + self.index[self.array.offset] += self.step return next else: raise OperationError(space.w_ValueError, @@ -70,6 +81,7 @@ self.order = order self.offset = offset + #print "__init__(self=%s, slice_starts=%s)" % (self, slice_starts) # XXX: DEBUG self.slice_starts = slice_starts[:] for i in range(len(shape) - len(slice_starts)): self.slice_starts.append(0) @@ -95,7 +107,7 @@ self.data = null_data def descr_len(self, space): - return space.wrap(self.shape[0]) + return space.wrap(self.shape[self.offset]) descr_len.unwrap_spec = ['self', ObjSpace] def getitem(self, space, offset): @@ -112,12 +124,24 @@ dtype = self.dtype.dtype #XXX: kinda ugly dtype.w_setitem(space, self.data, index, w_value) #maybe hang onto w_dtype separately? + def flatten_slice_starts(self, slice_starts): + """Computes offset into subarray from all information. + Gives offset into subarray, not into data.""" + offset = 0 + for i in range(len(index)): + offset += (self.slice_steps[i] * index[i]) * self.strides[i] + return offset + def flatten_index(self, index): """Computes offset into subarray from all information. Gives offset into subarray, not into data.""" offset = 0 + + #print 'strides(%s) * index(%s)' % (self.strides, index) # XXX: DEBUG for i in range(len(index)): - offset += (self.slice_starts[i] + self.slice_steps[i] * index[i]) * self.strides[i] + #offset += (self.slice_starts[i] + self.slice_steps[i] * index[i]) * self.strides[i] + offset += (self.slice_steps[i] * index[i]) * self.strides[i] + #print "offset=%d" % offset # XXX: DEBUG return offset def stride(self, i): @@ -126,8 +150,7 @@ elif self.order == 'F': return stride_column(self.shape, i) # else: - raise OperationError(space.w_NotImplementedError, - space.wrap("Unknown order: '%s'" % self.order)) + raise NotImplementedError("Unknown order: '%s'" % self.order) def opposite_stride(self, i): if self.order == 'C': # C for C not Column, but this is opposite @@ -135,39 +158,45 @@ elif self.order == 'F': return stride_row(self.shape, i) else: - raise OperationError(space.w_NotImplementedError, - space.wrap("Unknown order: '%s'" % self.order)) + raise NotImplementedError("Unknown order: '%s'" % self.order) def index2slices(self, space, w_index): dtype = self.dtype.dtype + + slice_starts = self.slice_starts[:] + shape = self.shape[:] + slice_steps = self.slice_steps[:] + try: index = space.int_w(space.index(w_index)) - slice_starts = self.slice_starts[:] - slice_starts[0] += index - return slice_starts, self.shape[1:], self.slice_steps[1:] + # Normalize + if index < 0: + index += self.shape[self.offset] + + slice_starts[self.offset] += index * self.slice_steps[self.offset] + shape[self.offset] = 1 + return slice_starts, shape, slice_steps except OperationError, e: if e.match(space, space.w_TypeError): pass else:raise + if isinstance(w_index, W_SliceObject): + # TODO: fast path for slice + w_index = space.newlist([w_index]) + elif space.is_w(w_index, space.w_Ellipsis): + return slice_starts, shape, slice_steps + try: indices = space.fixedview(w_index) indexlen = len(indices) - slack = len(self.shape) - indexlen - - assert slack >= 0 - - slice_starts = self.slice_starts[:] - slice_steps = self.slice_steps[:] - strides = self.strides[:] - shape = self.shape[:] for i in range(indexlen): w_index = indices[i] try: index = space.int_w(space.index(w_index)) - slice_starts[i] += index - shape[i] = 1 + slice_starts[self.offset + i] += index + shape[self.offset + i] = 1 continue except OperationError, e: @@ -176,71 +205,78 @@ if isinstance(w_index, W_SliceObject): start, stop, step, length = w_index.indices4(space, self.shape[i]) - slice_starts[i] += start - shape[i] = length - slice_steps[i] *= step + #print "start=%d, stop=%d, step=%d, length=%d" % (start, stop, step, length) # XXX: DEBUG + slice_starts[self.offset + i] += start * slice_steps[self.offset + i] + shape[self.offset + i] = length + slice_steps[self.offset + i] *= step elif space.is_w(w_index, space.w_Ellipsis): pass # I can't think of anything we need to do else: - # XXX: no exception handling because we *want* to blow up - print 'type(w_index) = %s, w_index = %s' % (type(w_index), w_index) index = space.str(w_index) - raise OperationError(space.w_NotImplementedError, - space.wrap("Don't support records yet.")) # XXX: and we may never + raise OperationError(space.w_ValueError, + space.wrap("Don't support records," + " so pretend we don't have this field")) + raise OperationError(space.w_NotImplementedError, # this is the correct error + space.wrap("Don't support records yet.")) # for what we actually do try: - slice_starts = normalize_slice_starts(slice_starts, self.shape) # XXX: could make in-place + normalize_slice_starts(slice_starts, self.shape) # XXX: in-place operation except IndexError, e: raise OperationError(space.w_IndexError, space.wrap("invalid index")) - - return slice_starts, shape, slice_steps - finally: pass + #print "slice_starts=%s, shape=%s, slice_steps=%s" % (slice_starts, shape, slice_steps) # XXX: DEBUG + return slice_starts, shape, slice_steps + def descr_getitem(self, space, w_index): slice_starts, shape, slice_steps = self.index2slices(space, w_index) size = size_from_shape(shape) if size == 1: + #print 'getting item with slice_starts=%s' % slice_starts # XXX: DEBUG + #print 'flattened=%d' % self.flatten_index(slice_starts) # XXX: DEBUG return self.getitem(space, - self.offset + self.flatten_index(slice_starts)) + self.flatten_index(slice_starts)) else: + prefix = shape_prefix(shape) ar = MicroArray(shape, - dtype=dtype, + dtype=self.dtype, parent=self, - offset=self.offset + offset, # XXX: need to figure out how to name things better - slice_starts=slice_starts, # XXX: begin renaming slice_starts + offset=prefix, # XXX: what do we do about shapes that needs to be squeezed out? + slice_starts=slice_starts, slice_steps=slice_steps) + #print 'start:', ar.slice_starts # XXX: DEBUG + #print 'stop:', ar.shape # XXX: DEBUG + #print 'step:', ar.slice_steps # XXX: DEBUG return space.wrap(ar) descr_getitem.unwrap_spec = ['self', ObjSpace, W_Root] - def set_slice_single_value(self, space, offset, shape, offsets, steps, w_value): - index = offsets + def set_slice_single_value(self, space, slice_starts, shape, slice_steps, w_value): + index = slice_starts[:] if len(shape) > 1: - for i in shape[0]: - self.set_slice_single_value(space, offset, shape[1:], index, steps[1:], w_value) - index[len(index) - len(shape)] += steps[0] # XXX: reason + for i in range(shape[0]): + self.set_slice_single_value(space, index, shape[1:], slice_steps[1:], w_value) + index[len(index) - len(shape)] += slice_steps[0] # XXX: reason else: - dtype = self.dtype.dtype - for i in range(0, shape[0]): - self.set_slice_single_value(self, space, - offset, shape, index, steps, - w_value) - index[len(index)-1] += steps[0] # XXX: don't do steps in range + for i in range(shape[0]): + self.setitem(space, self.flatten_index(index), w_value) + index[len(index)-1] += slice_steps[0] # XXX: don't do steps in range - def set_slice(self, space, shape, offsets, steps, w_value): - if self.dtype.w_is_compatible(space, w_value): - self.set_slice_single_value(space, shape, offsets, steps, w_value) - else: + def set_slice(self, space, slice_starts, shape, slice_steps, w_value): + try: length = space.int_w(space.len(w_value)) if length == 1: - self.set_slice_single_value(space, shape, offsets, steps, w_value) + self.set_slice_single_value(space, slice_starts, shape, slice_steps, w_value) else: raise OperationError(space.w_NotImplementedError, space.wrap("TODO")) + except OperationError, e: + if e.match(space, space.w_TypeError): + self.set_slice_single_value(space, slice_starts, shape, slice_steps, w_value) + else: raise def descr_setitem(self, space, w_index, w_value): dtype = self.dtype.dtype @@ -249,32 +285,36 @@ size = size_from_shape(shape) + try: + if space.int_w(space.len(w_value)) == 1: + w_value = space.getitem(w_value, space.wrap(0)) + value_shape = infer_shape(space, w_value) + value_size = size_from_shape(value_shape) + + except OperationError, e: + if e.match(space, space.w_TypeError): + value_size = 1 + value_shape = [1] + else: raise + + if squeeze_shape(value_shape) != squeeze_shape(shape): + raise OperationError(space.w_ValueError, + space.wrap("shape mismatch: objects cannot" + " be broadcast to a single shape")) + if size == 1: self.setitem(space, - self.offset + self.flatten_index(slice_starts), + self.flatten_index(slice_starts), w_value) else: - try: - if space.int_w(space.len(w_value)) == 1: - w_value = space.getitem(w_value, space.wrap(0)) - value_shape = infer_shape(space, w_value) - value_size = size_from_shape(value_shape) - - except OperationError, e: - if e.match(space, space.w_TypeError): - value_size = 1 - else: raise - - if value_size == 1: - pass # TODO: set one value many times - - if value_shape != shape: - raise OperationError(space.w_ValueError, - space.wrap("shape mismatch: objects cannot" - " be broadcast to a single shape")) - else: - self.set_slice(space, offset, shape, slice_starts, slice_steps, w_value) - + if value_size == 1: + self.set_slice_single_value(space, + slice_starts, shape, slice_steps, + w_value) + else: + self.set_slice(space, + slice_starts, shape, slice_steps, + w_value) descr_setitem.unwrap_spec = ['self', ObjSpace, W_Root, W_Root] def descr_repr(self, space): @@ -361,7 +401,7 @@ return space.wrap(self.dtype) def descr_get_shape(space, self): - return space.newtuple([space.wrap(x) for x in self.shape]) + return space.newtuple([space.wrap(x) for x in self.shape[self.offset:]]) #TODO: add to typedef when ready def descr_new(space, w_cls, w_shape, w_dtype=NoneNotWrapped, @@ -393,13 +433,16 @@ return a app_fill_array = gateway.applevel(""" - def fill_array(start, a, b): + def fill_array(array, data): + _fill_array([], array, data) + + def _fill_array(start, array, data): i = 0 - for x in b: + for element in data: try: - fill_array(start + [i], a, x) + _fill_array(start + [i], array, element) except TypeError, e: - a[start + [i]] = x + array[start + [i]] = element i += 1 """) @@ -424,7 +467,7 @@ w_ar = space.wrap(ar) fill_array(space, - space.newlist([]), w_ar, w_xs) + w_ar, w_xs) return w_ar array.unwrap_spec = [ObjSpace, W_Root, W_Root, bool, str, bool, W_Root] Modified: pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/test/test_numpy.py Sat Aug 14 12:17:10 2010 @@ -104,7 +104,6 @@ ar = zeros(3, dtype=int) assert ar[0] == data_type(0.0) - @py.test.mark.xfail def test_setitem_getitem(self): from micronumpy import zeros compare = self.compare @@ -113,7 +112,7 @@ assert ar[0] == 0 ar[1] = 3 assert ar[1] == 3 - raises((IndexError, ValueError), ar.__getitem__, 'xyz') #FIXME: why TypeError? + raises((IndexError, ValueError), ar.__getitem__, 'xyz') raises(IndexError, ar.__getitem__, 38) assert ar[-2] == 0 assert ar[-7] == 3 @@ -134,6 +133,7 @@ raises(ValueError, ar.__setitem__, 0, 'f') raises(ValueError, ar.__setitem__, slice(2, 3), [4, 5, 6]) + @py.test.mark.xfail def test_minimum(self): from micronumpy import zeros, minimum @@ -246,8 +246,26 @@ for i in range(3): for j in range(3): assert ar[i, j] == i*3+j + + def test_iteration(self): + from micronumpy import array + ar = array([1, 2, 3, 4]) + + for x, y in zip(ar, [1, 2, 3, 4]): + assert x == y + + ar2 = ar[0:] + for x, y in zip(ar2, [1, 2, 3, 4]): + assert x == y + + ar2 = ar[1:] + for x, y in zip(ar2, [2, 3, 4]): + assert x == y + + ar2 = ar[2:4] + for x, y, in zip(ar2, [3, 4]): + assert x == y - @py.test.mark.xfail def test_get_set_slices(self): from micronumpy import array gen_array = self.gen_array @@ -298,6 +316,14 @@ ar3[1:3] = [[0, 1, 2], [3, 4, 5]] assert compare(ar3[1, 2], [2, 2, 2]) + def test_subarray(self): + from micronumpy import array + gen_array = self.gen_array + + ar = array(gen_array((7, 5, 3, 2))) + + assert ar[0][0][0][0] == ar[0, 0, 0, 0] + @py.test.mark.xfail def test_newaxis(self): from micronumpy import array @@ -430,6 +456,36 @@ assert start == [0, 2, 0, 0] assert shape == [5, 1, 3, 2] assert step == [1, 1, 1, 1] + + w_index = space.wrap(slice(1, 3)) + start, shape, step = ar.index2slices(space, w_index) + assert start[0] == 1 + assert shape[0] == 2 + assert step[0] == 1 + + def test_slice_setting(self, space): + from pypy.module.micronumpy.array import size_from_shape + from pypy.module.micronumpy.microarray import MicroArray + from pypy.module.micronumpy.dtype import w_int_descr + + ar = MicroArray(shape=[7, 5, 3, 2], + dtype=w_int_descr) + + ar.set_slice_single_value(space, + slice_starts=[5, 4, 2, 1], shape=[1, 1, 1, 1], slice_steps=[1, 1, 1, 1], + w_value=space.wrap(3)) + + assert space.is_true(space.eq(ar.getitem(space, ar.offset + ar.flatten_index([5, 4, 2, 1])), space.wrap(3))) + + ar.set_slice_single_value(space, + slice_starts=[0, 0, 0, 0], shape=[7, 5, 3, 2], slice_steps=[1, 1, 1, 1], + w_value=space.wrap(97)) + + for i in range(size_from_shape(ar.shape)): + array_element = space.unwrap(ar.getitem(space, i)) # ugly, but encapsulates everything + assert array_element == 97 + + # TODO: test more def test_strides(self, space): from pypy.module.micronumpy.array import stride_row, stride_column Modified: pypy/branch/micronumpy/pypy/tool/convolve.py ============================================================================== --- pypy/branch/micronumpy/pypy/tool/convolve.py (original) +++ pypy/branch/micronumpy/pypy/tool/convolve.py Sat Aug 14 12:17:10 2010 @@ -25,8 +25,10 @@ # Allocate result image. h = np.zeros([xmax, ymax], dtype=f.dtype) # Do convolution + print "Herp" for x in range(xmax): for y in range(ymax): + print "<%d,%d>" % (x, y) # Calculate pixel value for h at (x,y). Sum one component # for each pixel (s, t) of the filter g. s_from = max(smid - x, -smid) Modified: pypy/branch/micronumpy/pypy/tool/numpybench.py ============================================================================== --- pypy/branch/micronumpy/pypy/tool/numpybench.py (original) +++ pypy/branch/micronumpy/pypy/tool/numpybench.py Sat Aug 14 12:17:10 2010 @@ -44,6 +44,7 @@ image = generate_image(width, height) kernel = generate_kernel(kwidth, kheight) + print "Timing" from timeit import Timer convolve_timer = Timer('naive_convolve(image, kernel)', 'from convolve import naive_convolve; from __main__ import image, kernel; gc.enable()') print "%.5f sec/pass" % (convolve_timer.timeit(number=count)/count) From hakanardo at codespeak.net Sat Aug 14 14:12:22 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Sat, 14 Aug 2010 14:12:22 +0200 (CEST) Subject: [pypy-svn] r76625 - in pypy/branch/jit-bounds/pypy/jit/metainterp: . test Message-ID: <20100814121222.9E334282B90@codespeak.net> Author: hakanardo Date: Sat Aug 14 14:12:20 2010 New Revision: 76625 Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py Log: attempt to lower the call overhead by removing framestackdepth + 1 - 1 Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py Sat Aug 14 14:12:20 2010 @@ -745,6 +745,15 @@ # a real GUARD_VALUE. Make it use one counter per value. descr.make_a_counter_per_value(op) + def make_args_key(self, op): + args = op.args[:] + for i in range(len(args)): + arg = args[i] + if arg in self.values: + args[i] = self.values[arg].get_key_box() + args.append(ConstInt(op.opnum)) + return args + def optimize_default(self, op): if op.is_always_pure(): for arg in op.args: @@ -759,12 +768,7 @@ return # did we do the exact same operation already? - args = op.args[:] - for i in range(len(args)): - arg = args[i] - if arg in self.values: - args[i] = self.values[arg].get_key_box() - args.append(ConstInt(op.opnum)) + args = self.make_args_key(op) oldop = self.pure_operations.get(args, None) if oldop is not None and oldop.descr is op.descr: assert oldop.opnum == op.opnum @@ -1224,6 +1228,15 @@ r = self.getvalue(op.result) r.intbound.intersect(v1.intbound.add_bound(v2.intbound)) + # Synthesize the reverse op for optimize_default to reuse + revop = ResOperation(rop.INT_SUB, (op.result, op.args[1]), \ + op.args[0], op.descr) + self.pure_operations[self.make_args_key(revop)] = revop + revop = ResOperation(rop.INT_SUB, (op.result, op.args[0]), \ + op.args[1], op.descr) + self.pure_operations[self.make_args_key(revop)] = revop + + def optimize_INT_LT(self, op): v1 = self.getvalue(op.args[0]) v2 = self.getvalue(op.args[1]) Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py Sat Aug 14 14:12:20 2010 @@ -3292,6 +3292,37 @@ """ self.optimize_loop(ops, 'Not', expected) + def test_framestackdepth_overhead(self): + ops = """ + [p0, i22] + i1 = getfield_gc(p0, descr=valuedescr) + i2 = int_gt(i1, i22) + guard_false(i2) [] + i3 = int_add(i1, 1) + setfield_gc(p0, i3, descr=valuedescr) + i4 = int_sub(i3, 1) + setfield_gc(p0, i4, descr=valuedescr) + i5 = int_gt(i4, i22) + guard_false(i5) [] + i6 = int_add(i4, 1) + i331 = force_token() + i7 = int_sub(i6, 1) + setfield_gc(p0, i7, descr=valuedescr) + jump(p0, i22) + """ + expected = """ + [p0, i22] + i1 = getfield_gc(p0, descr=valuedescr) + i2 = int_gt(i1, i22) + guard_false(i2) [] + i3 = int_add(i1, 1) + i331 = force_token() + setfield_gc(p0, i1, descr=valuedescr) + jump(p0, i22) + """ + self.optimize_loop(ops, 'Not, Not', expected) + + ##class TestOOtype(BaseTestOptimizeOpt, OOtypeMixin): From arigo at codespeak.net Sat Aug 14 17:09:13 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 14 Aug 2010 17:09:13 +0200 (CEST) Subject: [pypy-svn] r76626 - pypy/branch/kill-caninline/pypy/module/pypyjit Message-ID: <20100814150913.0868E282B90@codespeak.net> Author: arigo Date: Sat Aug 14 17:09:11 2010 New Revision: 76626 Modified: pypy/branch/kill-caninline/pypy/module/pypyjit/interp_jit.py Log: Cancel the hacking for generators. Will likely reintroduce it, but for now I want to measure the effects of one change at a time. Modified: pypy/branch/kill-caninline/pypy/module/pypyjit/interp_jit.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/pypyjit/interp_jit.py (original) +++ pypy/branch/kill-caninline/pypy/module/pypyjit/interp_jit.py Sat Aug 14 17:09:11 2010 @@ -20,7 +20,6 @@ 'fastlocals_w[*]', 'last_exception', 'lastblock', - 'last_yield', ] JUMP_ABSOLUTE = opmap['JUMP_ABSOLUTE'] @@ -60,14 +59,7 @@ set_jitcell_at = set_jitcell_at, confirm_enter_jit = confirm_enter_jit) -PyFrame_execute_generator_frame = PyFrame.execute_generator_frame - class __extend__(PyFrame): - last_yield = -1 - - def execute_generator_frame(self, w_inputvalue, ex=False): - self.last_yield = self.last_instr - return PyFrame_execute_generator_frame(self, w_inputvalue, ex) def dispatch(self, pycode, next_instr, ec): self = hint(self, access_directly=True) @@ -82,22 +74,11 @@ except ExitFrame: return self.popvalue() - def jump_absolute(self, jumpto, _, ec): - # Custom logic. A JUMP_ABSOLUTE closes a loop, except when it is - # a jump over the most recent YIELD_VALUE. - if jumpto <= self.last_yield: - self.last_yield = -1 - return jumpto - # In jitted mode, we call bytecode_trace() only once per loop, - # instead of every N instructions. + def jump_absolute(self, jumpto, _, ec=None): if we_are_jitted(): self.last_instr = intmask(jumpto) ec.bytecode_trace(self) jumpto = r_uint(self.last_instr) - # Call the main hook on pypyjitdriver. In the non-JITted version - # of this code, this becomes a call to a profiling helper that may - # jump to the JITted version. In the JITted version, it marks the - # start and the end of all loops. pypyjitdriver.can_enter_jit(frame=self, ec=ec, next_instr=jumpto, pycode=self.getcode()) return jumpto From arigo at codespeak.net Sat Aug 14 17:22:46 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 14 Aug 2010 17:22:46 +0200 (CEST) Subject: [pypy-svn] r76627 - pypy/extradoc/planning Message-ID: <20100814152246.C157C282B90@codespeak.net> Author: arigo Date: Sat Aug 14 17:22:45 2010 New Revision: 76627 Modified: pypy/extradoc/planning/jit.txt Log: Update the status of the kill-caninline branch. Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Sat Aug 14 17:22:45 2010 @@ -10,11 +10,13 @@ - trace into functions even if they have a loop. only if the loop is actually hit, a residual portal call is produced (status: kill-caninline branch, - buggy) + DONE) - generators are not really fast ? maybe add a JUMP_ABSOLUTE_GENERATOR that does not call can_enter_jit after an iteration in which there was a yield. - obviously. (status: kill-caninline branch) + obviously. (status: unclear -- the frame of the generator cannot be a + virtual, so should we somehow make it a virtualizable? e.g. by never inlining + generators?) - think again about perfect specialization. Check if we loose anything if we turn it off. Another approach to specialization: specialize things From arigo at codespeak.net Sat Aug 14 17:32:13 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 14 Aug 2010 17:32:13 +0200 (CEST) Subject: [pypy-svn] r76628 - pypy/extradoc/planning Message-ID: <20100814153213.C318B282B90@codespeak.net> Author: arigo Date: Sat Aug 14 17:32:12 2010 New Revision: 76628 Modified: pypy/extradoc/planning/jit.txt Log: Potentially solve this issue (private discussion with antocuni on irc). Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Sat Aug 14 17:32:12 2010 @@ -5,6 +5,10 @@ traced after the outer one, the call is slow. Might be solved easily if we implement full out-of-line guards (e.g. by invalidating the outer function when the inner one gets compiled) + --- + More thoughts (antonio, arigo): by the way we detect traces too long, + we can always immediately compile the subfunction. If we do that then + there is no risk of generating a slow call. - have benchmarks for jit compile time and jit memory usage From hakanardo at codespeak.net Sun Aug 15 21:44:30 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Sun, 15 Aug 2010 21:44:30 +0200 (CEST) Subject: [pypy-svn] r76629 - in pypy/branch/jit-bounds/pypy: jit/metainterp module/pypyjit/test Message-ID: <20100815194430.AF882282C34@codespeak.net> Author: hakanardo Date: Sun Aug 15 21:44:28 2010 New Revision: 76629 Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py pypy/branch/jit-bounds/pypy/module/pypyjit/test/test_pypy_c.py Log: rpythonized, pypy-c tests Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py Sun Aug 15 21:44:28 2010 @@ -18,7 +18,6 @@ from pypy.rpython.lltypesystem import lltype from pypy.jit.metainterp.history import AbstractDescr, make_hashable_int from pypy.rlib.rarithmetic import ovfcheck -from copy import copy def optimize_loop_1(metainterp_sd, loop): """Optimize loop.operations to make it match the input of loop.specnodes @@ -40,6 +39,8 @@ # ____________________________________________________________ class IntBound(object): + _attrs_ = ('has_upper', 'has_lower', 'upper', 'lower') + def __init__(self, lower, upper): self.has_upper = True self.has_lower = True @@ -103,7 +104,7 @@ return r def add(self, offset): - res = copy(self) + res = self.copy() try: res.lower = ovfcheck(res.lower + offset) except OverflowError: @@ -115,7 +116,7 @@ return res def add_bound(self, other): - res = copy(self) + res = self.copy() if other.has_upper: res.upper += other.upper else: @@ -127,7 +128,7 @@ return res def sub_bound(self, other): - res = copy(self) + res = self.copy() if other.has_upper: res.lower -= other.upper else: @@ -156,6 +157,11 @@ u = 'Inf' return '%s <= x <= %s' % (l, u) + def copy(self): + res = IntBound(self.lower, self.upper) + res.has_lower = self.has_lower + res.has_upper = self.has_upper + return res @@ -186,15 +192,15 @@ LEVEL_CONSTANT = '\x03' class OptValue(object): - _attrs_ = ('box', 'known_class', 'last_guard_index', 'level') + _attrs_ = ('box', 'known_class', 'last_guard_index', 'level', 'intbound') last_guard_index = -1 level = LEVEL_UNKNOWN known_class = None + intbound = None - def __init__(self, box, producer=None): + def __init__(self, box): self.box = box - self.producer = producer self.intbound = IntUnbounded() if isinstance(box, Const): self.make_constant(box) @@ -1229,10 +1235,10 @@ r.intbound.intersect(v1.intbound.add_bound(v2.intbound)) # Synthesize the reverse op for optimize_default to reuse - revop = ResOperation(rop.INT_SUB, (op.result, op.args[1]), \ + revop = ResOperation(rop.INT_SUB, [op.result, op.args[1]], \ op.args[0], op.descr) self.pure_operations[self.make_args_key(revop)] = revop - revop = ResOperation(rop.INT_SUB, (op.result, op.args[0]), \ + revop = ResOperation(rop.INT_SUB, [op.result, op.args[0]], \ op.args[1], op.descr) self.pure_operations[self.make_args_key(revop)] = revop Modified: pypy/branch/jit-bounds/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/jit-bounds/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/jit-bounds/pypy/module/pypyjit/test/test_pypy_c.py Sun Aug 15 21:44:28 2010 @@ -118,6 +118,7 @@ assert result.splitlines()[-1].strip() == 'OK :-)' self.parse_loops(logfilepath) self.print_loops() + print logfilepath if self.total_ops > expected_max_ops: assert 0, "too many operations: got %d, expected maximum %d" % ( self.total_ops, expected_max_ops) @@ -846,6 +847,112 @@ return intimg[i - 1] ''', maxops, ([tc], res)) + def test_intbound(self): + ops = ('<', '>', '<=', '>=') + nbr = (3, 7) + for o1 in ops: + for o2 in ops: + for n1 in nbr: + for n2 in nbr: + src = ''' + def f(i): + a, b = 3, 3 + if i %s %d: + a = 0 + else: + a = 1 + if i %s %d: + b = 0 + else: + b = 1 + return a + b * 2 + + def main(): + res = [0] * 4 + for i in range(15) * 1500: + res[f(i)] += 1 + return res + + ''' % (o1, n1, o2, n2) + + exec(str(py.code.Source(src))) + res = [0] * 4 + for i in range(15): + res[f(i)] += 1500 + self.run_source(src, 220, ([], res)) + + def test_intbound_addsub(self): + tests = ('i > 4', 'i > 2', 'i + 1 > 2', '1 + i > 4', + 'i - 1 > 1', '1 - i > 1', '1 - i < -3') + for t1 in tests: + for t2 in tests: + src = ''' + def f(i): + a, b = 3, 3 + if %s: + a = 0 + else: + a = 1 + if %s: + b = 0 + else: + b = 1 + return a + b * 2 + + def main(): + res = [0] * 4 + for i in range(15) * 1500: + res[f(i)] += 1 + return res + + ''' % (t1, t2) + + exec(str(py.code.Source(src))) + res = [0] * 4 + for i in range(15): + res[f(i)] += 1500 + self.run_source(src, 232, ([], res)) + + def test_intbound_gt(self): + self.run_source(''' + def main(): + i, a, b = 0, 0, 0 + while i < 2000: + if i > -1: + a += 1 + if i > -2: + b += 1 + i += 1 + return (a, b) + ''', 48, ([], (2000, 2000))) + + + def test_intbound_addsub_ge(self): + self.run_source(''' + def main(): + i, a, b = 0, 0, 0 + while i < 2000: + if i + 5 >= 5: + a += 1 + #if i - 1 >= -1: + if i - 1 >= -1: + b += 1 + i += 1 + return (a, b) + ''', 0, ([], (2000, 2000))) + + def test_intbound_sub_lt(self): + self.run_source(''' + def main(): + i, a, b = 0, 0, 0 + while i < 2000: + if i - 10 < 1995: + a += 1 + i += 1 + return (a, b) + ''', 0, ([], (2000, 0))) + + class AppTestJIT(PyPyCJITTests): def setup_class(cls): if not option.runappdirect: From dan at codespeak.net Mon Aug 16 04:52:06 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Mon, 16 Aug 2010 04:52:06 +0200 (CEST) Subject: [pypy-svn] r76630 - in pypy/branch/micronumpy/pypy: config doc doc/config interpreter interpreter/astcompiler interpreter/astcompiler/tools interpreter/test jit/backend jit/backend/llsupport jit/backend/llsupport/test jit/backend/test jit/backend/x86 jit/backend/x86/test jit/backend/x86/tool jit/metainterp jit/metainterp/doc jit/metainterp/test jit/tl module/__builtin__ module/__builtin__/test module/_ast module/_ast/test module/_codecs/test module/_demo module/_file module/_file/test module/_rawffi/test module/_socket/test module/_sre/test module/_stackless module/array module/array/benchmark module/array/test module/cpyext module/fcntl/test module/marshal/test module/micronumpy module/posix module/posix/test module/pypyjit module/pypyjit/test module/signal module/test_lib_pypy module/thread/test objspace/std objspace/std/test rlib rlib/test rpython rpython/lltypesystem rpython/module rpython/module/test rpython/test rpython/tool tool tool/release tool/release/test translator/c translator/goal translator/goal/test2 translator/platform Message-ID: <20100816025206.F1E60282B9C@codespeak.net> Author: dan Date: Mon Aug 16 04:52:00 2010 New Revision: 76630 Added: pypy/branch/micronumpy/pypy/doc/config/objspace.usemodules.array.txt - copied unchanged from r76624, pypy/trunk/pypy/doc/config/objspace.usemodules.array.txt pypy/branch/micronumpy/pypy/jit/backend/x86/arch.py - copied unchanged from r76624, pypy/trunk/pypy/jit/backend/x86/arch.py pypy/branch/micronumpy/pypy/jit/backend/x86/regloc.py - copied unchanged from r76624, pypy/trunk/pypy/jit/backend/x86/regloc.py pypy/branch/micronumpy/pypy/jit/backend/x86/rx86.py - copied unchanged from r76624, pypy/trunk/pypy/jit/backend/x86/rx86.py pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_regloc.py - copied unchanged from r76624, pypy/trunk/pypy/jit/backend/x86/test/test_regloc.py pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_rx86.py - copied unchanged from r76624, pypy/trunk/pypy/jit/backend/x86/test/test_rx86.py pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py - copied unchanged from r76624, pypy/trunk/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py - copied unchanged from r76624, pypy/trunk/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py pypy/branch/micronumpy/pypy/jit/backend/x86/tool/instruction_encoding.sh - copied unchanged from r76624, pypy/trunk/pypy/jit/backend/x86/tool/instruction_encoding.sh pypy/branch/micronumpy/pypy/module/array/ (props changed) - copied from r76624, pypy/trunk/pypy/module/array/ pypy/branch/micronumpy/pypy/rlib/test/test_rposix.py - copied unchanged from r76624, pypy/trunk/pypy/rlib/test/test_rposix.py pypy/branch/micronumpy/pypy/rpython/module/ll_win32file.py - copied unchanged from r76624, pypy/trunk/pypy/rpython/module/ll_win32file.py Removed: pypy/branch/micronumpy/pypy/jit/backend/x86/ri386.py pypy/branch/micronumpy/pypy/jit/backend/x86/ri386setup.py pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_ri386.py pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py pypy/branch/micronumpy/pypy/jit/metainterp/doc/ pypy/branch/micronumpy/pypy/module/test_lib_pypy/test_array.py Modified: pypy/branch/micronumpy/pypy/config/pypyoption.py pypy/branch/micronumpy/pypy/config/translationoption.py pypy/branch/micronumpy/pypy/doc/faq.txt pypy/branch/micronumpy/pypy/interpreter/astcompiler/assemble.py pypy/branch/micronumpy/pypy/interpreter/astcompiler/ast.py pypy/branch/micronumpy/pypy/interpreter/astcompiler/codegen.py pypy/branch/micronumpy/pypy/interpreter/astcompiler/consts.py pypy/branch/micronumpy/pypy/interpreter/astcompiler/symtable.py pypy/branch/micronumpy/pypy/interpreter/astcompiler/tools/asdl_py.py pypy/branch/micronumpy/pypy/interpreter/baseobjspace.py pypy/branch/micronumpy/pypy/interpreter/error.py pypy/branch/micronumpy/pypy/interpreter/gateway.py pypy/branch/micronumpy/pypy/interpreter/pyopcode.py pypy/branch/micronumpy/pypy/interpreter/test/test_compiler.py pypy/branch/micronumpy/pypy/interpreter/test/test_gateway.py pypy/branch/micronumpy/pypy/jit/backend/detect_cpu.py pypy/branch/micronumpy/pypy/jit/backend/llsupport/llmodel.py pypy/branch/micronumpy/pypy/jit/backend/llsupport/regalloc.py pypy/branch/micronumpy/pypy/jit/backend/llsupport/test/test_descr.py pypy/branch/micronumpy/pypy/jit/backend/llsupport/test/test_regalloc.py pypy/branch/micronumpy/pypy/jit/backend/test/runner_test.py pypy/branch/micronumpy/pypy/jit/backend/x86/assembler.py pypy/branch/micronumpy/pypy/jit/backend/x86/codebuf.py pypy/branch/micronumpy/pypy/jit/backend/x86/jump.py pypy/branch/micronumpy/pypy/jit/backend/x86/regalloc.py pypy/branch/micronumpy/pypy/jit/backend/x86/runner.py pypy/branch/micronumpy/pypy/jit/backend/x86/test/conftest.py pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_assembler.py pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_basic.py pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_gc_integration.py pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_jump.py pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_recompilation.py pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_regalloc.py pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_regalloc2.py pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_runner.py pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_symbolic_x86.py pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_zll_random.py pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_zrpy_gc.py pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_ztranslation.py pypy/branch/micronumpy/pypy/jit/backend/x86/tool/viewcode.py pypy/branch/micronumpy/pypy/jit/metainterp/executor.py pypy/branch/micronumpy/pypy/jit/metainterp/optimizeopt.py pypy/branch/micronumpy/pypy/jit/metainterp/resoperation.py pypy/branch/micronumpy/pypy/jit/metainterp/test/test_executor.py pypy/branch/micronumpy/pypy/jit/metainterp/test/test_loop.py pypy/branch/micronumpy/pypy/jit/metainterp/test/test_loop_spec.py pypy/branch/micronumpy/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/micronumpy/pypy/jit/tl/pypyjit.py pypy/branch/micronumpy/pypy/jit/tl/pypyjit_demo.py pypy/branch/micronumpy/pypy/module/__builtin__/compiling.py pypy/branch/micronumpy/pypy/module/__builtin__/test/test_buffer.py pypy/branch/micronumpy/pypy/module/_ast/__init__.py pypy/branch/micronumpy/pypy/module/_ast/test/test_ast.py pypy/branch/micronumpy/pypy/module/_codecs/test/test_codecs.py pypy/branch/micronumpy/pypy/module/_demo/demo.py pypy/branch/micronumpy/pypy/module/_file/interp_file.py pypy/branch/micronumpy/pypy/module/_file/test/test_file.py pypy/branch/micronumpy/pypy/module/_file/test/test_file_extra.py pypy/branch/micronumpy/pypy/module/_rawffi/test/test__rawffi.py pypy/branch/micronumpy/pypy/module/_socket/test/test_sock_app.py pypy/branch/micronumpy/pypy/module/_sre/test/test_app_sre.py pypy/branch/micronumpy/pypy/module/_stackless/interp_coroutine.py pypy/branch/micronumpy/pypy/module/array/benchmark/ (props changed) pypy/branch/micronumpy/pypy/module/array/test/ (props changed) pypy/branch/micronumpy/pypy/module/cpyext/api.py pypy/branch/micronumpy/pypy/module/cpyext/methodobject.py pypy/branch/micronumpy/pypy/module/fcntl/test/test_fcntl.py pypy/branch/micronumpy/pypy/module/marshal/test/test_marshalimpl.py pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py pypy/branch/micronumpy/pypy/module/posix/interp_posix.py pypy/branch/micronumpy/pypy/module/posix/test/test_posix2.py pypy/branch/micronumpy/pypy/module/pypyjit/policy.py pypy/branch/micronumpy/pypy/module/pypyjit/test/test_pypy_c.py pypy/branch/micronumpy/pypy/module/signal/interp_signal.py pypy/branch/micronumpy/pypy/module/thread/test/test_gil.py pypy/branch/micronumpy/pypy/objspace/std/callmethod.py pypy/branch/micronumpy/pypy/objspace/std/itertype.py pypy/branch/micronumpy/pypy/objspace/std/model.py pypy/branch/micronumpy/pypy/objspace/std/test/test_callmethod.py pypy/branch/micronumpy/pypy/rlib/objectmodel.py pypy/branch/micronumpy/pypy/rlib/rmmap.py pypy/branch/micronumpy/pypy/rlib/rposix.py pypy/branch/micronumpy/pypy/rlib/rwin32.py pypy/branch/micronumpy/pypy/rlib/streamio.py pypy/branch/micronumpy/pypy/rlib/test/test_objectmodel.py pypy/branch/micronumpy/pypy/rpython/extfunc.py pypy/branch/micronumpy/pypy/rpython/lltypesystem/ll2ctypes.py pypy/branch/micronumpy/pypy/rpython/lltypesystem/rffi.py pypy/branch/micronumpy/pypy/rpython/lltypesystem/rstr.py pypy/branch/micronumpy/pypy/rpython/module/ll_os.py pypy/branch/micronumpy/pypy/rpython/module/ll_os_stat.py pypy/branch/micronumpy/pypy/rpython/module/test/test_ll_os_stat.py pypy/branch/micronumpy/pypy/rpython/rstr.py pypy/branch/micronumpy/pypy/rpython/test/test_extfunc.py pypy/branch/micronumpy/pypy/rpython/tool/rfficache.py pypy/branch/micronumpy/pypy/tool/convolve.py pypy/branch/micronumpy/pypy/tool/numpybench.py pypy/branch/micronumpy/pypy/tool/release/package.py pypy/branch/micronumpy/pypy/tool/release/test/test_package.py pypy/branch/micronumpy/pypy/translator/c/genc.py pypy/branch/micronumpy/pypy/translator/c/node.py pypy/branch/micronumpy/pypy/translator/goal/app_main.py pypy/branch/micronumpy/pypy/translator/goal/test2/test_app_main.py pypy/branch/micronumpy/pypy/translator/platform/posix.py Log: After the merge. Modified: pypy/branch/micronumpy/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/micronumpy/pypy/config/pypyoption.py (original) +++ pypy/branch/micronumpy/pypy/config/pypyoption.py Mon Aug 16 04:52:00 2010 @@ -30,7 +30,7 @@ "rctime" , "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios", "zlib", "struct", "md5", "sha", "bz2", "_minimal_curses", "cStringIO", - "thread", "itertools", "pyexpat", "_ssl", "cpyext"] + "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array"] )) working_oo_modules = default_modules.copy() Modified: pypy/branch/micronumpy/pypy/config/translationoption.py ============================================================================== --- pypy/branch/micronumpy/pypy/config/translationoption.py (original) +++ pypy/branch/micronumpy/pypy/config/translationoption.py Mon Aug 16 04:52:00 2010 @@ -342,6 +342,10 @@ 'jit': 'hybrid extraopts jit', } +# For now, 64-bit JIT requires boehm +if IS_64_BITS: + OPT_TABLE['jit'] = OPT_TABLE['jit'].replace('hybrid', 'boehm') + def set_opt_level(config, level): """Apply optimization suggestions on the 'config'. The optimizations depend on the selected level and possibly on the backend. Modified: pypy/branch/micronumpy/pypy/doc/faq.txt ============================================================================== --- pypy/branch/micronumpy/pypy/doc/faq.txt (original) +++ pypy/branch/micronumpy/pypy/doc/faq.txt Mon Aug 16 04:52:00 2010 @@ -47,8 +47,8 @@ There is also an experimental support for CPython extension modules, so they'll run without change (from current observation, rather with little -change) on trunk. It has not been released yet, although it should be a major -point of the next pypy release. +change) on trunk. It has been a part of 1.3 release, but support is still +in alpha phase. .. _`extension modules`: cpython_differences.html#extension-modules .. _`cpython_differences`: cpython_differences.html @@ -373,7 +373,7 @@ -------------------------------------------- No. PyPy always runs your code in its own interpreter, which is a -full and compliant Python 2.4 interpreter. RPython_ is only the +full and compliant Python 2.5 interpreter. RPython_ is only the language in which parts of PyPy itself are written and extension modules for it. The answer to whether something needs to be written as an extension module, apart from the "gluing to external libraries" reason, will Modified: pypy/branch/micronumpy/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/micronumpy/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/micronumpy/pypy/interpreter/astcompiler/assemble.py Mon Aug 16 04:52:00 2010 @@ -566,22 +566,22 @@ return (oparg % 256) + 2 * (oparg / 256) def _compute_CALL_FUNCTION(arg): - return _num_args(arg) + return -_num_args(arg) def _compute_CALL_FUNCTION_VAR(arg): - return _num_args(arg) - 1 + return -_num_args(arg) - 1 def _compute_CALL_FUNCTION_KW(arg): - return _num_args(arg) - 1 + return -_num_args(arg) - 1 def _compute_CALL_FUNCTION_VAR_KW(arg): - return _num_args(arg) - 2 + return -_num_args(arg) - 2 def _compute_CALL_LIKELY_BUILTIN(arg): return -(arg & 0xFF) + 1 def _compute_CALL_METHOD(arg): - return -arg - 1 + return -_num_args(arg) - 1 _stack_effect_computers = {} Modified: pypy/branch/micronumpy/pypy/interpreter/astcompiler/ast.py ============================================================================== --- pypy/branch/micronumpy/pypy/interpreter/astcompiler/ast.py (original) +++ pypy/branch/micronumpy/pypy/interpreter/astcompiler/ast.py Mon Aug 16 04:52:00 2010 @@ -230,6 +230,7 @@ visitor.visit_FunctionDef(self) def mutate_over(self, visitor): + self.args = self.args.mutate_over(visitor) if self.body: visitor._mutate_sequence(self.body) if self.decorators: @@ -784,6 +785,8 @@ def mutate_over(self, visitor): if self.body: visitor._mutate_sequence(self.body) + if self.handlers: + visitor._mutate_sequence(self.handlers) if self.orelse: visitor._mutate_sequence(self.orelse) return visitor.visit_TryExcept(self) @@ -927,6 +930,8 @@ visitor.visit_Import(self) def mutate_over(self, visitor): + if self.names: + visitor._mutate_sequence(self.names) return visitor.visit_Import(self) def sync_app_attrs(self, space): @@ -965,6 +970,8 @@ visitor.visit_ImportFrom(self) def mutate_over(self, visitor): + if self.names: + visitor._mutate_sequence(self.names) return visitor.visit_ImportFrom(self) def sync_app_attrs(self, space): @@ -1282,6 +1289,7 @@ visitor.visit_Lambda(self) def mutate_over(self, visitor): + self.args = self.args.mutate_over(visitor) self.body = self.body.mutate_over(visitor) return visitor.visit_Lambda(self) @@ -1398,6 +1406,8 @@ def mutate_over(self, visitor): self.elt = self.elt.mutate_over(visitor) + if self.generators: + visitor._mutate_sequence(self.generators) return visitor.visit_ListComp(self) def sync_app_attrs(self, space): @@ -1437,6 +1447,8 @@ def mutate_over(self, visitor): self.elt = self.elt.mutate_over(visitor) + if self.generators: + visitor._mutate_sequence(self.generators) return visitor.visit_GeneratorExp(self) def sync_app_attrs(self, space): @@ -1562,6 +1574,8 @@ self.func = self.func.mutate_over(visitor) if self.args: visitor._mutate_sequence(self.args) + if self.keywords: + visitor._mutate_sequence(self.keywords) if self.starargs: self.starargs = self.starargs.mutate_over(visitor) if self.kwargs: @@ -2293,6 +2307,13 @@ self.w_ifs = None self.initialization_state = 7 + def mutate_over(self, visitor): + self.target = self.target.mutate_over(visitor) + self.iter = self.iter.mutate_over(visitor) + if self.ifs: + visitor._mutate_sequence(self.ifs) + return visitor.visit_comprehension(self) + def walkabout(self, visitor): visitor.visit_comprehension(self) @@ -2327,6 +2348,15 @@ self.col_offset = col_offset self.initialization_state = 31 + def mutate_over(self, visitor): + if self.type: + self.type = self.type.mutate_over(visitor) + if self.name: + self.name = self.name.mutate_over(visitor) + if self.body: + visitor._mutate_sequence(self.body) + return visitor.visit_excepthandler(self) + def walkabout(self, visitor): visitor.visit_excepthandler(self) @@ -2366,6 +2396,13 @@ self.w_defaults = None self.initialization_state = 15 + def mutate_over(self, visitor): + if self.args: + visitor._mutate_sequence(self.args) + if self.defaults: + visitor._mutate_sequence(self.defaults) + return visitor.visit_arguments(self) + def walkabout(self, visitor): visitor.visit_arguments(self) @@ -2407,6 +2444,10 @@ self.value = value self.initialization_state = 3 + def mutate_over(self, visitor): + self.value = self.value.mutate_over(visitor) + return visitor.visit_keyword(self) + def walkabout(self, visitor): visitor.visit_keyword(self) @@ -2426,6 +2467,9 @@ self.asname = asname self.initialization_state = 3 + def mutate_over(self, visitor): + return visitor.visit_alias(self) + def walkabout(self, visitor): visitor.visit_alias(self) Modified: pypy/branch/micronumpy/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/micronumpy/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/micronumpy/pypy/interpreter/astcompiler/codegen.py Mon Aug 16 04:52:00 2010 @@ -258,9 +258,11 @@ # Load decorators first, but apply them after the function is created. if func.decorators: self.visit_sequence(func.decorators) - if func.args.defaults: - self.visit_sequence(func.args.defaults) - num_defaults = len(func.args.defaults) + args = func.args + assert isinstance(args, ast.arguments) + if args.defaults: + self.visit_sequence(args.defaults) + num_defaults = len(args.defaults) else: num_defaults = 0 code = self.sub_scope(FunctionCodeGenerator, func.name, func, @@ -274,9 +276,11 @@ def visit_Lambda(self, lam): self.update_position(lam.lineno) - if lam.args.defaults: - self.visit_sequence(lam.args.defaults) - default_count = len(lam.args.defaults) + args = lam.args + assert isinstance(args, ast.arguments) + if args.defaults: + self.visit_sequence(args.defaults) + default_count = len(args.defaults) else: default_count = 0 code = self.sub_scope(LambdaCodeGenerator, "", lam, lam.lineno) @@ -956,9 +960,12 @@ elif call_type == 3: op = ops.CALL_FUNCTION_VAR_KW self.emit_op_arg(op, arg) + + def _call_has_no_star_args(self, call): + return not call.starargs and not call.kwargs def _call_has_simple_args(self, call): - return not call.starargs and not call.kwargs and not call.keywords + return self._call_has_no_star_args(call) and not call.keywords def _optimize_builtin_call(self, call): if not self.space.config.objspace.opcodes.CALL_LIKELY_BUILTIN or \ @@ -984,7 +991,7 @@ def _optimize_method_call(self, call): if not self.space.config.objspace.opcodes.CALL_METHOD or \ - not self._call_has_simple_args(call) or \ + not self._call_has_no_star_args(call) or \ not isinstance(call.func, ast.Attribute): return False attr_lookup = call.func @@ -996,7 +1003,12 @@ arg_count = len(call.args) else: arg_count = 0 - self.emit_op_arg(ops.CALL_METHOD, arg_count) + if call.keywords: + self.visit_sequence(call.keywords) + kwarg_count = len(call.keywords) + else: + kwarg_count = 0 + self.emit_op_arg(ops.CALL_METHOD, (kwarg_count << 8) | arg_count) return True def _listcomp_generator(self, list_name, gens, gen_index, elt): @@ -1275,9 +1287,11 @@ else: self.add_const(self.space.w_None) start = 0 - if func.args.args: - self._handle_nested_args(func.args.args) - self.argcount = len(func.args.args) + args = func.args + assert isinstance(args, ast.arguments) + if args.args: + self._handle_nested_args(args.args) + self.argcount = len(args.args) for i in range(start, len(func.body)): func.body[i].walkabout(self) @@ -1286,9 +1300,11 @@ def _compile(self, lam): assert isinstance(lam, ast.Lambda) - if lam.args.args: - self._handle_nested_args(lam.args.args) - self.argcount = len(lam.args.args) + args = lam.args + assert isinstance(args, ast.arguments) + if args.args: + self._handle_nested_args(args.args) + self.argcount = len(args.args) # Prevent a string from being the first constant and thus a docstring. self.add_const(self.space.w_None) lam.body.walkabout(self) Modified: pypy/branch/micronumpy/pypy/interpreter/astcompiler/consts.py ============================================================================== --- pypy/branch/micronumpy/pypy/interpreter/astcompiler/consts.py (original) +++ pypy/branch/micronumpy/pypy/interpreter/astcompiler/consts.py Mon Aug 16 04:52:00 2010 @@ -18,4 +18,4 @@ PyCF_SOURCE_IS_UTF8 = 0x0100 PyCF_DONT_IMPLY_DEDENT = 0x0200 -PyCF_AST_ONLY = 0x0400 +PyCF_ONLY_AST = 0x0400 Modified: pypy/branch/micronumpy/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/micronumpy/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/micronumpy/pypy/interpreter/astcompiler/symtable.py Mon Aug 16 04:52:00 2010 @@ -353,8 +353,10 @@ def visit_FunctionDef(self, func): self.note_symbol(func.name, SYM_ASSIGNED) # Function defaults and decorators happen in the outer scope. - if func.args.defaults: - self.visit_sequence(func.args.defaults) + args = func.args + assert isinstance(args, ast.arguments) + if args.defaults: + self.visit_sequence(args.defaults) if func.decorators: self.visit_sequence(func.decorators) new_scope = FunctionScope(func.name, func.lineno, func.col_offset) @@ -420,8 +422,10 @@ self.note_symbol(name, SYM_GLOBAL) def visit_Lambda(self, lamb): - if lamb.args.defaults: - self.visit_sequence(lamb.args.defaults) + args = lamb.args + assert isinstance(args, ast.arguments) + if args.defaults: + self.visit_sequence(args.defaults) new_scope = FunctionScope("lambda", lamb.lineno, lamb.col_offset) self.push_scope(new_scope, lamb) lamb.args.walkabout(self) Modified: pypy/branch/micronumpy/pypy/interpreter/astcompiler/tools/asdl_py.py ============================================================================== --- pypy/branch/micronumpy/pypy/interpreter/astcompiler/tools/asdl_py.py (original) +++ pypy/branch/micronumpy/pypy/interpreter/astcompiler/tools/asdl_py.py Mon Aug 16 04:52:00 2010 @@ -100,6 +100,7 @@ self.emit("") self.make_constructor(product.fields, product) self.emit("") + self.make_mutate_over(product, name) self.emit("def walkabout(self, visitor):", 1) self.emit("visitor.visit_%s(self)" % (name,), 2) self.emit("") @@ -183,6 +184,26 @@ have_everything = self.data.required_masks[node] | \ self.data.optional_masks[node] self.emit("self.initialization_state = %i" % (have_everything,), 2) + + def make_mutate_over(self, cons, name): + self.emit("def mutate_over(self, visitor):", 1) + for field in cons.fields: + if (field.type.value not in asdl.builtin_types and + field.type.value not in self.data.simple_types): + if field.opt or field.seq: + level = 3 + self.emit("if self.%s:" % (field.name,), 2) + else: + level = 2 + if field.seq: + sub = (field.name,) + self.emit("visitor._mutate_sequence(self.%s)" % sub, level) + else: + sub = (field.name, field.name) + self.emit("self.%s = self.%s.mutate_over(visitor)" % sub, + level) + self.emit("return visitor.visit_%s(self)" % (name,), 2) + self.emit("") def visitConstructor(self, cons, base, extra_attributes): self.emit("class %s(%s):" % (cons.name, base)) @@ -199,24 +220,7 @@ self.emit("def walkabout(self, visitor):", 1) self.emit("visitor.visit_%s(self)" % (cons.name,), 2) self.emit("") - self.emit("def mutate_over(self, visitor):", 1) - for field in cons.fields: - if field.type.value not in asdl.builtin_types and \ - field.type.value not in self.data.prod_simple: - if field.opt or field.seq: - level = 3 - self.emit("if self.%s:" % (field.name,), 2) - else: - level = 2 - if field.seq: - sub = (field.name,) - self.emit("visitor._mutate_sequence(self.%s)" % sub, level) - else: - sub = (field.name, field.name) - self.emit("self.%s = self.%s.mutate_over(visitor)" % sub, - level) - self.emit("return visitor.visit_%s(self)" % (cons.name,), 2) - self.emit("") + self.make_mutate_over(cons, cons.name) self.make_var_syncer(cons.fields + self.data.cons_attributes[cons], cons, cons.name) Modified: pypy/branch/micronumpy/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/micronumpy/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/micronumpy/pypy/interpreter/baseobjspace.py Mon Aug 16 04:52:00 2010 @@ -1102,17 +1102,6 @@ self.wrap('argument must be a unicode')) return self.unicode_w(w_obj) - def path_w(self, w_obj): - """ Like str_w, but if the object is unicode, encode it using - filesystemencoding - """ - filesystemencoding = self.sys.filesystemencoding - if (filesystemencoding and - self.is_true(self.isinstance(w_obj, self.w_unicode))): - w_obj = self.call_method(w_obj, "encode", - self.wrap(filesystemencoding)) - return self.str_w(w_obj) - def bool_w(self, w_obj): # Unwraps a bool, also accepting an int for compatibility. # This is here mostly just for gateway.int_unwrapping_space_method(). Modified: pypy/branch/micronumpy/pypy/interpreter/error.py ============================================================================== --- pypy/branch/micronumpy/pypy/interpreter/error.py (original) +++ pypy/branch/micronumpy/pypy/interpreter/error.py Mon Aug 16 04:52:00 2010 @@ -344,7 +344,7 @@ else: _WINDOWS = True - def wrap_windowserror(space, e, filename=None): + def wrap_windowserror(space, e, w_filename=None): from pypy.rlib import rwin32 winerror = e.winerror @@ -353,19 +353,19 @@ except ValueError: msg = 'Windows Error %d' % winerror exc = space.w_WindowsError - if filename is not None: + if w_filename is not None: w_error = space.call_function(exc, space.wrap(winerror), - space.wrap(msg), space.wrap(filename)) + space.wrap(msg), w_filename) else: w_error = space.call_function(exc, space.wrap(winerror), space.wrap(msg)) return OperationError(exc, w_error) -def wrap_oserror(space, e, filename=None, exception_name='w_OSError'): +def wrap_oserror2(space, e, w_filename=None, exception_name='w_OSError'): assert isinstance(e, OSError) if _WINDOWS and isinstance(e, WindowsError): - return wrap_windowserror(space, e, filename) + return wrap_windowserror(space, e, w_filename) errno = e.errno try: @@ -373,10 +373,21 @@ except ValueError: msg = 'error %d' % errno exc = getattr(space, exception_name) - if filename is not None: + if w_filename is not None: w_error = space.call_function(exc, space.wrap(errno), - space.wrap(msg), space.wrap(filename)) + space.wrap(msg), w_filename) else: - w_error = space.call_function(exc, space.wrap(errno), space.wrap(msg)) + w_error = space.call_function(exc, space.wrap(errno), + space.wrap(msg)) return OperationError(exc, w_error) +wrap_oserror2._annspecialcase_ = 'specialize:arg(3)' + +def wrap_oserror(space, e, filename=None, exception_name='w_OSError'): + if filename is not None: + return wrap_oserror2(space, e, space.wrap(filename), + exception_name=exception_name) + else: + return wrap_oserror2(space, e, None, + exception_name=exception_name) wrap_oserror._annspecialcase_ = 'specialize:arg(3)' + Modified: pypy/branch/micronumpy/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/micronumpy/pypy/interpreter/gateway.py (original) +++ pypy/branch/micronumpy/pypy/interpreter/gateway.py Mon Aug 16 04:52:00 2010 @@ -137,9 +137,6 @@ def visit_c_nonnegint(self, el, app_sig): self.checked_space_method(el, app_sig) - def visit_path(self, el, app_sig): - self.checked_space_method(el, app_sig) - def visit__Wrappable(self, el, app_sig): name = el.__name__ argname = self.orig_arg() @@ -241,9 +238,6 @@ def visit_bufferstr(self, typ): self.run_args.append("space.bufferstr_w(%s)" % (self.scopenext(),)) - def visit_path(self, typ): - self.run_args.append("space.path_w(%s)" % (self.scopenext(),)) - def visit_nonnegint(self, typ): self.run_args.append("space.nonnegint_w(%s)" % (self.scopenext(),)) @@ -371,9 +365,6 @@ def visit_bufferstr(self, typ): self.unwrap.append("space.bufferstr_w(%s)" % (self.nextarg(),)) - def visit_path(self, typ): - self.unwrap.append("space.path_w(%s)" % (self.nextarg(),)) - def visit_nonnegint(self, typ): self.unwrap.append("space.nonnegint_w(%s)" % (self.nextarg(),)) Modified: pypy/branch/micronumpy/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/micronumpy/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/micronumpy/pypy/interpreter/pyopcode.py Mon Aug 16 04:52:00 2010 @@ -988,23 +988,9 @@ self.dropvalues(nargs) self.pushvalue(w_result) - def LOOKUP_METHOD(self, nameindex, next_instr): - # overridden by faster version in the standard object space. - space = self.space - w_obj = self.popvalue() - w_name = self.getname_w(nameindex) - w_value = space.getattr(w_obj, w_name) - self.pushvalue(w_value) - - def CALL_METHOD(self, nargs, next_instr): - # overridden by faster version in the standard object space. - # 'nargs' is the argument count excluding the implicit 'self' - w_callable = self.peekvalue(nargs) - try: - w_result = self.space.call_valuestack(w_callable, nargs, self) - finally: - self.dropvalues(nargs + 1) - self.pushvalue(w_result) + # overridden by faster version in the standard object space. + LOOKUP_METHOD = LOAD_ATTR + CALL_METHOD = CALL_FUNCTION def MISSING_OPCODE(self, oparg, next_instr): ofs = self.last_instr Modified: pypy/branch/micronumpy/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/branch/micronumpy/pypy/interpreter/test/test_compiler.py (original) +++ pypy/branch/micronumpy/pypy/interpreter/test/test_compiler.py Mon Aug 16 04:52:00 2010 @@ -4,6 +4,7 @@ from pypy.interpreter.pycode import PyCode from pypy.interpreter.error import OperationError from pypy.interpreter.argument import Arguments +from pypy.conftest import gettestobjspace class BaseTestCompiler: def setup_method(self, method): @@ -838,6 +839,47 @@ sys.stdout = save_stdout output = s.getvalue() assert "STOP_CODE" not in output + + def test_optimize_list_comp(self): + source = """def _f(a): + return [x for x in a if None] + """ + exec source + code = _f.func_code + + import StringIO, sys, dis + s = StringIO.StringIO() + out = sys.stdout + sys.stdout = s + try: + dis.dis(code) + finally: + sys.stdout = out + output = s.getvalue() + assert "LOAD_GLOBAL" not in output + +class AppTestCallMethod(object): + def setup_class(cls): + cls.space = gettestobjspace(**{'objspace.opcodes.CALL_METHOD': True}) + + def test_call_method_kwargs(self): + source = """def _f(a): + return a.f(a=a) + """ + exec source + code = _f.func_code + + import StringIO, sys, dis + s = StringIO.StringIO() + out = sys.stdout + sys.stdout = s + try: + dis.dis(code) + finally: + sys.stdout = out + output = s.getvalue() + assert "CALL_METHOD" in output + class AppTestExceptions: def test_indentation_error(self): Modified: pypy/branch/micronumpy/pypy/interpreter/test/test_gateway.py ============================================================================== --- pypy/branch/micronumpy/pypy/interpreter/test/test_gateway.py (original) +++ pypy/branch/micronumpy/pypy/interpreter/test/test_gateway.py Mon Aug 16 04:52:00 2010 @@ -454,16 +454,6 @@ assert len(l) == 1 assert space.eq_w(l[0], w("foo")) - def test_interp2app_unwrap_spec_path(self, monkeypatch): - space = self.space - def g(space, p): - return p - - app_g = gateway.interp2app(g, unwrap_spec=[gateway.ObjSpace, 'path']) - w_app_g = space.wrap(app_g) - monkeypatch.setattr(space.sys, "filesystemencoding", "utf-8") - w_res = space.call_function(w_app_g, space.wrap(u"?")) - def test_interp2app_classmethod(self): space = self.space w = space.wrap Modified: pypy/branch/micronumpy/pypy/jit/backend/detect_cpu.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/backend/detect_cpu.py (original) +++ pypy/branch/micronumpy/pypy/jit/backend/detect_cpu.py Mon Aug 16 04:52:00 2010 @@ -56,6 +56,8 @@ return "pypy.jit.backend.x86.runner", "CPU" elif backend_name == 'x86-without-sse2': return "pypy.jit.backend.x86.runner", "CPU386_NO_SSE2" + elif backend_name == 'x86_64': + return "pypy.jit.backend.x86.runner", "CPU_X86_64" elif backend_name == 'cli': return "pypy.jit.backend.cli.runner", "CliCPU" elif backend_name == 'llvm': Modified: pypy/branch/micronumpy/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/micronumpy/pypy/jit/backend/llsupport/llmodel.py Mon Aug 16 04:52:00 2010 @@ -315,11 +315,64 @@ items[itemindex] = newvalue # --- end of GC unsafe code --- - bh_setarrayitem_raw_i = bh_setarrayitem_gc_i - bh_setarrayitem_raw_f = bh_setarrayitem_gc_f + @specialize.argtype(2) + def bh_getarrayitem_raw_i(self, arraydescr, gcref, itemindex): + size = arraydescr.get_item_size(self.translate_support_code) + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), 0) + for TYPE, itemsize in unroll_basic_sizes: + if size == itemsize: + items = rffi.cast(rffi.CArrayPtr(TYPE), items) + val = items[itemindex] + # --- end of GC unsafe code --- + return rffi.cast(lltype.Signed, val) + else: + raise NotImplementedError("size = %d" % size) + + def bh_getarrayitem_raw_r(self, arraydescr, gcref, itemindex): + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items) + pval = self._cast_int_to_gcref(items[itemindex]) + # --- end of GC unsafe code --- + return pval + + @specialize.argtype(2) + def bh_getarrayitem_raw_f(self, arraydescr, gcref, itemindex): + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), 0) + items = rffi.cast(rffi.CArrayPtr(lltype.Float), items) + fval = items[itemindex] + # --- end of GC unsafe code --- + return fval - bh_getarrayitem_raw_i = bh_getarrayitem_gc_i - bh_getarrayitem_raw_f = bh_getarrayitem_gc_f + @specialize.argtype(2) + def bh_setarrayitem_raw_i(self, arraydescr, gcref, itemindex, newvalue): + size = arraydescr.get_item_size(self.translate_support_code) + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), 0) + for TYPE, itemsize in unroll_basic_sizes: + if size == itemsize: + items = rffi.cast(rffi.CArrayPtr(TYPE), items) + items[itemindex] = rffi.cast(TYPE, newvalue) + # --- end of GC unsafe code --- + return + else: + raise NotImplementedError("size = %d" % size) + + def bh_setarrayitem_raw_r(self, arraydescr, gcref, itemindex, newvalue): + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), 0) + items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items) + items[itemindex] = self.cast_gcref_to_int(newvalue) + # --- end of GC unsafe code --- + + @specialize.argtype(2) + def bh_setarrayitem_raw_f(self, arraydescr, gcref, itemindex, newvalue): + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), 0) + items = rffi.cast(rffi.CArrayPtr(lltype.Float), items) + items[itemindex] = newvalue + # --- end of GC unsafe code --- def bh_strlen(self, string): s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string) Modified: pypy/branch/micronumpy/pypy/jit/backend/llsupport/regalloc.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/backend/llsupport/regalloc.py (original) +++ pypy/branch/micronumpy/pypy/jit/backend/llsupport/regalloc.py Mon Aug 16 04:52:00 2010 @@ -22,18 +22,19 @@ def get(self, box): return self.frame_bindings.get(box, None) - def loc(self, box, size): + def loc(self, box): res = self.get(box) if res is not None: return res - newloc = self.frame_pos(self.frame_depth, size) + newloc = self.frame_pos(self.frame_depth, box.type) self.frame_bindings[box] = newloc - self.frame_depth += size + # Objects returned by frame_pos must support frame_size() + self.frame_depth += newloc.frame_size() return newloc # abstract methods that need to be overwritten for specific assemblers @staticmethod - def frame_pos(loc, size): + def frame_pos(loc, type): raise NotImplementedError("Purely abstract") class RegisterManager(object): @@ -43,7 +44,6 @@ all_regs = [] no_lower_byte_regs = [] save_around_call_regs = [] - reg_width = 1 # in terms of stack space eaten def __init__(self, longevity, frame_manager=None, assembler=None): self.free_regs = self.all_regs[:] @@ -148,7 +148,7 @@ loc = self.reg_bindings[v_to_spill] del self.reg_bindings[v_to_spill] if self.frame_manager.get(v_to_spill) is None: - newloc = self.frame_manager.loc(v_to_spill, self.reg_width) + newloc = self.frame_manager.loc(v_to_spill) self.assembler.regalloc_mov(loc, newloc) return loc @@ -204,7 +204,7 @@ try: return self.reg_bindings[box] except KeyError: - return self.frame_manager.loc(box, self.reg_width) + return self.frame_manager.loc(box) def return_constant(self, v, forbidden_vars=[], selected_reg=None, imm_fine=True): @@ -260,7 +260,7 @@ self.reg_bindings[v] = loc self.assembler.regalloc_mov(prev_loc, loc) else: - loc = self.frame_manager.loc(v, self.reg_width) + loc = self.frame_manager.loc(v) self.assembler.regalloc_mov(prev_loc, loc) def force_result_in_reg(self, result_v, v, forbidden_vars=[]): @@ -280,7 +280,7 @@ self.free_regs = [reg for reg in self.free_regs if reg is not loc] return loc if v not in self.reg_bindings: - prev_loc = self.frame_manager.loc(v, self.reg_width) + prev_loc = self.frame_manager.loc(v) loc = self.force_allocate_reg(v, forbidden_vars) self.assembler.regalloc_mov(prev_loc, loc) assert v in self.reg_bindings @@ -300,7 +300,7 @@ def _sync_var(self, v): if not self.frame_manager.get(v): reg = self.reg_bindings[v] - to = self.frame_manager.loc(v, self.reg_width) + to = self.frame_manager.loc(v) self.assembler.regalloc_mov(reg, to) # otherwise it's clean Modified: pypy/branch/micronumpy/pypy/jit/backend/llsupport/test/test_descr.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/backend/llsupport/test/test_descr.py (original) +++ pypy/branch/micronumpy/pypy/jit/backend/llsupport/test/test_descr.py Mon Aug 16 04:52:00 2010 @@ -228,11 +228,11 @@ # descr2 = get_field_descr(c0, S, 'y') o, _ = symbolic.get_field_token(S, 'y', False) - assert descr2.repr_of_descr() == '' % o + assert descr2.repr_of_descr() == '' % o # descr2i = get_field_descr(c0, S, 'x') o, _ = symbolic.get_field_token(S, 'x', False) - assert descr2i.repr_of_descr() == '' % o + assert descr2i.repr_of_descr() == '' % o # descr3 = get_array_descr(c0, lltype.GcArray(lltype.Ptr(S))) assert descr3.repr_of_descr() == '' Modified: pypy/branch/micronumpy/pypy/jit/backend/llsupport/test/test_regalloc.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/backend/llsupport/test/test_regalloc.py (original) +++ pypy/branch/micronumpy/pypy/jit/backend/llsupport/test/test_regalloc.py Mon Aug 16 04:52:00 2010 @@ -1,5 +1,5 @@ -from pypy.jit.metainterp.history import BoxInt, ConstInt, BoxFloat +from pypy.jit.metainterp.history import BoxInt, ConstInt, BoxFloat, INT, FLOAT from pypy.jit.backend.llsupport.regalloc import FrameManager from pypy.jit.backend.llsupport.regalloc import RegisterManager as BaseRegMan @@ -26,9 +26,20 @@ def convert_to_imm(self, v): return v +class FakeFramePos(object): + def __init__(self, pos, box_type): + self.pos = pos + self.box_type = box_type + + def frame_size(self): + if self.box_type == FLOAT: + return 2 + else: + return 1 + class TFrameManager(FrameManager): - def frame_pos(self, i, size): - return i + def frame_pos(self, i, box_type): + return FakeFramePos(i, box_type) class MockAsm(object): def __init__(self): @@ -146,8 +157,8 @@ rm.next_instruction() # allocate a stack position b0, b1, b2, b3, b4 = boxes - sp = fm.loc(b0, 1) - assert sp == 0 + sp = fm.loc(b0) + assert sp.pos == 0 loc = rm.make_sure_var_in_reg(b0) assert isinstance(loc, FakeReg) rm._check_invariants() @@ -207,13 +218,13 @@ asm = MockAsm() rm = RegisterManager(longevity, frame_manager=fm, assembler=asm) rm.next_instruction() - fm.loc(b0, 1) + fm.loc(b0) rm.force_result_in_reg(b1, b0) rm._check_invariants() loc = rm.loc(b1) assert isinstance(loc, FakeReg) loc = rm.loc(b0) - assert isinstance(loc, int) + assert isinstance(loc, FakeFramePos) assert len(asm.moves) == 1 def test_return_constant(self): @@ -304,7 +315,7 @@ def test_different_frame_width(self): class XRegisterManager(RegisterManager): - reg_width = 2 + pass fm = TFrameManager() b0 = BoxInt() Modified: pypy/branch/micronumpy/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/micronumpy/pypy/jit/backend/test/runner_test.py Mon Aug 16 04:52:00 2010 @@ -461,6 +461,25 @@ [funcbox] + args, 'float', descr=calldescr) assert abs(res.value - 4.6) < 0.0001 + + def test_call_many_arguments(self): + # Test calling a function with a large number of arguments (more than + # 6, which will force passing some arguments on the stack on 64-bit) + + def func(*args): + assert len(args) == 16 + # Try to sum up args in a way that would probably detect a + # transposed argument + return sum(arg * (2**i) for i, arg in enumerate(args)) + + FUNC = self.FuncType([lltype.Signed]*16, lltype.Signed) + FPTR = self.Ptr(FUNC) + calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + func_ptr = llhelper(FPTR, func) + args = range(16) + funcbox = self.get_funcbox(self.cpu, func_ptr) + res = self.execute_operation(rop.CALL, [funcbox] + map(BoxInt, args), 'int', descr=calldescr) + assert res.value == func(*args) def test_call_stack_alignment(self): # test stack alignment issues, notably for Mac OS/X. @@ -638,6 +657,21 @@ assert r.value == 1 def test_array_basic(self): + a_box, A = self.alloc_array_of(rffi.SHORT, 342) + arraydescr = self.cpu.arraydescrof(A) + assert not arraydescr.is_array_of_pointers() + # + r = self.execute_operation(rop.ARRAYLEN_GC, [a_box], + 'int', descr=arraydescr) + assert r.value == 342 + r = self.execute_operation(rop.SETARRAYITEM_GC, [a_box, BoxInt(310), + BoxInt(744)], + 'void', descr=arraydescr) + assert r is None + r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(310)], + 'int', descr=arraydescr) + assert r.value == 744 + a_box, A = self.alloc_array_of(lltype.Signed, 342) arraydescr = self.cpu.arraydescrof(A) assert not arraydescr.is_array_of_pointers() @@ -978,6 +1012,8 @@ else: assert 0 operations.append(ResOperation(opnum, boxargs, boxres)) + # Unique-ify inputargs + inputargs = list(set(inputargs)) faildescr = BasicFailDescr(1) operations.append(ResOperation(rop.FINISH, [], None, descr=faildescr)) @@ -1050,9 +1086,11 @@ descr=BasicFailDescr(5))] operations[1].fail_args = [] looptoken = LoopToken() - self.cpu.compile_loop(list(testcase), operations, + # Use "set" to unique-ify inputargs + unique_testcase_list = list(set(testcase)) + self.cpu.compile_loop(unique_testcase_list, operations, looptoken) - for i, box in enumerate(testcase): + for i, box in enumerate(unique_testcase_list): self.cpu.set_future_value_float(i, box.value) fail = self.cpu.execute_token(looptoken) if fail.identifier != 5 - (expected_id^expected): @@ -1695,7 +1733,7 @@ def test_assembler_call(self): called = [] def assembler_helper(failindex, virtualizable): - assert self.cpu.get_latest_value_int(0) == 10 + assert self.cpu.get_latest_value_int(0) == 97 called.append(failindex) return 4 + 9 @@ -1708,33 +1746,41 @@ _assembler_helper_ptr) ops = ''' - [i0, i1] - i2 = int_add(i0, i1) - finish(i2)''' + [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] + i10 = int_add(i0, i1) + i11 = int_add(i10, i2) + i12 = int_add(i11, i3) + i13 = int_add(i12, i4) + i14 = int_add(i13, i5) + i15 = int_add(i14, i6) + i16 = int_add(i15, i7) + i17 = int_add(i16, i8) + i18 = int_add(i17, i9) + finish(i18)''' loop = parse(ops) looptoken = LoopToken() looptoken.outermost_jitdriver_sd = FakeJitDriverSD() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) - ARGS = [lltype.Signed, lltype.Signed] + ARGS = [lltype.Signed] * 10 RES = lltype.Signed self.cpu.portal_calldescr = self.cpu.calldescrof( lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES) - self.cpu.set_future_value_int(0, 1) - self.cpu.set_future_value_int(1, 2) + for i in range(10): + self.cpu.set_future_value_int(i, i+1) res = self.cpu.execute_token(looptoken) - assert self.cpu.get_latest_value_int(0) == 3 + assert self.cpu.get_latest_value_int(0) == 55 ops = ''' - [i4, i5] - i6 = int_add(i4, 1) - i3 = call_assembler(i6, i5, descr=looptoken) + [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] + i10 = int_add(i0, 42) + i11 = call_assembler(i10, i1, i2, i3, i4, i5, i6, i7, i8, i9, descr=looptoken) guard_not_forced()[] - finish(i3) + finish(i11) ''' loop = parse(ops, namespace=locals()) othertoken = LoopToken() self.cpu.compile_loop(loop.inputargs, loop.operations, othertoken) - self.cpu.set_future_value_int(0, 4) - self.cpu.set_future_value_int(1, 5) + for i in range(10): + self.cpu.set_future_value_int(i, i+1) res = self.cpu.execute_token(othertoken) assert self.cpu.get_latest_value_int(0) == 13 assert called Modified: pypy/branch/micronumpy/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/micronumpy/pypy/jit/backend/x86/assembler.py Mon Aug 16 04:52:00 2010 @@ -1,4 +1,4 @@ -import sys +import sys, os from pypy.jit.backend.llsupport import symbolic from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxFloat from pypy.jit.metainterp.history import AbstractFailDescr, INT, REF, FLOAT,\ @@ -7,23 +7,35 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.annlowlevel import llhelper from pypy.tool.uid import fixid -from pypy.jit.backend.x86.regalloc import RegAlloc, WORD,\ - X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs, FRAME_FIXED_SIZE,\ - FORCE_INDEX_OFS +from pypy.jit.backend.x86.regalloc import RegAlloc, \ + X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs + +from pypy.jit.backend.x86.arch import FRAME_FIXED_SIZE, FORCE_INDEX_OFS, WORD, IS_X86_32, IS_X86_64 + +from pypy.jit.backend.x86.regloc import (eax, ecx, edx, ebx, + esp, ebp, esi, edi, + xmm0, xmm1, xmm2, xmm3, + xmm4, xmm5, xmm6, xmm7, + r8, r9, r10, r11, + r12, r13, r14, r15, + X86_64_SCRATCH_REG, + X86_64_XMM_SCRATCH_REG, + RegLoc, StackLoc, ConstFloatLoc, + ImmedLoc, AddressLoc, imm) + from pypy.rlib.objectmodel import we_are_translated, specialize -from pypy.jit.backend.x86 import codebuf -from pypy.jit.backend.x86.ri386 import * -from pypy.jit.metainterp.resoperation import rop +from pypy.jit.backend.x86 import rx86, regloc, codebuf +from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.backend.x86.support import values_array from pypy.rlib.debug import debug_print from pypy.rlib import rgc - -# our calling convention - we pass first 6 args in registers -# and the rest stays on the stack +from pypy.jit.backend.x86.jump import remap_frame_layout +from pypy.rlib.streamio import open_file_as_stream +from pypy.jit.metainterp.history import ConstInt, BoxInt # darwin requires the stack to be 16 bytes aligned on calls. Same for gcc 4.5.0, # better safe than sorry -CALL_ALIGN = 4 +CALL_ALIGN = 16 // WORD def align_stack_words(words): return (words + CALL_ALIGN - 1) & ~(CALL_ALIGN-1) @@ -31,16 +43,36 @@ class MachineCodeBlockWrapper(object): MC_DEFAULT_SIZE = 1024*1024 - def __init__(self, bigsize, profile_agent=None): + def __init__(self, assembler, bigsize, profile_agent=None): + self.assembler = assembler self.old_mcs = [] # keepalive self.bigsize = bigsize self._mc = self._instantiate_mc() self.function_name = None self.profile_agent = profile_agent + self.reset_reserved_bytes() def _instantiate_mc(self): # hook for testing return codebuf.MachineCodeBlock(self.bigsize) + def ensure_bytes_available(self, num_bytes): + if self.bytes_free() <= (self._reserved_bytes + num_bytes): + self.make_new_mc() + + def reserve_bytes(self, num_bytes): + self.ensure_bytes_available(num_bytes) + self._reserved_bytes += num_bytes + + def reset_reserved_bytes(self): + # XXX er.... pretty random number, just to be sure + # not to write half-instruction + self._reserved_bytes = 64 + + def get_relative_pos(self): + return self._mc.get_relative_pos() + + def overwrite(self, pos, listofchars): + return self._mc.overwrite(pos, listofchars) def bytes_free(self): return self._mc._size - self._mc.get_relative_pos() @@ -62,12 +94,25 @@ def make_new_mc(self): new_mc = self._instantiate_mc() debug_print('[new machine code block at', new_mc.tell(), ']') - self._mc.JMP(rel32(new_mc.tell())) + + if IS_X86_64: + # The scratch register is sometimes used as a temporary + # register, but the JMP below might clobber it. Rather than risk + # subtle bugs, we preserve the scratch register across the jump. + self._mc.PUSH_r(X86_64_SCRATCH_REG.value) + + self._mc.JMP(imm(new_mc.tell())) + + if IS_X86_64: + # Restore scratch reg + new_mc.POP_r(X86_64_SCRATCH_REG.value) if self.function_name is not None: self.end_function(done=False) self.start_pos = new_mc.get_relative_pos() + self.assembler.write_pending_failure_recoveries() + self._mc.done() self.old_mcs.append(self._mc) self._mc = new_mc @@ -81,24 +126,39 @@ def _new_method(name): def method(self, *args): - # XXX er.... pretty random number, just to be sure - # not to write half-instruction - if self.bytes_free() < 64: + if self.bytes_free() < self._reserved_bytes: self.make_new_mc() getattr(self._mc, name)(*args) method.func_name = name return method +for _name in rx86.all_instructions + regloc.all_extra_instructions: + setattr(MachineCodeBlockWrapper, _name, _new_method(_name)) + for name in dir(codebuf.MachineCodeBlock): if name.upper() == name or name == "writechr": setattr(MachineCodeBlockWrapper, name, _new_method(name)) +class GuardToken(object): + def __init__(self, faildescr, failargs, fail_locs, exc, desc_bytes): + self.faildescr = faildescr + self.failargs = failargs + self.fail_locs = fail_locs + self.exc = exc + self.desc_bytes = desc_bytes + + def recovery_stub_size(self): + # XXX: 32 is pulled out of the air + return 32 + len(self.desc_bytes) + +DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed)) + class Assembler386(object): mc = None - mc2 = None mc_size = MachineCodeBlockWrapper.MC_DEFAULT_SIZE _float_constants = None _regalloc = None + _output_loop_log = None def __init__(self, cpu, translate_support_code=False, failargs_limit=1000): @@ -113,18 +173,26 @@ self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) self.fail_boxes_float = values_array(lltype.Float, failargs_limit) self.fail_ebp = 0 - self.loc_float_const_neg = None - self.loc_float_const_abs = None + self.loop_run_counters = [] + # if we have 10000 loops, we have some other problems I guess + self.float_const_neg_addr = 0 + self.float_const_abs_addr = 0 self.malloc_fixedsize_slowpath1 = 0 self.malloc_fixedsize_slowpath2 = 0 + self.pending_guard_tokens = None self.setup_failure_recovery() + self._debug = False + self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') def leave_jitted_hook(self): ptrs = self.fail_boxes_ptr.ar llop.gc_assume_young_pointers(lltype.Void, llmemory.cast_ptr_to_adr(ptrs)) - def make_sure_mc_exists(self): + def set_debug(self, v): + self._debug = v + + def setup(self): if self.mc is None: # the address of the function called by 'new' gc_ll_descr = self.cpu.gc_ll_descr @@ -143,11 +211,7 @@ ll_new_unicode = gc_ll_descr.get_funcptr_for_newunicode() self.malloc_unicode_func_addr = rffi.cast(lltype.Signed, ll_new_unicode) - # done - # we generate the loop body in 'mc' - # 'mc2' is for guard recovery code - self.mc = MachineCodeBlockWrapper(self.mc_size, self.cpu.profile_agent) - self.mc2 = MachineCodeBlockWrapper(self.mc_size) + self.mc = MachineCodeBlockWrapper(self, self.mc_size, self.cpu.profile_agent) self._build_failure_recovery(False) self._build_failure_recovery(True) if self.cpu.supports_floats: @@ -157,46 +221,72 @@ self._build_float_constants() if hasattr(gc_ll_descr, 'get_malloc_fixedsize_slowpath_addr'): self._build_malloc_fixedsize_slowpath() + s = os.environ.get('PYPYLOG') + if s: + if s.find(':') != -1: + s = s.split(':')[-1] + self.set_debug(True) + self._output_loop_log = s + ".count" + # Intialize here instead of __init__ to prevent + # pending_guard_tokens from being considered a prebuilt object, + # which sometimes causes memory leaks since the prebuilt list is + # still considered a GC root after we re-assign + # pending_guard_tokens in write_pending_failure_recoveries + self.pending_guard_tokens = [] + + def finish_once(self): + if self._debug: + output_log = self._output_loop_log + assert output_log is not None + f = open_file_as_stream(output_log, "w") + for i in range(len(self.loop_run_counters)): + name, struct = self.loop_run_counters[i] + f.write(name + ":" + str(struct.i) + "\n") + f.close() def _build_float_constants(self): - # 11 words: 8 words for the data, and up to 3 words for alignment - addr = lltype.malloc(rffi.CArray(lltype.Signed), 11, flavor='raw') + # 44 bytes: 32 bytes for the data, and up to 12 bytes for alignment + addr = lltype.malloc(rffi.CArray(lltype.Char), 44, flavor='raw') if not we_are_translated(): self._keepalive_malloced_float_consts = addr float_constants = rffi.cast(lltype.Signed, addr) float_constants = (float_constants + 15) & ~15 # align to 16 bytes - addr = rffi.cast(rffi.CArrayPtr(lltype.Signed), float_constants) - addr[0] = 0 # \ - addr[1] = -2147483648 # / for neg - addr[2] = 0 # - addr[3] = 0 # - addr[4] = -1 # \ - addr[5] = 2147483647 # / for abs - addr[6] = 0 # - addr[7] = 0 # - self.loc_float_const_neg = heap64(float_constants) - self.loc_float_const_abs = heap64(float_constants + 16) + addr = rffi.cast(rffi.CArrayPtr(lltype.Char), float_constants) + qword_padding = '\x00\x00\x00\x00\x00\x00\x00\x00' + # 0x8000000000000000 + neg_const = '\x00\x00\x00\x00\x00\x00\x00\x80' + # 0x7FFFFFFFFFFFFFFF + abs_const = '\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x7F' + data = neg_const + qword_padding + abs_const + qword_padding + for i in range(len(data)): + addr[i] = data[i] + self.float_const_neg_addr = float_constants + self.float_const_abs_addr = float_constants + 16 def _build_malloc_fixedsize_slowpath(self): - mc = self.mc2._mc # ---------- first helper for the slow path of malloc ---------- - self.malloc_fixedsize_slowpath1 = mc.tell() + self.malloc_fixedsize_slowpath1 = self.mc.tell() if self.cpu.supports_floats: # save the XMM registers in - for i in range(8): # the *caller* frame, from esp+8 - mc.MOVSD(mem64(esp, 8+8*i), xmm_registers[i]) - mc.SUB(edx, eax) # compute the size we want - mc.MOV(mem(esp, 4), edx) # save it as the new argument + for i in range(self.cpu.NUM_REGS):# the *caller* frame, from esp+8 + self.mc.MOVSD_sx((WORD*2)+8*i, i) + self.mc.SUB_rr(edx.value, eax.value) # compute the size we want + if IS_X86_32: + self.mc.MOV_sr(WORD, edx.value) # save it as the new argument + elif IS_X86_64: + # FIXME: We can't just clobber rdi like this, can we? + self.mc.MOV_rr(edi.value, edx.value) + addr = self.cpu.gc_ll_descr.get_malloc_fixedsize_slowpath_addr() - mc.JMP(rel32(addr)) # tail call to the real malloc + self.mc.JMP(imm(addr)) # tail call to the real malloc # ---------- second helper for the slow path of malloc ---------- - self.malloc_fixedsize_slowpath2 = mc.tell() + self.malloc_fixedsize_slowpath2 = self.mc.tell() if self.cpu.supports_floats: # restore the XMM registers - for i in range(8): # from where they were saved - mc.MOVSD(xmm_registers[i], mem64(esp, 8+8*i)) + for i in range(self.cpu.NUM_REGS):# from where they were saved + self.mc.MOVSD_xs(i, (WORD*2)+8*i) nursery_free_adr = self.cpu.gc_ll_descr.get_nursery_free_addr() - mc.MOV(edx, heap(nursery_free_adr)) # load this in EDX - mc.RET() - self.mc2.done() + self.mc.MOV(edx, heap(nursery_free_adr)) # load this in EDX + self.mc.RET() + self.mc.done() def assemble_loop(self, inputargs, operations, looptoken): """adds the following attributes to looptoken: @@ -207,15 +297,18 @@ _x86_param_depth _x86_arglocs """ + if not we_are_translated(): + # Arguments should be unique + assert len(set(inputargs)) == len(inputargs) + + self.setup() funcname = self._find_debug_merge_point(operations) - self.make_sure_mc_exists() + regalloc = RegAlloc(self, self.cpu.translate_support_code) + operations = self._inject_debugging_code(operations) arglocs = regalloc.prepare_loop(inputargs, operations, looptoken) looptoken._x86_arglocs = arglocs - needed_mem = len(arglocs[0]) * 16 + 16 - if needed_mem >= self.mc.bytes_free(): - self.mc.make_new_mc() # profile support name = "Loop # %s: %s" % (looptoken.number, funcname) @@ -230,29 +323,30 @@ self._patch_stackadjust(adr_stackadjust, frame_depth+param_depth) looptoken._x86_frame_depth = frame_depth looptoken._x86_param_depth = param_depth - # we need to make sure here that we don't overload an mc badly. - # a safe estimate is that we need at most 16 bytes per arg - needed_mem = len(arglocs[0]) * 16 + 16 - if needed_mem >= self.mc.bytes_free(): - self.mc.make_new_mc() + looptoken._x86_direct_bootstrap_code = self.mc.tell() self._assemble_bootstrap_direct_call(arglocs, curadr, frame_depth+param_depth) debug_print("Loop #", looptoken.number, "has address", looptoken._x86_loop_code, "to", self.mc.tell()) self.mc.end_function() + self.write_pending_failure_recoveries() - def assemble_bridge(self, faildescr, inputargs, operations): + if not we_are_translated(): + # Arguments should be unique + assert len(set(inputargs)) == len(inputargs) + + self.setup() funcname = self._find_debug_merge_point(operations) - self.make_sure_mc_exists() arglocs = self.rebuild_faillocs_from_descr( faildescr._x86_failure_recovery_bytecode) if not we_are_translated(): assert ([loc.assembler() for loc in arglocs] == [loc.assembler() for loc in faildescr._x86_debug_faillocs]) regalloc = RegAlloc(self, self.cpu.translate_support_code) + operations = self._inject_debugging_code(operations) fail_depths = faildescr._x86_current_depths regalloc.prepare_bridge(fail_depths, inputargs, arglocs, operations) @@ -276,25 +370,81 @@ descr_number, "has address", adr_bridge, "to", self.mc.tell()) self.mc.end_function() + self.write_pending_failure_recoveries() + + def write_pending_failure_recoveries(self): + for tok in self.pending_guard_tokens: + # Okay to write to _mc because we've already made sure that + # there's enough space by "reserving" bytes. + addr = self.generate_quick_failure(self.mc._mc, tok.faildescr, tok.failargs, tok.fail_locs, tok.exc, tok.desc_bytes) + tok.faildescr._x86_adr_recovery_stub = addr + self.patch_jump_for_descr(tok.faildescr, addr) + + self.pending_guard_tokens = [] + self.mc.reset_reserved_bytes() + self.mc.done() def _find_debug_merge_point(self, operations): + for op in operations: if op.opnum == rop.DEBUG_MERGE_POINT: - return op.args[0]._get_str() - return "" + funcname = op.args[0]._get_str() + break + else: + funcname = "" % len(self.loop_run_counters) + # invent the counter, so we don't get too confused + if self._debug: + struct = lltype.malloc(DEBUG_COUNTER, flavor='raw') + struct.i = 0 + self.loop_run_counters.append((funcname, struct)) + return funcname def patch_jump_for_descr(self, faildescr, adr_new_target): adr_jump_offset = faildescr._x86_adr_jump_offset - mc = codebuf.InMemoryCodeBuilder(adr_jump_offset, adr_jump_offset + 4) - mc.write(packimm32(adr_new_target - adr_jump_offset - 4)) + adr_recovery_stub = faildescr._x86_adr_recovery_stub + offset = adr_new_target - (adr_jump_offset + 4) + # If the new target fits within a rel32 of the jump, just patch + # that. Otherwise, leave the original rel32 to the recovery stub in + # place, but clobber the recovery stub with a jump to the real + # target. + if rx86.fits_in_32bits(offset): + mc = codebuf.InMemoryCodeBuilder(adr_jump_offset, adr_jump_offset + 4) + mc.writeimm32(offset) + else: + # "mov r11, addr; jmp r11" is 13 bytes + mc = codebuf.InMemoryCodeBuilder(adr_recovery_stub, adr_recovery_stub + 13) + mc.MOV_ri(X86_64_SCRATCH_REG.value, adr_new_target) + mc.JMP_r(X86_64_SCRATCH_REG.value) + mc.valgrind_invalidated() mc.done() + def _inject_debugging_code(self, operations): + if self._debug: + # before doing anything, let's increase a counter + c_adr = ConstInt(rffi.cast(lltype.Signed, + self.loop_run_counters[-1][1])) + box = BoxInt() + box2 = BoxInt() + ops = [ResOperation(rop.GETFIELD_RAW, [c_adr], + box, descr=self.debug_counter_descr), + ResOperation(rop.INT_ADD, [box, ConstInt(1)], box2), + ResOperation(rop.SETFIELD_RAW, [c_adr, box2], + None, descr=self.debug_counter_descr)] + operations = ops + operations + # # we need one register free (a bit of a hack, but whatever) + # self.mc.PUSH(eax) + # adr = rffi.cast(lltype.Signed, self.loop_run_counters[-1][1]) + # self.mc.MOV(eax, heap(adr)) + # self.mc.ADD(eax, imm(1)) + # self.mc.MOV(heap(adr), eax) + # self.mc.POP(eax) + return operations + def _assemble(self, regalloc, operations): self._regalloc = regalloc regalloc.walk_operations(operations) self.mc.done() - self.mc2.done() if we_are_translated() or self.cpu.dont_keepalive_stuff: self._regalloc = None # else keep it around for debugging frame_depth = regalloc.fm.frame_depth @@ -309,7 +459,7 @@ def _patchable_stackadjust(self): # stack adjustment LEA - self.mc.LEA(esp, fixedsize_ebp_ofs(0)) + self.mc.LEA32_rb(esp.value, 0) return self.mc.tell() - 4 def _patch_stackadjust(self, adr_lea, reserved_depth): @@ -318,23 +468,34 @@ # Compute the correct offset for the instruction LEA ESP, [EBP-4*words]. # Given that [EBP] is where we saved EBP, i.e. in the last word # of our fixed frame, then the 'words' value is: - words = (FRAME_FIXED_SIZE - 1) + reserved_depth + words = (self.cpu.FRAME_FIXED_SIZE - 1) + reserved_depth # align, e.g. for Mac OS X aligned_words = align_stack_words(words+2)-2 # 2 = EIP+EBP - mc.write(packimm32(-WORD * aligned_words)) + mc.writeimm32(-WORD * aligned_words) mc.done() def _call_header(self): - self.mc.PUSH(ebp) - self.mc.MOV(ebp, esp) - self.mc.PUSH(ebx) - self.mc.PUSH(esi) - self.mc.PUSH(edi) + self.mc.PUSH_r(ebp.value) + self.mc.MOV_rr(ebp.value, esp.value) + for regloc in self.cpu.CALLEE_SAVE_REGISTERS: + self.mc.PUSH_r(regloc.value) + # NB. the shape of the frame is hard-coded in get_basic_shape() too. # Also, make sure this is consistent with FRAME_FIXED_SIZE. return self._patchable_stackadjust() + def _call_footer(self): + self.mc.LEA_rb(esp.value, -len(self.cpu.CALLEE_SAVE_REGISTERS) * WORD) + + for i in range(len(self.cpu.CALLEE_SAVE_REGISTERS)-1, -1, -1): + self.mc.POP_r(self.cpu.CALLEE_SAVE_REGISTERS[i].value) + + self.mc.POP_r(ebp.value) + self.mc.RET() + def _assemble_bootstrap_direct_call(self, arglocs, jmpadr, stackdepth): + if IS_X86_64: + return self._assemble_bootstrap_direct_call_64(arglocs, jmpadr, stackdepth) # XXX pushing ebx esi and edi is a bit pointless, since we store # all regsiters anyway, for the case of guard_not_forced # XXX this can be improved greatly. Right now it'll behave like @@ -345,23 +506,81 @@ self._patch_stackadjust(adr_stackadjust, stackdepth) for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] - if isinstance(loc, REG): - self.mc.MOV(loc, mem(ebp, (2 + i) * WORD)) + if isinstance(loc, RegLoc): + assert not loc.is_xmm + self.mc.MOV_rb(loc.value, (2 + i) * WORD) loc = floatlocs[i] - if isinstance(loc, XMMREG): - self.mc.MOVSD(loc, mem64(ebp, (1 + i) * 2 * WORD)) + if isinstance(loc, RegLoc): + assert loc.is_xmm + self.mc.MOVSD_xb(loc.value, (1 + i) * 2 * WORD) tmp = eax xmmtmp = xmm0 for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] - if loc is not None and not isinstance(loc, REG): - self.mc.MOV(tmp, mem(ebp, (2 + i) * WORD)) + if loc is not None and not isinstance(loc, RegLoc): + self.mc.MOV_rb(tmp.value, (2 + i) * WORD) self.mc.MOV(loc, tmp) loc = floatlocs[i] - if loc is not None and not isinstance(loc, XMMREG): - self.mc.MOVSD(xmmtmp, mem64(ebp, (1 + i) * 2 * WORD)) - self.mc.MOVSD(loc, xmmtmp) - self.mc.JMP(rel32(jmpadr)) + if loc is not None and not isinstance(loc, RegLoc): + self.mc.MOVSD_xb(xmmtmp.value, (1 + i) * 2 * WORD) + assert isinstance(loc, StackLoc) + self.mc.MOVSD_bx(loc.value, xmmtmp.value) + self.mc.JMP_l(jmpadr) + return adr_stackadjust + + def _assemble_bootstrap_direct_call_64(self, arglocs, jmpadr, stackdepth): + # XXX: Very similar to _emit_call_64 + + src_locs = [] + dst_locs = [] + xmm_src_locs = [] + xmm_dst_locs = [] + get_from_stack = [] + + # In reverse order for use with pop() + unused_gpr = [r9, r8, ecx, edx, esi, edi] + unused_xmm = [xmm7, xmm6, xmm5, xmm4, xmm3, xmm2, xmm1, xmm0] + + nonfloatlocs, floatlocs = arglocs + adr_stackadjust = self._call_header() + self._patch_stackadjust(adr_stackadjust, stackdepth) + + # The lists are padded with Nones + assert len(nonfloatlocs) == len(floatlocs) + + for i in range(len(nonfloatlocs)): + loc = nonfloatlocs[i] + if loc is not None: + if len(unused_gpr) > 0: + src_locs.append(unused_gpr.pop()) + dst_locs.append(loc) + else: + get_from_stack.append((loc, False)) + + floc = floatlocs[i] + if floc is not None: + if len(unused_xmm) > 0: + xmm_src_locs.append(unused_xmm.pop()) + xmm_dst_locs.append(floc) + else: + get_from_stack.append((floc, True)) + + remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG) + remap_frame_layout(self, xmm_src_locs, xmm_dst_locs, X86_64_XMM_SCRATCH_REG) + + for i in range(len(get_from_stack)): + loc, is_xmm = get_from_stack[i] + if is_xmm: + self.mc.MOVSD_xb(X86_64_XMM_SCRATCH_REG.value, (2 + i) * WORD) + self.mc.MOVSD(loc, X86_64_XMM_SCRATCH_REG) + else: + self.mc.MOV_rb(X86_64_SCRATCH_REG.value, (2 + i) * WORD) + # XXX: We're assuming that "loc" won't require regloc to + # clobber the scratch register + self.mc.MOV(loc, X86_64_SCRATCH_REG) + + self.mc.JMP(imm(jmpadr)) + return adr_stackadjust def _assemble_bootstrap_code(self, inputargs, arglocs): @@ -369,11 +588,12 @@ adr_stackadjust = self._call_header() tmp = X86RegisterManager.all_regs[0] xmmtmp = X86XMMRegisterManager.all_regs[0] + self.mc._mc.begin_reuse_scratch_register() for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] if loc is None: continue - if isinstance(loc, REG): + if isinstance(loc, RegLoc): target = loc else: target = tmp @@ -387,17 +607,20 @@ adr = self.fail_boxes_int.get_addr_for_num(i) self.mc.MOV(target, heap(adr)) if target is not loc: - self.mc.MOV(loc, target) + assert isinstance(loc, StackLoc) + self.mc.MOV_br(loc.value, target.value) for i in range(len(floatlocs)): loc = floatlocs[i] if loc is None: continue adr = self.fail_boxes_float.get_addr_for_num(i) - if isinstance(loc, REG): - self.mc.MOVSD(loc, heap64(adr)) + if isinstance(loc, RegLoc): + self.mc.MOVSD(loc, heap(adr)) else: - self.mc.MOVSD(xmmtmp, heap64(adr)) - self.mc.MOVSD(loc, xmmtmp) + self.mc.MOVSD(xmmtmp, heap(adr)) + assert isinstance(loc, StackLoc) + self.mc.MOVSD_bx(loc.value, xmmtmp.value) + self.mc._mc.end_reuse_scratch_register() return adr_stackadjust def dump(self, text): @@ -410,27 +633,10 @@ finally: Box._extended_display = _prev - def _start_block(self): - # Return a 'mc' that can be used to write an "atomic" block, - # i.e. one that will not contain any JMP. - mc = self.mc._mc - if not we_are_translated(): - self._block_started_mc = (self.mc, mc.tell()) - self.mc = "block started" - return mc - - def _stop_block(self): - if not we_are_translated(): - assert self.mc == "block started" - self.mc, orgpos = self._block_started_mc - assert 0 <= self.mc._mc.tell() - orgpos <= 58, ( - "too many bytes in _start_block/_stop_block pair") - del self._block_started_mc - # ------------------------------------------------------------ def mov(self, from_loc, to_loc): - if isinstance(from_loc, XMMREG) or isinstance(to_loc, XMMREG): + if (isinstance(from_loc, RegLoc) and from_loc.is_xmm) or (isinstance(to_loc, RegLoc) and to_loc.is_xmm): self.mc.MOVSD(to_loc, from_loc) else: self.mc.MOV(to_loc, from_loc) @@ -438,24 +644,24 @@ regalloc_mov = mov # legacy interface def regalloc_push(self, loc): - if isinstance(loc, XMMREG): - self.mc.SUB(esp, imm(2*WORD)) - self.mc.MOVSD(mem64(esp, 0), loc) - elif isinstance(loc, MODRM64): + if isinstance(loc, RegLoc) and loc.is_xmm: + self.mc.SUB_ri(esp.value, 2*WORD) + self.mc.MOVSD_sx(0, loc.value) + elif WORD == 4 and isinstance(loc, StackLoc) and loc.width == 8: # XXX evil trick - self.mc.PUSH(mem(ebp, get_ebp_ofs(loc.position))) - self.mc.PUSH(mem(ebp, get_ebp_ofs(loc.position + 1))) + self.mc.PUSH_b(get_ebp_ofs(loc.position)) + self.mc.PUSH_b(get_ebp_ofs(loc.position + 1)) else: self.mc.PUSH(loc) def regalloc_pop(self, loc): - if isinstance(loc, XMMREG): - self.mc.MOVSD(loc, mem64(esp, 0)) - self.mc.ADD(esp, imm(2*WORD)) - elif isinstance(loc, MODRM64): + if isinstance(loc, RegLoc) and loc.is_xmm: + self.mc.MOVSD_xs(loc.value, 0) + self.mc.ADD_ri(esp.value, 2*WORD) + elif WORD == 4 and isinstance(loc, StackLoc) and loc.width == 8: # XXX evil trick - self.mc.POP(mem(ebp, get_ebp_ofs(loc.position + 1))) - self.mc.POP(mem(ebp, get_ebp_ofs(loc.position))) + self.mc.POP_b(get_ebp_ofs(loc.position + 1)) + self.mc.POP_b(get_ebp_ofs(loc.position)) else: self.mc.POP(loc) @@ -472,14 +678,14 @@ faildescr._x86_current_depths = current_depths failargs = guard_op.fail_args guard_opnum = guard_op.opnum - failaddr = self.implement_guard_recovery(guard_opnum, - faildescr, failargs, - faillocs) + guard_token = self.implement_guard_recovery(guard_opnum, + faildescr, failargs, + faillocs) if op is None: dispatch_opnum = guard_opnum else: dispatch_opnum = op.opnum - res = genop_guard_list[dispatch_opnum](self, op, guard_op, failaddr, + res = genop_guard_list[dispatch_opnum](self, op, guard_op, guard_token, arglocs, resloc) faildescr._x86_adr_jump_offset = res @@ -506,103 +712,161 @@ rl = result_loc.lowest8bits() if isinstance(op.args[0], Const): self.mc.CMP(arglocs[1], arglocs[0]) - getattr(self.mc, 'SET' + rev_cond)(rl) + self.mc.SET_ir(rx86.Conditions[rev_cond], rl.value) else: self.mc.CMP(arglocs[0], arglocs[1]) - getattr(self.mc, 'SET' + cond)(rl) - self.mc.MOVZX(result_loc, rl) + self.mc.SET_ir(rx86.Conditions[cond], rl.value) + self.mc.MOVZX8_rr(result_loc.value, rl.value) return genop_cmp def _cmpop_float(cond, is_ne=False): def genop_cmp(self, op, arglocs, result_loc): self.mc.UCOMISD(arglocs[0], arglocs[1]) - rl = result_loc.lowest8bits() - rh = result_loc.higher8bits() - getattr(self.mc, 'SET' + cond)(rl) + tmp1 = result_loc.lowest8bits() + if IS_X86_32: + tmp2 = result_loc.higher8bits() + elif IS_X86_64: + tmp2 = X86_64_SCRATCH_REG.lowest8bits() + + self.mc.SET_ir(rx86.Conditions[cond], tmp1.value) if is_ne: - self.mc.SETP(rh) - self.mc.OR(rl, rh) + self.mc.SET_ir(rx86.Conditions['P'], tmp2.value) + self.mc.OR8_rr(tmp1.value, tmp2.value) else: - self.mc.SETNP(rh) - self.mc.AND(rl, rh) - self.mc.MOVZX(result_loc, rl) + self.mc.SET_ir(rx86.Conditions['NP'], tmp2.value) + self.mc.AND8_rr(tmp1.value, tmp2.value) + self.mc.MOVZX8_rr(result_loc.value, tmp1.value) return genop_cmp def _cmpop_guard(cond, rev_cond, false_cond, false_rev_cond): - def genop_cmp_guard(self, op, guard_op, addr, arglocs, result_loc): + def genop_cmp_guard(self, op, guard_op, guard_token, arglocs, result_loc): guard_opnum = guard_op.opnum if isinstance(op.args[0], Const): self.mc.CMP(arglocs[1], arglocs[0]) if guard_opnum == rop.GUARD_FALSE: - name = 'J' + rev_cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(guard_token, rev_cond) else: - name = 'J' + false_rev_cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(guard_token, false_rev_cond) else: self.mc.CMP(arglocs[0], arglocs[1]) if guard_opnum == rop.GUARD_FALSE: - name = 'J' + cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(guard_token, cond) else: - name = 'J' + false_cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(guard_token, false_cond) return genop_cmp_guard def _cmpop_guard_float(cond, false_cond, need_jp): - def genop_cmp_guard_float(self, op, guard_op, addr, arglocs, + def genop_cmp_guard_float(self, op, guard_op, guard_token, arglocs, result_loc): guard_opnum = guard_op.opnum self.mc.UCOMISD(arglocs[0], arglocs[1]) + # 16 is enough space for the rel8 jumps below and the rel32 + # jump in implement_guard + self.mc.ensure_bytes_available(16 + guard_token.recovery_stub_size()) if guard_opnum == rop.GUARD_FALSE: - mc = self.mc._mc - name = 'J' + cond if need_jp: - mc.JP(rel8(6)) - getattr(mc, name)(rel32(addr)) - return mc.tell() - 4 + self.mc.J_il8(rx86.Conditions['P'], 6) + return self.implement_guard(guard_token, cond) else: if need_jp: - mc = self.mc._mc - mc.JP(rel8(2)) - getattr(mc, 'J' + cond)(rel8(5)) - return self.implement_guard(addr, mc.JMP) - name = 'J' + false_cond - return self.implement_guard(addr, getattr(self.mc, name)) + self.mc.J_il8(rx86.Conditions['P'], 2) + self.mc.J_il8(rx86.Conditions[cond], 5) + return self.implement_guard(guard_token) + return self.implement_guard(guard_token, false_cond) return genop_cmp_guard_float - @specialize.arg(5) - def _emit_call(self, x, arglocs, start=0, tmp=eax, force_mc=False, - mc=None): - if not force_mc: - mc = self.mc + def _emit_call(self, x, arglocs, start=0, tmp=eax): + if IS_X86_64: + return self._emit_call_64(x, arglocs, start) + p = 0 n = len(arglocs) for i in range(start, n): loc = arglocs[i] - if isinstance(loc, REG): - if isinstance(loc, XMMREG): - mc.MOVSD(mem64(esp, p), loc) + if isinstance(loc, RegLoc): + if loc.is_xmm: + self.mc.MOVSD_sx(p, loc.value) else: - mc.MOV(mem(esp, p), loc) + self.mc.MOV_sr(p, loc.value) p += round_up_to_4(loc.width) p = 0 for i in range(start, n): loc = arglocs[i] - if not isinstance(loc, REG): - if isinstance(loc, MODRM64): - mc.MOVSD(xmm0, loc) - mc.MOVSD(mem64(esp, p), xmm0) + if not isinstance(loc, RegLoc): + if loc.width == 8: + self.mc.MOVSD(xmm0, loc) + self.mc.MOVSD_sx(p, xmm0.value) else: - mc.MOV(tmp, loc) - mc.MOV(mem(esp, p), tmp) + self.mc.MOV(tmp, loc) + self.mc.MOV_sr(p, tmp.value) p += round_up_to_4(loc.width) self._regalloc.reserve_param(p//WORD) - mc.CALL(x) + # x is a location + self.mc.CALL(x) self.mark_gc_roots() + + def _emit_call_64(self, x, arglocs, start=0): + src_locs = [] + dst_locs = [] + xmm_src_locs = [] + xmm_dst_locs = [] + pass_on_stack = [] + + # In reverse order for use with pop() + unused_gpr = [r9, r8, ecx, edx, esi, edi] + unused_xmm = [xmm7, xmm6, xmm5, xmm4, xmm3, xmm2, xmm1, xmm0] + + for i in range(start, len(arglocs)): + loc = arglocs[i] + # XXX: Should be much simplier to tell whether a location is a + # float! It's so ugly because we have to "guard" the access to + # .type with isinstance, since not all AssemblerLocation classes + # are "typed" + if ((isinstance(loc, RegLoc) and loc.is_xmm) or + (isinstance(loc, StackLoc) and loc.type == FLOAT) or + (isinstance(loc, ConstFloatLoc))): + if len(unused_xmm) > 0: + xmm_src_locs.append(loc) + xmm_dst_locs.append(unused_xmm.pop()) + else: + pass_on_stack.append(loc) + else: + if len(unused_gpr) > 0: + src_locs.append(loc) + dst_locs.append(unused_gpr.pop()) + else: + pass_on_stack.append(loc) + # Emit instructions to pass the stack arguments + # XXX: Would be nice to let remap_frame_layout take care of this, but + # we'd need to create something like StackLoc, but relative to esp, + # and I don't know if it's worth it. + for i in range(len(pass_on_stack)): + loc = pass_on_stack[i] + if not isinstance(loc, RegLoc): + if isinstance(loc, StackLoc) and loc.type == FLOAT: + self.mc.MOVSD(X86_64_XMM_SCRATCH_REG, loc) + self.mc.MOVSD_sx(i*WORD, X86_64_XMM_SCRATCH_REG.value) + else: + self.mc.MOV(X86_64_SCRATCH_REG, loc) + self.mc.MOV_sr(i*WORD, X86_64_SCRATCH_REG.value) + else: + # It's a register + if loc.is_xmm: + self.mc.MOVSD_sx(i*WORD, loc.value) + else: + self.mc.MOV_sr(i*WORD, loc.value) + + # Handle register arguments + remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG) + remap_frame_layout(self, xmm_src_locs, xmm_dst_locs, X86_64_XMM_SCRATCH_REG) + + self._regalloc.reserve_param(len(pass_on_stack)) + self.mc.CALL(x) + self.mark_gc_roots() + def call(self, addr, args, res): - self._emit_call(rel32(addr), args) + self._emit_call(imm(addr), args) assert res is eax genop_int_neg = _unaryop("NEG") @@ -613,6 +877,9 @@ genop_int_and = _binaryop("AND", True) genop_int_or = _binaryop("OR", True) genop_int_xor = _binaryop("XOR", True) + genop_int_lshift = _binaryop("SHL") + genop_int_rshift = _binaryop("SAR") + genop_uint_rshift = _binaryop("SHR") genop_float_add = _binaryop("ADDSD", True) genop_float_sub = _binaryop('SUBSD') genop_float_mul = _binaryop('MULSD', True) @@ -659,26 +926,27 @@ genop_guard_float_gt = _cmpop_guard_float("A", "BE", False) genop_guard_float_ge = _cmpop_guard_float("AE", "B", False) - def genop_guard_float_ne(self, op, guard_op, addr, arglocs, result_loc): + def genop_guard_float_ne(self, op, guard_op, guard_token, arglocs, result_loc): guard_opnum = guard_op.opnum self.mc.UCOMISD(arglocs[0], arglocs[1]) - mc = self.mc._mc + # 16 is enough space for the rel8 jumps below and the rel32 + # jump in implement_guard + self.mc.ensure_bytes_available(16 + guard_token.recovery_stub_size()) if guard_opnum == rop.GUARD_TRUE: - mc.JP(rel8(6)) - mc.JE(rel32(addr)) - return mc.tell() - 4 - else: - mc.JP(rel8(2)) - mc.JE(rel8(5)) - return self.implement_guard(addr, mc.JMP) + self.mc.J_il8(rx86.Conditions['P'], 6) + return self.implement_guard(guard_token, 'E') + else: + self.mc.J_il8(rx86.Conditions['P'], 2) + self.mc.J_il8(rx86.Conditions['E'], 5) + return self.implement_guard(guard_token) def genop_float_neg(self, op, arglocs, resloc): # Following what gcc does: res = x ^ 0x8000000000000000 - self.mc.XORPD(arglocs[0], self.loc_float_const_neg) + self.mc.XORPD(arglocs[0], heap(self.float_const_neg_addr)) def genop_float_abs(self, op, arglocs, resloc): # Following what gcc does: res = x & 0x7FFFFFFFFFFFFFFF - self.mc.ANDPD(arglocs[0], self.loc_float_const_abs) + self.mc.ANDPD(arglocs[0], heap(self.float_const_abs_addr)) def genop_cast_float_to_int(self, op, arglocs, resloc): self.mc.CVTTSD2SI(resloc, arglocs[0]) @@ -686,70 +954,56 @@ def genop_cast_int_to_float(self, op, arglocs, resloc): self.mc.CVTSI2SD(resloc, arglocs[0]) - def genop_int_lshift(self, op, arglocs, resloc): - loc, loc2 = arglocs - if loc2 is ecx: - loc2 = cl - self.mc.SHL(loc, loc2) - - def genop_int_rshift(self, op, arglocs, resloc): - loc, loc2 = arglocs - if loc2 is ecx: - loc2 = cl - self.mc.SAR(loc, loc2) - - def genop_uint_rshift(self, op, arglocs, resloc): - loc, loc2 = arglocs - if loc2 is ecx: - loc2 = cl - self.mc.SHR(loc, loc2) - - def genop_guard_int_is_true(self, op, guard_op, addr, arglocs, resloc): + def genop_guard_int_is_true(self, op, guard_op, guard_token, arglocs, resloc): guard_opnum = guard_op.opnum - self.mc.CMP(arglocs[0], imm8(0)) + self.mc.CMP(arglocs[0], imm(0)) if guard_opnum == rop.GUARD_TRUE: - return self.implement_guard(addr, self.mc.JZ) + return self.implement_guard(guard_token, 'Z') else: - return self.implement_guard(addr, self.mc.JNZ) + return self.implement_guard(guard_token, 'NZ') def genop_int_is_true(self, op, arglocs, resloc): - self.mc.CMP(arglocs[0], imm8(0)) + self.mc.CMP(arglocs[0], imm(0)) rl = resloc.lowest8bits() - self.mc.SETNE(rl) - self.mc.MOVZX(resloc, rl) + self.mc.SET_ir(rx86.Conditions['NE'], rl.value) + self.mc.MOVZX8(resloc, rl) - def genop_guard_int_is_zero(self, op, guard_op, addr, arglocs, resloc): + def genop_guard_int_is_zero(self, op, guard_op, guard_token, arglocs, resloc): guard_opnum = guard_op.opnum - self.mc.CMP(arglocs[0], imm8(0)) + self.mc.CMP(arglocs[0], imm(0)) if guard_opnum == rop.GUARD_TRUE: - return self.implement_guard(addr, self.mc.JNZ) + return self.implement_guard(guard_token, 'NZ') else: - return self.implement_guard(addr, self.mc.JZ) + return self.implement_guard(guard_token, 'Z') def genop_int_is_zero(self, op, arglocs, resloc): - self.mc.CMP(arglocs[0], imm8(0)) + self.mc.CMP(arglocs[0], imm(0)) rl = resloc.lowest8bits() - self.mc.SETE(rl) - self.mc.MOVZX(resloc, rl) + self.mc.SET_ir(rx86.Conditions['E'], rl.value) + self.mc.MOVZX8(resloc, rl) def genop_same_as(self, op, arglocs, resloc): self.mov(arglocs[0], resloc) #genop_cast_ptr_to_int = genop_same_as def genop_int_mod(self, op, arglocs, resloc): - self.mc.CDQ() - self.mc.IDIV(ecx) + if IS_X86_32: + self.mc.CDQ() + elif IS_X86_64: + self.mc.CQO() + + self.mc.IDIV_r(ecx.value) genop_int_floordiv = genop_int_mod def genop_uint_floordiv(self, op, arglocs, resloc): - self.mc.XOR(edx, edx) - self.mc.DIV(ecx) + self.mc.XOR_rr(edx.value, edx.value) + self.mc.DIV_r(ecx.value) def genop_new_with_vtable(self, op, arglocs, result_loc): assert result_loc is eax loc_vtable = arglocs[-1] - assert isinstance(loc_vtable, IMM32) + assert isinstance(loc_vtable, ImmedLoc) arglocs = arglocs[:-1] self.call(self.malloc_func_addr, arglocs, eax) # xxx ignore NULL returns for now @@ -757,7 +1011,9 @@ def set_vtable(self, loc, loc_vtable): if self.cpu.vtable_offset is not None: - self.mc.MOV(mem(loc, self.cpu.vtable_offset), loc_vtable) + assert isinstance(loc, RegLoc) + assert isinstance(loc_vtable, ImmedLoc) + self.mc.MOV_mi((loc.value, self.cpu.vtable_offset), loc_vtable.value) # XXX genop_new is abused for all varsized mallocs with Boehm, for now # (instead of genop_new_array, genop_newstr, genop_newunicode) @@ -779,16 +1035,19 @@ def genop_getfield_gc(self, op, arglocs, resloc): base_loc, ofs_loc, size_loc = arglocs - assert isinstance(size_loc, IMM32) + assert isinstance(size_loc, ImmedLoc) + assert isinstance(resloc, RegLoc) size = size_loc.value - if size == 1: - self.mc.MOVZX(resloc, addr8_add(base_loc, ofs_loc)) + + source_addr = AddressLoc(base_loc, ofs_loc) + if resloc.is_xmm: + self.mc.MOVSD(resloc, source_addr) + elif size == 1: + self.mc.MOVZX8(resloc, source_addr) elif size == 2: - self.mc.MOVZX(resloc, addr_add(base_loc, ofs_loc)) + self.mc.MOVZX16(resloc, source_addr) elif size == WORD: - self.mc.MOV(resloc, addr_add(base_loc, ofs_loc)) - elif size == 8: - self.mc.MOVSD(resloc, addr64_add(base_loc, ofs_loc)) + self.mc.MOV(resloc, source_addr) else: raise NotImplementedError("getfield size = %d" % size) @@ -798,20 +1057,23 @@ def genop_getarrayitem_gc(self, op, arglocs, resloc): base_loc, ofs_loc, scale, ofs = arglocs - assert isinstance(ofs, IMM32) - assert isinstance(scale, IMM32) + assert isinstance(ofs, ImmedLoc) + assert isinstance(scale, ImmedLoc) if op.result.type == FLOAT: - self.mc.MOVSD(resloc, addr64_add(base_loc, ofs_loc, ofs.value, + self.mc.MOVSD(resloc, addr_add(base_loc, ofs_loc, ofs.value, scale.value)) else: if scale.value == 0: - self.mc.MOVZX(resloc, addr8_add(base_loc, ofs_loc, ofs.value, + self.mc.MOVZX8(resloc, addr_add(base_loc, ofs_loc, ofs.value, scale.value)) - elif scale.value == 2: + elif scale.value == 1: + self.mc.MOVZX16(resloc, addr_add(base_loc, ofs_loc, ofs.value, + scale.value)) + elif (1 << scale.value) == WORD: self.mc.MOV(resloc, addr_add(base_loc, ofs_loc, ofs.value, scale.value)) else: - print "[asmgen]setarrayitem unsupported size: %d" % scale.value + print "[asmgen]getarrayitem unsupported size: %d" % scale.value raise NotImplementedError() genop_getarrayitem_gc_pure = genop_getarrayitem_gc @@ -819,34 +1081,35 @@ def genop_discard_setfield_gc(self, op, arglocs): base_loc, ofs_loc, size_loc, value_loc = arglocs - assert isinstance(size_loc, IMM32) + assert isinstance(size_loc, ImmedLoc) size = size_loc.value - if size == WORD * 2: - self.mc.MOVSD(addr64_add(base_loc, ofs_loc), value_loc) + dest_addr = AddressLoc(base_loc, ofs_loc) + if isinstance(value_loc, RegLoc) and value_loc.is_xmm: + self.mc.MOVSD(dest_addr, value_loc) elif size == WORD: - self.mc.MOV(addr_add(base_loc, ofs_loc), value_loc) + self.mc.MOV(dest_addr, value_loc) elif size == 2: - self.mc.MOV16(addr_add(base_loc, ofs_loc), value_loc) + self.mc.MOV16(dest_addr, value_loc) elif size == 1: - self.mc.MOV(addr8_add(base_loc, ofs_loc), value_loc.lowest8bits()) + self.mc.MOV8(dest_addr, value_loc.lowest8bits()) else: print "[asmgen]setfield addr size %d" % size raise NotImplementedError("Addr size %d" % size) def genop_discard_setarrayitem_gc(self, op, arglocs): base_loc, ofs_loc, value_loc, scale_loc, baseofs = arglocs - assert isinstance(baseofs, IMM32) - assert isinstance(scale_loc, IMM32) + assert isinstance(baseofs, ImmedLoc) + assert isinstance(scale_loc, ImmedLoc) + dest_addr = AddressLoc(base_loc, ofs_loc, scale_loc.value, baseofs.value) if op.args[2].type == FLOAT: - self.mc.MOVSD(addr64_add(base_loc, ofs_loc, baseofs.value, - scale_loc.value), value_loc) + self.mc.MOVSD(dest_addr, value_loc) else: - if scale_loc.value == 2: - self.mc.MOV(addr_add(base_loc, ofs_loc, baseofs.value, - scale_loc.value), value_loc) + if (1 << scale_loc.value) == WORD: + self.mc.MOV(dest_addr, value_loc) + elif scale_loc.value == 1: + self.mc.MOV16(dest_addr, value_loc) elif scale_loc.value == 0: - self.mc.MOV(addr8_add(base_loc, ofs_loc, baseofs.value, - scale_loc.value), value_loc.lowest8bits()) + self.mc.MOV8(dest_addr, value_loc.lowest8bits()) else: raise NotImplementedError("scale = %d" % scale_loc.value) @@ -855,17 +1118,17 @@ basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) assert itemsize == 1 - self.mc.MOV(addr8_add(base_loc, ofs_loc, basesize), - val_loc.lowest8bits()) + dest_addr = AddressLoc(base_loc, ofs_loc, 0, basesize) + self.mc.MOV8(dest_addr, val_loc.lowest8bits()) def genop_discard_unicodesetitem(self, op, arglocs): base_loc, ofs_loc, val_loc = arglocs basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) if itemsize == 4: - self.mc.MOV(addr_add(base_loc, ofs_loc, basesize, 2), val_loc) + self.mc.MOV32(AddressLoc(base_loc, ofs_loc, 2, basesize), val_loc) elif itemsize == 2: - self.mc.MOV16(addr_add(base_loc, ofs_loc, basesize, 1), val_loc) + self.mc.MOV16(AddressLoc(base_loc, ofs_loc, 1, basesize), val_loc) else: assert 0, itemsize @@ -886,7 +1149,7 @@ def genop_arraylen_gc(self, op, arglocs, resloc): base_loc, ofs_loc = arglocs - assert isinstance(ofs_loc, IMM32) + assert isinstance(ofs_loc, ImmedLoc) self.mc.MOV(resloc, addr_add_const(base_loc, ofs_loc.value)) def genop_strgetitem(self, op, arglocs, resloc): @@ -894,83 +1157,83 @@ basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) assert itemsize == 1 - self.mc.MOVZX(resloc, addr8_add(base_loc, ofs_loc, basesize)) + self.mc.MOVZX8(resloc, AddressLoc(base_loc, ofs_loc, 0, basesize)) def genop_unicodegetitem(self, op, arglocs, resloc): base_loc, ofs_loc = arglocs basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) if itemsize == 4: - self.mc.MOV(resloc, addr_add(base_loc, ofs_loc, basesize, 2)) + self.mc.MOV32(resloc, AddressLoc(base_loc, ofs_loc, 2, basesize)) elif itemsize == 2: - self.mc.MOVZX(resloc, addr_add(base_loc, ofs_loc, basesize, 1)) + self.mc.MOVZX16(resloc, AddressLoc(base_loc, ofs_loc, 1, basesize)) else: assert 0, itemsize - def genop_guard_guard_true(self, ign_1, guard_op, addr, locs, ign_2): + def genop_guard_guard_true(self, ign_1, guard_op, guard_token, locs, ign_2): loc = locs[0] self.mc.TEST(loc, loc) - return self.implement_guard(addr, self.mc.JZ) + return self.implement_guard(guard_token, 'Z') genop_guard_guard_nonnull = genop_guard_guard_true - def genop_guard_guard_no_exception(self, ign_1, guard_op, addr, + def genop_guard_guard_no_exception(self, ign_1, guard_op, guard_token, locs, ign_2): self.mc.CMP(heap(self.cpu.pos_exception()), imm(0)) - return self.implement_guard(addr, self.mc.JNZ) + return self.implement_guard(guard_token, 'NZ') - def genop_guard_guard_exception(self, ign_1, guard_op, addr, + def genop_guard_guard_exception(self, ign_1, guard_op, guard_token, locs, resloc): loc = locs[0] loc1 = locs[1] self.mc.MOV(loc1, heap(self.cpu.pos_exception())) self.mc.CMP(loc1, loc) - addr = self.implement_guard(addr, self.mc.JNE) + addr = self.implement_guard(guard_token, 'NE') if resloc is not None: self.mc.MOV(resloc, heap(self.cpu.pos_exc_value())) self.mc.MOV(heap(self.cpu.pos_exception()), imm(0)) self.mc.MOV(heap(self.cpu.pos_exc_value()), imm(0)) return addr - def _gen_guard_overflow(self, guard_op, addr): + def _gen_guard_overflow(self, guard_op, guard_token): guard_opnum = guard_op.opnum if guard_opnum == rop.GUARD_NO_OVERFLOW: - return self.implement_guard(addr, self.mc.JO) + return self.implement_guard(guard_token, 'O') elif guard_opnum == rop.GUARD_OVERFLOW: - return self.implement_guard(addr, self.mc.JNO) + return self.implement_guard(guard_token, 'NO') else: print "int_xxx_ovf followed by", guard_op.getopname() raise AssertionError - def genop_guard_int_add_ovf(self, op, guard_op, addr, arglocs, result_loc): + def genop_guard_int_add_ovf(self, op, guard_op, guard_token, arglocs, result_loc): self.genop_int_add(op, arglocs, result_loc) - return self._gen_guard_overflow(guard_op, addr) + return self._gen_guard_overflow(guard_op, guard_token) - def genop_guard_int_sub_ovf(self, op, guard_op, addr, arglocs, result_loc): + def genop_guard_int_sub_ovf(self, op, guard_op, guard_token, arglocs, result_loc): self.genop_int_sub(op, arglocs, result_loc) - return self._gen_guard_overflow(guard_op, addr) + return self._gen_guard_overflow(guard_op, guard_token) - def genop_guard_int_mul_ovf(self, op, guard_op, addr, arglocs, result_loc): + def genop_guard_int_mul_ovf(self, op, guard_op, guard_token, arglocs, result_loc): self.genop_int_mul(op, arglocs, result_loc) - return self._gen_guard_overflow(guard_op, addr) + return self._gen_guard_overflow(guard_op, guard_token) - def genop_guard_guard_false(self, ign_1, guard_op, addr, locs, ign_2): + def genop_guard_guard_false(self, ign_1, guard_op, guard_token, locs, ign_2): loc = locs[0] self.mc.TEST(loc, loc) - return self.implement_guard(addr, self.mc.JNZ) + return self.implement_guard(guard_token, 'NZ') genop_guard_guard_isnull = genop_guard_guard_false - def genop_guard_guard_value(self, ign_1, guard_op, addr, locs, ign_2): + def genop_guard_guard_value(self, ign_1, guard_op, guard_token, locs, ign_2): if guard_op.args[0].type == FLOAT: assert guard_op.args[1].type == FLOAT self.mc.UCOMISD(locs[0], locs[1]) else: self.mc.CMP(locs[0], locs[1]) - return self.implement_guard(addr, self.mc.JNE) + return self.implement_guard(guard_token, 'NE') - def _cmp_guard_class(self, mc, locs): + def _cmp_guard_class(self, locs): offset = self.cpu.vtable_offset if offset is not None: - mc.CMP(mem(locs[0], offset), locs[1]) + self.mc.CMP(mem(locs[0], offset), locs[1]) else: # XXX hard-coded assumption: to go from an object to its class # we use the following algorithm: @@ -979,7 +1242,7 @@ # - multiply by 4 and use it as an offset in type_info_group # - add 16 bytes, to go past the TYPE_INFO structure loc = locs[1] - assert isinstance(loc, IMM32) + assert isinstance(loc, ImmedLoc) classptr = loc.value # here, we have to go back from 'classptr' to the value expected # from reading the 16 bits in the object header @@ -988,37 +1251,37 @@ type_info_group = llop.gc_get_type_info_group(llmemory.Address) type_info_group = rffi.cast(lltype.Signed, type_info_group) expected_typeid = (classptr - sizeof_ti - type_info_group) >> 2 - mc.CMP16(mem(locs[0], 0), imm32(expected_typeid)) + self.mc.CMP16(mem(locs[0], 0), ImmedLoc(expected_typeid)) - def genop_guard_guard_class(self, ign_1, guard_op, addr, locs, ign_2): - mc = self._start_block() - self._cmp_guard_class(mc, locs) - self._stop_block() - return self.implement_guard(addr, self.mc.JNE) + def genop_guard_guard_class(self, ign_1, guard_op, guard_token, locs, ign_2): + self.mc.ensure_bytes_available(256) + self._cmp_guard_class(locs) + return self.implement_guard(guard_token, 'NE') def genop_guard_guard_nonnull_class(self, ign_1, guard_op, - addr, locs, ign_2): - mc = self._start_block() - mc.CMP(locs[0], imm8(1)) - mc.JB(rel8_patched_later) - jb_location = mc.get_relative_pos() - self._cmp_guard_class(mc, locs) + guard_token, locs, ign_2): + self.mc.ensure_bytes_available(256) + self.mc.CMP(locs[0], imm(1)) + # Patched below + self.mc.J_il8(rx86.Conditions['B'], 0) + jb_location = self.mc.get_relative_pos() + self._cmp_guard_class(locs) # patch the JB above - offset = mc.get_relative_pos() - jb_location + offset = self.mc.get_relative_pos() - jb_location assert 0 < offset <= 127 - mc.overwrite(jb_location-1, [chr(offset)]) - self._stop_block() + self.mc.overwrite(jb_location-1, [chr(offset)]) # - return self.implement_guard(addr, self.mc.JNE) + return self.implement_guard(guard_token, 'NE') def implement_guard_recovery(self, guard_opnum, faildescr, failargs, fail_locs): exc = (guard_opnum == rop.GUARD_EXCEPTION or guard_opnum == rop.GUARD_NO_EXCEPTION or guard_opnum == rop.GUARD_NOT_FORCED) - return self.generate_quick_failure(faildescr, failargs, fail_locs, exc) + desc_bytes = self.failure_recovery_description(failargs, fail_locs) + return GuardToken(faildescr, failargs, fail_locs, exc, desc_bytes) - def generate_quick_failure(self, faildescr, failargs, fail_locs, exc): + def generate_quick_failure(self, mc, faildescr, failargs, fail_locs, exc, desc_bytes): """Generate the initial code for handling a failure. We try to keep it as compact as possible. The idea is that this code is executed at most once (and very often, zero times); when @@ -1026,38 +1289,43 @@ really handle recovery from this particular failure. """ fail_index = self.cpu.get_fail_descr_number(faildescr) - bytes_needed = 20 + 5 * len(failargs) # conservative estimate - if self.mc2.bytes_free() < bytes_needed: - self.mc2.make_new_mc() - mc = self.mc2._mc addr = mc.tell() withfloats = False for box in failargs: if box is not None and box.type == FLOAT: withfloats = True break - mc.CALL(rel32(self.failure_recovery_code[exc + 2 * withfloats])) + mc.CALL(imm(self.failure_recovery_code[exc + 2 * withfloats])) # write tight data that describes the failure recovery faildescr._x86_failure_recovery_bytecode = mc.tell() - self.write_failure_recovery_description(mc, failargs, fail_locs) + for byte in desc_bytes: + mc.writechr(ord(byte)) # write the fail_index too - mc.write(packimm32(fail_index)) + mc.writeimm32(fail_index) # for testing the decoding, write a final byte 0xCC if not we_are_translated(): mc.writechr(0xCC) faildescr._x86_debug_faillocs = [loc for loc in fail_locs if loc is not None] + + # Make sure the recovery stub is at least 16 bytes long (for the + # case where we overwrite the recovery stub with a 64-bit absolute + # jump) + while mc.tell() - addr < 16: + mc.writechr(0x00) return addr DESCR_REF = 0x00 DESCR_INT = 0x01 DESCR_FLOAT = 0x02 DESCR_SPECIAL = 0x03 - CODE_FROMSTACK = 4*8 + # XXX: 4*8 works on i386, should we optimize for that case? + CODE_FROMSTACK = 4*16 CODE_STOP = 0 | DESCR_SPECIAL CODE_HOLE = 4 | DESCR_SPECIAL - def write_failure_recovery_description(self, mc, failargs, locs): + def failure_recovery_description(self, failargs, locs): + desc_bytes = [] for i in range(len(failargs)): arg = failargs[i] if arg is not None: @@ -1070,24 +1338,30 @@ else: raise AssertionError("bogus kind") loc = locs[i] - if isinstance(loc, MODRM): + if isinstance(loc, StackLoc): n = self.CODE_FROMSTACK//4 + loc.position else: - assert isinstance(loc, REG) - n = loc.op + assert isinstance(loc, RegLoc) + n = loc.value n = kind + 4*n while n > 0x7F: - mc.writechr((n & 0x7F) | 0x80) + desc_bytes.append(chr((n & 0x7F) | 0x80)) n >>= 7 else: n = self.CODE_HOLE - mc.writechr(n) - mc.writechr(self.CODE_STOP) + desc_bytes.append(chr(n)) + desc_bytes.append(chr(self.CODE_STOP)) # assert that the fail_boxes lists are big enough assert len(failargs) <= self.fail_boxes_int.SIZE + return desc_bytes + + def write_failure_recovery_description(self, mc, failargs, locs): + for byte in self.failure_recovery_description(failargs, locs): + mc.writechr(ord(byte)) def rebuild_faillocs_from_descr(self, bytecode): from pypy.jit.backend.x86.regalloc import X86FrameManager + descr_to_box_type = [REF, INT, FLOAT] bytecode = rffi.cast(rffi.UCHARP, bytecode) arglocs = [] while 1: @@ -1112,7 +1386,7 @@ size = 2 else: size = 1 - loc = X86FrameManager.frame_pos(code, size) + loc = X86FrameManager.frame_pos(code, descr_to_box_type[kind]) elif code == self.CODE_STOP: break elif code == self.CODE_HOLE: @@ -1122,16 +1396,16 @@ kind = code & 3 code >>= 2 if kind == self.DESCR_FLOAT: - loc = xmm_registers[code] + loc = regloc.XMMREGLOCS[code] else: - loc = registers[code] + loc = regloc.REGLOCS[code] arglocs.append(loc) return arglocs[:] @rgc.no_collect def grab_frame_values(self, bytecode, frame_addr, allregisters): # no malloc allowed here!! - self.fail_ebp = allregisters[16 + ebp.op] + self.fail_ebp = allregisters[16 + ebp.value] num = 0 value_hi = 0 while 1: @@ -1154,7 +1428,7 @@ code = (code - self.CODE_FROMSTACK) >> 2 stackloc = frame_addr + get_ebp_ofs(code) value = rffi.cast(rffi.LONGP, stackloc)[0] - if kind == self.DESCR_FLOAT: + if kind == self.DESCR_FLOAT and WORD == 4: value_hi = value value = rffi.cast(rffi.LONGP, stackloc - 4)[0] else: @@ -1168,8 +1442,11 @@ break code >>= 2 if kind == self.DESCR_FLOAT: - value = allregisters[2*code] - value_hi = allregisters[2*code + 1] + if WORD == 4: + value = allregisters[2*code] + value_hi = allregisters[2*code + 1] + else: + value = allregisters[code] else: value = allregisters[16 + code] @@ -1180,7 +1457,8 @@ tgt = self.fail_boxes_ptr.get_addr_for_num(num) elif kind == self.DESCR_FLOAT: tgt = self.fail_boxes_float.get_addr_for_num(num) - rffi.cast(rffi.LONGP, tgt)[1] = value_hi + if WORD == 4: + rffi.cast(rffi.LONGP, tgt)[1] = value_hi else: assert 0, "bogus kind" rffi.cast(rffi.LONGP, tgt)[0] = value @@ -1189,7 +1467,8 @@ if not we_are_translated(): assert bytecode[4] == 0xCC self.fail_boxes_count = num - fail_index = rffi.cast(rffi.LONGP, bytecode)[0] + fail_index = rffi.cast(rffi.INTP, bytecode)[0] + fail_index = rffi.cast(lltype.Signed, fail_index) return fail_index def setup_failure_recovery(self): @@ -1200,8 +1479,8 @@ # original value of the registers, optionally the original # value of XMM registers, and finally a reference to the # recovery bytecode. See _build_failure_recovery() for details. - stack_at_ebp = registers[ebp.op] - bytecode = rffi.cast(rffi.UCHARP, registers[8]) + stack_at_ebp = registers[ebp.value] + bytecode = rffi.cast(rffi.UCHARP, registers[self.cpu.NUM_REGS]) allregisters = rffi.ptradd(registers, -16) return self.grab_frame_values(bytecode, stack_at_ebp, allregisters) @@ -1216,23 +1495,23 @@ self.failure_recovery_func) failure_recovery_func = rffi.cast(lltype.Signed, failure_recovery_func) - mc = self.mc2._mc + mc = self.mc._mc # Assume that we are called at the beginning, when there is no risk # that 'mc' runs out of space. Checked by asserts in mc.write(). recovery_addr = mc.tell() - mc.PUSH(edi) - mc.PUSH(esi) - mc.PUSH(ebp) - mc.PUSH(esp) # <-- not really used, but needed to take up the space - mc.PUSH(ebx) - mc.PUSH(edx) - mc.PUSH(ecx) - mc.PUSH(eax) - mc.MOV(esi, esp) + + # Push all general purpose registers + for gpr in range(self.cpu.NUM_REGS-1, -1, -1): + mc.PUSH_r(gpr) + + # ebx/rbx is callee-save in both i386 and x86-64 + mc.MOV_rr(ebx.value, esp.value) + if withfloats: - mc.SUB(esp, imm(8*8)) - for i in range(8): - mc.MOVSD(mem64(esp, 8*i), xmm_registers[i]) + # Push all float registers + mc.SUB_ri(esp.value, self.cpu.NUM_REGS*8) + for i in range(self.cpu.NUM_REGS): + mc.MOVSD_sx(8*i, i) # we call a provided function that will # - call our on_leave_jitted_hook which will mark @@ -1240,7 +1519,7 @@ # avoid unwarranted freeing # - optionally save exception depending on the flag addr = self.cpu.get_on_leave_jitted_int(save_exception=exc) - mc.CALL(rel32(addr)) + mc.CALL(imm(addr)) # the following call saves all values from the stack and from # registers to the right 'fail_boxes_' location. @@ -1250,50 +1529,58 @@ # bytecode, pushed just before by the CALL instruction written by # generate_quick_failure(). XXX misaligned stack in the call, but # it's ok because failure_recovery_func is not calling anything more - mc.PUSH(esi) - mc.CALL(rel32(failure_recovery_func)) + + # XXX + if IS_X86_32: + mc.PUSH_r(ebx.value) + elif IS_X86_64: + mc.MOV_rr(edi.value, ebx.value) + # XXX: Correct to only align the stack on 64-bit? + mc.AND_ri(esp.value, -16) + else: + raise AssertionError("Shouldn't happen") + + mc.CALL(imm(failure_recovery_func)) # returns in eax the fail_index # now we return from the complete frame, which starts from - # _assemble_bootstrap_code(). The LEA below throws away most - # of the frame, including all the PUSHes that we did just above. - mc.LEA(esp, addr_add(ebp, imm(-3 * WORD))) - mc.POP(edi) # [ebp-12] - mc.POP(esi) # [ebp-8] - mc.POP(ebx) # [ebp-4] - mc.POP(ebp) # [ebp] - mc.RET() - self.mc2.done() + # _assemble_bootstrap_code(). The LEA in _call_footer below throws + # away most of the frame, including all the PUSHes that we did just + # above. + + self._call_footer() + self.mc.done() self.failure_recovery_code[exc + 2 * withfloats] = recovery_addr def generate_failure(self, fail_index, locs, exc, locs_are_ref): - mc = self.mc + self.mc._mc.begin_reuse_scratch_register() for i in range(len(locs)): loc = locs[i] - if isinstance(loc, REG): - if loc.width == 8: + if isinstance(loc, RegLoc): + if loc.is_xmm: adr = self.fail_boxes_float.get_addr_for_num(i) - mc.MOVSD(heap64(adr), loc) + self.mc.MOVSD(heap(adr), loc) else: if locs_are_ref[i]: adr = self.fail_boxes_ptr.get_addr_for_num(i) else: adr = self.fail_boxes_int.get_addr_for_num(i) - mc.MOV(heap(adr), loc) + self.mc.MOV(heap(adr), loc) for i in range(len(locs)): loc = locs[i] - if not isinstance(loc, REG): - if loc.width == 8: - mc.MOVSD(xmm0, loc) + if not isinstance(loc, RegLoc): + if isinstance(loc, StackLoc) and loc.type == FLOAT: + self.mc.MOVSD_xb(xmm0.value, loc.value) adr = self.fail_boxes_float.get_addr_for_num(i) - mc.MOVSD(heap64(adr), xmm0) + self.mc.MOVSD(heap(adr), xmm0) else: if locs_are_ref[i]: adr = self.fail_boxes_ptr.get_addr_for_num(i) else: adr = self.fail_boxes_int.get_addr_for_num(i) - mc.MOV(eax, loc) - mc.MOV(heap(adr), eax) + self.mc.MOV(eax, loc) + self.mc.MOV(heap(adr), eax) + self.mc._mc.end_reuse_scratch_register() # we call a provided function that will # - call our on_leave_jitted_hook which will mark @@ -1301,28 +1588,31 @@ # avoid unwarranted freeing # - optionally save exception depending on the flag addr = self.cpu.get_on_leave_jitted_int(save_exception=exc) - mc.CALL(rel32(addr)) + self.mc.CALL(imm(addr)) + + self.mc.MOV_ri(eax.value, fail_index) + + # exit function + self._call_footer() - mc.LEA(esp, addr_add(ebp, imm(-3 * WORD))) - mc.MOV(eax, imm(fail_index)) - mc.POP(edi) # [ebp-12] - mc.POP(esi) # [ebp-8] - mc.POP(ebx) # [ebp-4] - mc.POP(ebp) # [ebp] - mc.RET() - - @specialize.arg(2) - def implement_guard(self, addr, emit_jump): - emit_jump(rel32(addr)) + def implement_guard(self, guard_token, condition=None): + self.mc.reserve_bytes(guard_token.recovery_stub_size()) + self.pending_guard_tokens.append(guard_token) + # XXX: These jumps are patched later, the self.mc.tell() are just + # dummy values + if condition: + self.mc.J_il(rx86.Conditions[condition], self.mc.tell()) + else: + self.mc.JMP_l(self.mc.tell()) return self.mc.tell() - 4 def genop_call(self, op, arglocs, resloc): sizeloc = arglocs[0] - assert isinstance(sizeloc, IMM32) + assert isinstance(sizeloc, ImmedLoc) size = sizeloc.value if isinstance(op.args[0], Const): - x = rel32(op.args[0].getint()) + x = imm(op.args[0].getint()) else: x = arglocs[1] if x is eax: @@ -1332,35 +1622,35 @@ self._emit_call(x, arglocs, 2, tmp=tmp) - if isinstance(resloc, MODRM64): - self.mc.FSTP(resloc) + if isinstance(resloc, StackLoc) and resloc.width == 8 and IS_X86_32: + self.mc.FSTP_b(resloc.value) elif size == 1: - self.mc.AND(eax, imm(0xff)) + self.mc.AND_ri(eax.value, 0xff) elif size == 2: - self.mc.AND(eax, imm(0xffff)) + self.mc.AND_ri(eax.value, 0xffff) - def genop_guard_call_may_force(self, op, guard_op, addr, + def genop_guard_call_may_force(self, op, guard_op, guard_token, arglocs, result_loc): faildescr = guard_op.descr fail_index = self.cpu.get_fail_descr_number(faildescr) - self.mc.MOV(mem(ebp, FORCE_INDEX_OFS), imm(fail_index)) + self.mc.MOV_bi(FORCE_INDEX_OFS, fail_index) self.genop_call(op, arglocs, result_loc) - self.mc.CMP(mem(ebp, FORCE_INDEX_OFS), imm(0)) - return self.implement_guard(addr, self.mc.JL) + self.mc.CMP_bi(FORCE_INDEX_OFS, 0) + return self.implement_guard(guard_token, 'L') - def genop_guard_call_assembler(self, op, guard_op, addr, + def genop_guard_call_assembler(self, op, guard_op, guard_token, arglocs, result_loc): faildescr = guard_op.descr fail_index = self.cpu.get_fail_descr_number(faildescr) - self.mc.MOV(mem(ebp, FORCE_INDEX_OFS), imm(fail_index)) + self.mc.MOV_bi(FORCE_INDEX_OFS, fail_index) descr = op.descr assert isinstance(descr, LoopToken) assert len(arglocs) - 2 == len(descr._x86_arglocs[0]) # # Write a call to the direct_bootstrap_code of the target assembler - self._emit_call(rel32(descr._x86_direct_bootstrap_code), arglocs, 2, + self._emit_call(imm(descr._x86_direct_bootstrap_code), arglocs, 2, tmp=eax) - mc = self._start_block() + self.mc.ensure_bytes_available(256) if op.result is None: assert result_loc is None value = self.cpu.done_with_this_frame_void_v @@ -1376,26 +1666,27 @@ value = self.cpu.done_with_this_frame_float_v else: raise AssertionError(kind) - mc.CMP(eax, imm(value)) - mc.JE(rel8_patched_later) # goto B if we get 'done_with_this_frame' - je_location = mc.get_relative_pos() + self.mc.CMP_ri(eax.value, value) + # patched later + self.mc.J_il8(rx86.Conditions['E'], 0) # goto B if we get 'done_with_this_frame' + je_location = self.mc.get_relative_pos() # # Path A: use assembler_helper_adr jd = descr.outermost_jitdriver_sd assert jd is not None asm_helper_adr = self.cpu.cast_adr_to_int(jd.assembler_helper_adr) - self._emit_call(rel32(asm_helper_adr), [eax, arglocs[1]], 0, - tmp=ecx, force_mc=True, mc=mc) - if isinstance(result_loc, MODRM64): - mc.FSTP(result_loc) + self._emit_call(imm(asm_helper_adr), [eax, arglocs[1]], 0, + tmp=ecx) + if IS_X86_32 and isinstance(result_loc, StackLoc) and result_loc.type == FLOAT: + self.mc.FSTP_b(result_loc.value) #else: result_loc is already either eax or None, checked below - mc.JMP(rel8_patched_later) # done - jmp_location = mc.get_relative_pos() + self.mc.JMP_l8(0) # jump to done, patched later + jmp_location = self.mc.get_relative_pos() # # Path B: fast path. Must load the return value, and reset the token offset = jmp_location - je_location assert 0 < offset <= 127 - mc.overwrite(je_location - 1, [chr(offset)]) + self.mc.overwrite(je_location - 1, [chr(offset)]) # # Reset the vable token --- XXX really too much special logic here:-( if jd.index_of_virtualizable >= 0: @@ -1403,8 +1694,8 @@ fielddescr = jd.vable_token_descr assert isinstance(fielddescr, BaseFieldDescr) ofs = fielddescr.offset - mc.MOV(eax, arglocs[1]) - mc.MOV(addr_add(eax, imm(ofs)), imm(0)) + self.mc.MOV(eax, arglocs[1]) + self.mc.MOV_mi((eax.value, ofs), 0) # in the line above, TOKEN_NONE = 0 # if op.result is not None: @@ -1413,27 +1704,26 @@ if kind == FLOAT: xmmtmp = X86XMMRegisterManager.all_regs[0] adr = self.fail_boxes_float.get_addr_for_num(0) - mc.MOVSD(xmmtmp, heap64(adr)) - mc.MOVSD(result_loc, xmmtmp) + self.mc.MOVSD(xmmtmp, heap(adr)) + self.mc.MOVSD(result_loc, xmmtmp) else: assert result_loc is eax if kind == INT: adr = self.fail_boxes_int.get_addr_for_num(0) - mc.MOV(eax, heap(adr)) + self.mc.MOV(eax, heap(adr)) elif kind == REF: adr = self.fail_boxes_ptr.get_addr_for_num(0) - mc.XOR(eax, eax) - mc.XCHG(eax, heap(adr)) + self.mc.XOR_rr(eax.value, eax.value) + self.mc.XCHG(eax, heap(adr)) else: raise AssertionError(kind) # # Here we join Path A and Path B again - offset = mc.get_relative_pos() - jmp_location + offset = self.mc.get_relative_pos() - jmp_location assert 0 <= offset <= 127 - mc.overwrite(jmp_location - 1, [chr(offset)]) - self._stop_block() - self.mc.CMP(mem(ebp, FORCE_INDEX_OFS), imm(0)) - return self.implement_guard(addr, self.mc.JL) + self.mc.overwrite(jmp_location - 1, [chr(offset)]) + self.mc.CMP_bi(FORCE_INDEX_OFS, 0) + return self.implement_guard(guard_token, 'L') def genop_discard_cond_call_gc_wb(self, op, arglocs): # use 'mc._mc' directly instead of 'mc', to avoid @@ -1443,31 +1733,51 @@ cls = self.cpu.gc_ll_descr.has_write_barrier_class() assert cls is not None and isinstance(descr, cls) loc_base = arglocs[0] - mc = self._start_block() - mc.TEST(mem8(loc_base, descr.jit_wb_if_flag_byteofs), - imm8(descr.jit_wb_if_flag_singlebyte)) - mc.JZ(rel8_patched_later) - jz_location = mc.get_relative_pos() + self.mc.ensure_bytes_available(256) + self.mc.TEST8_mi((loc_base.value, descr.jit_wb_if_flag_byteofs), + descr.jit_wb_if_flag_singlebyte) + self.mc.J_il8(rx86.Conditions['Z'], 0) # patched later + jz_location = self.mc.get_relative_pos() # the following is supposed to be the slow path, so whenever possible # we choose the most compact encoding over the most efficient one. for i in range(len(arglocs)-1, -1, -1): - mc.PUSH(arglocs[i]) + loc = arglocs[i] + if isinstance(loc, RegLoc): + self.mc.PUSH_r(loc.value) + else: + if IS_X86_64: + self.mc.MOV_ri(X86_64_SCRATCH_REG.value, loc.getint()) + self.mc.PUSH_r(X86_64_SCRATCH_REG.value) + else: + self.mc.PUSH_i32(loc.getint()) + + if IS_X86_64: + # We clobber these registers to pass the arguments, but that's + # okay, because consider_cond_call_gc_wb makes sure that any + # caller-save registers with values in them are present in arglocs, + # so they are saved on the stack above and restored below + self.mc.MOV_rs(edi.value, 0) + self.mc.MOV_rs(esi.value, 8) + # misaligned stack in the call, but it's ok because the write barrier # is not going to call anything more. Also, this assumes that the # write barrier does not touch the xmm registers. - mc.CALL(rel32(descr.get_write_barrier_fn(self.cpu))) + self.mc.CALL(imm(descr.get_write_barrier_fn(self.cpu))) for i in range(len(arglocs)): loc = arglocs[i] - assert isinstance(loc, REG) - mc.POP(loc) + if isinstance(loc, RegLoc): + self.mc.POP_r(loc.value) + else: + self.mc.ADD_ri(esp.value, WORD) # ignore the pushed constant # patch the JZ above - offset = mc.get_relative_pos() - jz_location + offset = self.mc.get_relative_pos() - jz_location assert 0 < offset <= 127 - mc.overwrite(jz_location-1, [chr(offset)]) - self._stop_block() + self.mc.overwrite(jz_location-1, [chr(offset)]) def genop_force_token(self, op, arglocs, resloc): - self.mc.LEA(resloc, mem(ebp, FORCE_INDEX_OFS)) + # RegAlloc.consider_force_token ensures this: + assert isinstance(resloc, RegLoc) + self.mc.LEA_rb(resloc.value, FORCE_INDEX_OFS) def not_implemented_op_discard(self, op, arglocs): msg = "not implemented operation: %s" % op.getopname() @@ -1495,17 +1805,16 @@ return loop_token._x86_arglocs def closing_jump(self, loop_token): - self.mc.JMP(rel32(loop_token._x86_loop_code)) + self.mc.JMP(imm(loop_token._x86_loop_code)) def malloc_cond_fixedsize(self, nursery_free_adr, nursery_top_adr, size, tid): - # don't use self.mc - mc = self._start_block() - mc.MOV(eax, heap(nursery_free_adr)) - mc.LEA(edx, addr_add(eax, imm(size))) - mc.CMP(edx, heap(nursery_top_adr)) - mc.JNA(rel8_patched_later) - jmp_adr = mc.get_relative_pos() + self.mc.ensure_bytes_available(256) + self.mc.MOV(eax, heap(nursery_free_adr)) + self.mc.LEA_rm(edx.value, (eax.value, size)) + self.mc.CMP(edx, heap(nursery_top_adr)) + self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later + jmp_adr = self.mc.get_relative_pos() # See comments in _build_malloc_fixedsize_slowpath for the # details of the two helper functions that we are calling below. @@ -1521,17 +1830,16 @@ # reserve room for the argument to the real malloc and the # 8 saved XMM regs self._regalloc.reserve_param(1+16) - mc.CALL(rel32(slowpath_addr1)) + self.mc.CALL(imm(slowpath_addr1)) self.mark_gc_roots() slowpath_addr2 = self.malloc_fixedsize_slowpath2 - mc.CALL(rel32(slowpath_addr2)) + self.mc.CALL(imm(slowpath_addr2)) - offset = mc.get_relative_pos() - jmp_adr + offset = self.mc.get_relative_pos() - jmp_adr assert 0 < offset <= 127 - mc.overwrite(jmp_adr-1, [chr(offset)]) - mc.MOV(addr_add(eax, imm(0)), imm(tid)) - mc.MOV(heap(nursery_free_adr), edx) - self._stop_block() + self.mc.overwrite(jmp_adr-1, [chr(offset)]) + self.mc.MOV_mi((eax.value, 0), tid) + self.mc.MOV(heap(nursery_free_adr), edx) genop_discard_list = [Assembler386.not_implemented_op_discard] * rop._LAST genop_list = [Assembler386.not_implemented_op] * rop._LAST @@ -1551,32 +1859,20 @@ num = getattr(rop, opname.upper()) genop_list[num] = value -def new_addr_add(heap, mem, memsib): - def addr_add(reg_or_imm1, reg_or_imm2, offset=0, scale=0): - if isinstance(reg_or_imm1, IMM32): - if isinstance(reg_or_imm2, IMM32): - return heap(reg_or_imm1.value + offset + - (reg_or_imm2.value << scale)) - else: - return memsib(None, reg_or_imm2, scale, reg_or_imm1.value + offset) - else: - if isinstance(reg_or_imm2, IMM32): - return mem(reg_or_imm1, offset + (reg_or_imm2.value << scale)) - else: - return memsib(reg_or_imm1, reg_or_imm2, scale, offset) - return addr_add - -addr8_add = new_addr_add(heap8, mem8, memSIB8) -addr_add = new_addr_add(heap, mem, memSIB) -addr64_add = new_addr_add(heap64, mem64, memSIB64) - -def addr_add_const(reg_or_imm1, offset): - if isinstance(reg_or_imm1, IMM32): - return heap(reg_or_imm1.value + offset) - else: - return mem(reg_or_imm1, offset) - def round_up_to_4(size): if size < 4: return 4 return size + +# XXX: ri386 migration shims: +def addr_add(reg_or_imm1, reg_or_imm2, offset=0, scale=0): + return AddressLoc(reg_or_imm1, reg_or_imm2, scale, offset) + +def addr_add_const(reg_or_imm1, offset): + return AddressLoc(reg_or_imm1, ImmedLoc(0), 0, offset) + +def mem(loc, offset): + return AddressLoc(loc, ImmedLoc(0), 0, offset) + +def heap(addr): + return AddressLoc(ImmedLoc(addr), ImmedLoc(0), 0, 0) Modified: pypy/branch/micronumpy/pypy/jit/backend/x86/codebuf.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/backend/x86/codebuf.py (original) +++ pypy/branch/micronumpy/pypy/jit/backend/x86/codebuf.py Mon Aug 16 04:52:00 2010 @@ -2,12 +2,20 @@ import os, sys from pypy.rpython.lltypesystem import lltype, rffi from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.jit.backend.x86.ri386 import I386CodeBuilder +from pypy.jit.backend.x86.rx86 import X86_32_CodeBuilder, X86_64_CodeBuilder +from pypy.jit.backend.x86.regloc import LocationCodeBuilder from pypy.rlib.rmmap import PTR, alloc, free from pypy.rlib.debug import make_sure_not_resized +from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 +from pypy.rlib.objectmodel import we_are_translated +# XXX: Seems nasty to change the superclass of InMemoryCodeBuilder like this +if IS_X86_32: + codebuilder_cls = X86_32_CodeBuilder +elif IS_X86_64: + codebuilder_cls = X86_64_CodeBuilder -class InMemoryCodeBuilder(I386CodeBuilder): +class InMemoryCodeBuilder(codebuilder_cls, LocationCodeBuilder): _last_dump_start = 0 def __init__(self, start, end): @@ -31,13 +39,15 @@ def write(self, listofchars): self._pos = self.overwrite(self._pos, listofchars) - def writechr(self, n): - # purely for performance: don't make the one-element list [chr(n)] + def writechar(self, char): pos = self._pos assert pos + 1 <= self._size - self._data[pos] = chr(n) + self._data[pos] = char self._pos = pos + 1 + def writechr(self, n): + self.writechar(chr(n)) + def get_relative_pos(self): return self._pos @@ -50,11 +60,6 @@ self._pos = pos self._last_dump_start = pos - def execute(self, arg1, arg2): - # XXX old testing stuff - fnptr = rffi.cast(lltype.Ptr(BINARYFN), self._data) - return fnptr(arg1, arg2) - def done(self): # normally, no special action is needed here if machine_code_dumper.enabled: @@ -77,9 +82,6 @@ valgrind.discard_translations(self._data, self._size) -BINARYFN = lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed) - - class MachineCodeDumper: enabled = True log_fd = -1 @@ -107,7 +109,10 @@ return False # log the executable name from pypy.jit.backend.hlinfo import highleveljitinfo - os.write(self.log_fd, 'BACKEND i386\n') + if IS_X86_32: + os.write(self.log_fd, 'BACKEND x86\n') + elif IS_X86_64: + os.write(self.log_fd, 'BACKEND x86_64\n') if highleveljitinfo.sys_executable: os.write(self.log_fd, 'SYS_EXECUTABLE %s\n' % ( highleveljitinfo.sys_executable,)) @@ -137,6 +142,12 @@ def __init__(self, map_size): data = alloc(map_size) + if IS_X86_64 and not we_are_translated(): + # Hack to make sure that mcs are not within 32-bits of one + # another for testing purposes + from pypy.rlib.rmmap import hint + hint.pos += 0xFFFFFFFF + self._init(data, map_size) def __del__(self): Modified: pypy/branch/micronumpy/pypy/jit/backend/x86/jump.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/backend/x86/jump.py (original) +++ pypy/branch/micronumpy/pypy/jit/backend/x86/jump.py Mon Aug 16 04:52:00 2010 @@ -1,23 +1,6 @@ import sys from pypy.tool.pairtype import extendabletype -from pypy.jit.backend.x86.ri386 import * - -class __extend__(OPERAND): - __metaclass__ = extendabletype - def _getregkey(self): - raise AssertionError("should only happen to registers and frame " - "positions") - -class __extend__(REG): - __metaclass__ = extendabletype - def _getregkey(self): - return ~self.op - -class __extend__(MODRM): - __metaclass__ = extendabletype - def _getregkey(self): - return self.position - +from pypy.jit.backend.x86.regloc import ImmedLoc, StackLoc def remap_frame_layout(assembler, src_locations, dst_locations, tmpreg): pending_dests = len(dst_locations) @@ -27,7 +10,7 @@ srccount[dst._getregkey()] = 0 for i in range(len(dst_locations)): src = src_locations[i] - if isinstance(src, IMM32): + if isinstance(src, ImmedLoc): continue key = src._getregkey() if key in srccount: @@ -46,7 +29,7 @@ srccount[key] = -1 # means "it's done" pending_dests -= 1 src = src_locations[i] - if not isinstance(src, IMM32): + if not isinstance(src, ImmedLoc): key = src._getregkey() if key in srccount: srccount[key] -= 1 @@ -80,7 +63,7 @@ assert pending_dests == 0 def _move(assembler, src, dst, tmpreg): - if isinstance(dst, MODRM) and isinstance(src, MODRM): + if dst.is_memory_reference() and src.is_memory_reference(): assembler.regalloc_mov(src, tmpreg) src = tmpreg assembler.regalloc_mov(src, dst) Modified: pypy/branch/micronumpy/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/micronumpy/pypy/jit/backend/x86/regalloc.py Mon Aug 16 04:52:00 2010 @@ -5,7 +5,7 @@ from pypy.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr, ResOperation, BoxPtr, LoopToken, INT, REF, FLOAT) -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.regloc import * from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi, rstr from pypy.rlib.objectmodel import we_are_translated from pypy.rlib import rgc @@ -17,16 +17,7 @@ from pypy.jit.backend.llsupport.descr import BaseCallDescr, BaseSizeDescr from pypy.jit.backend.llsupport.regalloc import FrameManager, RegisterManager,\ TempBox - -WORD = 4 -FRAME_FIXED_SIZE = 5 # ebp + ebx + esi + edi + force_index = 5 words -FORCE_INDEX_OFS = -4*WORD - -width_of_type = { - INT : 1, - REF : 1, - FLOAT : 2, - } +from pypy.jit.backend.x86.arch import WORD, FRAME_FIXED_SIZE, IS_X86_32, IS_X86_64 class X86RegisterManager(RegisterManager): @@ -50,12 +41,19 @@ print "convert_to_imm: got a %s" % c raise AssertionError +class X86_64_RegisterManager(X86RegisterManager): + # r11 omitted because it's used as scratch + all_regs = [eax, ecx, edx, ebx, esi, edi, r8, r9, r10, r12, r13, r14, r15] + no_lower_byte_regs = [] + save_around_call_regs = [eax, ecx, edx, esi, edi, r8, r9, r10] + class FloatConstants(object): BASE_CONSTANT_SIZE = 1000 def __init__(self): self.cur_array_free = 0 + self.const_id = 0 def _get_new_array(self): n = self.BASE_CONSTANT_SIZE @@ -71,7 +69,8 @@ n = self.cur_array_free - 1 arr[n] = floatval self.cur_array_free = n - return rffi.cast(lltype.Signed, arr) + n * 8 + self.const_id += 1 + return (self.const_id, rffi.cast(lltype.Signed, arr) + n * 8) class X86XMMRegisterManager(RegisterManager): @@ -80,7 +79,6 @@ all_regs = [xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7] # we never need lower byte I hope save_around_call_regs = all_regs - reg_width = 2 def __init__(self, longevity, frame_manager=None, assembler=None): RegisterManager.__init__(self, longevity, frame_manager=frame_manager, @@ -93,28 +91,36 @@ self.float_constants = assembler._float_constants def convert_to_imm(self, c): - adr = self.float_constants.record_float(c.getfloat()) - return heap64(adr) + const_id, adr = self.float_constants.record_float(c.getfloat()) + return ConstFloatLoc(adr, const_id) def after_call(self, v): # the result is stored in st0, but we don't have this around, # so genop_call will move it to some frame location immediately # after the call - return self.frame_manager.loc(v, 2) + return self.frame_manager.loc(v) -class X86FrameManager(FrameManager): +class X86_64_XMMRegisterManager(X86XMMRegisterManager): + # xmm15 reserved for scratch use + all_regs = [xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14] + save_around_call_regs = all_regs + + def call_result_location(self, v): + return xmm0 + + def after_call(self, v): + # We use RegisterManager's implementation, since X86XMMRegisterManager + # places the result on the stack, which we don't need to do when the + # calling convention places the result in xmm0 + return RegisterManager.after_call(self, v) +class X86FrameManager(FrameManager): @staticmethod - def frame_pos(i, size): - if size == 1: - res = mem(ebp, get_ebp_ofs(i)) - elif size == 2: - res = mem64(ebp, get_ebp_ofs(i + 1)) - else: - print "Unimplemented size %d" % i - raise NotImplementedError("unimplemented size %d" % i) - res.position = i - return res + def frame_pos(i, box_type): + if IS_X86_32 and box_type == FLOAT: + return StackLoc(i, get_ebp_ofs(i+1), 2, box_type) + else: + return StackLoc(i, get_ebp_ofs(i), 1, box_type) class RegAlloc(object): exc = False @@ -135,11 +141,21 @@ # compute longevity of variables longevity = self._compute_vars_longevity(inputargs, operations) self.longevity = longevity - self.rm = X86RegisterManager(longevity, - frame_manager = self.fm, - assembler = self.assembler) - self.xrm = X86XMMRegisterManager(longevity, frame_manager = self.fm, - assembler = self.assembler) + # XXX + if cpu.WORD == 4: + gpr_reg_mgr_cls = X86RegisterManager + xmm_reg_mgr_cls = X86XMMRegisterManager + elif cpu.WORD == 8: + gpr_reg_mgr_cls = X86_64_RegisterManager + xmm_reg_mgr_cls = X86_64_XMMRegisterManager + else: + raise AssertionError("Word size should be 4 or 8") + + self.rm = gpr_reg_mgr_cls(longevity, + frame_manager = self.fm, + assembler = self.assembler) + self.xrm = xmm_reg_mgr_cls(longevity, frame_manager = self.fm, + assembler = self.assembler) def prepare_loop(self, inputargs, operations, looptoken): self._prepare(inputargs, operations) @@ -184,7 +200,7 @@ if reg: loc = reg else: - loc = self.fm.loc(arg, width_of_type[arg.type]) + loc = self.fm.loc(arg) if arg.type == FLOAT: floatlocs[i] = loc else: @@ -252,23 +268,23 @@ arg = inputargs[i] i += 1 if arg.type == FLOAT: - if isinstance(loc, REG): + if isinstance(loc, RegLoc): self.xrm.reg_bindings[arg] = loc used[loc] = None else: self.fm.frame_bindings[arg] = loc else: - if isinstance(loc, REG): + if isinstance(loc, RegLoc): self.rm.reg_bindings[arg] = loc used[loc] = None else: self.fm.frame_bindings[arg] = loc self.rm.free_regs = [] - for reg in X86RegisterManager.all_regs: + for reg in self.rm.all_regs: if reg not in used: self.rm.free_regs.append(reg) self.xrm.free_regs = [] - for reg in X86XMMRegisterManager.all_regs: + for reg in self.xrm.all_regs: if reg not in used: self.xrm.free_regs.append(reg) # note: we need to make a copy of inputargs because possibly_free_vars @@ -646,7 +662,7 @@ vable_index = jd.index_of_virtualizable if vable_index >= 0: self.rm._sync_var(op.args[vable_index]) - vable = self.fm.loc(op.args[vable_index], 1) + vable = self.fm.loc(op.args[vable_index]) else: vable = imm(0) self._call(op, [imm(size), vable] + @@ -655,13 +671,12 @@ def consider_cond_call_gc_wb(self, op): assert op.result is None + loc_newvalue = self.rm.make_sure_var_in_reg(op.args[1], op.args) + # ^^^ we force loc_newvalue in a reg (unless it's a Const), + # because it will be needed anyway by the following setfield_gc. + # It avoids loading it twice from the memory. loc_base = self.rm.make_sure_var_in_reg(op.args[0], op.args, imm_fine=False) - loc_newvalue = self.rm.make_sure_var_in_reg(op.args[1], op.args, - imm_fine=False) - # ^^^ we also force loc_newvalue in a reg, because it will be needed - # anyway by the following setfield_gc. It avoids loading it twice - # from the memory. arglocs = [loc_base, loc_newvalue] # add eax, ecx and edx as extra "arguments" to ensure they are # saved and restored. Fish in self.rm to know which of these @@ -670,7 +685,7 @@ # function, a GC write barrier, is known not to touch them. # See remember_young_pointer() in rpython/memory/gc/generation.py. for v, reg in self.rm.reg_bindings.items(): - if ((reg is eax or reg is ecx or reg is edx) + if (reg in self.rm.save_around_call_regs and self.rm.stays_alive(v)): arglocs.append(reg) self.PerformDiscard(op, arglocs) @@ -809,7 +824,7 @@ def consider_setfield_gc(self, op): ofs_loc, size_loc, ptr = self._unpack_fielddescr(op.descr) - assert isinstance(size_loc, IMM32) + assert isinstance(size_loc, ImmedLoc) if size_loc.value == 1: need_lower_byte = True else: @@ -950,7 +965,7 @@ shape = gcrootmap.get_basic_shape() for v, val in self.fm.frame_bindings.items(): if (isinstance(v, BoxPtr) and self.rm.stays_alive(v)): - assert isinstance(val, MODRM) + assert isinstance(val, StackLoc) gcrootmap.add_ebp_offset(shape, get_ebp_ofs(val.position)) for v, reg in self.rm.reg_bindings.items(): if reg is eax: Modified: pypy/branch/micronumpy/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/micronumpy/pypy/jit/backend/x86/runner.py Mon Aug 16 04:52:00 2010 @@ -4,11 +4,13 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.jit.metainterp import history, compile from pypy.jit.backend.x86.assembler import Assembler386 -from pypy.jit.backend.x86.regalloc import FORCE_INDEX_OFS +from pypy.jit.backend.x86.arch import FORCE_INDEX_OFS from pypy.jit.backend.x86.profagent import ProfileAgent from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU +from pypy.jit.backend.x86 import regloc +import sys -class CPU386(AbstractLLCPU): +class AbstractX86CPU(AbstractLLCPU): debug = True supports_floats = True @@ -44,6 +46,7 @@ self.profile_agent.startup() def finish_once(self): + self.assembler.finish_once() self.profile_agent.shutdown() def compile_loop(self, inputargs, operations, looptoken): @@ -131,10 +134,28 @@ assert fail_index == fail_index_2 return faildescr +class CPU386(AbstractX86CPU): + WORD = 4 + NUM_REGS = 8 + CALLEE_SAVE_REGISTERS = [regloc.ebx, regloc.esi, regloc.edi] + FRAME_FIXED_SIZE = len(CALLEE_SAVE_REGISTERS) + 2 + + def __init__(self, *args, **kwargs): + assert sys.maxint == (2**31 - 1) + super(CPU386, self).__init__(*args, **kwargs) class CPU386_NO_SSE2(CPU386): supports_floats = False +class CPU_X86_64(AbstractX86CPU): + WORD = 8 + NUM_REGS = 16 + CALLEE_SAVE_REGISTERS = [regloc.ebx, regloc.r12, regloc.r13, regloc.r14, regloc.r15] + FRAME_FIXED_SIZE = len(CALLEE_SAVE_REGISTERS) + 2 + + def __init__(self, *args, **kwargs): + assert sys.maxint == (2**63 - 1) + super(CPU_X86_64, self).__init__(*args, **kwargs) CPU = CPU386 Modified: pypy/branch/micronumpy/pypy/jit/backend/x86/test/conftest.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/backend/x86/test/conftest.py (original) +++ pypy/branch/micronumpy/pypy/jit/backend/x86/test/conftest.py Mon Aug 16 04:52:00 2010 @@ -3,6 +3,5 @@ cpu = detect_cpu.autodetect() def pytest_runtest_setup(item): - if cpu != 'x86': - py.test.skip("x86 directory skipped: cpu is %r" % (cpu,)) - + if cpu not in ('x86', 'x86_64'): + py.test.skip("x86/x86_64 tests skipped: cpu is %r" % (cpu,)) Modified: pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_assembler.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_assembler.py (original) +++ pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_assembler.py Mon Aug 16 04:52:00 2010 @@ -1,14 +1,22 @@ -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.regloc import * from pypy.jit.backend.x86.assembler import Assembler386, MachineCodeBlockWrapper from pypy.jit.backend.x86.regalloc import X86FrameManager, get_ebp_ofs -from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat +from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, INT, REF, FLOAT from pypy.rlib.rarithmetic import intmask from pypy.rpython.lltypesystem import lltype, llmemory, rffi +from pypy.jit.backend.x86.arch import WORD, IS_X86_32, IS_X86_64 +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.regalloc import X86RegisterManager, X86_64_RegisterManager, X86XMMRegisterManager, X86_64_XMMRegisterManager +ACTUAL_CPU = getcpuclass() class FakeCPU: rtyper = None supports_floats = True + NUM_REGS = ACTUAL_CPU.NUM_REGS + + def fielddescrof(self, STRUCT, name): + return 42 class FakeMC: def __init__(self, base_address=0): @@ -25,7 +33,14 @@ self.content.append(("JMP", args)) def done(self): pass + def PUSH_r(self, reg): + pass + def POP_r(self, reg): + pass +class FakeAssembler: + def write_pending_failure_recoveries(self): + pass def test_write_failure_recovery_description(): assembler = Assembler386(FakeCPU()) @@ -33,12 +48,12 @@ failargs = [BoxInt(), BoxPtr(), BoxFloat()] * 3 failargs.insert(6, None) failargs.insert(7, None) - locs = [X86FrameManager.frame_pos(0, 1), - X86FrameManager.frame_pos(1, 1), - X86FrameManager.frame_pos(10, 2), - X86FrameManager.frame_pos(100, 1), - X86FrameManager.frame_pos(101, 1), - X86FrameManager.frame_pos(110, 2), + locs = [X86FrameManager.frame_pos(0, INT), + X86FrameManager.frame_pos(1, REF), + X86FrameManager.frame_pos(10, FLOAT), + X86FrameManager.frame_pos(100, INT), + X86FrameManager.frame_pos(101, REF), + X86FrameManager.frame_pos(110, FLOAT), None, None, ebx, @@ -46,17 +61,17 @@ xmm2] assert len(failargs) == len(locs) assembler.write_failure_recovery_description(mc, failargs, locs) - nums = [Assembler386.DESCR_INT + 4*(8+0), - Assembler386.DESCR_REF + 4*(8+1), - Assembler386.DESCR_FLOAT + 4*(8+10), - Assembler386.DESCR_INT + 4*(8+100), - Assembler386.DESCR_REF + 4*(8+101), - Assembler386.DESCR_FLOAT + 4*(8+110), + nums = [Assembler386.DESCR_INT + 4*(16+0), + Assembler386.DESCR_REF + 4*(16+1), + Assembler386.DESCR_FLOAT + 4*(16+10), + Assembler386.DESCR_INT + 4*(16+100), + Assembler386.DESCR_REF + 4*(16+101), + Assembler386.DESCR_FLOAT + 4*(16+110), Assembler386.CODE_HOLE, Assembler386.CODE_HOLE, - Assembler386.DESCR_INT + 4*ebx.op, - Assembler386.DESCR_REF + 4*esi.op, - Assembler386.DESCR_FLOAT + 4*xmm2.op] + Assembler386.DESCR_INT + 4*ebx.value, + Assembler386.DESCR_REF + 4*esi.value, + Assembler386.DESCR_FLOAT + 4*xmm2.value] double_byte_nums = [] for num in nums[3:6]: double_byte_nums.append((num & 0x7F) | 0x80) @@ -94,6 +109,9 @@ return lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S)) def get_random_float(): + # Returns , , + # NB: on 64-bit, will be the entire float and + # will be random garbage from malloc! assert withfloats value = random.random() - 0.5 # make sure it fits into 64 bits @@ -101,9 +119,16 @@ rffi.cast(rffi.DOUBLEP, tmp)[0] = value return rffi.cast(rffi.DOUBLEP, tmp)[0], tmp[0], tmp[1] + if IS_X86_32: + main_registers = X86RegisterManager.all_regs + xmm_registers = X86XMMRegisterManager.all_regs + elif IS_X86_64: + main_registers = X86_64_RegisterManager.all_regs + xmm_registers = X86_64_XMMRegisterManager.all_regs + # memory locations: 26 integers, 26 pointers, 26 floats # main registers: half of them as signed and the other half as ptrs - # xmm registers: all floats, from xmm0 to xmm7 + # xmm registers: all floats, from xmm0 to xmm(7|15) # holes: 8 locations = [] baseloc = 4 @@ -117,18 +142,17 @@ content = ([('int', locations.pop()) for _ in range(26)] + [('ptr', locations.pop()) for _ in range(26)] + [(['int', 'ptr'][random.randrange(0, 2)], reg) - for reg in [eax, ecx, edx, ebx, esi, edi]]) + for reg in main_registers]) if withfloats: content += ([('float', locations.pop()) for _ in range(26)] + - [('float', reg) for reg in [xmm0, xmm1, xmm2, xmm3, - xmm4, xmm5, xmm6, xmm7]]) + [('float', reg) for reg in xmm_registers]) for i in range(8): content.append(('hole', None)) random.shuffle(content) # prepare the expected target arrays, the descr_bytecode, # the 'registers' and the 'stack' arrays according to 'content' - xmmregisters = lltype.malloc(rffi.LONGP.TO, 16+9, flavor='raw') + xmmregisters = lltype.malloc(rffi.LONGP.TO, 16+ACTUAL_CPU.NUM_REGS+1, flavor='raw') registers = rffi.ptradd(xmmregisters, 16) stacklen = baseloc + 10 stack = lltype.malloc(rffi.LONGP.TO, stacklen, flavor='raw') @@ -140,8 +164,8 @@ assert loc >= 0 ofs = get_ebp_ofs(loc) assert ofs < 0 - assert (ofs % 4) == 0 - stack[stacklen + ofs//4] = value + assert (ofs % WORD) == 0 + stack[stacklen + ofs//WORD] = value descr_bytecode = [] for i, (kind, loc) in enumerate(content): @@ -152,12 +176,18 @@ value, lo, hi = get_random_float() expected_floats[i] = value kind = Assembler386.DESCR_FLOAT - if isinstance(loc, REG): - xmmregisters[2*loc.op] = lo - xmmregisters[2*loc.op+1] = hi + if isinstance(loc, RegLoc): + if WORD == 4: + xmmregisters[2*loc.value] = lo + xmmregisters[2*loc.value+1] = hi + elif WORD == 8: + xmmregisters[loc.value] = lo else: - write_in_stack(loc, hi) - write_in_stack(loc+1, lo) + if WORD == 4: + write_in_stack(loc, hi) + write_in_stack(loc+1, lo) + elif WORD == 8: + write_in_stack(loc, lo) else: if kind == 'int': value = get_random_int() @@ -170,15 +200,15 @@ value = rffi.cast(rffi.LONG, value) else: assert 0, kind - if isinstance(loc, REG): - registers[loc.op] = value + if isinstance(loc, RegLoc): + registers[loc.value] = value else: write_in_stack(loc, value) - if isinstance(loc, REG): - num = kind + 4*loc.op + if isinstance(loc, RegLoc): + num = kind + 4*loc.value else: - num = kind + 4*(8+loc) + num = kind + Assembler386.CODE_FROMSTACK + (4*loc) while num >= 0x80: descr_bytecode.append((num & 0x7F) | 0x80) num >>= 7 @@ -195,8 +225,8 @@ for i in range(len(descr_bytecode)): assert 0 <= descr_bytecode[i] <= 255 descr_bytes[i] = rffi.cast(rffi.UCHAR, descr_bytecode[i]) - registers[8] = rffi.cast(rffi.LONG, descr_bytes) - registers[ebp.op] = rffi.cast(rffi.LONG, stack) + 4*stacklen + registers[ACTUAL_CPU.NUM_REGS] = rffi.cast(rffi.LONG, descr_bytes) + registers[ebp.value] = rffi.cast(rffi.LONG, stack) + WORD*stacklen # run! assembler = Assembler386(FakeCPU()) @@ -237,7 +267,8 @@ def test_mc_wrapper_profile_agent(): agent = FakeProfileAgent() - mc = FakeMCWrapper(100, agent) + assembler = FakeAssembler() + mc = FakeMCWrapper(assembler, 100, agent) mc.start_function("abc") mc.writechr("x") mc.writechr("x") Modified: pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_basic.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_basic.py (original) +++ pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_basic.py Mon Aug 16 04:52:00 2010 @@ -1,5 +1,5 @@ import py -from pypy.jit.backend.x86.runner import CPU386 +from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.metainterp.warmspot import ll_meta_interp from pypy.jit.metainterp.test import test_basic from pypy.jit.codewriter.policy import StopAtXPolicy @@ -7,7 +7,7 @@ class Jit386Mixin(test_basic.LLJitMixin): type_system = 'lltype' - CPUClass = CPU386 + CPUClass = getcpuclass() def check_jumps(self, maxcount): pass Modified: pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_gc_integration.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_gc_integration.py (original) +++ pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_gc_integration.py Mon Aug 16 04:52:00 2010 @@ -9,13 +9,13 @@ from pypy.jit.codewriter import heaptracker from pypy.jit.backend.llsupport.descr import GcCache from pypy.jit.backend.llsupport.gc import GcLLDescription -from pypy.jit.backend.x86.runner import CPU -from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, FRAME_FIXED_SIZE +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.regalloc import RegAlloc +from pypy.jit.backend.x86.arch import WORD, FRAME_FIXED_SIZE from pypy.jit.metainterp.test.oparser import parse from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.annlowlevel import llhelper from pypy.rpython.lltypesystem import rclass, rstr -from pypy.jit.backend.x86.ri386 import * from pypy.jit.backend.llsupport.gc import GcLLDescr_framework, GcRefList, GcPtrFieldDescr from pypy.jit.backend.x86.test.test_regalloc import MockAssembler @@ -23,6 +23,8 @@ from pypy.jit.backend.x86.regalloc import X86RegisterManager, X86FrameManager,\ X86XMMRegisterManager +CPU = getcpuclass() + class MockGcRootMap(object): def get_basic_shape(self): return ['shape'] @@ -84,7 +86,7 @@ mark = regalloc.get_mark_gc_roots(cpu.gc_ll_descr.gcrootmap) assert mark[0] == 'compressed' base = -WORD * FRAME_FIXED_SIZE - expected = ['ebx', 'esi', 'edi', base, base-4, base-8] + expected = ['ebx', 'esi', 'edi', base, base-WORD, base-WORD*2] assert dict.fromkeys(mark[1:]) == dict.fromkeys(expected) class TestRegallocGcIntegration(BaseTestRegalloc): @@ -175,7 +177,7 @@ self.addrs[1] = self.addrs[0] + 64 # 64 bytes def malloc_slowpath(size): - assert size == 8 + assert size == WORD*2 nadr = rffi.cast(lltype.Signed, self.nursery) self.addrs[0] = nadr + size return nadr @@ -199,7 +201,7 @@ return rffi.cast(lltype.Signed, self.addrs) def get_nursery_top_addr(self): - return rffi.cast(lltype.Signed, self.addrs) + 4 + return rffi.cast(lltype.Signed, self.addrs) + WORD def get_malloc_fixedsize_slowpath_addr(self): fptr = llhelper(lltype.Ptr(self.MALLOC_SLOWPATH), self.malloc_slowpath) @@ -213,7 +215,7 @@ def setup_method(self, method): cpu = CPU(None, None) - cpu.vtable_offset = 4 + cpu.vtable_offset = WORD cpu.gc_ll_descr = GCDescrFastpathMalloc() NODE = lltype.Struct('node', ('tid', lltype.Signed), @@ -249,7 +251,7 @@ assert gc_ll_descr.nursery[0] == self.nodedescr.tid assert gc_ll_descr.nursery[1] == 42 nurs_adr = rffi.cast(lltype.Signed, gc_ll_descr.nursery) - assert gc_ll_descr.addrs[0] == nurs_adr + 8 + assert gc_ll_descr.addrs[0] == nurs_adr + (WORD*2) def test_malloc_slowpath(self): ops = ''' @@ -269,7 +271,7 @@ # this should call slow path once gc_ll_descr = self.cpu.gc_ll_descr nadr = rffi.cast(lltype.Signed, gc_ll_descr.nursery) - assert gc_ll_descr.addrs[0] == nadr + 8 + assert gc_ll_descr.addrs[0] == nadr + (WORD*2) def test_new_with_vtable(self): ops = ''' @@ -284,4 +286,4 @@ assert gc_ll_descr.nursery[0] == self.descrsize.tid assert gc_ll_descr.nursery[1] == self.vtable_int nurs_adr = rffi.cast(lltype.Signed, gc_ll_descr.nursery) - assert gc_ll_descr.addrs[0] == nurs_adr + 12 + assert gc_ll_descr.addrs[0] == nurs_adr + (WORD*3) Modified: pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_jump.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_jump.py (original) +++ pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_jump.py Mon Aug 16 04:52:00 2010 @@ -1,6 +1,7 @@ -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.regloc import * from pypy.jit.backend.x86.regalloc import X86FrameManager from pypy.jit.backend.x86.jump import remap_frame_layout +from pypy.jit.metainterp.history import INT frame_pos = X86FrameManager.frame_pos @@ -25,7 +26,7 @@ continue assert len(op1) == len(op2) for x, y in zip(op1, op2): - if isinstance(x, MODRM) and isinstance(y, MODRM): + if isinstance(x, StackLoc) and isinstance(y, MODRM): assert x.byte == y.byte assert x.extradata == y.extradata else: @@ -41,9 +42,9 @@ remap_frame_layout(assembler, [eax, ebx, ecx, edx, esi, edi], [eax, ebx, ecx, edx, esi, edi], '?') assert assembler.ops == [] - s8 = frame_pos(1, 1) - s12 = frame_pos(31, 1) - s20 = frame_pos(6, 1) + s8 = frame_pos(1, INT) + s12 = frame_pos(31, INT) + s20 = frame_pos(6, INT) remap_frame_layout(assembler, [eax, ebx, ecx, s20, s8, edx, s12, esi, edi], [eax, ebx, ecx, s20, s8, edx, s12, esi, edi], '?') @@ -58,10 +59,10 @@ def test_simple_framelocs(): assembler = MockAssembler() - s8 = frame_pos(0, 1) - s12 = frame_pos(13, 1) - s20 = frame_pos(20, 1) - s24 = frame_pos(221, 1) + s8 = frame_pos(0, INT) + s12 = frame_pos(13, INT) + s20 = frame_pos(20, INT) + s24 = frame_pos(221, INT) remap_frame_layout(assembler, [s8, eax, s12], [s20, s24, edi], edx) assert assembler.ops == [('mov', s8, edx), ('mov', edx, s20), @@ -70,10 +71,10 @@ def test_reordering(): assembler = MockAssembler() - s8 = frame_pos(8, 1) - s12 = frame_pos(12, 1) - s20 = frame_pos(19, 1) - s24 = frame_pos(1, 1) + s8 = frame_pos(8, INT) + s12 = frame_pos(12, INT) + s20 = frame_pos(19, INT) + s24 = frame_pos(1, INT) remap_frame_layout(assembler, [eax, s8, s20, ebx], [s8, ebx, eax, edi], '?') assert assembler.got([('mov', ebx, edi), @@ -83,10 +84,10 @@ def test_cycle(): assembler = MockAssembler() - s8 = frame_pos(8, 1) - s12 = frame_pos(12, 1) - s20 = frame_pos(19, 1) - s24 = frame_pos(1, 1) + s8 = frame_pos(8, INT) + s12 = frame_pos(12, INT) + s20 = frame_pos(19, INT) + s24 = frame_pos(1, INT) remap_frame_layout(assembler, [eax, s8, s20, ebx], [s8, ebx, eax, s20], '?') assert assembler.got([('push', s8), @@ -97,12 +98,12 @@ def test_cycle_2(): assembler = MockAssembler() - s8 = frame_pos(8, 1) - s12 = frame_pos(12, 1) - s20 = frame_pos(19, 1) - s24 = frame_pos(1, 1) - s2 = frame_pos(2, 1) - s3 = frame_pos(3, 1) + s8 = frame_pos(8, INT) + s12 = frame_pos(12, INT) + s20 = frame_pos(19, INT) + s24 = frame_pos(1, INT) + s2 = frame_pos(2, INT) + s3 = frame_pos(3, INT) remap_frame_layout(assembler, [eax, s8, edi, s20, eax, s20, s24, esi, s2, s3], [s8, s20, edi, eax, edx, s24, ebx, s12, s3, s2], @@ -127,14 +128,14 @@ remap_frame_layout(assembler, [c3], [eax], '?') assert assembler.ops == [('mov', c3, eax)] assembler = MockAssembler() - s12 = frame_pos(12, 1) + s12 = frame_pos(12, INT) remap_frame_layout(assembler, [c3], [s12], '?') assert assembler.ops == [('mov', c3, s12)] def test_constants_and_cycle(): assembler = MockAssembler() c3 = imm(3) - s12 = frame_pos(13, 1) + s12 = frame_pos(13, INT) remap_frame_layout(assembler, [ebx, c3, s12], [s12, eax, ebx], edi) assert assembler.ops == [('mov', c3, eax), Modified: pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_recompilation.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_recompilation.py (original) +++ pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_recompilation.py Mon Aug 16 04:52:00 2010 @@ -1,6 +1,5 @@ - -from pypy.jit.backend.x86.runner import CPU from pypy.jit.backend.x86.test.test_regalloc import BaseTestRegalloc +from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 class TestRecompilation(BaseTestRegalloc): def test_compile_bridge_not_deeper(self): @@ -51,7 +50,9 @@ descr = loop.operations[2].descr new = descr._x86_bridge_frame_depth assert descr._x86_bridge_param_depth == 0 - assert new > previous + # XXX: Maybe add enough ops to force stack on 64-bit as well? + if IS_X86_32: + assert new > previous self.cpu.set_future_value_int(0, 0) fail = self.run(loop) assert fail.identifier == 2 @@ -111,7 +112,9 @@ guard_op = loop.operations[5] loop_frame_depth = loop.token._x86_frame_depth assert loop.token._x86_param_depth == 0 - assert guard_op.descr._x86_bridge_frame_depth > loop_frame_depth + # XXX: Maybe add enough ops to force stack on 64-bit as well? + if IS_X86_32: + assert guard_op.descr._x86_bridge_frame_depth > loop_frame_depth assert guard_op.descr._x86_bridge_param_depth == 0 self.cpu.set_future_value_int(0, 0) self.cpu.set_future_value_int(1, 0) Modified: pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_regalloc.py Mon Aug 16 04:52:00 2010 @@ -7,15 +7,17 @@ BoxPtr, ConstPtr, LoopToken, BasicFailDescr from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.backend.llsupport.descr import GcCache -from pypy.jit.backend.x86.runner import CPU -from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, X86RegisterManager,\ +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.regalloc import RegAlloc, X86RegisterManager,\ FloatConstants +from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 from pypy.jit.metainterp.test.oparser import parse from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.annlowlevel import llhelper from pypy.rpython.lltypesystem import rclass, rstr -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.rx86 import * +CPU = getcpuclass() class MockGcDescr(GcCache): def get_funcptr_for_new(self): return 123 @@ -92,13 +94,20 @@ def f2(x, y): return x*y + def f10(*args): + assert len(args) == 10 + return sum(args) + F1PTR = lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Signed)) F2PTR = lltype.Ptr(lltype.FuncType([lltype.Signed]*2, lltype.Signed)) + F10PTR = lltype.Ptr(lltype.FuncType([lltype.Signed]*10, lltype.Signed)) f1ptr = llhelper(F1PTR, f1) f2ptr = llhelper(F2PTR, f2) + f10ptr = llhelper(F10PTR, f10) f1_calldescr = cpu.calldescrof(F1PTR.TO, F1PTR.TO.ARGS, F1PTR.TO.RESULT) f2_calldescr = cpu.calldescrof(F2PTR.TO, F2PTR.TO.ARGS, F2PTR.TO.RESULT) + f10_calldescr = cpu.calldescrof(F10PTR.TO, F10PTR.TO.ARGS, F10PTR.TO.RESULT) namespace = locals().copy() type_system = 'lltype' @@ -541,6 +550,12 @@ assert self.getints(9) == [0, 1, 1, 1, 1, 1, 1, 1, 1] class TestRegAllocCallAndStackDepth(BaseTestRegalloc): + def expected_param_depth(self, num_args): + # Assumes the arguments are all non-float + if IS_X86_32: + return num_args + elif IS_X86_64: + return max(num_args - 6, 0) def test_one_call(self): ops = ''' @@ -550,7 +565,7 @@ ''' loop = self.interpret(ops, [4, 7, 9, 9 ,9, 9, 9, 9, 9, 9, 9]) assert self.getints(11) == [5, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9] - assert loop.token._x86_param_depth == 1 + assert loop.token._x86_param_depth == self.expected_param_depth(1) def test_two_calls(self): ops = ''' @@ -561,8 +576,21 @@ ''' loop = self.interpret(ops, [4, 7, 9, 9 ,9, 9, 9, 9, 9, 9, 9]) assert self.getints(11) == [5*7, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9] - assert loop.token._x86_param_depth == 2 - + assert loop.token._x86_param_depth == self.expected_param_depth(2) + + def test_call_many_arguments(self): + # NB: The first and last arguments in the call are constants. This + # is primarily for x86-64, to ensure that loading a constant to an + # argument register or to the stack works correctly + ops = ''' + [i0, i1, i2, i3, i4, i5, i6, i7] + i8 = call(ConstClass(f10ptr), 1, i0, i1, i2, i3, i4, i5, i6, i7, 10, descr=f10_calldescr) + finish(i8) + ''' + loop = self.interpret(ops, [2, 3, 4, 5, 6, 7, 8, 9]) + assert self.getint(0) == 55 + assert loop.token._x86_param_depth == self.expected_param_depth(10) + def test_bridge_calls_1(self): ops = ''' [i0, i1] @@ -579,7 +607,7 @@ ''' bridge = self.attach_bridge(ops, loop, -2) - assert loop.operations[-2].descr._x86_bridge_param_depth == 2 + assert loop.operations[-2].descr._x86_bridge_param_depth == self.expected_param_depth(2) self.cpu.set_future_value_int(0, 4) self.cpu.set_future_value_int(1, 7) @@ -602,7 +630,7 @@ ''' bridge = self.attach_bridge(ops, loop, -2) - assert loop.operations[-2].descr._x86_bridge_param_depth == 2 + assert loop.operations[-2].descr._x86_bridge_param_depth == self.expected_param_depth(2) self.cpu.set_future_value_int(0, 4) self.cpu.set_future_value_int(1, 7) Modified: pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_regalloc2.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_regalloc2.py (original) +++ pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_regalloc2.py Mon Aug 16 04:52:00 2010 @@ -2,7 +2,9 @@ from pypy.jit.metainterp.history import ResOperation, BoxInt, ConstInt,\ BoxPtr, ConstPtr, BasicFailDescr, LoopToken from pypy.jit.metainterp.resoperation import rop -from pypy.jit.backend.x86.runner import CPU +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.arch import WORD +CPU = getcpuclass() def test_bug_rshift(): v1 = BoxInt() @@ -281,5 +283,8 @@ assert cpu.get_latest_value_int(16) == -57344 assert cpu.get_latest_value_int(17) == 1 assert cpu.get_latest_value_int(18) == -1 - assert cpu.get_latest_value_int(19) == -2147483648 + if WORD == 4: + assert cpu.get_latest_value_int(19) == -2147483648 + elif WORD == 8: + assert cpu.get_latest_value_int(19) == 19327352832 assert cpu.get_latest_value_int(20) == -49 Modified: pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_runner.py Mon Aug 16 04:52:00 2010 @@ -1,16 +1,22 @@ import py from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rstr, rclass +from pypy.rpython.annlowlevel import llhelper from pypy.jit.metainterp.history import ResOperation, LoopToken -from pypy.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstPtr, - Box, BasicFailDescr) -from pypy.jit.backend.x86.runner import CPU -from pypy.jit.backend.x86.regalloc import WORD +from pypy.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstFloat, + ConstPtr, Box, BoxFloat, BasicFailDescr) +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.x86.arch import WORD from pypy.jit.backend.llsupport import symbolic from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.executor import execute from pypy.jit.backend.test.runner_test import LLtypeBackendTest +from pypy.jit.metainterp.test.oparser import parse +from pypy.tool.udir import udir import ctypes import sys +import os + +CPU = getcpuclass() class FakeStats(object): pass @@ -56,7 +62,7 @@ assert u.chars[3] == u'd' @staticmethod - def _resbuf(res, item_tp=ctypes.c_int): + def _resbuf(res, item_tp=ctypes.c_long): return ctypes.cast(res.value._obj.intval, ctypes.POINTER(item_tp)) def test_allocations(self): @@ -71,8 +77,11 @@ return ctypes.cast(buf, ctypes.c_void_p).value func = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int)(f) addr = ctypes.cast(func, ctypes.c_void_p).value + # ctypes produces an unsigned value. We need it to be signed for, eg, + # relative addressing to work properly. + addr = rffi.cast(lltype.Signed, addr) - self.cpu.assembler.make_sure_mc_exists() + self.cpu.assembler.setup() self.cpu.assembler.malloc_func_addr = addr ofs = symbolic.get_field_token(rstr.STR, 'chars', False)[0] @@ -357,7 +366,9 @@ self.cpu.compile_bridge(faildescr1, [i1b], bridge) name, address, size = agent.functions[1] assert name == "Bridge # 0: bye" - assert address == loopaddress + loopsize + # Would be exactly ==, but there are some guard failure recovery + # stubs in-between + assert address >= loopaddress + loopsize assert size >= 10 # randomish number self.cpu.set_future_value_int(0, 2) @@ -366,6 +377,19 @@ res = self.cpu.get_latest_value_int(0) assert res == 20 + def test_call_with_const_floats(self): + def func(f1, f2): + return f1 + f2 + + FUNC = self.FuncType([lltype.Float, lltype.Float], lltype.Float) + FPTR = self.Ptr(FUNC) + calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + func_ptr = llhelper(FPTR, func) + funcbox = self.get_funcbox(self.cpu, func_ptr) + res = self.execute_operation(rop.CALL, [funcbox, ConstFloat(1.5), ConstFloat(2.5)], 'float', descr=calldescr) + assert res.value == 4.0 + + class TestX86OverflowMC(TestX86): def setup_method(self, meth): @@ -383,10 +407,100 @@ ops.append(ResOperation(rop.FINISH, [v], None, descr=BasicFailDescr())) looptoken = LoopToken() - self.cpu.assembler.make_sure_mc_exists() + self.cpu.assembler.setup() old_mc_mc = self.cpu.assembler.mc._mc self.cpu.compile_loop([base_v], ops, looptoken) assert self.cpu.assembler.mc._mc != old_mc_mc # overflowed self.cpu.set_future_value_int(0, base_v.value) self.cpu.execute_token(looptoken) assert self.cpu.get_latest_value_int(0) == 1024 + + def test_overflow_guard_float_cmp(self): + # The float comparisons on x86 tend to use small relative jumps, + # which may run into trouble if they fall on the edge of a + # MachineCodeBlock change. + a = BoxFloat(1.0) + b = BoxFloat(2.0) + failed = BoxInt(41) + finished = BoxInt(42) + + # We select guards that will always succeed, so that execution will + # continue through the entire set of comparisions + ops_to_test = ( + (rop.FLOAT_LT, [a, b], rop.GUARD_TRUE), + (rop.FLOAT_LT, [b, a], rop.GUARD_FALSE), + + (rop.FLOAT_LE, [a, a], rop.GUARD_TRUE), + (rop.FLOAT_LE, [a, b], rop.GUARD_TRUE), + (rop.FLOAT_LE, [b, a], rop.GUARD_FALSE), + + (rop.FLOAT_EQ, [a, a], rop.GUARD_TRUE), + (rop.FLOAT_EQ, [a, b], rop.GUARD_FALSE), + + (rop.FLOAT_NE, [a, b], rop.GUARD_TRUE), + (rop.FLOAT_NE, [a, a], rop.GUARD_FALSE), + + (rop.FLOAT_GT, [b, a], rop.GUARD_TRUE), + (rop.FLOAT_GT, [a, b], rop.GUARD_FALSE), + + (rop.FLOAT_GE, [a, a], rop.GUARD_TRUE), + (rop.FLOAT_GE, [b, a], rop.GUARD_TRUE), + (rop.FLOAT_GE, [a, b], rop.GUARD_FALSE), + ) + + for float_op, args, guard_op in ops_to_test: + ops = [] + + for i in range(200): + cmp_result = BoxInt() + ops.append(ResOperation(float_op, args, cmp_result)) + ops.append(ResOperation(guard_op, [cmp_result], None, descr=BasicFailDescr())) + ops[-1].fail_args = [failed] + + ops.append(ResOperation(rop.FINISH, [finished], None, descr=BasicFailDescr())) + + looptoken = LoopToken() + self.cpu.compile_loop([a, b, failed, finished], ops, looptoken) + self.cpu.set_future_value_float(0, a.value) + self.cpu.set_future_value_float(1, b.value) + self.cpu.set_future_value_int(2, failed.value) + self.cpu.set_future_value_int(3, finished.value) + self.cpu.execute_token(looptoken) + + # Really just a sanity check. We're actually interested in + # whether the test segfaults. + assert self.cpu.get_latest_value_int(0) == finished.value + + +class TestDebuggingAssembler(object): + def setup_method(self, meth): + self.pypylog = os.environ.get('PYPYLOG', None) + self.logfile = str(udir.join('x86_runner.log')) + os.environ['PYPYLOG'] = "mumble:" + self.logfile + self.cpu = CPU(rtyper=None, stats=FakeStats()) + + def teardown_method(self, meth): + if self.pypylog is not None: + os.environ['PYPYLOG'] = self.pypylog + + def test_debugger_on(self): + loop = """ + [i0] + debug_merge_point('xyz') + i1 = int_add(i0, 1) + i2 = int_ge(i1, 10) + guard_false(i2) [] + jump(i1) + """ + ops = parse(loop) + self.cpu.assembler.set_debug(True) + self.cpu.compile_loop(ops.inputargs, ops.operations, ops.token) + self.cpu.set_future_value_int(0, 0) + self.cpu.execute_token(ops.token) + # check debugging info + name, struct = self.cpu.assembler.loop_run_counters[0] + assert name == 'xyz' + assert struct.i == 10 + self.cpu.finish_once() + lines = py.path.local(self.logfile + ".count").readlines() + assert lines[0] == 'xyz:10\n' Modified: pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_symbolic_x86.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_symbolic_x86.py (original) +++ pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_symbolic_x86.py Mon Aug 16 04:52:00 2010 @@ -1,6 +1,7 @@ import py from pypy.jit.backend.llsupport.symbolic import * from pypy.rpython.lltypesystem import lltype, rffi +from pypy.jit.backend.x86.arch import WORD # This test file is here and not in llsupport/test/ because it checks # that we get correct numbers for a 32-bit machine. @@ -19,32 +20,32 @@ ofs_z, size_z = get_field_token(S, 'z', False) # ofs_x might be 0 or not, depending on how we count the headers # but the rest should be as expected for a 386 machine - assert size_x == size_y == size_z == 4 + assert size_x == size_y == size_z == WORD assert ofs_x >= 0 - assert ofs_y == ofs_x + 4 - assert ofs_z == ofs_x + 8 + assert ofs_y == ofs_x + WORD + assert ofs_z == ofs_x + (WORD*2) def test_struct_size(): ofs_z, size_z = get_field_token(S, 'z', False) totalsize = get_size(S, False) - assert totalsize == ofs_z + 4 + assert totalsize == ofs_z + WORD def test_primitive_size(): - assert get_size(lltype.Signed, False) == 4 + assert get_size(lltype.Signed, False) == WORD assert get_size(lltype.Char, False) == 1 - assert get_size(lltype.Ptr(S), False) == 4 + assert get_size(lltype.Ptr(S), False) == WORD def test_array_token(): A = lltype.GcArray(lltype.Char) basesize, itemsize, ofs_length = get_array_token(A, False) - assert basesize >= 4 # at least the 'length', maybe some gc headers + assert basesize >= WORD # at least the 'length', maybe some gc headers assert itemsize == 1 - assert ofs_length == basesize - 4 + assert ofs_length == basesize - WORD A = lltype.GcArray(lltype.Signed) basesize, itemsize, ofs_length = get_array_token(A, False) - assert basesize >= 4 # at least the 'length', maybe some gc headers - assert itemsize == 4 - assert ofs_length == basesize - 4 + assert basesize >= WORD # at least the 'length', maybe some gc headers + assert itemsize == WORD + assert ofs_length == basesize - WORD def test_varsized_struct_size(): S1 = lltype.GcStruct('S1', ('parent', S), @@ -54,9 +55,9 @@ ofs_extra, size_extra = get_field_token(S1, 'extra', False) basesize, itemsize, ofs_length = get_array_token(S1, False) assert size_parent == ofs_extra - assert size_extra == 4 - assert ofs_length == ofs_extra + 4 - assert basesize == ofs_length + 4 + assert size_extra == WORD + assert ofs_length == ofs_extra + WORD + assert basesize == ofs_length + WORD assert itemsize == 1 def test_string(): Modified: pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_zll_random.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_zll_random.py (original) +++ pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_zll_random.py Mon Aug 16 04:52:00 2010 @@ -1,9 +1,11 @@ from pypy.jit.backend.test.test_random import check_random_function, Random from pypy.jit.backend.test.test_ll_random import LLtypeOperationBuilder -from pypy.jit.backend.x86.runner import CPU386 +from pypy.jit.backend.detect_cpu import getcpuclass + +CPU = getcpuclass() def test_stress(): - cpu = CPU386(None, None) + cpu = CPU(None, None) r = Random() for i in range(1000): check_random_function(cpu, LLtypeOperationBuilder, r, i, 1000) Modified: pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_zrpy_gc.py Mon Aug 16 04:52:00 2010 @@ -17,6 +17,8 @@ from pypy.jit.backend.llsupport.gc import GcRefList, GcRootMap_asmgcc from pypy.jit.backend.llsupport.gc import GcLLDescr_framework from pypy.tool.udir import udir +from pypy.jit.backend.x86.arch import IS_X86_64 +import py.test class X(object): def __init__(self, x=0): @@ -126,6 +128,10 @@ class TestCompileHybrid(object): def setup_class(cls): + if IS_X86_64: + # No hybrid GC on 64-bit for the time being + py.test.skip() + funcs = [] name_to_func = {} for fullname in dir(cls): Modified: pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_ztranslation.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_ztranslation.py (original) +++ pypy/branch/micronumpy/pypy/jit/backend/x86/test/test_ztranslation.py Mon Aug 16 04:52:00 2010 @@ -3,13 +3,14 @@ from pypy.rlib.jit import JitDriver, OPTIMIZER_FULL, unroll_parameters from pypy.rlib.jit import PARAMETERS, dont_look_inside from pypy.jit.metainterp.jitprof import Profiler -from pypy.jit.backend.x86.runner import CPU386 +from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.backend.test.support import CCompiledMixin from pypy.jit.codewriter.policy import StopAtXPolicy from pypy.translator.translator import TranslationContext +from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 class TestTranslationX86(CCompiledMixin): - CPUClass = CPU386 + CPUClass = getcpuclass() def _check_cbuilder(self, cbuilder): # We assume here that we have sse2. If not, the CPUClass @@ -114,7 +115,7 @@ class TestTranslationRemoveTypePtrX86(CCompiledMixin): - CPUClass = CPU386 + CPUClass = getcpuclass() def _get_TranslationContext(self): t = TranslationContext() @@ -125,6 +126,10 @@ return t def test_external_exception_handling_translates(self): + # FIXME + if IS_X86_64: + import py.test; py.test.skip() + jitdriver = JitDriver(greens = [], reds = ['n', 'total']) class ImDone(Exception): Modified: pypy/branch/micronumpy/pypy/jit/backend/x86/tool/viewcode.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/backend/x86/tool/viewcode.py (original) +++ pypy/branch/micronumpy/pypy/jit/backend/x86/tool/viewcode.py Mon Aug 16 04:52:00 2010 @@ -31,16 +31,23 @@ if sys.platform == "win32": XXX # lots more in Psyco -def machine_code_dump(data, originaddr): - # the disassembler to use. 'objdump' writes GNU-style instructions. - # 'ndisasm' would use Intel syntax, but you need to fix the output parsing. - objdump = ('objdump -M intel -b binary -m i386 ' +def machine_code_dump(data, originaddr, backend_name): + objdump_backend_option = { + 'x86': 'i386', + 'x86_64': 'x86-64', + 'i386': 'i386', + } + objdump = ('objdump -M intel,%(backend)s -b binary -m i386 ' '--adjust-vma=%(origin)d -D %(file)s') # f = open(tmpfile, 'wb') f.write(data) f.close() - g = os.popen(objdump % {'file': tmpfile, 'origin': originaddr}, 'r') + g = os.popen(objdump % { + 'file': tmpfile, + 'origin': originaddr, + 'backend': objdump_backend_option[backend_name], + }, 'r') result = g.readlines() g.close() return result[6:] # drop some objdump cruft @@ -126,7 +133,7 @@ def disassemble(self): if not hasattr(self, 'text'): - lines = machine_code_dump(self.data, self.addr) + lines = machine_code_dump(self.data, self.addr, self.world.backend_name) # instead of adding symbol names in the dumps we could # also make the 0xNNNNNNNN addresses be red and show the # symbol name when the mouse is over them @@ -171,10 +178,13 @@ self.jumps = {} self.symbols = {} self.logentries = {} + self.backend_name = None def parse(self, f, textonly=True): for line in f: - if line.startswith('CODE_DUMP '): + if line.startswith('BACKEND '): + self.backend_name = line.split(' ')[1].strip() + elif line.startswith('CODE_DUMP '): pieces = line.split() assert pieces[1].startswith('@') assert pieces[2].startswith('+') Modified: pypy/branch/micronumpy/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/micronumpy/pypy/jit/metainterp/executor.py Mon Aug 16 04:52:00 2010 @@ -91,7 +91,7 @@ return BoxInt(cpu.bh_getarrayitem_gc_i(arraydescr, array, index)) def do_getarrayitem_raw(cpu, _, arraybox, indexbox, arraydescr): - array = arraybox.getref_base() + array = arraybox.getint() index = indexbox.getint() assert not arraydescr.is_array_of_pointers() if arraydescr.is_array_of_floats(): Modified: pypy/branch/micronumpy/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/micronumpy/pypy/jit/metainterp/optimizeopt.py Mon Aug 16 04:52:00 2010 @@ -1,7 +1,7 @@ from pypy.jit.metainterp.history import Box, BoxInt, LoopToken, BoxFloat,\ ConstFloat from pypy.jit.metainterp.history import Const, ConstInt, ConstPtr, ConstObj, REF -from pypy.jit.metainterp.resoperation import rop, ResOperation +from pypy.jit.metainterp.resoperation import rop, ResOperation, opboolinvers, opboolreflex from pypy.jit.metainterp import jitprof from pypy.jit.metainterp.executor import execute_nonspec from pypy.jit.metainterp.specnode import SpecNode, NotSpecNode, ConstantSpecNode @@ -18,6 +18,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.jit.metainterp.history import AbstractDescr, make_hashable_int + def optimize_loop_1(metainterp_sd, loop): """Optimize loop.operations to make it match the input of loop.specnodes and to remove internal overheadish operations. Note that loop.specnodes @@ -611,12 +612,59 @@ assert oldop.opnum == op.opnum self.make_equal_to(op.result, self.getvalue(oldop.result)) return + elif self.find_rewritable_bool(op, args): + return else: self.pure_operations[args] = op # otherwise, the operation remains self.emit_operation(op) + + def try_boolinvers(self, op, targs): + oldop = self.pure_operations.get(targs, None) + if oldop is not None and oldop.descr is op.descr: + value = self.getvalue(oldop.result) + if value.is_constant(): + if value.box is CONST_1: + self.make_constant(op.result, CONST_0) + return True + elif value.box is CONST_0: + self.make_constant(op.result, CONST_1) + return True + return False + + + def find_rewritable_bool(self, op, args): + try: + oldopnum = opboolinvers[op.opnum] + targs = [args[0], args[1], ConstInt(oldopnum)] + if self.try_boolinvers(op, targs): + return True + except KeyError: + pass + + try: + oldopnum = opboolreflex[op.opnum] + targs = [args[1], args[0], ConstInt(oldopnum)] + oldop = self.pure_operations.get(targs, None) + if oldop is not None and oldop.descr is op.descr: + self.make_equal_to(op.result, self.getvalue(oldop.result)) + return True + except KeyError: + pass + + try: + oldopnum = opboolinvers[opboolreflex[op.opnum]] + targs = [args[1], args[0], ConstInt(oldopnum)] + if self.try_boolinvers(op, targs): + return True + except KeyError: + pass + + return False + + def optimize_JUMP(self, op): orgop = self.loop.operations[-1] exitargs = [] @@ -992,6 +1040,25 @@ self.make_equal_to(op.result, v1) else: self.optimize_default(op) + + def optimize_INT_SUB(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + if v2.is_constant() and v2.box.getint() == 0: + self.make_equal_to(op.result, v1) + else: + return self.optimize_default(op) + + def optimize_INT_ADD(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + # If one side of the op is 0 the result is the other side. + if v1.is_constant() and v1.box.getint() == 0: + self.make_equal_to(op.result, v2) + elif v2.is_constant() and v2.box.getint() == 0: + self.make_equal_to(op.result, v1) + else: + self.optimize_default(op) optimize_ops = _findall(Optimizer, 'optimize_') Modified: pypy/branch/micronumpy/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/micronumpy/pypy/jit/metainterp/resoperation.py Mon Aug 16 04:52:00 2010 @@ -274,3 +274,51 @@ setup(__name__ == '__main__') # print out the table when run directly del _oplist + +opboolinvers = { + rop.INT_EQ: rop.INT_NE, + rop.INT_NE: rop.INT_EQ, + rop.INT_LT: rop.INT_GE, + rop.INT_GE: rop.INT_LT, + rop.INT_GT: rop.INT_LE, + rop.INT_LE: rop.INT_GT, + + rop.UINT_LT: rop.UINT_GE, + rop.UINT_GE: rop.UINT_LT, + rop.UINT_GT: rop.UINT_LE, + rop.UINT_LE: rop.UINT_GT, + + rop.FLOAT_EQ: rop.FLOAT_NE, + rop.FLOAT_NE: rop.FLOAT_EQ, + rop.FLOAT_LT: rop.FLOAT_GE, + rop.FLOAT_GE: rop.FLOAT_LT, + rop.FLOAT_GT: rop.FLOAT_LE, + rop.FLOAT_LE: rop.FLOAT_GT, + + rop.PTR_EQ: rop.PTR_NE, + rop.PTR_NE: rop.PTR_EQ, + } + +opboolreflex = { + rop.INT_EQ: rop.INT_EQ, + rop.INT_NE: rop.INT_NE, + rop.INT_LT: rop.INT_GT, + rop.INT_GE: rop.INT_LE, + rop.INT_GT: rop.INT_LT, + rop.INT_LE: rop.INT_GE, + + rop.UINT_LT: rop.UINT_GT, + rop.UINT_GE: rop.UINT_LE, + rop.UINT_GT: rop.UINT_LT, + rop.UINT_LE: rop.UINT_GE, + + rop.FLOAT_EQ: rop.FLOAT_EQ, + rop.FLOAT_NE: rop.FLOAT_NE, + rop.FLOAT_LT: rop.FLOAT_GT, + rop.FLOAT_GE: rop.FLOAT_LE, + rop.FLOAT_GT: rop.FLOAT_LT, + rop.FLOAT_LE: rop.FLOAT_GE, + + rop.PTR_EQ: rop.PTR_EQ, + rop.PTR_NE: rop.PTR_NE, + } Modified: pypy/branch/micronumpy/pypy/jit/metainterp/test/test_executor.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/metainterp/test/test_executor.py (original) +++ pypy/branch/micronumpy/pypy/jit/metainterp/test/test_executor.py Mon Aug 16 04:52:00 2010 @@ -4,14 +4,14 @@ from pypy.rpython.lltypesystem import lltype, llmemory from pypy.jit.metainterp.executor import execute from pypy.jit.metainterp.executor import execute_varargs, execute_nonspec -from pypy.jit.metainterp.resoperation import rop +from pypy.jit.metainterp.resoperation import rop, opboolinvers, opboolreflex, opname from pypy.jit.metainterp.history import BoxInt, ConstInt from pypy.jit.metainterp.history import BoxPtr, ConstPtr from pypy.jit.metainterp.history import BoxFloat, ConstFloat from pypy.jit.metainterp.history import AbstractDescr, Box from pypy.jit.metainterp import history from pypy.jit.backend.model import AbstractCPU - +from pypy.rpython.lltypesystem import llmemory, rffi class FakeDescr(AbstractDescr): pass @@ -312,3 +312,40 @@ assert box.getint() == retvalue else: assert 0, "rettype is %r" % (rettype,) + +def make_args_for_op(op, a, b): + n=opname[op] + if n[0:3] == 'INT' or n[0:4] == 'UINT': + arg1 = ConstInt(a) + arg2 = ConstInt(b) + elif n[0:5] == 'FLOAT': + arg1 = ConstFloat(float(a)) + arg2 = ConstFloat(float(b)) + elif n[0:3] == 'PTR': + arg1 = ConstPtr(rffi.cast(llmemory.GCREF, a)) + arg2 = ConstPtr(rffi.cast(llmemory.GCREF, b)) + else: + raise NotImplementedError( + "Don't know how to make args for " + n) + return arg1, arg2 + + +def test_opboolinvers(): + cpu = FakeCPU() + for op1, op2 in opboolinvers.items(): + for a in (1,2,3): + for b in (1,2,3): + arg1, arg2 = make_args_for_op(op1, a, b) + box1 = execute(cpu, None, op1, None, arg1, arg2) + box2 = execute(cpu, None, op2, None, arg1, arg2) + assert box1.value == (not box2.value) + +def test_opboolreflex(): + cpu = FakeCPU() + for op1, op2 in opboolreflex.items(): + for a in (1,2,3): + for b in (1,2,3): + arg1, arg2 = make_args_for_op(op1, a, b) + box1 = execute(cpu, None, op1, None, arg1, arg2) + box2 = execute(cpu, None, op2, None, arg2, arg1) + assert box1.value == box2.value Modified: pypy/branch/micronumpy/pypy/jit/metainterp/test/test_loop.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/metainterp/test/test_loop.py (original) +++ pypy/branch/micronumpy/pypy/jit/metainterp/test/test_loop.py Mon Aug 16 04:52:00 2010 @@ -9,6 +9,10 @@ class LoopTest(object): optimizer = OPTIMIZER_SIMPLE + automatic_promotion_result = { + 'int_add' : 6, 'int_gt' : 1, 'guard_false' : 1, 'jump' : 1, + 'guard_value' : 3 + } def meta_interp(self, f, args, policy=None): return ll_meta_interp(f, args, optimizer=self.optimizer, @@ -477,9 +481,9 @@ res = self.meta_interp(main_interpreter_loop, [1]) assert res == main_interpreter_loop(1) self.check_loop_count(1) - # XXX maybe later optimize guard_value away - self.check_loops({'int_add' : 6, 'int_gt' : 1, - 'guard_false' : 1, 'jump' : 1, 'guard_value' : 3}) + # These loops do different numbers of ops based on which optimizer we + # are testing with. + self.check_loops(self.automatic_promotion_result) def test_can_enter_jit_outside_main_loop(self): myjitdriver = JitDriver(greens=[], reds=['i', 'j', 'a']) Modified: pypy/branch/micronumpy/pypy/jit/metainterp/test/test_loop_spec.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/metainterp/test/test_loop_spec.py (original) +++ pypy/branch/micronumpy/pypy/jit/metainterp/test/test_loop_spec.py Mon Aug 16 04:52:00 2010 @@ -5,6 +5,10 @@ class LoopSpecTest(test_loop.LoopTest): optimizer = OPTIMIZER_FULL + automatic_promotion_result = { + 'int_add' : 3, 'int_gt' : 1, 'guard_false' : 1, 'jump' : 1, + 'guard_value' : 1 + } # ====> test_loop.py Modified: pypy/branch/micronumpy/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/micronumpy/pypy/jit/metainterp/test/test_optimizeopt.py Mon Aug 16 04:52:00 2010 @@ -364,6 +364,74 @@ """ self.optimize_loop(ops, 'Not', expected) + def test_constant_boolrewrite_lt(self): + ops = """ + [i0] + i1 = int_lt(i0, 0) + guard_true(i1) [] + i2 = int_ge(i0, 0) + guard_false(i2) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_lt(i0, 0) + guard_true(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_constant_boolrewrite_gt(self): + ops = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + i2 = int_le(i0, 0) + guard_false(i2) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_constant_boolrewrite_reflex(self): + ops = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + i2 = int_lt(0, i0) + guard_true(i2) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_constant_boolrewrite_reflex_invers(self): + ops = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + i2 = int_ge(0, i0) + guard_false(i2) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_gt(i0, 0) + guard_true(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + def test_remove_consecutive_guard_value_constfold(self): ops = """ [] @@ -411,7 +479,6 @@ self.optimize_loop(ops, 'Not', expected) def test_int_is_true_1(self): - py.test.skip("XXX implement me") ops = """ [i0] i1 = int_is_true(i0) @@ -806,16 +873,10 @@ guard_nonnull(p0) [] i7 = ptr_ne(p0, p1) guard_true(i7) [] - i8 = ptr_eq(p0, p1) - guard_false(i8) [] i9 = ptr_ne(p0, p2) guard_true(i9) [] - i10 = ptr_eq(p0, p2) - guard_false(i10) [] i11 = ptr_ne(p2, p1) guard_true(i11) [] - i12 = ptr_eq(p2, p1) - guard_false(i12) [] jump(p0, p1, p2) """ self.optimize_loop(ops, 'Not, Not, Not', expected2) @@ -2051,8 +2112,41 @@ jump(i1, i0) """ self.optimize_loop(ops, 'Not, Not', expected) - - + + def test_fold_partially_constant_ops(self): + ops = """ + [i0] + i1 = int_sub(i0, 0) + jump(i1) + """ + expected = """ + [i0] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + ops = """ + [i0] + i1 = int_add(i0, 0) + jump(i1) + """ + expected = """ + [i0] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + ops = """ + [i0] + i1 = int_add(0, i0) + jump(i1) + """ + expected = """ + [i0] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + # ---------- def make_fail_descr(self): Modified: pypy/branch/micronumpy/pypy/jit/tl/pypyjit.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/tl/pypyjit.py (original) +++ pypy/branch/micronumpy/pypy/jit/tl/pypyjit.py Mon Aug 16 04:52:00 2010 @@ -37,6 +37,7 @@ set_opt_level(config, level='jit') config.objspace.allworkingmodules = False config.objspace.usemodules.pypyjit = True +config.objspace.usemodules.array = True config.objspace.usemodules._weakref = False config.objspace.usemodules._sre = False set_pypy_opt_level(config, level='jit') Modified: pypy/branch/micronumpy/pypy/jit/tl/pypyjit_demo.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/tl/pypyjit_demo.py (original) +++ pypy/branch/micronumpy/pypy/jit/tl/pypyjit_demo.py Mon Aug 16 04:52:00 2010 @@ -1,38 +1,64 @@ -base = object +## base = object -class Number(base): - __slots__ = ('val', ) - def __init__(self, val=0): - self.val = val - - def __add__(self, other): - if not isinstance(other, int): - other = other.val - return Number(val=self.val + other) +## class Number(base): +## __slots__ = ('val', ) +## def __init__(self, val=0): +## self.val = val + +## def __add__(self, other): +## if not isinstance(other, int): +## other = other.val +## return Number(val=self.val + other) - def __cmp__(self, other): - val = self.val - if not isinstance(other, int): - other = other.val - return cmp(val, other) - - def __nonzero__(self): - return bool(self.val) - -def g(x, inc=2): - return x + inc - -def f(n, x, inc): - while x < n: - x = g(x, inc=1) - return x - -import time -#t1 = time.time() -#f(10000000, Number(), 1) -#t2 = time.time() -#print t2 - t1 -t1 = time.time() -f(10000000, 0, 1) -t2 = time.time() -print t2 - t1 +## def __cmp__(self, other): +## val = self.val +## if not isinstance(other, int): +## other = other.val +## return cmp(val, other) + +## def __nonzero__(self): +## return bool(self.val) + +## def g(x, inc=2): +## return x + inc + +## def f(n, x, inc): +## while x < n: +## x = g(x, inc=1) +## return x + +## import time +## #t1 = time.time() +## #f(10000000, Number(), 1) +## #t2 = time.time() +## #print t2 - t1 +## t1 = time.time() +## f(10000000, 0, 1) +## t2 = time.time() +## print t2 - t1 + +try: + from array import array + def f(img): + i=0 + sa=0 + while i < img.__len__(): + sa+=img[i] + i+=1 + return sa + + img=array('h',(1,2,3,4)) + print f(img) +except Exception, e: + print "Exception: ", type(e) + print e + +## def f(): +## a=7 +## i=0 +## while i<4: +## if i<0: break +## if i<0: break +## i+=1 + +## f() Modified: pypy/branch/micronumpy/pypy/module/__builtin__/compiling.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/__builtin__/compiling.py (original) +++ pypy/branch/micronumpy/pypy/module/__builtin__/compiling.py Mon Aug 16 04:52:00 2010 @@ -38,7 +38,7 @@ str_ = space.str_w(w_source) ec = space.getexecutioncontext() - if flags & ~(ec.compiler.compiler_flags | consts.PyCF_AST_ONLY | + if flags & ~(ec.compiler.compiler_flags | consts.PyCF_ONLY_AST | consts.PyCF_DONT_IMPLY_DEDENT | consts.PyCF_SOURCE_IS_UTF8): raise OperationError(space.w_ValueError, space.wrap("compile() unrecognized flags")) @@ -53,7 +53,7 @@ "or 'eval' or 'single'")) if ast_node is None: - if flags & consts.PyCF_AST_ONLY: + if flags & consts.PyCF_ONLY_AST: mod = ec.compiler.compile_to_ast(str_, filename, mode, flags) return space.wrap(mod) else: Modified: pypy/branch/micronumpy/pypy/module/__builtin__/test/test_buffer.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/__builtin__/test/test_buffer.py (original) +++ pypy/branch/micronumpy/pypy/module/__builtin__/test/test_buffer.py Mon Aug 16 04:52:00 2010 @@ -1,8 +1,11 @@ """Tests some behaviour of the buffer type that is not tested in lib-python/2.5.2/test/test_types.py where the stdlib buffer tests live.""" import autopath +from pypy.conftest import gettestobjspace class AppTestBuffer: + def setup_class(cls): + cls.space = gettestobjspace(usemodules=('array',)) def test_unicode_buffer(self): import sys Modified: pypy/branch/micronumpy/pypy/module/_ast/__init__.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/_ast/__init__.py (original) +++ pypy/branch/micronumpy/pypy/module/_ast/__init__.py Mon Aug 16 04:52:00 2010 @@ -5,7 +5,7 @@ class Module(MixedModule): interpleveldefs = { - "PyCF_AST_ONLY" : "space.wrap(%s)" % consts.PyCF_AST_ONLY + "PyCF_ONLY_AST" : "space.wrap(%s)" % consts.PyCF_ONLY_AST } appleveldefs = {} Modified: pypy/branch/micronumpy/pypy/module/_ast/test/test_ast.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/_ast/test/test_ast.py (original) +++ pypy/branch/micronumpy/pypy/module/_ast/test/test_ast.py Mon Aug 16 04:52:00 2010 @@ -10,7 +10,7 @@ cls.w_get_ast = cls.space.appexec([], """(): def get_ast(source, mode="exec"): import _ast as ast - mod = compile(source, "", mode, ast.PyCF_AST_ONLY) + mod = compile(source, "", mode, ast.PyCF_ONLY_AST) assert isinstance(mod, ast.mod) return mod return get_ast""") Modified: pypy/branch/micronumpy/pypy/module/_codecs/test/test_codecs.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/_codecs/test/test_codecs.py (original) +++ pypy/branch/micronumpy/pypy/module/_codecs/test/test_codecs.py Mon Aug 16 04:52:00 2010 @@ -123,6 +123,10 @@ class AppTestPartialEvaluation: + def setup_class(cls): + space = gettestobjspace(usemodules=('array',)) + cls.space = space + def test_partial_utf8(self): import _codecs encoding = 'utf-8' Modified: pypy/branch/micronumpy/pypy/module/_demo/demo.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/_demo/demo.py (original) +++ pypy/branch/micronumpy/pypy/module/_demo/demo.py Mon Aug 16 04:52:00 2010 @@ -4,11 +4,14 @@ from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.tool import rffi_platform +from pypy.translator.tool.cbuild import ExternalCompilationInfo import sys, math time_t = rffi_platform.getsimpletype('time_t', '#include ', rffi.LONG) -time = rffi.llexternal('time', [rffi.VOIDP], time_t, includes=['time.h']) +eci = ExternalCompilationInfo(includes=['time.h']) +time = rffi.llexternal('time', [int], time_t, + compilation_info=eci) def get(space, name): w_module = space.getbuiltinmodule('_demo') @@ -20,10 +23,10 @@ w_DemoError = get(space, 'DemoError') msg = "repetition count must be > 0" raise OperationError(w_DemoError, space.wrap(msg)) - starttime = time(None) + starttime = time(0) for i in range(repetitions): space.call_function(w_callable) - endtime = time(None) + endtime = time(0) return space.wrap(endtime - starttime) measuretime.unwrap_spec = [ObjSpace, int, W_Root] @@ -62,11 +65,16 @@ self.x = space.int_w(w_value) def mytype_new(space, w_subtype, x): + if x == 3: + return space.wrap(MySubType(space, x)) return space.wrap(W_MyType(space, x)) mytype_new.unwrap_spec = [ObjSpace, W_Root, int] getset_x = GetSetProperty(W_MyType.fget_x, W_MyType.fset_x, cls=W_MyType) +class MySubType(W_MyType): + pass + W_MyType.typedef = TypeDef('MyType', __new__ = interp2app(mytype_new), x = getset_x, Modified: pypy/branch/micronumpy/pypy/module/_file/interp_file.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/_file/interp_file.py (original) +++ pypy/branch/micronumpy/pypy/module/_file/interp_file.py Mon Aug 16 04:52:00 2010 @@ -4,6 +4,7 @@ from pypy.rlib.rarithmetic import r_longlong from pypy.module._file.interp_stream import W_AbstractStream from pypy.module._file.interp_stream import StreamErrors, wrap_streamerror +from pypy.module.posix.interp_posix import dispatch_filename from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import ObjSpace, W_Root, Arguments from pypy.interpreter.typedef import TypeDef, GetSetProperty @@ -81,11 +82,11 @@ # file lock. They don't convert StreamErrors to OperationErrors, too. def direct___init__(self, w_name, mode='r', buffering=-1): - name = self.space.str_w(w_name) self.direct_close() self.w_name = w_name self.check_mode_ok(mode) - stream = streamio.open_file_as_stream(name, mode, buffering) + stream = dispatch_filename(streamio.open_file_as_stream)( + self.space, w_name, mode, buffering) fd = stream.try_to_find_file_descriptor() self.fdopenstream(stream, fd, mode) Modified: pypy/branch/micronumpy/pypy/module/_file/test/test_file.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/_file/test/test_file.py (original) +++ pypy/branch/micronumpy/pypy/module/_file/test/test_file.py Mon Aug 16 04:52:00 2010 @@ -125,6 +125,15 @@ assert type(res) is str f.close() + def test_unicode_filename(self): + import sys + try: + u'\xe9'.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + skip("encoding not good enough") + f = self.file(self.temppath + u'\xe9', "w") + f.close() + def test_oserror_has_filename(self): try: f = self.file("file that is clearly not there") Modified: pypy/branch/micronumpy/pypy/module/_file/test/test_file_extra.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/_file/test/test_file_extra.py (original) +++ pypy/branch/micronumpy/pypy/module/_file/test/test_file_extra.py Mon Aug 16 04:52:00 2010 @@ -353,6 +353,10 @@ class AppTestAFewExtra: + def setup_class(cls): + space = gettestobjspace(usemodules=('array',)) + cls.space = space + def setup_method(self, method): fn = str(udir.join('temptestfile')) self.w_temptestfile = self.space.wrap(fn) Modified: pypy/branch/micronumpy/pypy/module/_rawffi/test/test__rawffi.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/_rawffi/test/test__rawffi.py (original) +++ pypy/branch/micronumpy/pypy/module/_rawffi/test/test__rawffi.py Mon Aug 16 04:52:00 2010 @@ -677,7 +677,12 @@ a = A(1) a[0] = -1234 a.free() - + + def test_long_with_fromaddress(self): + import _rawffi + addr = -1 + raises(ValueError, _rawffi.Array('u').fromaddress, addr, 100) + def test_passing_raw_pointers(self): import _rawffi lib = _rawffi.CDLL(self.lib_name) Modified: pypy/branch/micronumpy/pypy/module/_socket/test/test_sock_app.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/_socket/test/test_sock_app.py (original) +++ pypy/branch/micronumpy/pypy/module/_socket/test/test_sock_app.py Mon Aug 16 04:52:00 2010 @@ -4,7 +4,7 @@ from pypy.tool.udir import udir def setup_module(mod): - mod.space = gettestobjspace(usemodules=['_socket']) + mod.space = gettestobjspace(usemodules=['_socket', 'array']) global socket import socket mod.w_socket = space.appexec([], "(): import _socket as m; return m") Modified: pypy/branch/micronumpy/pypy/module/_sre/test/test_app_sre.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/_sre/test/test_app_sre.py (original) +++ pypy/branch/micronumpy/pypy/module/_sre/test/test_app_sre.py Mon Aug 16 04:52:00 2010 @@ -87,7 +87,9 @@ class AppTestSreMatch: - + def setup_class(cls): + cls.space = gettestobjspace(usemodules=('array', )) + def test_copy(self): import re # copy support is disabled by default in _sre.c Modified: pypy/branch/micronumpy/pypy/module/_stackless/interp_coroutine.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/_stackless/interp_coroutine.py (original) +++ pypy/branch/micronumpy/pypy/module/_stackless/interp_coroutine.py Mon Aug 16 04:52:00 2010 @@ -265,10 +265,14 @@ instr += 1 oparg = ord(code[instr]) | ord(code[instr + 1]) << 8 nargs = oparg & 0xff + nkwds = (oparg >> 8) & 0xff if space.config.objspace.opcodes.CALL_METHOD and opcode == map['CALL_METHOD']: - chain = resume_state_create(chain, 'CALL_METHOD', frame, - nargs) - elif opcode == map['CALL_FUNCTION'] and (oparg >> 8) & 0xff == 0: + if nkwds == 0: # only positional arguments + chain = resume_state_create(chain, 'CALL_METHOD', frame, + nargs) + else: # includes keyword arguments + chain = resume_state_create(chain, 'CALL_METHOD_KW', frame) + elif opcode == map['CALL_FUNCTION'] and nkwds == 0: # Only positional arguments # case1: ("CALL_FUNCTION", f, nargs, returns=w_result) chain = resume_state_create(chain, 'CALL_FUNCTION', frame, Modified: pypy/branch/micronumpy/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/cpyext/api.py (original) +++ pypy/branch/micronumpy/pypy/module/cpyext/api.py Mon Aug 16 04:52:00 2010 @@ -1,5 +1,5 @@ import ctypes -import sys +import sys, os import atexit import py @@ -896,6 +896,8 @@ initfunctype = lltype.Ptr(lltype.FuncType([], lltype.Void)) @unwrap_spec(ObjSpace, str, str) def load_extension_module(space, path, name): + if os.sep not in path: + path = os.curdir + os.sep + path # force a '/' in the path state = space.fromcache(State) state.package_context = name try: Modified: pypy/branch/micronumpy/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/cpyext/methodobject.py (original) +++ pypy/branch/micronumpy/pypy/module/cpyext/methodobject.py Mon Aug 16 04:52:00 2010 @@ -100,7 +100,11 @@ return generic_cpy_call(space, self.ml.c_ml_meth, w_self, w_arg) def get_doc(space, self): - return space.wrap(rffi.charp2str(self.ml.c_ml_doc)) + doc = self.ml.c_ml_doc + if doc: + return space.wrap(rffi.charp2str(doc)) + else: + return space.w_None class W_PyCMethodObject(W_PyCFunctionObject): Modified: pypy/branch/micronumpy/pypy/module/fcntl/test/test_fcntl.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/fcntl/test/test_fcntl.py (original) +++ pypy/branch/micronumpy/pypy/module/fcntl/test/test_fcntl.py Mon Aug 16 04:52:00 2010 @@ -13,7 +13,7 @@ class AppTestFcntl: def setup_class(cls): - space = gettestobjspace(usemodules=('fcntl',)) + space = gettestobjspace(usemodules=('fcntl', 'array')) cls.space = space tmpprefix = str(udir.ensure('test_fcntl', dir=1).join('tmp_')) cls.w_tmp = space.wrap(tmpprefix) Modified: pypy/branch/micronumpy/pypy/module/marshal/test/test_marshalimpl.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/marshal/test/test_marshalimpl.py (original) +++ pypy/branch/micronumpy/pypy/module/marshal/test/test_marshalimpl.py Mon Aug 16 04:52:00 2010 @@ -1,9 +1,13 @@ from pypy.module.marshal import interp_marshal from pypy.interpreter.error import OperationError +from pypy.conftest import gettestobjspace import sys class AppTestMarshalMore: + def setup_class(cls): + space = gettestobjspace(usemodules=('array',)) + cls.space = space def test_long_0(self): import marshal Modified: pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py Mon Aug 16 04:52:00 2010 @@ -18,18 +18,12 @@ from pypy.module.micronumpy.array import squeeze_slice, squeeze_shape from pypy.module.micronumpy.array import shape_prefix -def index_w(space, w_index): - return space.int_w(space.index(w_index)) - class MicroIter(Wrappable): - _immutable_fields_ = ['array', 'step', 'stop', 'ndim'] + _immutable_fields_ = ['step', 'stop', 'ndim'] # XXX: removed array def __init__(self, array): - #print "MicroIter.__init__(array=%s)" % array # XXX: DEBUG self.array = array self.i = 0 - #self.i = array.slice_starts[array.offset] # XXX: DEBUG self.index = array.slice_starts[:] - #print "slice_starts:", self.index # XXX: DEBUG self.step = array.slice_steps[array.offset] self.stop = array.shape[array.offset] self.ndim = len(array.shape[array.offset:]) @@ -52,9 +46,7 @@ self.index[self.array.offset] += self.step return space.wrap(ar) elif self.ndim == 1: - #print "getting at index:", self.index # XXX: DEBUG next = self.array.getitem(space, self.array.flatten_index(self.index)) - #print "done (got '%s')" % next # XXX: DEBUG self.i += 1 self.index[self.array.offset] += self.step return next @@ -71,7 +63,7 @@ ) class MicroArray(BaseNumArray): - _immutable_fields_ = ['parent', 'shape', 'strides', 'offset', 'slice_starts'] + _immutable_fields_ = ['shape', 'strides', 'offset', 'slice_starts'] # XXX: removed parent def __init__(self, shape, dtype, order='C', strides=[], parent=None, offset=0, slice_starts=[], slice_steps=[]): assert dtype is not None @@ -81,7 +73,6 @@ self.order = order self.offset = offset - #print "__init__(self=%s, slice_starts=%s)" % (self, slice_starts) # XXX: DEBUG self.slice_starts = slice_starts[:] for i in range(len(shape) - len(slice_starts)): self.slice_starts.append(0) @@ -90,17 +81,15 @@ for i in range(len(shape) - len(slice_steps)): self.slice_steps.append(1) - dtype = dtype.dtype #XXX: ugly - size = size_from_shape(shape) self.strides = strides[:] stridelen = len(self.strides) for i in range(len(self.shape) - stridelen): - self.strides.append(self.stride(stridelen + i)) # XXX calling self.stride repeatedly is a bit wasteful + self.strides.append(self.stride(stridelen + i)) if size > 0 and parent is None: - self.data = dtype.alloc(size) + self.data = dtype.dtype.alloc(size) elif parent is not None: self.data = parent.data else: @@ -114,49 +103,33 @@ """Helper function. Grabs a value at an offset into the data.""" try: - dtype = self.dtype.dtype #XXX: kinda ugly - return dtype.w_getitem(space, self.data, offset) + return self.dtype.dtype.w_getitem(space, self.data, offset) except IndexError, e: raise OperationError(space.w_IndexError, space.wrap("index out of bounds")) def setitem(self, space, index, w_value): - dtype = self.dtype.dtype #XXX: kinda ugly - dtype.w_setitem(space, self.data, index, w_value) #maybe hang onto w_dtype separately? + try: + self.dtype.dtype.w_setitem(space, self.data, index, w_value) + except IndexError, e: + raise OperationError(space.w_IndexError, + space.wrap("index out of bounds")) def flatten_slice_starts(self, slice_starts): """Computes offset into subarray from all information. Gives offset into subarray, not into data.""" offset = 0 - for i in range(len(index)): - offset += (self.slice_steps[i] * index[i]) * self.strides[i] + for i in range(len(slice_starts)): + offset += (self.slice_steps[i] * slice_starts[i]) * self.strides[i] return offset - def flatten_index(self, index): - """Computes offset into subarray from all information. - Gives offset into subarray, not into data.""" - offset = 0 - - #print 'strides(%s) * index(%s)' % (self.strides, index) # XXX: DEBUG - for i in range(len(index)): - #offset += (self.slice_starts[i] + self.slice_steps[i] * index[i]) * self.strides[i] - offset += (self.slice_steps[i] * index[i]) * self.strides[i] - #print "offset=%d" % offset # XXX: DEBUG - return offset + flatten_index = flatten_slice_starts # TODO: migrate to slice_starts for name? def stride(self, i): if self.order == 'C': return stride_row(self.shape, i) # row order for C elif self.order == 'F': - return stride_column(self.shape, i) # - else: - raise NotImplementedError("Unknown order: '%s'" % self.order) - - def opposite_stride(self, i): - if self.order == 'C': # C for C not Column, but this is opposite return stride_column(self.shape, i) - elif self.order == 'F': - return stride_row(self.shape, i) else: raise NotImplementedError("Unknown order: '%s'" % self.order) @@ -181,8 +154,11 @@ else:raise if isinstance(w_index, W_SliceObject): - # TODO: fast path for slice - w_index = space.newlist([w_index]) + start, stop, step, length = w_index.indices4(space, self.shape[0]) + slice_starts[self.offset] += start * slice_steps[self.offset] + shape[self.offset] = length + slice_steps[self.offset] *= step + return slice_starts, shape, slice_steps elif space.is_w(w_index, space.w_Ellipsis): return slice_starts, shape, slice_steps @@ -205,7 +181,6 @@ if isinstance(w_index, W_SliceObject): start, stop, step, length = w_index.indices4(space, self.shape[i]) - #print "start=%d, stop=%d, step=%d, length=%d" % (start, stop, step, length) # XXX: DEBUG slice_starts[self.offset + i] += start * slice_steps[self.offset + i] shape[self.offset + i] = length slice_steps[self.offset + i] *= step @@ -226,7 +201,6 @@ space.wrap("invalid index")) finally: pass - #print "slice_starts=%s, shape=%s, slice_steps=%s" % (slice_starts, shape, slice_steps) # XXX: DEBUG return slice_starts, shape, slice_steps def descr_getitem(self, space, w_index): @@ -235,8 +209,6 @@ size = size_from_shape(shape) if size == 1: - #print 'getting item with slice_starts=%s' % slice_starts # XXX: DEBUG - #print 'flattened=%d' % self.flatten_index(slice_starts) # XXX: DEBUG return self.getitem(space, self.flatten_index(slice_starts)) else: @@ -247,9 +219,6 @@ offset=prefix, # XXX: what do we do about shapes that needs to be squeezed out? slice_starts=slice_starts, slice_steps=slice_steps) - #print 'start:', ar.slice_starts # XXX: DEBUG - #print 'stop:', ar.shape # XXX: DEBUG - #print 'step:', ar.slice_steps # XXX: DEBUG return space.wrap(ar) descr_getitem.unwrap_spec = ['self', ObjSpace, W_Root] @@ -258,11 +227,11 @@ if len(shape) > 1: for i in range(shape[0]): self.set_slice_single_value(space, index, shape[1:], slice_steps[1:], w_value) - index[len(index) - len(shape)] += slice_steps[0] # XXX: reason + index[len(index) - len(shape)] += slice_steps[0] else: for i in range(shape[0]): self.setitem(space, self.flatten_index(index), w_value) - index[len(index)-1] += slice_steps[0] # XXX: don't do steps in range + index[len(index)-1] += slice_steps[0] def set_slice(self, space, slice_starts, shape, slice_steps, w_value): try: @@ -271,7 +240,7 @@ if length == 1: self.set_slice_single_value(space, slice_starts, shape, slice_steps, w_value) else: - raise OperationError(space.w_NotImplementedError, + raise OperationError(space.w_NotImplementedError, # XXX: TODO space.wrap("TODO")) except OperationError, e: if e.match(space, space.w_TypeError): @@ -407,11 +376,23 @@ def descr_new(space, w_cls, w_shape, w_dtype=NoneNotWrapped, w_buffer=NoneNotWrapped, w_offset=NoneNotWrapped, w_strides=NoneNotWrapped, order='C'): - from pypy.module.micronumpy import dtype - shape_w = unpack_shape(space, w_shape) - dtype_w = dtype.get(space, w_dtype) - result = MicroArray(shape_w, dtype_w) - #TODO: load from buffer + from pypy.module import micronumpy + shape_w = space.fixedview(w_shape) + dtype_w = micronumpy.dtype.get(space, w_dtype) + + if w_strides is not None: + strides_w = space.fixedview(w_strides) + else: + strides_w = [] + + if w_offset is not None: + offset_w = space.int_w(w_offset) + else: + offset_w = 0 + + result = MicroArray(shape_w, dtype_w, + order=order, + strides=strides_w) return space.wrap(result) descr_new.unwrap_spec = [ObjSpace, W_Root, W_Root, W_Root, W_Root, W_Root, @@ -428,10 +409,6 @@ __iter__ = interp2app(MicroArray.descr_iter), ) -def reconcile_shapes(space, a, b): - assert a == b, "Invalid assertion I think" # FIXME - return a - app_fill_array = gateway.applevel(""" def fill_array(array, data): _fill_array([], array, data) Modified: pypy/branch/micronumpy/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/micronumpy/pypy/module/posix/interp_posix.py Mon Aug 16 04:52:00 2010 @@ -1,8 +1,9 @@ from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped from pypy.rlib import rposix +from pypy.rlib.objectmodel import specialize from pypy.rlib.rarithmetic import r_longlong from pypy.rlib.unroll import unrolling_iterable -from pypy.interpreter.error import OperationError, wrap_oserror +from pypy.interpreter.error import OperationError, wrap_oserror, wrap_oserror2 from pypy.rpython.module.ll_os import RegisterOs from pypy.rpython.module import ll_os_stat from pypy.rpython.lltypesystem import rffi, lltype @@ -12,15 +13,78 @@ import os, sys _WIN = sys.platform == 'win32' -def open(space, fname, flag, mode=0777): +class FileEncoder: + def __init__(self, space, w_obj): + self.space = space + self.w_obj = w_obj + + def as_bytes(self): + from pypy.module.sys.interp_encoding import getfilesystemencoding + space = self.space + w_bytes = space.call_method(self.w_obj, 'encode', + getfilesystemencoding(space)) + return space.str_w(w_bytes) + + def as_unicode(self): + return self.space.unicode_w(self.w_obj) + +class FileDecoder: + def __init__(self, space, w_obj): + self.space = space + self.w_obj = w_obj + + def as_bytes(self): + return self.space.str_w(self.w_obj) + + def as_unicode(self): + from pypy.module.sys.interp_encoding import getfilesystemencoding + space = self.space + w_unicode = space.call_method(self.w_obj, 'decode', + getfilesystemencoding(space)) + return space.unicode_w(w_unicode) + + at specialize.memo() +def dispatch_filename(func, tag=0): + def dispatch(space, w_fname, *args): + if space.isinstance_w(w_fname, space.w_unicode): + fname = FileEncoder(space, w_fname) + return func(fname, *args) + else: + fname = space.str_w(w_fname) + return func(fname, *args) + return dispatch + + at specialize.memo() +def dispatch_filename_2(func): + def dispatch(space, w_fname1, w_fname2, *args): + if space.isinstance_w(w_fname1, space.w_unicode): + fname1 = FileEncoder(space, w_fname1) + if space.isinstance_w(w_fname2, space.w_unicode): + fname2 = FileEncoder(space, w_fname2) + return func(fname1, fname2, *args) + else: + fname2 = FileDecoder(space, w_fname2) + return func(fname1, fname2, *args) + else: + fname1 = FileDecoder(space, w_fname1) + if space.isinstance_w(w_fname2, space.w_unicode): + fname2 = FileEncoder(space, w_fname2) + return func(fname1, fname2, *args) + else: + fname2 = FileDecoder(space, w_fname2) + return func(fname1, fname2, *args) + return dispatch + +def open(space, w_fname, flag, mode=0777): """Open a file (for low level IO). Return a file descriptor (a small integer).""" - try: - fd = os.open(fname, flag, mode) + try: + fd = dispatch_filename(rposix.open)( + space, w_fname, flag, mode) except OSError, e: - raise wrap_oserror(space, e, fname) + raise wrap_oserror2(space, e, w_fname) return space.wrap(fd) -open.unwrap_spec = [ObjSpace, 'path', "c_int", "c_int"] +open.unwrap_spec = [ObjSpace, W_Root, "c_int", "c_int"] def lseek(space, fd, pos, how): """Set the current position of a file descriptor. Return the new position. @@ -159,7 +223,7 @@ return build_stat_result(space, st) fstat.unwrap_spec = [ObjSpace, "c_int"] -def stat(space, path): +def stat(space, w_path): """Perform a stat system call on the given path. Return an object with (at least) the following attributes: st_mode @@ -175,22 +239,22 @@ """ try: - st = os.stat(path) + st = dispatch_filename(rposix.stat)(space, w_path) except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) else: return build_stat_result(space, st) -stat.unwrap_spec = [ObjSpace, 'path'] +stat.unwrap_spec = [ObjSpace, W_Root] -def lstat(space, path): +def lstat(space, w_path): "Like stat(path), but do no follow symbolic links." try: - st = os.lstat(path) + st = dispatch_filename(rposix.lstat)(space, w_path) except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) else: return build_stat_result(space, st) -lstat.unwrap_spec = [ObjSpace, 'path'] +lstat.unwrap_spec = [ObjSpace, W_Root] class StatState(object): def __init__(self, space): @@ -231,7 +295,7 @@ raise wrap_oserror(space, e) dup2.unwrap_spec = [ObjSpace, "c_int", "c_int"] -def access(space, path, mode): +def access(space, w_path, mode): """ access(path, mode) -> 1 if granted, 0 otherwise @@ -242,12 +306,12 @@ existence, or the inclusive-OR of R_OK, W_OK, and X_OK. """ try: - ok = os.access(path, mode) - except OSError, e: - raise wrap_oserror(space, e, path) + ok = dispatch_filename(rposix.access)(space, w_path, mode) + except OSError, e: + raise wrap_oserror2(space, e, w_path) else: return space.wrap(ok) -access.unwrap_spec = [ObjSpace, str, "c_int"] +access.unwrap_spec = [ObjSpace, W_Root, "c_int"] def times(space): @@ -278,32 +342,38 @@ return space.wrap(rc) system.unwrap_spec = [ObjSpace, str] -def unlink(space, path): +def unlink(space, w_path): """Remove a file (same as remove(path)).""" try: - os.unlink(path) - except OSError, e: - raise wrap_oserror(space, e, path) -unlink.unwrap_spec = [ObjSpace, 'path'] + dispatch_filename(rposix.unlink)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +unlink.unwrap_spec = [ObjSpace, W_Root] -def remove(space, path): +def remove(space, w_path): """Remove a file (same as unlink(path)).""" try: - os.unlink(path) - except OSError, e: - raise wrap_oserror(space, e, path) -remove.unwrap_spec = [ObjSpace, 'path'] + dispatch_filename(rposix.unlink)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +remove.unwrap_spec = [ObjSpace, W_Root] -def _getfullpathname(space, path): +def _getfullpathname(space, w_path): """helper for ntpath.abspath """ - posix = __import__(os.name) # nt specific try: - fullpath = posix._getfullpathname(path) + if space.isinstance_w(w_path, space.w_unicode): + path = FileEncoder(space, w_path) + fullpath = rposix._getfullpathname(path) + w_fullpath = space.wrap(fullpath) + else: + path = space.str_w(w_path) + fullpath = rposix._getfullpathname(path) + w_fullpath = space.wrap(fullpath) except OSError, e: - raise wrap_oserror(space, e, path) - else: - return space.wrap(fullpath) -_getfullpathname.unwrap_spec = [ObjSpace, str] + raise wrap_oserror2(space, e, w_path) + else: + return w_fullpath +_getfullpathname.unwrap_spec = [ObjSpace, W_Root] def getcwd(space): """Return the current working directory.""" @@ -315,35 +385,46 @@ return space.wrap(cur) getcwd.unwrap_spec = [ObjSpace] -def getcwdu(space): - """Return the current working directory as a unicode string.""" - # XXX ascii encoding for now - return space.call_method(getcwd(space), 'decode') +if sys.platform == 'win32': + def getcwdu(space): + """Return the current working directory as a unicode string.""" + try: + cur = os.getcwdu() + except OSError, e: + raise wrap_oserror(space, e) + else: + return space.wrap(cur) +else: + def getcwdu(space): + """Return the current working directory as a unicode string.""" + filesystemencoding = space.sys.filesystemencoding + return space.call_method(getcwd(space), 'decode', + space.wrap(filesystemencoding)) getcwdu.unwrap_spec = [ObjSpace] -def chdir(space, path): +def chdir(space, w_path): """Change the current working directory to the specified path.""" try: - os.chdir(path) - except OSError, e: - raise wrap_oserror(space, e, path) -chdir.unwrap_spec = [ObjSpace, str] + dispatch_filename(rposix.chdir)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +chdir.unwrap_spec = [ObjSpace, W_Root] -def mkdir(space, path, mode=0777): +def mkdir(space, w_path, mode=0777): """Create a directory.""" try: - os.mkdir(path, mode) - except OSError, e: - raise wrap_oserror(space, e, path) -mkdir.unwrap_spec = [ObjSpace, str, "c_int"] + dispatch_filename(rposix.mkdir)(space, w_path, mode) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +mkdir.unwrap_spec = [ObjSpace, W_Root, "c_int"] -def rmdir(space, path): +def rmdir(space, w_path): """Remove a directory.""" try: - os.rmdir(path) - except OSError, e: - raise wrap_oserror(space, e, path) -rmdir.unwrap_spec = [ObjSpace, str] + dispatch_filename(rposix.rmdir)(space, w_path) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +rmdir.unwrap_spec = [ObjSpace, W_Root] def strerror(space, errno): """Translate an error code to a message string.""" @@ -410,7 +491,7 @@ unsetenv.unwrap_spec = [ObjSpace, str] -def listdir(space, dirname): +def listdir(space, w_dirname): """Return a list containing the names of the entries in the directory. \tpath: path of directory to list @@ -418,12 +499,18 @@ The list is in arbitrary order. It does not include the special entries '.' and '..' even if they are present in the directory.""" try: - result = os.listdir(dirname) + if space.isinstance_w(w_dirname, space.w_unicode): + dirname = FileEncoder(space, w_dirname) + result = rposix.listdir(dirname) + result_w = [space.wrap(s) for s in result] + else: + dirname = space.str_w(w_dirname) + result = rposix.listdir(dirname) + result_w = [space.wrap(s) for s in result] except OSError, e: - raise wrap_oserror(space, e, dirname) - result_w = [space.wrap(s) for s in result] + raise wrap_oserror2(space, e, w_dirname) return space.newlist(result_w) -listdir.unwrap_spec = [ObjSpace, str] +listdir.unwrap_spec = [ObjSpace, W_Root] def pipe(space): "Create a pipe. Returns (read_end, write_end)." @@ -434,21 +521,21 @@ return space.newtuple([space.wrap(fd1), space.wrap(fd2)]) pipe.unwrap_spec = [ObjSpace] -def chmod(space, path, mode): +def chmod(space, w_path, mode): "Change the access permissions of a file." - try: - os.chmod(path, mode) - except OSError, e: - raise wrap_oserror(space, e, path) -chmod.unwrap_spec = [ObjSpace, str, "c_int"] + try: + dispatch_filename(rposix.chmod)(space, w_path, mode) + except OSError, e: + raise wrap_oserror2(space, e, w_path) +chmod.unwrap_spec = [ObjSpace, W_Root, "c_int"] -def rename(space, old, new): +def rename(space, w_old, w_new): "Rename a file or directory." - try: - os.rename(old, new) - except OSError, e: + try: + dispatch_filename_2(rposix.rename)(space, w_old, w_new) + except OSError, e: raise wrap_oserror(space, e) -rename.unwrap_spec = [ObjSpace, str, str] +rename.unwrap_spec = [ObjSpace, W_Root, W_Root] def umask(space, mask): "Set the current numeric umask and return the previous umask." @@ -576,7 +663,7 @@ raise wrap_oserror(space, e) execve.unwrap_spec = [ObjSpace, str, W_Root, W_Root] -def utime(space, path, w_tuple): +def utime(space, w_path, w_tuple): """ utime(path, (atime, mtime)) utime(path, None) @@ -585,10 +672,10 @@ """ if space.is_w(w_tuple, space.w_None): try: - os.utime(path, None) + dispatch_filename(rposix.utime, 1)(space, w_path, None) return except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) try: msg = "utime() arg 2 must be a tuple (atime, mtime) or None" args_w = space.fixedview(w_tuple) @@ -596,14 +683,14 @@ raise OperationError(space.w_TypeError, space.wrap(msg)) actime = space.float_w(args_w[0]) modtime = space.float_w(args_w[1]) - os.utime(path, (actime, modtime)) + dispatch_filename(rposix.utime, 2)(space, w_path, (actime, modtime)) except OSError, e: - raise wrap_oserror(space, e, path) + raise wrap_oserror2(space, e, w_path) except OperationError, e: if not e.match(space, space.w_TypeError): raise raise OperationError(space.w_TypeError, space.wrap(msg)) -utime.unwrap_spec = [ObjSpace, str, W_Root] +utime.unwrap_spec = [ObjSpace, W_Root, W_Root] def setsid(space): """setsid() -> pid Modified: pypy/branch/micronumpy/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/posix/test/test_posix2.py (original) +++ pypy/branch/micronumpy/pypy/module/posix/test/test_posix2.py Mon Aug 16 04:52:00 2010 @@ -32,6 +32,9 @@ # even when running on top of CPython 2.4. os.stat_float_times(True) + # Initialize sys.filesystemencoding + space.call_method(space.getbuiltinmodule('sys'), 'getfilesystemencoding') + def need_sparse_files(): if sys.platform == 'darwin': py.test.skip("no sparse files on default Mac OS X file system") @@ -706,6 +709,28 @@ except OSError: pass +class AppTestUnicodeFilename: + def setup_class(cls): + ufilename = (unicode(udir.join('test_unicode_filename_')) + + u'\u65e5\u672c.txt') # "Japan" + try: + f = file(ufilename, 'w') + except UnicodeEncodeError: + py.test.skip("encoding not good enough") + f.write("test") + f.close() + cls.space = space + cls.w_filename = space.wrap(ufilename) + cls.w_posix = space.appexec([], GET_POSIX) + + def test_open(self): + fd = self.posix.open(self.filename, self.posix.O_RDONLY) + try: + content = self.posix.read(fd, 50) + finally: + self.posix.close(fd) + assert content == "test" + class TestPexpect(object): # XXX replace with AppExpectTest class as soon as possible Modified: pypy/branch/micronumpy/pypy/module/pypyjit/policy.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/pypyjit/policy.py (original) +++ pypy/branch/micronumpy/pypy/module/pypyjit/policy.py Mon Aug 16 04:52:00 2010 @@ -11,7 +11,7 @@ if '.' in modname: modname, _ = modname.split('.', 1) if modname in ['pypyjit', 'signal', 'micronumpy', 'math', 'exceptions', - 'imp', 'sys']: + 'imp', 'sys', 'array']: return True return False Modified: pypy/branch/micronumpy/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/micronumpy/pypy/module/pypyjit/test/test_pypy_c.py Mon Aug 16 04:52:00 2010 @@ -615,6 +615,237 @@ return total ''', 170, ([], 4999450000L)) + def test_boolrewrite_invers(self): + for a, b, res, ops in (('2000', '2000', 20001000, 51), + ( '500', '500', 15001500, 81), + ( '300', '600', 16001700, 83), + ( 'a', 'b', 16001700, 89), + ( 'a', 'a', 13001700, 85)): + + self.run_source(''' + def main(): + sa = 0 + a = 300 + b = 600 + for i in range(1000): + if i < %s: sa += 1 + else: sa += 2 + if i >= %s: sa += 10000 + else: sa += 20000 + return sa + '''%(a, b), ops, ([], res)) + + def test_boolrewrite_reflex(self): + for a, b, res, ops in (('2000', '2000', 10001000, 51), + ( '500', '500', 15001500, 81), + ( '300', '600', 14001700, 83), + ( 'a', 'b', 14001700, 89), + ( 'a', 'a', 17001700, 85)): + + self.run_source(''' + def main(): + sa = 0 + a = 300 + b = 600 + for i in range(1000): + if i < %s: sa += 1 + else: sa += 2 + if %s > i: sa += 10000 + else: sa += 20000 + return sa + '''%(a, b), ops, ([], res)) + + + def test_boolrewrite_correct_invers(self): + def opval(i, op, a): + if eval('%d %s %d' % (i, op, a)): return 1 + return 2 + + ops = ('<', '>', '<=', '>=', '==', '!=') + for op1 in ops: + for op2 in ops: + for a,b in ((500, 500), (300, 600)): + res = 0 + res += opval(a-1, op1, a) * (a) + res += opval( a, op1, a) + res += opval(a+1, op1, a) * (1000 - a - 1) + res += opval(b-1, op2, b) * 10000 * (b) + res += opval( b, op2, b) * 10000 + res += opval(b+1, op2, b) * 10000 * (1000 - b - 1) + + self.run_source(''' + def main(): + sa = 0 + for i in range(1000): + if i %s %d: sa += 1 + else: sa += 2 + if i %s %d: sa += 10000 + else: sa += 20000 + return sa + '''%(op1, a, op2, b), 83, ([], res)) + + self.run_source(''' + def main(): + sa = 0 + i = 0.0 + while i < 250.0: + if i %s %f: sa += 1 + else: sa += 2 + if i %s %f: sa += 10000 + else: sa += 20000 + i += 0.25 + return sa + '''%(op1, float(a)/4.0, op2, float(b)/4.0), 109, ([], res)) + + + def test_boolrewrite_correct_reflex(self): + def opval(i, op, a): + if eval('%d %s %d' % (i, op, a)): return 1 + return 2 + + ops = ('<', '>', '<=', '>=', '==', '!=') + for op1 in ops: + for op2 in ops: + for a,b in ((500, 500), (300, 600)): + res = 0 + res += opval(a-1, op1, a) * (a) + res += opval( a, op1, a) + res += opval(a+1, op1, a) * (1000 - a - 1) + res += opval(b, op2, b-1) * 10000 * (b) + res += opval(b, op2, b) * 10000 + res += opval(b, op2, b+1) * 10000 * (1000 - b - 1) + + self.run_source(''' + def main(): + sa = 0 + for i in range(1000): + if i %s %d: sa += 1 + else: sa += 2 + if %d %s i: sa += 10000 + else: sa += 20000 + return sa + '''%(op1, a, b, op2), 83, ([], res)) + + self.run_source(''' + def main(): + sa = 0 + i = 0.0 + while i < 250.0: + if i %s %f: sa += 1 + else: sa += 2 + if %f %s i: sa += 10000 + else: sa += 20000 + i += 0.25 + return sa + '''%(op1, float(a)/4.0, float(b)/4.0, op2), 109, ([], res)) + + def test_boolrewrite_ptr(self): + compares = ('a == b', 'b == a', 'a != b', 'b != a', 'a == c', 'c != b') + for e1 in compares: + for e2 in compares: + a, b, c = 1, 2, 3 + if eval(e1): res = 752 * 1 + else: res = 752 * 2 + if eval(e2): res += 752 * 10000 + else: res += 752 * 20000 + a = b + if eval(e1): res += 248 * 1 + else: res += 248 * 2 + if eval(e2): res += 248 * 10000 + else: res += 248 * 20000 + + + if 'c' in e1 or 'c' in e2: + n = 337 + else: + n = 215 + + self.run_source(''' + class tst: + pass + def main(): + a = tst() + b = tst() + c = tst() + sa = 0 + for i in range(1000): + if %s: sa += 1 + else: sa += 2 + if %s: sa += 10000 + else: sa += 20000 + if i > 750: a = b + return sa + '''%(e1, e2), n, ([], res)) + + def test_array_sum(self): + for tc, maxops in zip('bhilBHILfd', (38,) * 6 + (40, 40, 41, 38)): + res = 19352859 + if tc in 'IL': + res = long(res) + elif tc in 'fd': + res = float(res) + + self.run_source(''' + from array import array + + def main(): + img = array("%s", range(127) * 5) * 484 + l, i = 0, 0 + while i < 640 * 480: + l += img[i] + i += 1 + return l + ''' % tc, maxops, ([], res)) + + def test_array_sum_char(self): + self.run_source(''' + from array import array + + def main(): + img = array("c", "Hello") * 130 * 480 + l, i = 0, 0 + while i < 640 * 480: + l += ord(img[i]) + i += 1 + return l + ''', 60, ([], 30720000)) + + def test_array_sum_unicode(self): + self.run_source(''' + from array import array + + def main(): + img = array("u", u"Hello") * 130 * 480 + l, i = 0, 0 + while i < 640 * 480: + if img[i] == u"l": + l += 1 + i += 1 + return l + ''', 65, ([], 122880)) + + def test_array_intimg(self): + for tc, maxops in zip('ilILd', (67, 67, 69, 69, 61)): + res = 73574560 + if tc in 'IL': + res = long(res) + elif tc in 'fd': + res = float(res) + + self.run_source(''' + from array import array + + def main(tc): + img = array(tc, range(3)) * (350 * 480) + intimg = array(tc, (0,)) * (640 * 480) + l, i = 0, 640 + while i < 640 * 480: + l = l + img[i] + intimg[i] = (intimg[i-640] + l) + i += 1 + return intimg[i - 1] + ''', maxops, ([tc], res)) + class AppTestJIT(PyPyCJITTests): def setup_class(cls): if not option.runappdirect: @@ -637,6 +868,7 @@ cls.counter = 0 cls.pypy_c = option.pypy_c + def has_info(pypy_c, option): g = os.popen('"%s" --info' % pypy_c, 'r') lines = g.readlines() Modified: pypy/branch/micronumpy/pypy/module/signal/interp_signal.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/signal/interp_signal.py (original) +++ pypy/branch/micronumpy/pypy/module/signal/interp_signal.py Mon Aug 16 04:52:00 2010 @@ -7,6 +7,7 @@ from pypy.translator.tool.cbuild import ExternalCompilationInfo import py from pypy.tool import autopath +from pypy.rlib import jit def setup(): for key, value in cpy_signal.__dict__.items(): @@ -159,10 +160,12 @@ return space.wrap(SIG_DFL) getsignal.unwrap_spec = [ObjSpace, int] + at jit.dont_look_inside def alarm(space, timeout): return space.wrap(c_alarm(timeout)) alarm.unwrap_spec = [ObjSpace, int] + at jit.dont_look_inside def pause(space): c_pause() return space.w_None @@ -173,6 +176,7 @@ raise OperationError(space.w_ValueError, space.wrap("signal number out of range")) + at jit.dont_look_inside def signal(space, signum, w_handler): """ signal(sig, action) -> action Modified: pypy/branch/micronumpy/pypy/module/thread/test/test_gil.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/thread/test/test_gil.py (original) +++ pypy/branch/micronumpy/pypy/module/thread/test/test_gil.py Mon Aug 16 04:52:00 2010 @@ -1,7 +1,6 @@ import time from pypy.module.thread import gil from pypy.module.thread.test import test_ll_thread -from pypy.rpython.lltypesystem import rffi from pypy.module.thread import ll_thread as thread from pypy.rlib.objectmodel import we_are_translated Modified: pypy/branch/micronumpy/pypy/objspace/std/callmethod.py ============================================================================== --- pypy/branch/micronumpy/pypy/objspace/std/callmethod.py (original) +++ pypy/branch/micronumpy/pypy/objspace/std/callmethod.py Mon Aug 16 04:52:00 2010 @@ -12,7 +12,7 @@ from pypy.interpreter import function from pypy.objspace.descroperation import object_getattribute -from pypy.rlib import rstack # for resume points +from pypy.rlib import jit, rstack # for resume points # This module exports two extra methods for StdObjSpaceFrame implementing # the LOOKUP_METHOD and CALL_METHOD opcodes in an efficient way, as well @@ -56,16 +56,41 @@ f.pushvalue(w_value) f.pushvalue(None) -def CALL_METHOD(f, nargs, *ignored): - # 'nargs' is the argument count excluding the implicit 'self' - w_self = f.peekvalue(nargs) - w_callable = f.peekvalue(nargs + 1) - n = nargs + (w_self is not None) - try: - w_result = f.space.call_valuestack(w_callable, n, f) - rstack.resume_point("CALL_METHOD", f, nargs, returns=w_result) - finally: - f.dropvalues(nargs + 2) + at jit.unroll_safe +def CALL_METHOD(f, oparg, *ignored): + # opargs contains the arg, and kwarg count, excluding the implicit 'self' + n_args = oparg & 0xff + n_kwargs = (oparg >> 8) & 0xff + w_self = f.peekvalue(n_args + (2 * n_kwargs)) + n = n_args + (w_self is not None) + + if not n_kwargs: + w_callable = f.peekvalue(n_args + (2 * n_kwargs) + 1) + try: + w_result = f.space.call_valuestack(w_callable, n, f) + rstack.resume_point("CALL_METHOD", f, n_args, returns=w_result) + finally: + f.dropvalues(n_args + 2) + else: + keywords = [None] * n_kwargs + keywords_w = [None] * n_kwargs + while True: + n_kwargs -= 1 + if n_kwargs < 0: + break + w_value = f.popvalue() + w_key = f.popvalue() + key = f.space.str_w(w_key) + keywords[n_kwargs] = key + keywords_w[n_kwargs] = w_value + + arguments = f.popvalues(n) # includes w_self if it is not None + args = f.argument_factory(arguments, keywords, keywords_w, None, None) + if w_self is None: + f.popvalue() # removes w_self, which is None + w_callable = f.popvalue() + w_result = f.space.call_args(w_callable, args) + rstack.resume_point("CALL_METHOD_KW", f, returns=w_result) f.pushvalue(w_result) Modified: pypy/branch/micronumpy/pypy/objspace/std/itertype.py ============================================================================== --- pypy/branch/micronumpy/pypy/objspace/std/itertype.py (original) +++ pypy/branch/micronumpy/pypy/objspace/std/itertype.py Mon Aug 16 04:52:00 2010 @@ -1,5 +1,6 @@ from pypy.interpreter import gateway from pypy.objspace.std.stdtypedef import StdTypeDef +from pypy.interpreter.error import OperationError # ____________________________________________________________ @@ -8,6 +9,11 @@ XXX to do: remove this __reduce__ method and do a registration with copy_reg, instead. """ + + # cpython does not support pickling iterators but stackless python do + #msg = 'Pickling for iterators dissabled as cpython does not support it' + #raise OperationError(space.w_TypeError, space.wrap(msg)) + from pypy.objspace.std.iterobject import W_AbstractSeqIterObject assert isinstance(w_self, W_AbstractSeqIterObject) from pypy.interpreter.mixedmodule import MixedModule Modified: pypy/branch/micronumpy/pypy/objspace/std/model.py ============================================================================== --- pypy/branch/micronumpy/pypy/objspace/std/model.py (original) +++ pypy/branch/micronumpy/pypy/objspace/std/model.py Mon Aug 16 04:52:00 2010 @@ -88,6 +88,7 @@ import pypy.objspace.std.default # register a few catch-all multimethods import pypy.objspace.std.marshal_impl # install marshal multimethods + import pypy.module.array # the set of implementation types self.typeorder = { @@ -140,6 +141,8 @@ # check if we missed implementations for implcls in _registered_implementations: + if hasattr(implcls, 'register'): + implcls.register(self.typeorder) assert (implcls in self.typeorder or implcls in self.imported_but_not_registered), ( "please add %r in StdTypeModel.typeorder" % (implcls,)) Modified: pypy/branch/micronumpy/pypy/objspace/std/test/test_callmethod.py ============================================================================== --- pypy/branch/micronumpy/pypy/objspace/std/test/test_callmethod.py (original) +++ pypy/branch/micronumpy/pypy/objspace/std/test/test_callmethod.py Mon Aug 16 04:52:00 2010 @@ -106,6 +106,15 @@ else: raise Exception("did not raise?") """ + + def test_kwargs(self): + exec """if 1: + class C(object): + def f(self, a): + return a + 2 + + assert C().f(a=3) == 5 + """ class AppTestCallMethodWithGetattributeShortcut(AppTestCallMethod): Modified: pypy/branch/micronumpy/pypy/rlib/objectmodel.py ============================================================================== --- pypy/branch/micronumpy/pypy/rlib/objectmodel.py (original) +++ pypy/branch/micronumpy/pypy/rlib/objectmodel.py Mon Aug 16 04:52:00 2010 @@ -19,32 +19,80 @@ # def f(... # -class _AttachSpecialization(object): +class _Specialize(object): + def memo(self): + """ Specialize functions based on argument values. All arguments has + to be constant at the compile time. The whole function call is replaced + by a call result then. + """ + def decorated_func(func): + func._annspecialcase_ = 'specialize:memo' + return func + return decorated_func - def __init__(self, tag): - self.tag = tag + def arg(self, *args): + """ Specialize function based on values of given positions of arguments. + They must be compile-time constants in order to work. + + There will be a copy of provided function for each combination + of given arguments on positions in args (that can lead to + exponential behavior!). + """ + def decorated_func(func): + func._annspecialcase_ = 'specialize:arg' + self._wrap(args) + return func - def __call__(self, *args): - if not args: - args = "" - else: - args = "("+','.join([repr(arg) for arg in args]) +")" - specialcase = "specialize:%s%s" % (self.tag, args) - - def specialize_decorator(func): - "NOT_RPYTHON" - func._annspecialcase_ = specialcase + return decorated_func + + def argtype(self, *args): + """ Specialize function based on types of arguments on given positions. + + There will be a copy of provided function for each combination + of given arguments on positions in args (that can lead to + exponential behavior!). + """ + def decorated_func(func): + func._annspecialcase_ = 'specialize:argtype' + self._wrap(args) return func - return specialize_decorator - -class _Specialize(object): + return decorated_func - def __getattr__(self, name): - return _AttachSpecialization(name) + def ll(self): + """ This is version of argtypes that cares about low-level types + (so it'll get additional copies for two different types of pointers + for example). Same warnings about exponential behavior apply. + """ + def decorated_func(func): + func._annspecialcase_ = 'specialize:ll' + return func + + return decorated_func + + def ll_and_arg(self, arg): + """ XXX what does that do? + """ + def decorated_func(func): + func._annspecialcase_ = 'specialize:ll_and_arg(%d)' % arg + return func + + return decorated_func + + def _wrap(self, args): + return "("+','.join([repr(arg) for arg in args]) +")" specialize = _Specialize() +def enforceargs(*args): + """ Decorate a function with forcing of RPython-level types on arguments. + None means no enforcing. + + XXX shouldn't we also add asserts in function body? + """ + def decorator(f): + f._annenforceargs_ = args + return f + return decorator + # ____________________________________________________________ class Symbolic(object): Modified: pypy/branch/micronumpy/pypy/rlib/rmmap.py ============================================================================== --- pypy/branch/micronumpy/pypy/rlib/rmmap.py (original) +++ pypy/branch/micronumpy/pypy/rlib/rmmap.py Mon Aug 16 04:52:00 2010 @@ -646,8 +646,14 @@ hintp = rffi.cast(PTR, hint.pos) res = c_mmap_safe(hintp, map_size, prot, flags, -1, 0) if res == rffi.cast(PTR, -1): - raise MemoryError - hint.pos += map_size + # some systems (some versions of OS/X?) complain if they + # are passed a non-zero address. Try again. + hintp = rffi.cast(PTR, 0) + res = c_mmap_safe(hintp, map_size, prot, flags, -1, 0) + if res == rffi.cast(PTR, -1): + raise MemoryError + else: + hint.pos += map_size return res alloc._annenforceargs_ = (int,) Modified: pypy/branch/micronumpy/pypy/rlib/rposix.py ============================================================================== --- pypy/branch/micronumpy/pypy/rlib/rposix.py (original) +++ pypy/branch/micronumpy/pypy/rlib/rposix.py Mon Aug 16 04:52:00 2010 @@ -3,6 +3,7 @@ from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rlib.rarithmetic import intmask +from pypy.rlib.objectmodel import specialize class CConstantErrno(CConstant): # these accessors are used when calling get_errno() or set_errno() @@ -42,3 +43,102 @@ os.close(fd) except OSError: pass + +#___________________________________________________________________ +# Wrappers around posix functions, that accept either strings, or +# instances with a "as_bytes()" method. +# - pypy.modules.posix.interp_posix passes an object containing a unicode path +# which can encode itself with sys.filesystemencoding. +# - but pypy.rpython.module.ll_os.py on Windows will replace these functions +# with other wrappers that directly handle unicode strings. + at specialize.argtype(0) +def open(path, flags, mode): + if isinstance(path, str): + return os.open(path, flags, mode) + else: + return os.open(path.as_bytes(), flags, mode) + + at specialize.argtype(0) +def stat(path): + if isinstance(path, str): + return os.stat(path) + else: + return os.stat(path.as_bytes()) + + at specialize.argtype(0) +def lstat(path): + if isinstance(path, str): + return os.lstat(path) + else: + return os.lstat(path.as_bytes()) + + at specialize.argtype(0) +def unlink(path): + if isinstance(path, str): + return os.unlink(path) + else: + return os.unlink(path.as_bytes()) + + at specialize.argtype(0, 1) +def rename(path1, path2): + if isinstance(path1, str): + return os.rename(path1, path2) + else: + return os.rename(path1.as_bytes(), path2.as_bytes()) + + at specialize.argtype(0) +def listdir(dirname): + if isinstance(dirname, str): + return os.listdir(dirname) + else: + return os.listdir(dirname.as_bytes()) + + at specialize.argtype(0) +def access(path, mode): + if isinstance(path, str): + return os.access(path, mode) + else: + return os.access(path.as_bytes(), mode) + + at specialize.argtype(0) +def chmod(path, mode): + if isinstance(path, str): + return os.chmod(path, mode) + else: + return os.chmod(path.as_bytes(), mode) + + at specialize.argtype(0, 1) +def utime(path, times): + if isinstance(path, str): + return os.utime(path, times) + else: + return os.utime(path.as_bytes(), times) + + at specialize.argtype(0) +def chdir(path): + if isinstance(path, str): + return os.chdir(path) + else: + return os.chdir(path.as_bytes()) + + at specialize.argtype(0) +def mkdir(path, mode=0777): + if isinstance(path, str): + return os.mkdir(path, mode) + else: + return os.mkdir(path.as_bytes(), mode) + + at specialize.argtype(0) +def rmdir(path): + if isinstance(path, str): + return os.rmdir(path) + else: + return os.rmdir(path.as_bytes()) + +if os.name == 'nt': + import nt + def _getfullpathname(path): + if isinstance(path, str): + return nt._getfullpathname(path) + else: + return nt._getfullpathname(path.as_bytes()) Modified: pypy/branch/micronumpy/pypy/rlib/rwin32.py ============================================================================== --- pypy/branch/micronumpy/pypy/rlib/rwin32.py (original) +++ pypy/branch/micronumpy/pypy/rlib/rwin32.py Mon Aug 16 04:52:00 2010 @@ -31,6 +31,7 @@ DWORD = rffi_platform.SimpleType("DWORD", rffi.UINT) BOOL = rffi_platform.SimpleType("BOOL", rffi.LONG) BYTE = rffi_platform.SimpleType("BYTE", rffi.UCHAR) + WCHAR = rffi_platform.SimpleType("WCHAR", rffi.UCHAR) INT = rffi_platform.SimpleType("INT", rffi.INT) LONG = rffi_platform.SimpleType("LONG", rffi.LONG) PLONG = rffi_platform.SimpleType("PLONG", rffi.LONGP) @@ -38,6 +39,8 @@ LPCVOID = rffi_platform.SimpleType("LPCVOID", rffi.VOIDP) LPSTR = rffi_platform.SimpleType("LPSTR", rffi.CCHARP) LPCSTR = rffi_platform.SimpleType("LPCSTR", rffi.CCHARP) + LPWSTR = rffi_platform.SimpleType("LPWSTR", rffi.CWCHARP) + LPCWSTR = rffi_platform.SimpleType("LPCWSTR", rffi.CWCHARP) LPDWORD = rffi_platform.SimpleType("LPDWORD", rffi.INTP) SIZE_T = rffi_platform.SimpleType("SIZE_T", rffi.SIZE_T) ULONG_PTR = rffi_platform.SimpleType("ULONG_PTR", rffi.ULONG) @@ -87,6 +90,10 @@ GetLastError = winexternal('GetLastError', [], DWORD) SetLastError = winexternal('SetLastError', [DWORD], lltype.Void) + # In tests, the first call to GetLastError is always wrong, because error + # is hidden by operations in ll2ctypes. Call it now. + GetLastError() + LoadLibrary = winexternal('LoadLibraryA', [rffi.CCHARP], rffi.VOIDP) GetProcAddress = winexternal('GetProcAddress', [rffi.VOIDP, rffi.CCHARP], @@ -129,13 +136,29 @@ } return 0; }''') - exename = static_platform.compile( - [cfile], ExternalCompilationInfo(), - outputfilename = "dosmaperr", - standalone=True) - output = os.popen(str(exename)) - errors = dict(map(int, line.split()) - for line in output) + try: + exename = static_platform.compile( + [cfile], ExternalCompilationInfo(), + outputfilename = "dosmaperr", + standalone=True) + except WindowsError: + # Fallback for the mingw32 compiler + errors = { + 2: 2, 3: 2, 4: 24, 5: 13, 6: 9, 7: 12, 8: 12, 9: 12, 10: 7, + 11: 8, 15: 2, 16: 13, 17: 18, 18: 2, 19: 13, 20: 13, 21: 13, + 22: 13, 23: 13, 24: 13, 25: 13, 26: 13, 27: 13, 28: 13, + 29: 13, 30: 13, 31: 13, 32: 13, 33: 13, 34: 13, 35: 13, + 36: 13, 53: 2, 65: 13, 67: 2, 80: 17, 82: 13, 83: 13, 89: 11, + 108: 13, 109: 32, 112: 28, 114: 9, 128: 10, 129: 10, 130: 9, + 132: 13, 145: 41, 158: 13, 161: 2, 164: 11, 167: 13, 183: 17, + 188: 8, 189: 8, 190: 8, 191: 8, 192: 8, 193: 8, 194: 8, + 195: 8, 196: 8, 197: 8, 198: 8, 199: 8, 200: 8, 201: 8, + 202: 8, 206: 2, 215: 11, 1816: 12, + } + else: + output = os.popen(str(exename)) + errors = dict(map(int, line.split()) + for line in output) return errors, errno.EINVAL # A bit like strerror... Modified: pypy/branch/micronumpy/pypy/rlib/streamio.py ============================================================================== --- pypy/branch/micronumpy/pypy/rlib/streamio.py (original) +++ pypy/branch/micronumpy/pypy/rlib/streamio.py Mon Aug 16 04:52:00 2010 @@ -38,7 +38,9 @@ # import os, sys +from pypy.rlib.objectmodel import specialize from pypy.rlib.rarithmetic import r_longlong, intmask +from pypy.rlib import rposix from os import O_RDONLY, O_WRONLY, O_RDWR, O_CREAT, O_TRUNC O_BINARY = getattr(os, "O_BINARY", 0) @@ -71,6 +73,7 @@ return s.join(string.split(c)) + at specialize.argtype(0) def open_file_as_stream(path, mode="r", buffering=-1): os_flags, universal, reading, writing, basemode, binary = decode_mode(mode) stream = open_path_helper(path, os_flags, basemode == "a") @@ -89,9 +92,10 @@ return construct_stream_tower(stream, buffering, universal, reading, writing, binary) + at specialize.argtype(0) def open_path_helper(path, os_flags, append): # XXX for now always return DiskFile - fd = os.open(path, os_flags, 0666) + fd = rposix.open(path, os_flags, 0666) if append: try: os.lseek(fd, 0, 2) Modified: pypy/branch/micronumpy/pypy/rlib/test/test_objectmodel.py ============================================================================== --- pypy/branch/micronumpy/pypy/rlib/test/test_objectmodel.py (original) +++ pypy/branch/micronumpy/pypy/rlib/test/test_objectmodel.py Mon Aug 16 04:52:00 2010 @@ -404,6 +404,13 @@ assert f._annspecialcase_ == 'specialize:arg(1)' +def test_enforceargs_decorator(): + @enforceargs(int, str, None) + def f(a, b, c): + pass + + assert f._annenforceargs_ == (int, str, None) + def getgraph(f, argtypes): from pypy.translator.translator import TranslationContext, graphof from pypy.translator.backendopt.all import backend_optimizations Modified: pypy/branch/micronumpy/pypy/rpython/extfunc.py ============================================================================== --- pypy/branch/micronumpy/pypy/rpython/extfunc.py (original) +++ pypy/branch/micronumpy/pypy/rpython/extfunc.py Mon Aug 16 04:52:00 2010 @@ -52,7 +52,10 @@ return super(ExtRegistryEntry, self).__getattr__(attr) raise exc, exc_inst, tb -def registering(func): +def registering(func, condition=True): + if not condition: + return lambda method: None + def decorator(method): method._registering_func = func return method @@ -63,11 +66,9 @@ func = getattr(ns, name) except AttributeError: condition = False + func = None - if condition: - return registering(func) - else: - return lambda method: None + return registering(func, condition=condition) class LazyRegisteringMeta(type): def __new__(self, _name, _type, _vars): @@ -167,8 +168,6 @@ return signature_args def compute_result_annotation(self, *args_s): - if hasattr(self, 'ann_hook'): - self.ann_hook() self.normalize_args(*args_s) # check arguments return self.signature_result @@ -235,7 +234,6 @@ def register_external(function, args, result=None, export_name=None, llimpl=None, ooimpl=None, llfakeimpl=None, oofakeimpl=None, - annotation_hook=None, sandboxsafe=False): """ function: the RPython function that will be rendered as an external function (e.g.: math.floor) @@ -244,7 +242,6 @@ export_name: the name of the function as it will be seen by the backends llimpl, ooimpl: optional; if provided, these RPython functions are called instead of the target function llfakeimpl, oofakeimpl: optional; if provided, they are called by the llinterpreter - annotationhook: optional; a callable that is called during annotation, useful for genc hacks sandboxsafe: use True if the function performs no I/O (safe for --sandbox) """ @@ -271,8 +268,6 @@ lltypefakeimpl = staticmethod(llfakeimpl) if oofakeimpl: ootypefakeimpl = staticmethod(oofakeimpl) - if annotation_hook: - ann_hook = staticmethod(annotation_hook) if export_name: FunEntry.__name__ = export_name Modified: pypy/branch/micronumpy/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/micronumpy/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/micronumpy/pypy/rpython/lltypesystem/ll2ctypes.py Mon Aug 16 04:52:00 2010 @@ -24,6 +24,7 @@ from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE from pypy.rpython import raddress from pypy.translator.platform import platform +from array import array def uaddressof(obj): return fixid(ctypes.addressof(obj)) @@ -756,7 +757,15 @@ elif T is lltype.Char: llobj = chr(cobj) elif T is lltype.UniChar: - llobj = unichr(cobj) + try: + llobj = unichr(cobj) + except (ValueError, OverflowError): + for tc in 'HIL': + if array(tc).itemsize == array('u').itemsize: + llobj = array('u', array(tc, (cobj,)).tostring())[0] + break + else: + raise elif T is lltype.Signed: llobj = cobj elif T is lltype.Bool: Modified: pypy/branch/micronumpy/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/micronumpy/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/micronumpy/pypy/rpython/lltypesystem/rffi.py Mon Aug 16 04:52:00 2010 @@ -176,6 +176,15 @@ # XXX leaks if a str2charp() fails with MemoryError # and was not the first in this function freeme = arg + elif TARGET == CWCHARP: + if arg is None: + arg = lltype.nullptr(CWCHARP.TO) # None => (wchar_t*)NULL + freeme = arg + elif isinstance(arg, unicode): + arg = unicode2wcharp(arg) + # XXX leaks if a unicode2wcharp() fails with MemoryError + # and was not the first in this function + freeme = arg elif _isfunctype(TARGET) and not _isllptr(arg): # XXX pass additional arguments if invoke_around_handlers: Modified: pypy/branch/micronumpy/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/micronumpy/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/micronumpy/pypy/rpython/lltypesystem/rstr.py Mon Aug 16 04:52:00 2010 @@ -2,7 +2,7 @@ from pypy.tool.pairtype import pairtype from pypy.rpython.error import TyperError from pypy.rlib.objectmodel import malloc_zero_filled, we_are_translated -from pypy.rlib.objectmodel import _hash_string +from pypy.rlib.objectmodel import _hash_string, enforceargs from pypy.rlib.debug import ll_assert from pypy.rlib.jit import purefunction from pypy.rpython.robject import PyObjRepr, pyobj_repr @@ -56,6 +56,7 @@ llmemory.itemoffsetof(TP.chars, 0) + llmemory.sizeof(CHAR_TP) * item) + @enforceargs(None, None, int, int, int) def copy_string_contents(src, dst, srcstart, dststart, length): assert srcstart >= 0 assert dststart >= 0 @@ -674,6 +675,7 @@ res_index += item_len i += 1 return result + ll_join_strs._annenforceargs_ = [int, None] def ll_join_chars(length, chars): # no need to optimize this, will be replaced by string builder Modified: pypy/branch/micronumpy/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/micronumpy/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/micronumpy/pypy/rpython/module/ll_os.py Mon Aug 16 04:52:00 2010 @@ -6,12 +6,15 @@ # might be found in doc/rffi.txt import os, sys, errno +import py from pypy.rpython.module.support import ll_strcpy, OOSupport -from pypy.tool.sourcetools import func_with_new_name +from pypy.tool.sourcetools import func_with_new_name, func_renamer from pypy.rlib.rarithmetic import r_longlong -from pypy.rpython.extfunc import BaseLazyRegistering +from pypy.rpython.extfunc import ( + BaseLazyRegistering, lazy_register, register_external) from pypy.rpython.extfunc import registering, registering_if, extdef -from pypy.annotation.model import SomeInteger, SomeString, SomeTuple, SomeFloat +from pypy.annotation.model import ( + SomeInteger, SomeString, SomeTuple, SomeFloat, SomeUnicodeString) from pypy.annotation.model import s_ImpossibleValue, s_None, s_Bool from pypy.rpython.lltypesystem import rffi from pypy.rpython.lltypesystem import lltype @@ -26,7 +29,99 @@ from pypy.rpython.lltypesystem.rstr import STR from pypy.rpython.annlowlevel import llstr from pypy.rlib import rgc -from pypy.rlib.objectmodel import keepalive_until_here +from pypy.rlib.objectmodel import keepalive_until_here, specialize + +def monkeypatch_rposix(posixfunc, unicodefunc, signature): + func_name = posixfunc.__name__ + + if hasattr(signature, '_default_signature_'): + signature = signature._default_signature_ + arglist = ['arg%d' % (i,) for i in range(len(signature))] + transformed_arglist = arglist[:] + for i, arg in enumerate(signature): + if arg is unicode: + transformed_arglist[i] = transformed_arglist[i] + '.as_unicode()' + + args = ', '.join(arglist) + transformed_args = ', '.join(transformed_arglist) + main_arg = 'arg%d' % (signature.index(unicode),) + + source = py.code.Source(""" + def %(func_name)s(%(args)s): + if isinstance(%(main_arg)s, str): + return posixfunc(%(args)s) + else: + return unicodefunc(%(transformed_args)s) + """ % locals()) + miniglobals = {'posixfunc' : posixfunc, + 'unicodefunc': unicodefunc, + '__name__': __name__, # for module name propagation + } + exec source.compile() in miniglobals + new_func = miniglobals[func_name] + specialized_args = [i for i in range(len(signature)) + if signature[i] in (unicode, None)] + new_func = specialize.argtype(*specialized_args)(new_func) + + # Monkeypatch the function in pypy.rlib.rposix + setattr(rposix, func_name, new_func) + +class StringTraits: + str = str + CHAR = rffi.CHAR + CCHARP = rffi.CCHARP + charp2str = staticmethod(rffi.charp2str) + str2charp = staticmethod(rffi.str2charp) + free_charp = staticmethod(rffi.free_charp) + + @staticmethod + def posix_function_name(name): + return underscore_on_windows + name + + @staticmethod + def ll_os_name(name): + return 'll_os.ll_os_' + name + +class UnicodeTraits: + str = unicode + CHAR = rffi.WCHAR_T + CCHARP = rffi.CWCHARP + charp2str = staticmethod(rffi.wcharp2unicode) + str2charp = staticmethod(rffi.unicode2wcharp) + free_charp = staticmethod(rffi.free_wcharp) + + @staticmethod + def posix_function_name(name): + return underscore_on_windows + 'w' + name + + @staticmethod + def ll_os_name(name): + return 'll_os.ll_os_w' + name + +def registering_str_unicode(posixfunc, condition=True): + if not condition: + return registering(None, condition=False) + + func_name = posixfunc.__name__ + + def register_posixfunc(self, method): + val = method(self, StringTraits()) + register_external(posixfunc, *val.def_args, **val.def_kwds) + + if sys.platform == 'win32': + val = method(self, UnicodeTraits()) + @func_renamer(func_name + "_unicode") + def unicodefunc(*args): + return posixfunc(*args) + register_external(unicodefunc, *val.def_args, **val.def_kwds) + signature = val.def_args[0] + monkeypatch_rposix(posixfunc, unicodefunc, signature) + + def decorator(method): + decorated = lambda self: register_posixfunc(self, method) + decorated._registering_func = posixfunc + return decorated + return decorator posix = __import__(os.name) @@ -282,8 +377,8 @@ return extdef([int, int], s_None, llimpl=dup2_llimpl, export_name="ll_os.ll_os_dup2") - @registering(os.utime) - def register_os_utime(self): + @registering_str_unicode(os.utime) + def register_os_utime(self, traits): UTIMBUFP = lltype.Ptr(self.UTIMBUF) os_utime = self.llexternal('utime', [rffi.CCHARP, UTIMBUFP], rffi.INT) @@ -336,6 +431,9 @@ # tp is known to be None, and one version where it is known # to be a tuple of 2 floats. if not _WIN32: + assert traits.str is str + + @specialize.argtype(1) def os_utime_llimpl(path, tp): if tp is None: error = os_utime(path, lltype.nullptr(UTIMBUFP.TO)) @@ -346,85 +444,13 @@ if error == -1: raise OSError(rposix.get_errno(), "os_utime failed") else: - from pypy.rlib import rwin32 - from pypy.rpython.module.ll_os_stat import time_t_to_FILE_TIME - - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h'], - ) - - FILE_WRITE_ATTRIBUTES = platform.ConstantInteger( - 'FILE_WRITE_ATTRIBUTES') - OPEN_EXISTING = platform.ConstantInteger( - 'OPEN_EXISTING') - FILE_FLAG_BACKUP_SEMANTICS = platform.ConstantInteger( - 'FILE_FLAG_BACKUP_SEMANTICS') - globals().update(platform.configure(CConfig)) - - CreateFile = rffi.llexternal( - 'CreateFileA', - [rwin32.LPCSTR, rwin32.DWORD, rwin32.DWORD, - rwin32.LPSECURITY_ATTRIBUTES, rwin32.DWORD, rwin32.DWORD, - rwin32.HANDLE], - rwin32.HANDLE, - calling_conv='win') - - GetSystemTime = rffi.llexternal( - 'GetSystemTime', - [lltype.Ptr(rwin32.SYSTEMTIME)], - lltype.Void, - calling_conv='win') - - SystemTimeToFileTime = rffi.llexternal( - 'SystemTimeToFileTime', - [lltype.Ptr(rwin32.SYSTEMTIME), - lltype.Ptr(rwin32.FILETIME)], - rwin32.BOOL, - calling_conv='win') - - SetFileTime = rffi.llexternal( - 'SetFileTime', - [rwin32.HANDLE, - lltype.Ptr(rwin32.FILETIME), - lltype.Ptr(rwin32.FILETIME), - lltype.Ptr(rwin32.FILETIME)], - rwin32.BOOL, - calling_conv = 'win') + from pypy.rpython.module.ll_win32file import make_utime_impl + os_utime_llimpl = make_utime_impl(traits) - def os_utime_llimpl(path, tp): - hFile = CreateFile(path, - FILE_WRITE_ATTRIBUTES, 0, - None, OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS, 0) - if hFile == rwin32.INVALID_HANDLE_VALUE: - raise rwin32.lastWindowsError() - ctime = lltype.nullptr(rwin32.FILETIME) - atime = lltype.malloc(rwin32.FILETIME, flavor='raw') - mtime = lltype.malloc(rwin32.FILETIME, flavor='raw') - try: - if tp is None: - now = lltype.malloc(rwin32.SYSTEMTIME, flavor='raw') - try: - GetSystemTime(now) - if (not SystemTimeToFileTime(now, atime) or - not SystemTimeToFileTime(now, mtime)): - raise rwin32.lastWindowsError() - finally: - lltype.free(now, flavor='raw') - else: - actime, modtime = tp - time_t_to_FILE_TIME(actime, atime) - time_t_to_FILE_TIME(modtime, mtime) - if not SetFileTime(hFile, ctime, atime, mtime): - raise rwin32.lastWindowsError() - finally: - rwin32.CloseHandle(hFile) - lltype.free(atime, flavor='raw') - lltype.free(mtime, flavor='raw') - os_utime_llimpl._annspecialcase_ = 'specialize:argtype(1)' - - s_string = SomeString() + if traits.str is str: + s_string = SomeString() + else: + s_string = SomeUnicodeString() s_tuple_of_2_floats = SomeTuple([SomeFloat(), SomeFloat()]) def os_utime_normalize_args(s_path, s_times): @@ -445,12 +471,12 @@ else: raise Exception("os.utime() arg 2 must be None or a tuple of " "2 floats, got %s" % (s_times,)) + os_utime_normalize_args._default_signature_ = [traits.str, None] return extdef(os_utime_normalize_args, s_None, "ll_os.ll_os_utime", llimpl=os_utime_llimpl) - @registering(os.times) def register_os_times(self): if sys.platform.startswith('win'): @@ -687,22 +713,21 @@ def register_os_setsid(self): return self.extdef_for_os_function_returning_int('setsid') - @registering(os.open) - def register_os_open(self): - os_open = self.llexternal(underscore_on_windows+'open', - [rffi.CCHARP, rffi.INT, rffi.MODE_T], + @registering_str_unicode(os.open) + def register_os_open(self, traits): + os_open = self.llexternal(traits.posix_function_name('open'), + [traits.CCHARP, rffi.INT, rffi.MODE_T], rffi.INT) - def os_open_llimpl(path, flags, mode): result = rffi.cast(rffi.LONG, os_open(path, flags, mode)) if result == -1: raise OSError(rposix.get_errno(), "os_open failed") return result - def os_open_oofakeimpl(o_path, flags, mode): - return os.open(o_path._str, flags, mode) + def os_open_oofakeimpl(path, flags, mode): + return os.open(OOSupport.from_rstr(path), flags, mode) - return extdef([str, int, int], int, "ll_os.ll_os_open", + return extdef([traits.str, int, int], int, traits.ll_os_name('open'), llimpl=os_open_llimpl, oofakeimpl=os_open_oofakeimpl) # ------------------------------- os.read ------------------------------- @@ -862,10 +887,10 @@ llimpl=fdatasync_llimpl, export_name="ll_os.ll_os_fdatasync") - @registering(os.access) - def register_os_access(self): - os_access = self.llexternal(underscore_on_windows + 'access', - [rffi.CCHARP, rffi.INT], + @registering_str_unicode(os.access) + def register_os_access(self, traits): + os_access = self.llexternal(traits.posix_function_name('access'), + [traits.CCHARP, rffi.INT], rffi.INT) if sys.platform.startswith('win'): @@ -882,44 +907,22 @@ def os_access_oofakeimpl(path, mode): return os.access(OOSupport.from_rstr(path), mode) - return extdef([str, int], s_Bool, llimpl=access_llimpl, - export_name="ll_os.ll_os_access", + return extdef([traits.str, int], s_Bool, llimpl=access_llimpl, + export_name=traits.ll_os_name("access"), oofakeimpl=os_access_oofakeimpl) - @registering_if(posix, '_getfullpathname') - def register_posix__getfullpathname(self): - from pypy.rlib import rwin32 + @registering_str_unicode(getattr(posix, '_getfullpathname', None), + condition=sys.platform=='win32') + def register_posix__getfullpathname(self, traits): # this nt function is not exposed via os, but needed # to get a correct implementation of os.abspath - # XXX why do we ignore WINAPI conventions everywhere? - LPSTRP = rffi.CArrayPtr(rwin32.LPSTR) - # XXX unicode? - GetFullPathName = self.llexternal( - 'GetFullPathNameA', - [rwin32.LPCSTR, - rwin32.DWORD, - rwin32.LPSTR, - rffi.CArrayPtr(rwin32.LPSTR)], - rwin32.DWORD) - - def _getfullpathname_llimpl(lpFileName): - nBufferLength = rwin32.MAX_PATH + 1 - lpBuffer = lltype.malloc(rwin32.LPSTR.TO, nBufferLength, flavor='raw') - try: - res = GetFullPathName( - lpFileName, rffi.cast(rwin32.DWORD, nBufferLength), - lpBuffer, lltype.nullptr(LPSTRP.TO)) - if res == 0: - raise rwin32.lastWindowsError("_getfullpathname failed") - result = rffi.charp2str(lpBuffer) - return result - finally: - lltype.free(lpBuffer, flavor='raw') + from pypy.rpython.module.ll_win32file import make_getfullpathname_impl + getfullpathname_llimpl = make_getfullpathname_impl(traits) - return extdef([str], # a single argument which is a str - str, # returns a string - "ll_os.posix__getfullpathname", - llimpl=_getfullpathname_llimpl) + return extdef([traits.str], # a single argument which is a str + traits.str, # returns a string + traits.ll_os_name('_getfullpathname'), + llimpl=getfullpathname_llimpl) @registering(os.getcwd) def register_os_getcwd(self): @@ -953,71 +956,42 @@ "ll_os.ll_os_getcwd", llimpl=os_getcwd_llimpl, oofakeimpl=os_getcwd_oofakeimpl) - @registering(os.listdir) - def register_os_listdir(self): - # we need a different approach on Windows and on Posix - if sys.platform.startswith('win'): - from pypy.rlib import rwin32 - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h'] - ) - WIN32_FIND_DATA = platform.Struct('struct _WIN32_FIND_DATAA', - [('cFileName', lltype.FixedSizeArray(rffi.CHAR, 1))]) - ERROR_FILE_NOT_FOUND = platform.ConstantInteger( - 'ERROR_FILE_NOT_FOUND') - ERROR_NO_MORE_FILES = platform.ConstantInteger( - 'ERROR_NO_MORE_FILES') + @registering(os.getcwdu, condition=sys.platform=='win32') + def register_os_getcwdu(self): + os_wgetcwd = self.llexternal(underscore_on_windows + 'wgetcwd', + [rffi.CWCHARP, rffi.SIZE_T], + rffi.CWCHARP) - config = platform.configure(CConfig) - WIN32_FIND_DATA = config['WIN32_FIND_DATA'] - ERROR_FILE_NOT_FOUND = config['ERROR_FILE_NOT_FOUND'] - ERROR_NO_MORE_FILES = config['ERROR_NO_MORE_FILES'] - LPWIN32_FIND_DATA = lltype.Ptr(WIN32_FIND_DATA) - - FindFirstFile = self.llexternal('FindFirstFile', - [rwin32.LPCSTR, LPWIN32_FIND_DATA], - rwin32.HANDLE) - FindNextFile = self.llexternal('FindNextFile', - [rwin32.HANDLE, LPWIN32_FIND_DATA], - rwin32.BOOL) - FindClose = self.llexternal('FindClose', - [rwin32.HANDLE], - rwin32.BOOL) + def os_getcwd_llimpl(): + bufsize = 256 + while True: + buf = lltype.malloc(rffi.CWCHARP.TO, bufsize, flavor='raw') + res = os_wgetcwd(buf, rffi.cast(rffi.SIZE_T, bufsize)) + if res: + break # ok + error = rposix.get_errno() + lltype.free(buf, flavor='raw') + if error != errno.ERANGE: + raise OSError(error, "getcwd failed") + # else try again with a larger buffer, up to some sane limit + bufsize *= 4 + if bufsize > 1024*1024: # xxx hard-coded upper limit + raise OSError(error, "getcwd result too large") + result = rffi.wcharp2unicode(res) + lltype.free(buf, flavor='raw') + return result - def os_listdir_llimpl(path): - if path and path[-1] not in ('/', '\\', ':'): - path += '/' - path += '*.*' - filedata = lltype.malloc(WIN32_FIND_DATA, flavor='raw') - try: - result = [] - hFindFile = FindFirstFile(path, filedata) - if hFindFile == rwin32.INVALID_HANDLE_VALUE: - error = rwin32.GetLastError() - if error == ERROR_FILE_NOT_FOUND: - return result - else: - raise WindowsError(error, "FindFirstFile failed") - while True: - name = rffi.charp2str(rffi.cast(rffi.CCHARP, - filedata.c_cFileName)) - if name != "." and name != "..": # skip these - result.append(name) - if not FindNextFile(hFindFile, filedata): - break - # FindNextFile sets error to ERROR_NO_MORE_FILES if - # it got to the end of the directory - error = rwin32.GetLastError() - FindClose(hFindFile) - if error == ERROR_NO_MORE_FILES: - return result - else: - raise WindowsError(error, "FindNextFile failed") - finally: - lltype.free(filedata, flavor='raw') + return extdef([], unicode, + "ll_os.ll_os_wgetcwd", llimpl=os_getcwd_llimpl) + @registering_str_unicode(os.listdir) + def register_os_listdir(self, traits): + # we need a different approach on Windows and on Posix + if sys.platform.startswith('win'): + from pypy.rpython.module.ll_win32file import make_listdir_impl + os_listdir_llimpl = make_listdir_impl(traits) else: + assert traits.str is str compilation_info = ExternalCompilationInfo( includes = ['sys/types.h', 'dirent.h'] ) @@ -1057,9 +1031,9 @@ raise OSError(error, "os_readdir failed") return result - return extdef([str], # a single argument which is a str - [str], # returns a list of strings - "ll_os.ll_os_listdir", + return extdef([traits.str], # a single argument which is a str + [traits.str], # returns a list of strings + traits.ll_os_name('listdir'), llimpl=os_listdir_llimpl) @registering(os.pipe) @@ -1234,38 +1208,40 @@ return extdef([str], int, llimpl=system_llimpl, export_name="ll_os.ll_os_system") - @registering(os.unlink) - def register_os_unlink(self): - os_unlink = self.llexternal(underscore_on_windows+'unlink', [rffi.CCHARP], rffi.INT) + @registering_str_unicode(os.unlink) + def register_os_unlink(self, traits): + os_unlink = self.llexternal(traits.posix_function_name('unlink'), + [traits.CCHARP], rffi.INT) def unlink_llimpl(pathname): res = rffi.cast(lltype.Signed, os_unlink(pathname)) if res < 0: raise OSError(rposix.get_errno(), "os_unlink failed") - return extdef([str], s_None, llimpl=unlink_llimpl, - export_name="ll_os.ll_os_unlink") + return extdef([traits.str], s_None, llimpl=unlink_llimpl, + export_name=traits.ll_os_name('unlink')) - @registering(os.chdir) - def register_os_chdir(self): - os_chdir = self.llexternal(underscore_on_windows+'chdir', [rffi.CCHARP], rffi.INT) + @registering_str_unicode(os.chdir) + def register_os_chdir(self, traits): + os_chdir = self.llexternal(traits.posix_function_name('chdir'), + [traits.CCHARP], rffi.INT) def chdir_llimpl(path): res = rffi.cast(lltype.Signed, os_chdir(path)) if res < 0: raise OSError(rposix.get_errno(), "os_chdir failed") - return extdef([str], s_None, llimpl=chdir_llimpl, - export_name="ll_os.ll_os_chdir") + return extdef([traits.str], s_None, llimpl=chdir_llimpl, + export_name=traits.ll_os_name('chdir')) - @registering(os.mkdir) - def register_os_mkdir(self): + @registering_str_unicode(os.mkdir) + def register_os_mkdir(self, traits): if os.name == 'nt': ARG2 = [] # no 'mode' argument on Windows - just ignored else: ARG2 = [rffi.MODE_T] - os_mkdir = self.llexternal(underscore_on_windows+'mkdir', - [rffi.CCHARP]+ARG2, rffi.INT) + os_mkdir = self.llexternal(traits.posix_function_name('mkdir'), + [traits.CCHARP] + ARG2, rffi.INT) IGNORE_MODE = len(ARG2) == 0 def mkdir_llimpl(pathname, mode): @@ -1277,46 +1253,47 @@ if res < 0: raise OSError(rposix.get_errno(), "os_mkdir failed") - return extdef([str, int], s_None, llimpl=mkdir_llimpl, - export_name="ll_os.ll_os_mkdir") + return extdef([traits.str, int], s_None, llimpl=mkdir_llimpl, + export_name=traits.ll_os_name('mkdir')) - @registering(os.rmdir) - def register_os_rmdir(self): - os_rmdir = self.llexternal(underscore_on_windows+'rmdir', [rffi.CCHARP], rffi.INT) + @registering_str_unicode(os.rmdir) + def register_os_rmdir(self, traits): + os_rmdir = self.llexternal(traits.posix_function_name('rmdir'), + [traits.CCHARP], rffi.INT) def rmdir_llimpl(pathname): res = rffi.cast(lltype.Signed, os_rmdir(pathname)) if res < 0: raise OSError(rposix.get_errno(), "os_rmdir failed") - return extdef([str], s_None, llimpl=rmdir_llimpl, - export_name="ll_os.ll_os_rmdir") + return extdef([traits.str], s_None, llimpl=rmdir_llimpl, + export_name=traits.ll_os_name('rmdir')) - @registering(os.chmod) - def register_os_chmod(self): - os_chmod = self.llexternal(underscore_on_windows+'chmod', [rffi.CCHARP, rffi.MODE_T], - rffi.INT) + @registering_str_unicode(os.chmod) + def register_os_chmod(self, traits): + os_chmod = self.llexternal(traits.posix_function_name('chmod'), + [traits.CCHARP, rffi.MODE_T], rffi.INT) def chmod_llimpl(path, mode): res = rffi.cast(lltype.Signed, os_chmod(path, rffi.cast(rffi.MODE_T, mode))) if res < 0: raise OSError(rposix.get_errno(), "os_chmod failed") - return extdef([str, int], s_None, llimpl=chmod_llimpl, - export_name="ll_os.ll_os_chmod") + return extdef([traits.str, int], s_None, llimpl=chmod_llimpl, + export_name=traits.ll_os_name('chmod')) - @registering(os.rename) - def register_os_rename(self): - os_rename = self.llexternal('rename', [rffi.CCHARP, rffi.CCHARP], - rffi.INT) + @registering_str_unicode(os.rename) + def register_os_rename(self, traits): + os_rename = self.llexternal(traits.posix_function_name('rename'), + [traits.CCHARP, traits.CCHARP], rffi.INT) def rename_llimpl(oldpath, newpath): res = rffi.cast(lltype.Signed, os_rename(oldpath, newpath)) if res < 0: raise OSError(rposix.get_errno(), "os_rename failed") - return extdef([str, str], s_None, llimpl=rename_llimpl, - export_name="ll_os.ll_os_rename") + return extdef([traits.str, traits.str], s_None, llimpl=rename_llimpl, + export_name=traits.ll_os_name('rename')) @registering(os.umask) def register_os_umask(self): @@ -1425,17 +1402,17 @@ @registering(os.fstat) def register_os_fstat(self): from pypy.rpython.module import ll_os_stat - ll_os_stat.register_stat_variant('fstat') + return ll_os_stat.register_stat_variant('fstat', StringTraits()) - @registering(os.stat) - def register_os_stat(self): + @registering_str_unicode(os.stat) + def register_os_stat(self, traits): from pypy.rpython.module import ll_os_stat - ll_os_stat.register_stat_variant('stat') + return ll_os_stat.register_stat_variant('stat', traits) - @registering(os.lstat) - def register_os_lstat(self): + @registering_str_unicode(os.lstat) + def register_os_lstat(self, traits): from pypy.rpython.module import ll_os_stat - ll_os_stat.register_stat_variant('lstat') + return ll_os_stat.register_stat_variant('lstat', traits) # ------------------------------- os.W* --------------------------------- Modified: pypy/branch/micronumpy/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/branch/micronumpy/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/branch/micronumpy/pypy/rpython/module/ll_os_stat.py Mon Aug 16 04:52:00 2010 @@ -5,13 +5,14 @@ import os, sys from pypy.annotation import model as annmodel from pypy.tool.pairtype import pairtype -from pypy.tool.sourcetools import func_with_new_name +from pypy.tool.sourcetools import func_with_new_name, func_renamer from pypy.rpython import extregistry -from pypy.rpython.extfunc import register_external +from pypy.rpython.extfunc import register_external, extdef from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.tool import rffi_platform as platform from pypy.rpython.lltypesystem.rtupletype import TUPLE_TYPE from pypy.rlib import rposix +from pypy.rlib.objectmodel import specialize from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rpython.annlowlevel import hlstr @@ -211,13 +212,27 @@ return make_stat_result(result) -def register_stat_variant(name): - if sys.platform.startswith('win'): - _functions = {'stat': '_stati64', - 'fstat': '_fstati64', - 'lstat': '_stati64'} # no lstat on Windows - c_func_name = _functions[name] - elif sys.platform.startswith('linux'): +def register_stat_variant(name, traits): + if name != 'fstat': + arg_is_path = True + s_arg = traits.str + ARG1 = traits.CCHARP + else: + arg_is_path = False + s_arg = int + ARG1 = rffi.INT + + if sys.platform == 'win32': + # See Win32 implementation below + posix_stat_llimpl = make_win32_stat_impl(name, traits) + + return extdef( + [s_arg], s_StatResult, traits.ll_os_name(name), + llimpl=posix_stat_llimpl) + + assert traits.str is str + + if sys.platform.startswith('linux'): # because we always use _FILE_OFFSET_BITS 64 - this helps things work that are not a c compiler _functions = {'stat': 'stat64', 'fstat': 'fstat64', @@ -226,22 +241,26 @@ else: c_func_name = name - arg_is_path = (name != 'fstat') + posix_mystat = rffi.llexternal(c_func_name, + [ARG1, STAT_STRUCT], rffi.INT, + compilation_info=compilation_info) + @func_renamer('os_%s_llimpl' % (name,)) def posix_stat_llimpl(arg): stresult = lltype.malloc(STAT_STRUCT.TO, flavor='raw') try: if arg_is_path: - arg = rffi.str2charp(arg) + arg = traits.str2charp(arg) error = rffi.cast(rffi.LONG, posix_mystat(arg, stresult)) if arg_is_path: - rffi.free_charp(arg) + traits.free_charp(arg) if error != 0: raise OSError(rposix.get_errno(), "os_?stat failed") return build_stat_result(stresult) finally: lltype.free(stresult, flavor='raw') + @func_renamer('os_%s_fake' % (name,)) def posix_fakeimpl(arg): if s_arg == str: arg = hlstr(arg) @@ -259,40 +278,17 @@ setattr(ll_tup, 'item%d' % i, val) return ll_tup - if arg_is_path: - s_arg = str - ARG1 = rffi.CCHARP - else: - s_arg = int - ARG1 = rffi.INT + return extdef( + [s_arg], s_StatResult, "ll_os.ll_os_%s" % (name,), + llimpl=posix_stat_llimpl, llfakeimpl=posix_fakeimpl) - if sys.platform != 'win32': - posix_mystat = rffi.llexternal(c_func_name, - [ARG1, STAT_STRUCT], rffi.INT, - compilation_info=compilation_info) - - register_external( - getattr(os, name), [s_arg], s_StatResult, - "ll_os.ll_os_%s" % (name,), - llimpl=func_with_new_name(posix_stat_llimpl, - 'os_%s_llimpl' % (name,)), - llfakeimpl=func_with_new_name(posix_fakeimpl, - 'os_%s_fake' % (name,)), - ) - else: - # See Win32 implementation below - register_external( - getattr(os, name), [s_arg], s_StatResult, - "ll_os.ll_os_%s" % (name,), - llimpl=func_with_new_name(globals()['win32_%s_llimpl' % (name,)], - 'os_%s_llimpl' % (name,)), - ) +def make_win32_stat_impl(name, traits): + from pypy.rlib import rwin32 + from pypy.rpython.module.ll_win32file import make_win32_traits + win32traits = make_win32_traits(traits) -# ____________________________________________________________ -if sys.platform == 'win32': # The CRT of Windows has a number of flaws wrt. its stat() implementation: - # - for when we implement subsecond resolution in RPython, time stamps - # would be restricted to second resolution + # - time stamps are restricted to second resolution # - file modification times suffer from forth-and-back conversions between # UTC and local time # Therefore, we implement our own stat, based on the Win32 API directly. @@ -302,122 +298,18 @@ assert len(STAT_FIELDS) == 10 # no extra fields on Windows - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h', 'winbase.h', 'sys/stat.h'], - ) - - GetFileExInfoStandard = platform.ConstantInteger( - 'GetFileExInfoStandard') - FILE_ATTRIBUTE_DIRECTORY = platform.ConstantInteger( - 'FILE_ATTRIBUTE_DIRECTORY') - FILE_ATTRIBUTE_READONLY = platform.ConstantInteger( - 'FILE_ATTRIBUTE_READONLY') - ERROR_SHARING_VIOLATION = platform.ConstantInteger( - 'ERROR_SHARING_VIOLATION') - _S_IFDIR = platform.ConstantInteger('_S_IFDIR') - _S_IFREG = platform.ConstantInteger('_S_IFREG') - _S_IFCHR = platform.ConstantInteger('_S_IFCHR') - _S_IFIFO = platform.ConstantInteger('_S_IFIFO') - FILE_TYPE_UNKNOWN = platform.ConstantInteger('FILE_TYPE_UNKNOWN') - FILE_TYPE_CHAR = platform.ConstantInteger('FILE_TYPE_CHAR') - FILE_TYPE_PIPE = platform.ConstantInteger('FILE_TYPE_PIPE') - - WIN32_FILE_ATTRIBUTE_DATA = platform.Struct( - 'WIN32_FILE_ATTRIBUTE_DATA', - [('dwFileAttributes', rwin32.DWORD), - ('nFileSizeHigh', rwin32.DWORD), - ('nFileSizeLow', rwin32.DWORD), - ('ftCreationTime', rwin32.FILETIME), - ('ftLastAccessTime', rwin32.FILETIME), - ('ftLastWriteTime', rwin32.FILETIME)]) - - BY_HANDLE_FILE_INFORMATION = platform.Struct( - 'BY_HANDLE_FILE_INFORMATION', - [('dwFileAttributes', rwin32.DWORD), - ('nFileSizeHigh', rwin32.DWORD), - ('nFileSizeLow', rwin32.DWORD), - ('nNumberOfLinks', rwin32.DWORD), - ('nFileIndexHigh', rwin32.DWORD), - ('nFileIndexLow', rwin32.DWORD), - ('ftCreationTime', rwin32.FILETIME), - ('ftLastAccessTime', rwin32.FILETIME), - ('ftLastWriteTime', rwin32.FILETIME)]) - - WIN32_FIND_DATA = platform.Struct( - 'WIN32_FIND_DATAA', - # Only interesting fields - [('dwFileAttributes', rwin32.DWORD), - ('nFileSizeHigh', rwin32.DWORD), - ('nFileSizeLow', rwin32.DWORD), - ('ftCreationTime', rwin32.FILETIME), - ('ftLastAccessTime', rwin32.FILETIME), - ('ftLastWriteTime', rwin32.FILETIME)]) - - globals().update(platform.configure(CConfig)) - GET_FILEEX_INFO_LEVELS = rffi.ULONG # an enumeration - - GetFileAttributesEx = rffi.llexternal( - 'GetFileAttributesExA', - [rffi.CCHARP, GET_FILEEX_INFO_LEVELS, - lltype.Ptr(WIN32_FILE_ATTRIBUTE_DATA)], - rwin32.BOOL, - calling_conv='win') - - GetFileInformationByHandle = rffi.llexternal( - 'GetFileInformationByHandle', - [rwin32.HANDLE, lltype.Ptr(BY_HANDLE_FILE_INFORMATION)], - rwin32.BOOL, - calling_conv='win') - - GetFileType = rffi.llexternal( - 'GetFileType', - [rwin32.HANDLE], - rwin32.DWORD, - calling_conv='win') - - FindFirstFile = rffi.llexternal( - 'FindFirstFileA', - [rffi.CCHARP, lltype.Ptr(WIN32_FIND_DATA)], - rwin32.HANDLE, - calling_conv='win') - - FindClose = rffi.llexternal( - 'FindClose', - [rwin32.HANDLE], - rwin32.BOOL, - calling_conv='win') - def attributes_to_mode(attributes): m = 0 - if attributes & FILE_ATTRIBUTE_DIRECTORY: - m |= _S_IFDIR | 0111 # IFEXEC for user,group,other + if attributes & win32traits.FILE_ATTRIBUTE_DIRECTORY: + m |= win32traits._S_IFDIR | 0111 # IFEXEC for user,group,other else: - m |= _S_IFREG - if attributes & FILE_ATTRIBUTE_READONLY: + m |= win32traits._S_IFREG + if attributes & win32traits.FILE_ATTRIBUTE_READONLY: m |= 0444 else: m |= 0666 return m - def make_longlong(high, low): - return (lltype.r_longlong(high) << 32) + lltype.r_longlong(low) - - # Seconds between 1.1.1601 and 1.1.1970 - secs_between_epochs = lltype.r_longlong(11644473600) - - def FILE_TIME_to_time_t_nsec(filetime): - ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime) - # FILETIME is in units of 100 nsec - nsec = (ft % 10000000) * 100 - time = (ft / 10000000) - secs_between_epochs - return time, nsec - - def time_t_to_FILE_TIME(time, filetime): - ft = lltype.r_longlong((time + secs_between_epochs) * 10000000) - filetime.c_dwHighDateTime = lltype.r_uint(ft >> 32) - filetime.c_dwLowDateTime = lltype.r_uint(ft & ((1 << 32) - 1)) - def attribute_data_to_stat(info): st_mode = attributes_to_mode(info.c_dwFileAttributes) st_size = make_longlong(info.c_nFileSizeHigh, info.c_nFileSizeLow) @@ -456,65 +348,94 @@ return make_stat_result(result) def attributes_from_dir(l_path, data): - filedata = lltype.malloc(WIN32_FIND_DATA, flavor='raw') - hFindFile = FindFirstFile(l_path, filedata) - if hFindFile == rwin32.INVALID_HANDLE_VALUE: - return 0 - FindClose(hFindFile) - data.c_dwFileAttributes = filedata.c_dwFileAttributes - rffi.structcopy(data.c_ftCreationTime, filedata.c_ftCreationTime) - rffi.structcopy(data.c_ftLastAccessTime, filedata.c_ftLastAccessTime) - rffi.structcopy(data.c_ftLastWriteTime, filedata.c_ftLastWriteTime) - data.c_nFileSizeHigh = filedata.c_nFileSizeHigh - data.c_nFileSizeLow = filedata.c_nFileSizeLow - return 1 + filedata = lltype.malloc(win32traits.WIN32_FIND_DATA, flavor='raw') + try: + hFindFile = win32traits.FindFirstFile(l_path, filedata) + if hFindFile == rwin32.INVALID_HANDLE_VALUE: + return 0 + win32traits.FindClose(hFindFile) + data.c_dwFileAttributes = filedata.c_dwFileAttributes + rffi.structcopy(data.c_ftCreationTime, filedata.c_ftCreationTime) + rffi.structcopy(data.c_ftLastAccessTime, filedata.c_ftLastAccessTime) + rffi.structcopy(data.c_ftLastWriteTime, filedata.c_ftLastWriteTime) + data.c_nFileSizeHigh = filedata.c_nFileSizeHigh + data.c_nFileSizeLow = filedata.c_nFileSizeLow + return 1 + finally: + lltype.free(filedata, flavor='raw') def win32_stat_llimpl(path): - data = lltype.malloc(WIN32_FILE_ATTRIBUTE_DATA, flavor='raw') + data = lltype.malloc(win32traits.WIN32_FILE_ATTRIBUTE_DATA, flavor='raw') try: - l_path = rffi.str2charp(path) - res = GetFileAttributesEx(l_path, GetFileExInfoStandard, data) + l_path = traits.str2charp(path) + res = win32traits.GetFileAttributesEx(l_path, win32traits.GetFileExInfoStandard, data) errcode = rwin32.GetLastError() if res == 0: - if errcode == ERROR_SHARING_VIOLATION: + if errcode == win32traits.ERROR_SHARING_VIOLATION: res = attributes_from_dir(l_path, data) errcode = rwin32.GetLastError() - rffi.free_charp(l_path) + traits.free_charp(l_path) if res == 0: raise WindowsError(errcode, "os_stat failed") return attribute_data_to_stat(data) finally: lltype.free(data, flavor='raw') - win32_lstat_llimpl = win32_stat_llimpl def win32_fstat_llimpl(fd): handle = rwin32._get_osfhandle(fd) - filetype = GetFileType(handle) - if filetype == FILE_TYPE_CHAR: + filetype = win32traits.GetFileType(handle) + if filetype == win32traits.FILE_TYPE_CHAR: # console or LPT device - return make_stat_result((_S_IFCHR, + return make_stat_result((win32traits._S_IFCHR, 0, 0, 0, 0, 0, 0, 0, 0, 0)) - elif filetype == FILE_TYPE_PIPE: + elif filetype == win32traits.FILE_TYPE_PIPE: # socket or named pipe - return make_stat_result((_S_IFIFO, + return make_stat_result((win32traits._S_IFIFO, 0, 0, 0, 0, 0, 0, 0, 0, 0)) - elif filetype == FILE_TYPE_UNKNOWN: + elif filetype == win32traits.FILE_TYPE_UNKNOWN: error = rwin32.GetLastError() if error != 0: raise WindowsError(error, "os_fstat failed") # else: unknown but valid file # normal disk file (FILE_TYPE_DISK) - info = lltype.malloc(BY_HANDLE_FILE_INFORMATION, flavor='raw', - zero=True) + info = lltype.malloc(win32traits.BY_HANDLE_FILE_INFORMATION, + flavor='raw', zero=True) try: - res = GetFileInformationByHandle(handle, info) + res = win32traits.GetFileInformationByHandle(handle, info) if res == 0: raise WindowsError(rwin32.GetLastError(), "os_fstat failed") return by_handle_info_to_stat(info) finally: lltype.free(info, flavor='raw') + if name == 'fstat': + return win32_fstat_llimpl + else: + return win32_stat_llimpl + + +#__________________________________________________ +# Helper functions for win32 + +def make_longlong(high, low): + return (lltype.r_longlong(high) << 32) + lltype.r_longlong(low) + +# Seconds between 1.1.1601 and 1.1.1970 +secs_between_epochs = lltype.r_longlong(11644473600) + +def FILE_TIME_to_time_t_nsec(filetime): + ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime) + # FILETIME is in units of 100 nsec + nsec = (ft % 10000000) * 100 + time = (ft / 10000000) - secs_between_epochs + return time, nsec + +def time_t_to_FILE_TIME(time, filetime): + ft = lltype.r_longlong((time + secs_between_epochs) * 10000000) + filetime.c_dwHighDateTime = lltype.r_uint(ft >> 32) + filetime.c_dwLowDateTime = lltype.r_uint(ft & ((1 << 32) - 1)) + Modified: pypy/branch/micronumpy/pypy/rpython/module/test/test_ll_os_stat.py ============================================================================== --- pypy/branch/micronumpy/pypy/rpython/module/test/test_ll_os_stat.py (original) +++ pypy/branch/micronumpy/pypy/rpython/module/test/test_ll_os_stat.py Mon Aug 16 04:52:00 2010 @@ -1,4 +1,4 @@ -from pypy.rpython.module import ll_os_stat +from pypy.rpython.module import ll_os_stat, ll_os import sys, os import py @@ -8,14 +8,18 @@ py.test.skip("win32 specific tests") def test_stat(self): - stat = ll_os_stat.win32_stat_llimpl + stat = ll_os_stat.make_win32_stat_impl('stat', ll_os.StringTraits()) + wstat = ll_os_stat.make_win32_stat_impl('stat', ll_os.UnicodeTraits()) def check(f): - assert stat(f).st_mtime == os.stat(f).st_mtime + expected = os.stat(f).st_mtime + assert stat(f).st_mtime == expected + assert wstat(unicode(f)).st_mtime == expected check('c:/') check('c:/temp') check('c:/pagefile.sys') def test_fstat(self): - stat = ll_os_stat.win32_fstat_llimpl(0) # stdout + fstat = ll_os_stat.make_win32_stat_impl('fstat', ll_os.StringTraits()) + stat = fstat(0) # stdout assert stat.st_mode != 0 Modified: pypy/branch/micronumpy/pypy/rpython/rstr.py ============================================================================== --- pypy/branch/micronumpy/pypy/rpython/rstr.py (original) +++ pypy/branch/micronumpy/pypy/rpython/rstr.py Mon Aug 16 04:52:00 2010 @@ -288,8 +288,8 @@ if not hop.args_s[1].is_constant(): raise TyperError("encoding must be constant") encoding = hop.args_s[1].const - if encoding == "ascii": - expect = self.lowleveltype # can be a UniChar + if encoding == "ascii" and self.lowleveltype == UniChar: + expect = UniChar # only for unichar.encode('ascii') else: expect = self.repr # must be a regular unicode string v_self = hop.inputarg(expect, 0) Modified: pypy/branch/micronumpy/pypy/rpython/test/test_extfunc.py ============================================================================== --- pypy/branch/micronumpy/pypy/rpython/test/test_extfunc.py (original) +++ pypy/branch/micronumpy/pypy/rpython/test/test_extfunc.py Mon Aug 16 04:52:00 2010 @@ -6,150 +6,164 @@ from pypy.annotation.policy import AnnotatorPolicy from pypy.rpython.test.test_llinterp import interpret -def b(x): - return eval("x+40") +class TestExtFuncEntry: -class BTestFuncEntry(ExtFuncEntry): - _about_ = b - name = 'b' - signature_args = [annmodel.SomeInteger()] - signature_result = annmodel.SomeInteger() - -def test_annotation_b(): - def f(): - return b(1) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeInteger) - -def test_rtyping_b(): - def f(): - return b(2) - - res = interpret(f, []) - assert res == 42 - -def c(y, x): - yyy - -class CTestFuncEntry(ExtFuncEntry): - _about_ = c - name = 'ccc' - signature_args = [annmodel.SomeInteger()] * 2 - signature_result = annmodel.SomeInteger() - - def lltypeimpl(y, x): - return y + x - lltypeimpl = staticmethod(lltypeimpl) - -def test_interp_c(): - def f(): - return c(3, 4) - - res = interpret(f, []) - assert res == 7 - -def d(y): - return eval("y()") - -class DTestFuncEntry(ExtFuncEntry): - _about_ = d - name = 'd' - signature_args = [annmodel.SomeGenericCallable(args=[], result= - annmodel.SomeFloat())] - signature_result = annmodel.SomeFloat() - -def test_callback(): - def callback(): - return 2.5 - - def f(): - return d(callback) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeFloat) - assert a.translator._graphof(callback) - -def dd(): - pass - -register_external(dd, [int], int) - -def test_register_external_signature(): - def f(): - return dd(3) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeInteger) - - -def function_with_tuple_arg(): - """ - Dummy function which is declared via register_external to take a tuple as - an argument so that register_external's behavior for tuple-taking functions - can be verified. - """ -register_external(function_with_tuple_arg, [(int,)], int) - -def test_register_external_tuple_args(): - """ - Verify the annotation of a registered external function which takes a tuple - argument. - """ - def f(): - return function_with_tuple_arg((1,)) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - - # Not a very good assertion, but at least it means _something_ happened. - assert isinstance(s, annmodel.SomeInteger) - -def function_with_list(): - pass -register_external(function_with_list, [[int]], int) - -def function_returning_list(): - pass -register_external(function_returning_list, [], [int]) - -def test_register_external_return_goes_back(): - """ - Check whether it works to pass the same list from one external - fun to another - [bookkeeper and list joining issues] - """ - def f(): - return function_with_list(function_returning_list()) - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeInteger) - -def function_withspecialcase(arg): - return repr(arg) -register_external(function_withspecialcase, args=None, result=str) - -def test_register_external_specialcase(): - def f(): - x = function_withspecialcase - return x(33) + x("aaa") + x([]) + "\n" - - policy = AnnotatorPolicy() - policy.allow_someobjects = False - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, annmodel.SomeString) + def test_basic(self): + """ + A ExtFuncEntry provides an annotation for a function, no need to flow + its graph. + """ + def b(x): + "NOT_RPYTHON" + return eval("x+40") + + class BTestFuncEntry(ExtFuncEntry): + _about_ = b + name = 'b' + signature_args = [annmodel.SomeInteger()] + signature_result = annmodel.SomeInteger() + + def f(): + return b(2) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeInteger) + + res = interpret(f, []) + assert res == 42 + + def test_lltypeimpl(self): + """ + interpret() calls lltypeimpl instead of of the function/ + """ + def c(y, x): + yyy + + class CTestFuncEntry(ExtFuncEntry): + _about_ = c + name = 'ccc' + signature_args = [annmodel.SomeInteger()] * 2 + signature_result = annmodel.SomeInteger() + + def lltypeimpl(y, x): + return y + x + lltypeimpl = staticmethod(lltypeimpl) + + def f(): + return c(3, 4) + + res = interpret(f, []) + assert res == 7 + + def test_callback(self): + """ + Verify annotation when a callback function is in the arguments list. + """ + def d(y): + return eval("y()") + + class DTestFuncEntry(ExtFuncEntry): + _about_ = d + name = 'd' + signature_args = [annmodel.SomeGenericCallable(args=[], result= + annmodel.SomeFloat())] + signature_result = annmodel.SomeFloat() + + def callback(): + return 2.5 + + def f(): + return d(callback) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeFloat) + assert a.translator._graphof(callback) + + def test_register_external_signature(self): + """ + Test the standard interface for external functions. + """ + def dd(): + pass + register_external(dd, [int], int) + + def f(): + return dd(3) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeInteger) + + def test_register_external_tuple_args(self): + """ + Verify the annotation of a registered external function which takes a + tuple argument. + """ + + def function_with_tuple_arg(): + """ + Dummy function which is declared via register_external to take a + tuple as an argument so that register_external's behavior for + tuple-taking functions can be verified. + """ + register_external(function_with_tuple_arg, [(int,)], int) + + def f(): + return function_with_tuple_arg((1,)) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + + # Not a very good assertion, but at least it means _something_ happened. + assert isinstance(s, annmodel.SomeInteger) + + def test_register_external_return_goes_back(self): + """ + Check whether it works to pass the same list from one external + fun to another + [bookkeeper and list joining issues] + """ + def function_with_list(): + pass + register_external(function_with_list, [[int]], int) + + def function_returning_list(): + pass + register_external(function_returning_list, [], [int]) + + def f(): + return function_with_list(function_returning_list()) + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeInteger) + + def test_register_external_specialcase(self): + """ + When args=None, the external function accepts any arguments unmodified. + """ + def function_withspecialcase(arg): + return repr(arg) + register_external(function_withspecialcase, args=None, result=str) + + def f(): + x = function_withspecialcase + return x(33) + x("aaa") + x([]) + "\n" + + policy = AnnotatorPolicy() + policy.allow_someobjects = False + a = RPythonAnnotator(policy=policy) + s = a.build_types(f, []) + assert isinstance(s, annmodel.SomeString) Modified: pypy/branch/micronumpy/pypy/rpython/tool/rfficache.py ============================================================================== --- pypy/branch/micronumpy/pypy/rpython/tool/rfficache.py (original) +++ pypy/branch/micronumpy/pypy/rpython/tool/rfficache.py Mon Aug 16 04:52:00 2010 @@ -29,7 +29,7 @@ } ''' % (include_string, add_source, str(question))) c_file = udir.join("gcctest.c") - c_file.write(c_source) + c_file.write(str(c_source) + '\n') eci = ExternalCompilationInfo() return build_executable_cache([c_file], eci) Modified: pypy/branch/micronumpy/pypy/tool/convolve.py ============================================================================== --- pypy/branch/micronumpy/pypy/tool/convolve.py (original) +++ pypy/branch/micronumpy/pypy/tool/convolve.py Mon Aug 16 04:52:00 2010 @@ -25,10 +25,8 @@ # Allocate result image. h = np.zeros([xmax, ymax], dtype=f.dtype) # Do convolution - print "Herp" for x in range(xmax): for y in range(ymax): - print "<%d,%d>" % (x, y) # Calculate pixel value for h at (x,y). Sum one component # for each pixel (s, t) of the filter g. s_from = max(smid - x, -smid) @@ -41,5 +39,6 @@ v = x - smid + s w = y - tmid + t value += g[smid - s, tmid - t] * f[v, w] + print "h[%d, %d] = %s" % (x, y, value) h[x, y] = value return h Modified: pypy/branch/micronumpy/pypy/tool/numpybench.py ============================================================================== --- pypy/branch/micronumpy/pypy/tool/numpybench.py (original) +++ pypy/branch/micronumpy/pypy/tool/numpybench.py Mon Aug 16 04:52:00 2010 @@ -3,22 +3,22 @@ except ImportError, e: import micronumpy as numpy -def generate_image(width, height): - return numpy.array([[x + y for x in range(width)] for y in range(height)]) +def generate_image(width, height, dtype=float): + return numpy.array([[x + y for x in range(width)] for y in range(height)], dtype=dtype) -def generate_kernel(width, height): +def generate_kernel(width, height, dtype=float): from math import sin, pi - #kernel = numpy.zeros((width, height), dtype=int) # FIXME: micronumpy.zeros can't handle missing dtype - kernel = [[0] * width] * height + kernel = numpy.zeros((width, height), dtype=dtype) # FIXME: micronumpy.zeros can't handle missing dtype for i in range(width): for j in range(height): - u = i / float(width) - v = j / float(height) - kernel[j][i] = int((0.5 + sin(u * pi)) * (0.5 + sin(v * pi))) # DOUBLE FIXME: setitem doesn't coerce to array type + u = (i + 0.5) / float(width) + v = (j + 0.5) / float(height) + kernel[j,i] = int((0.5 + sin(u * pi)) * (0.5 + sin(v * pi))) # DOUBLE FIXME: setitem doesn't coerce to array type + print "kernel[%d,%d] = %s" % (j, i, kernel[j][i]) - #return kernel - return numpy.array(kernel) + print kernel + return kernel if __name__ == '__main__': from optparse import OptionParser @@ -41,8 +41,9 @@ kwidth, kheight = parse_dimension(options.kernel) count = int(options.count) - image = generate_image(width, height) - kernel = generate_kernel(kwidth, kheight) + dtype = float + image = generate_image(width, height, dtype) + kernel = generate_kernel(kwidth, kheight, dtype) print "Timing" from timeit import Timer Modified: pypy/branch/micronumpy/pypy/tool/release/package.py ============================================================================== --- pypy/branch/micronumpy/pypy/tool/release/package.py (original) +++ pypy/branch/micronumpy/pypy/tool/release/package.py Mon Aug 16 04:52:00 2010 @@ -11,11 +11,12 @@ import py import os import fnmatch -import tarfile from pypy.tool.udir import udir if sys.version_info < (2,6): py.test.skip("requires 2.6 so far") +USE_TARFILE_MODULE = sys.platform == 'win32' + def ignore_patterns(*patterns): """Function that can be used as copytree() ignore parameter. @@ -69,9 +70,17 @@ old_dir = os.getcwd() try: os.chdir(str(builddir)) - os.system("strip " + str(archive_pypy_c)) - os.system('tar cvjf ' + str(builddir.join(name + '.tar.bz2')) + - " " + name) + os.system("strip " + str(archive_pypy_c)) # ignore errors + if USE_TARFILE_MODULE: + import tarfile + tf = tarfile.open(str(builddir.join(name + '.tar.bz2')), 'w:bz2') + tf.add(name) + tf.close() + else: + e = os.system('tar cvjf ' + str(builddir.join(name + '.tar.bz2')) + + " " + name) + if e: + raise OSError('"tar" returned exit status %r' % e) finally: os.chdir(old_dir) if copy_to_dir is not None: Modified: pypy/branch/micronumpy/pypy/tool/release/test/test_package.py ============================================================================== --- pypy/branch/micronumpy/pypy/tool/release/test/test_package.py (original) +++ pypy/branch/micronumpy/pypy/tool/release/test/test_package.py Mon Aug 16 04:52:00 2010 @@ -5,7 +5,7 @@ from pypy.module.sys.version import CPYTHON_VERSION import tarfile, os -def test_dir_structure(): +def test_dir_structure(test='test'): # make sure we have sort of pypy-c pypy_c = py.path.local(pypydir).join('translator', 'goal', 'pypy-c') if not pypy_c.check(): @@ -14,8 +14,8 @@ else: fake_pypy_c = False try: - builddir = package(py.path.local(pypydir).dirpath(), 'test') - prefix = builddir.join('test') + builddir = package(py.path.local(pypydir).dirpath(), test) + prefix = builddir.join(test) cpyver = '%d.%d.%d' % CPYTHON_VERSION[:3] assert prefix.join('lib-python', cpyver, 'test').check() assert prefix.join('bin', 'pypy-c').check() @@ -24,18 +24,27 @@ assert not prefix.join('lib_pypy', 'ctypes_configure').check() assert prefix.join('LICENSE').check() assert prefix.join('README').check() - th = tarfile.open(str(builddir.join('test.tar.bz2'))) - assert th.getmember('test/lib_pypy/syslog.py') + th = tarfile.open(str(builddir.join('%s.tar.bz2' % test))) + assert th.getmember('%s/lib_pypy/syslog.py' % test) # the headers file could be not there, because they are copied into # trunk/include only during translation includedir = py.path.local(pypydir).dirpath().join('include') def check_include(name): if includedir.join(name).check(file=True): - assert th.getmember('test/include/%s' % name) + assert th.getmember('%s/include/%s' % (test, name)) check_include('Python.h') check_include('modsupport.inl') check_include('pypy_decl.h') finally: if fake_pypy_c: pypy_c.remove() + +def test_with_tarfile_module(): + from pypy.tool.release import package + prev = package.USE_TARFILE_MODULE + try: + package.USE_TARFILE_MODULE = True + test_dir_structure(test='testtarfile') + finally: + package.USE_TARFILE_MODULE = prev Modified: pypy/branch/micronumpy/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/micronumpy/pypy/translator/c/genc.py (original) +++ pypy/branch/micronumpy/pypy/translator/c/genc.py Mon Aug 16 04:52:00 2010 @@ -1,7 +1,6 @@ import autopath import py import sys, os -from pypy.translator.c.node import PyObjectNode, FuncNode from pypy.translator.c.database import LowLevelDatabase from pypy.translator.c.extfunc import pre_include_code_lines from pypy.translator.llsupport.wrapper import new_wrapper @@ -196,7 +195,7 @@ all = [] for node in self.db.globalcontainers(): - eci = getattr(node, 'compilation_info', None) + eci = node.compilation_info() if eci: all.append(eci) self.merge_eci(*all) @@ -222,7 +221,7 @@ graphs = db.all_graphs() db.gctransformer.prepare_inline_helpers(graphs) for node in db.containerlist: - if isinstance(node, FuncNode): + if hasattr(node, 'funcgens'): for funcgen in node.funcgens: funcgen.patch_graph(copy_graph=False) return db Modified: pypy/branch/micronumpy/pypy/translator/c/node.py ============================================================================== --- pypy/branch/micronumpy/pypy/translator/c/node.py (original) +++ pypy/branch/micronumpy/pypy/translator/c/node.py Mon Aug 16 04:52:00 2010 @@ -466,15 +466,15 @@ class ContainerNode(object): - if USESLOTS: - __slots__ = """db T obj + if USESLOTS: # keep the number of slots down! + __slots__ = """db obj typename implementationtypename - name ptrname compilation_info + name ptrname globalcontainer""".split() + eci_name = '_compilation_info' def __init__(self, db, T, obj): self.db = db - self.T = T self.obj = obj #self.dependencies = {} self.typename = db.gettype(T) #, who_asks=self) @@ -489,16 +489,22 @@ else: self.globalcontainer = False parentnode = db.getcontainernode(parent) - defnode = db.gettypedefnode(parentnode.T) + defnode = db.gettypedefnode(parentnode.getTYPE()) self.name = defnode.access_expr(parentnode.name, parentindex) if self.typename != self.implementationtypename: if db.gettypedefnode(T).extra_union_for_varlength: self.name += '.b' - self.compilation_info = getattr(obj, '_compilation_info', None) self.ptrname = '(&%s)' % self.name + def getTYPE(self): + return typeOf(self.obj) + def is_thread_local(self): - return hasattr(self.T, "_hints") and self.T._hints.get('thread_local') + T = self.getTYPE() + return hasattr(T, "_hints") and T._hints.get('thread_local') + + def compilation_info(self): + return getattr(self.obj, self.eci_name, None) def get_declaration(self): if self.name[-2:] == '.b': @@ -546,27 +552,31 @@ __slots__ = () def basename(self): - return self.T._name + T = self.getTYPE() + return T._name def enum_dependencies(self): - for name in self.T._names: + T = self.getTYPE() + for name in T._names: yield getattr(self.obj, name) def getlength(self): - if self.T._arrayfld is None: + T = self.getTYPE() + if T._arrayfld is None: return 1 else: - array = getattr(self.obj, self.T._arrayfld) + array = getattr(self.obj, T._arrayfld) return len(array.items) def initializationexpr(self, decoration=''): + T = self.getTYPE() is_empty = True yield '{' - defnode = self.db.gettypedefnode(self.T) + defnode = self.db.gettypedefnode(T) data = [] - if needs_gcheader(self.T): + if needs_gcheader(T): gc_init = self.db.gcpolicy.struct_gcheader_initdata(self) data.append(('gcheader', gc_init)) @@ -578,16 +588,16 @@ # '.fieldname = value'. But here we don't know which of the # fields need initialization, so XXX we pick the first one # arbitrarily. - if hasattr(self.T, "_hints") and self.T._hints.get('union'): + if hasattr(T, "_hints") and T._hints.get('union'): data = data[0:1] - if 'get_padding_drop' in self.T._hints: + if 'get_padding_drop' in T._hints: d = {} for name, _ in data: - T = defnode.c_struct_field_type(name) - typename = self.db.gettype(T) + T1 = defnode.c_struct_field_type(name) + typename = self.db.gettype(T1) d[name] = cdecl(typename, '') - padding_drop = self.T._hints['get_padding_drop'](d) + padding_drop = T._hints['get_padding_drop'](d) else: padding_drop = [] @@ -617,9 +627,10 @@ return 'struct _hashT_%s @' % self.name def forward_declaration(self): + T = self.getTYPE() assert self.typename == self.implementationtypename # no array part hash_typename = self.get_hash_typename() - hash_offset = self.db.gctransformer.get_hash_offset(self.T) + hash_offset = self.db.gctransformer.get_hash_offset(T) yield '%s {' % cdecl(hash_typename, '') yield '\tunion {' yield '\t\t%s;' % cdecl(self.implementationtypename, 'head') @@ -671,22 +682,23 @@ return len(self.obj.items) def initializationexpr(self, decoration=''): - defnode = self.db.gettypedefnode(self.T) + T = self.getTYPE() + defnode = self.db.gettypedefnode(T) yield '{' - if needs_gcheader(self.T): + if needs_gcheader(T): gc_init = self.db.gcpolicy.array_gcheader_initdata(self) lines = generic_initializationexpr(self.db, gc_init, 'gcheader', '%sgcheader' % (decoration,)) for line in lines: yield line - if self.T._hints.get('nolength', False): + if T._hints.get('nolength', False): length = '' else: length = '%d, ' % len(self.obj.items) - if self.T.OF is Void or len(self.obj.items) == 0: + if T.OF is Void or len(self.obj.items) == 0: yield '\t%s' % length.rstrip(', ') yield '}' - elif self.T.OF == Char: + elif T.OF == Char: if len(self.obj.items) and self.obj.items[0] is None: s = ''.join([self.obj.getitem(i) for i in range(len(self.obj.items))]) else: @@ -694,7 +706,7 @@ yield '\t%s%s' % (length, c_char_array_constant(s)) yield '}' else: - barebone = barebonearray(self.T) + barebone = barebonearray(T) if not barebone: yield '\t%s{' % length for j in range(len(self.obj.items)): @@ -722,7 +734,8 @@ self.ptrname = self.name def basename(self): - return self.T._name + T = self.getTYPE() + return T._name def enum_dependencies(self): for i in range(self.obj.getlength()): @@ -732,11 +745,12 @@ return 1 # not variable-sized! def initializationexpr(self, decoration=''): + T = self.getTYPE() assert self.typename == self.implementationtypename # not var-sized is_empty = True yield '{' # _names == ['item0', 'item1', ...] - for j, name in enumerate(self.T._names): + for j, name in enumerate(T._names): value = getattr(self.obj, name) lines = generic_initializationexpr(self.db, value, '%s[%d]' % (self.name, j), @@ -777,6 +791,7 @@ class FuncNode(ContainerNode): nodekind = 'func' + eci_name = 'compilation_info' # there not so many node of this kind, slots should not # be necessary @@ -794,7 +809,6 @@ else: self.name = (forcename or db.namespace.uniquename('g_' + self.basename())) - self.compilation_info = getattr(obj, 'compilation_info', None) self.make_funcgens() #self.dependencies = {} self.typename = db.gettype(T) #, who_asks=self) @@ -939,18 +953,20 @@ return [] def initializationexpr(self, decoration=''): - yield 'RPyOpaque_INITEXPR_%s' % (self.T.tag,) + T = self.getTYPE() + yield 'RPyOpaque_INITEXPR_%s' % (T.tag,) def startupcode(self): + T = self.getTYPE() args = [self.ptrname] # XXX how to make this code more generic? - if self.T.tag == 'ThreadLock': + if T.tag == 'ThreadLock': lock = self.obj.externalobj if lock.locked(): args.append('1') else: args.append('0') - yield 'RPyOpaque_SETUP_%s(%s);' % (self.T.tag, ', '.join(args)) + yield 'RPyOpaque_SETUP_%s(%s);' % (T.tag, ', '.join(args)) def opaquenode_factory(db, T, obj): Modified: pypy/branch/micronumpy/pypy/translator/goal/app_main.py ============================================================================== --- pypy/branch/micronumpy/pypy/translator/goal/app_main.py (original) +++ pypy/branch/micronumpy/pypy/translator/goal/app_main.py Mon Aug 16 04:52:00 2010 @@ -223,7 +223,6 @@ path = os.getenv('PYTHONPATH') if path: newpath = path.split(os.pathsep) + newpath - newpath.insert(0, '') # remove duplicates _seen = {} del sys.path[:] @@ -327,6 +326,10 @@ except: print >> sys.stderr, "'import site' failed" + # update sys.path *after* loading site.py, in case there is a + # "site.py" file in the script's directory. + sys.path.insert(0, '') + if warnoptions: sys.warnoptions.append(warnoptions) from warnings import _processoptions Modified: pypy/branch/micronumpy/pypy/translator/goal/test2/test_app_main.py ============================================================================== --- pypy/branch/micronumpy/pypy/translator/goal/test2/test_app_main.py (original) +++ pypy/branch/micronumpy/pypy/translator/goal/test2/test_app_main.py Mon Aug 16 04:52:00 2010 @@ -5,6 +5,7 @@ import sys, os, re import autopath from pypy.tool.udir import udir +from contextlib import contextmanager banner = sys.version.splitlines()[0] @@ -326,8 +327,9 @@ class TestNonInteractive: def run(self, cmdline, senddata='', expect_prompt=False, - expect_banner=False): - cmdline = '%s "%s" %s' % (sys.executable, app_main, cmdline) + expect_banner=False, python_flags=''): + cmdline = '%s %s "%s" %s' % (sys.executable, python_flags, + app_main, cmdline) print 'POPEN:', cmdline child_in, child_out_err = os.popen4(cmdline) child_in.write(senddata) @@ -449,6 +451,43 @@ assert data == '\x00(STDOUT)\n\x00' # from stdout child_out_err.close() + def test_proper_sys_path(self, tmpdir): + + @contextmanager + def chdir_and_unset_pythonpath(new_cwd): + old_cwd = new_cwd.chdir() + old_pythonpath = os.getenv('PYTHONPATH') + os.unsetenv('PYTHONPATH') + try: + yield + finally: + old_cwd.chdir() + os.putenv('PYTHONPATH', old_pythonpath) + + tmpdir.join('site.py').write('print "SHOULD NOT RUN"') + runme_py = tmpdir.join('runme.py') + runme_py.write('print "some text"') + + cmdline = str(runme_py) + + with chdir_and_unset_pythonpath(tmpdir): + data = self.run(cmdline, python_flags='-S') + + assert data == "some text\n" + + runme2_py = tmpdir.mkdir('otherpath').join('runme2.py') + runme2_py.write('print "some new text"\n' + 'import sys\n' + 'print sys.path\n') + + cmdline2 = str(runme2_py) + + with chdir_and_unset_pythonpath(tmpdir): + data = self.run(cmdline2, python_flags='-S') + + assert data.startswith("some new text\n") + assert repr(str(tmpdir.join('otherpath'))) in data + class AppTestAppMain: Modified: pypy/branch/micronumpy/pypy/translator/platform/posix.py ============================================================================== --- pypy/branch/micronumpy/pypy/translator/platform/posix.py (original) +++ pypy/branch/micronumpy/pypy/translator/platform/posix.py Mon Aug 16 04:52:00 2010 @@ -14,7 +14,10 @@ def __init__(self, cc=None): if cc is None: - cc = 'gcc' + try: + cc = os.environ['CC'] + except KeyError: + cc = 'gcc' self.cc = cc def _libs(self, libraries): From arigo at codespeak.net Mon Aug 16 14:47:05 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 16 Aug 2010 14:47:05 +0200 (CEST) Subject: [pypy-svn] r76631 - in pypy/branch/micronumpy/pypy/jit/backend: llsupport test Message-ID: <20100816124705.7FFAC282BD4@codespeak.net> Author: arigo Date: Mon Aug 16 14:47:03 2010 New Revision: 76631 Modified: pypy/branch/micronumpy/pypy/jit/backend/llsupport/descr.py pypy/branch/micronumpy/pypy/jit/backend/test/runner_test.py Log: Add a passing test. Check that we don't get plain lltype.Arrays. Modified: pypy/branch/micronumpy/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/branch/micronumpy/pypy/jit/backend/llsupport/descr.py Mon Aug 16 14:47:03 2010 @@ -195,9 +195,13 @@ try: return cache[ARRAY] except KeyError: + # we only support Arrays that are either GcArrays, or raw no-length + # non-gc Arrays. if ARRAY._hints.get('nolength', False): + assert not isinstance(ARRAY, lltype.GcArray) arraydescr = getArrayNoLengthDescrClass(ARRAY)() else: + assert isinstance(ARRAY, lltype.GcArray) arraydescr = getArrayDescrClass(ARRAY)() # verify basic assumption that all arrays' basesize and ofslength # are equal Modified: pypy/branch/micronumpy/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/micronumpy/pypy/jit/backend/test/runner_test.py Mon Aug 16 14:47:03 2010 @@ -1832,6 +1832,31 @@ assert self.cpu.get_latest_value_float(0) == 13.5 assert called + def test_raw_malloced_getarrayitem(self): + ARRAY = rffi.CArray(lltype.Signed) + descr = self.cpu.arraydescrof(ARRAY) + a = lltype.malloc(ARRAY, 10, flavor='raw') + a[7] = -4242 + addr = llmemory.cast_ptr_to_adr(a) + abox = BoxInt(heaptracker.adr2int(addr)) + r1 = self.execute_operation(rop.GETARRAYITEM_RAW, [abox, BoxInt(7)], + 'int', descr=descr) + assert r1.getint() == -4242 + lltype.free(a, flavor='raw') + + def test_raw_malloced_setarrayitem(self): + ARRAY = rffi.CArray(lltype.Signed) + descr = self.cpu.arraydescrof(ARRAY) + a = lltype.malloc(ARRAY, 10, flavor='raw') + addr = llmemory.cast_ptr_to_adr(a) + abox = BoxInt(heaptracker.adr2int(addr)) + self.execute_operation(rop.SETARRAYITEM_RAW, [abox, BoxInt(5), + BoxInt(12345)], + 'int', descr=descr) + assert a[5] == 12345 + lltype.free(a, flavor='raw') + + class OOtypeBackendTest(BaseBackendTest): type_system = 'ootype' From arigo at codespeak.net Mon Aug 16 14:47:54 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 16 Aug 2010 14:47:54 +0200 (CEST) Subject: [pypy-svn] r76632 - in pypy/trunk/pypy/jit/backend: llsupport test Message-ID: <20100816124754.470FF282BD4@codespeak.net> Author: arigo Date: Mon Aug 16 14:47:52 2010 New Revision: 76632 Modified: pypy/trunk/pypy/jit/backend/llsupport/descr.py pypy/trunk/pypy/jit/backend/test/runner_test.py Log: Merge r76631 from branch/micronumpy. Modified: pypy/trunk/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/descr.py Mon Aug 16 14:47:52 2010 @@ -198,9 +198,13 @@ try: return cache[ARRAY] except KeyError: + # we only support Arrays that are either GcArrays, or raw no-length + # non-gc Arrays. if ARRAY._hints.get('nolength', False): + assert not isinstance(ARRAY, lltype.GcArray) arraydescr = getArrayNoLengthDescrClass(ARRAY)() else: + assert isinstance(ARRAY, lltype.GcArray) arraydescr = getArrayDescrClass(ARRAY)() # verify basic assumption that all arrays' basesize and ofslength # are equal Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/runner_test.py (original) +++ pypy/trunk/pypy/jit/backend/test/runner_test.py Mon Aug 16 14:47:52 2010 @@ -1832,6 +1832,31 @@ assert self.cpu.get_latest_value_float(0) == 13.5 assert called + def test_raw_malloced_getarrayitem(self): + ARRAY = rffi.CArray(lltype.Signed) + descr = self.cpu.arraydescrof(ARRAY) + a = lltype.malloc(ARRAY, 10, flavor='raw') + a[7] = -4242 + addr = llmemory.cast_ptr_to_adr(a) + abox = BoxInt(heaptracker.adr2int(addr)) + r1 = self.execute_operation(rop.GETARRAYITEM_RAW, [abox, BoxInt(7)], + 'int', descr=descr) + assert r1.getint() == -4242 + lltype.free(a, flavor='raw') + + def test_raw_malloced_setarrayitem(self): + ARRAY = rffi.CArray(lltype.Signed) + descr = self.cpu.arraydescrof(ARRAY) + a = lltype.malloc(ARRAY, 10, flavor='raw') + addr = llmemory.cast_ptr_to_adr(a) + abox = BoxInt(heaptracker.adr2int(addr)) + self.execute_operation(rop.SETARRAYITEM_RAW, [abox, BoxInt(5), + BoxInt(12345)], + 'int', descr=descr) + assert a[5] == 12345 + lltype.free(a, flavor='raw') + + class OOtypeBackendTest(BaseBackendTest): type_system = 'ootype' From getxsick at codespeak.net Mon Aug 16 15:22:19 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Mon, 16 Aug 2010 15:22:19 +0200 (CEST) Subject: [pypy-svn] r76633 - in pypy/branch/fast-ctypes/pypy/module/_ctypes: . test Message-ID: <20100816132219.E7461282BD4@codespeak.net> Author: getxsick Date: Mon Aug 16 15:22:18 2010 New Revision: 76633 Added: pypy/branch/fast-ctypes/pypy/module/_ctypes/interp_dll.py (contents, props changed) pypy/branch/fast-ctypes/pypy/module/_ctypes/interp_test.py (contents, props changed) pypy/branch/fast-ctypes/pypy/module/_ctypes/test/test_dll.py (contents, props changed) Modified: pypy/branch/fast-ctypes/pypy/module/_ctypes/ (props changed) pypy/branch/fast-ctypes/pypy/module/_ctypes/__init__.py (contents, props changed) pypy/branch/fast-ctypes/pypy/module/_ctypes/test/ (props changed) pypy/branch/fast-ctypes/pypy/module/_ctypes/test/__init__.py (props changed) Log: add dlopen function (it's equivalent to CDLL from module/jitffi) Modified: pypy/branch/fast-ctypes/pypy/module/_ctypes/__init__.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/_ctypes/__init__.py (original) +++ pypy/branch/fast-ctypes/pypy/module/_ctypes/__init__.py Mon Aug 16 15:22:18 2010 @@ -1,5 +1,8 @@ from pypy.interpreter.mixedmodule import MixedModule class Module(MixedModule): - interpleveldefs = {} + interpleveldefs = { + 'dlopen' : 'interp_dll.W_CDLL', + 'Test' : 'interp_test.W_Test', + } appleveldefs = {} Added: pypy/branch/fast-ctypes/pypy/module/_ctypes/interp_dll.py ============================================================================== --- (empty file) +++ pypy/branch/fast-ctypes/pypy/module/_ctypes/interp_dll.py Mon Aug 16 15:22:18 2010 @@ -0,0 +1,144 @@ +from pypy.rlib import rjitffi +from pypy.interpreter.baseobjspace import ObjSpace, W_Root, Wrappable +from pypy.interpreter.error import OperationError, wrap_oserror +from pypy.interpreter.gateway import interp2app +from pypy.interpreter.typedef import TypeDef + +class W_CDLL(Wrappable): + def __init__(self, space, name, mode=0): + # XXX mode is ignored + if name is None: + return space.w_None # XXX this should return *all* loaded libs, dlopen(NULL) + self.space = space + self.rcdll = rjitffi.CDLL(name, load=False) + try: + self.lib_w = W_LibHandler(self.space, name) + except OSError, e: + raise OperationError(space.w_OSError, space.wrap(str(e))) + + def get_w(self, space, func, w_args_type, res_type='v'): + args_type_w = [ space.str_w(w_x) + for w_x in space.listview(w_args_type) ] + try: + ret = W_Get(space, self.rcdll.cpu, self.lib_w, + func, args_type_w, res_type) + except ValueError, e: + raise OperationError(space.w_ValueError, space.wrap(str(e))) + return space.wrap(ret) + +def W_CDLL___new__(space, w_type, name, mode=0): + try: + return space.wrap(W_CDLL(space, name, mode)) + except OSError, e: + raise wrap_oserror(space, e) + +W_CDLL.typedef = TypeDef( + 'CDLL', + __new__ = interp2app(W_CDLL___new__, + unwrap_spec=[ObjSpace, W_Root, str, int]), + get = interp2app(W_CDLL.get_w, + unwrap_spec=['self', ObjSpace, str, W_Root, str]), + __doc__ = """ C Dynamically loaded library +use CDLL(libname) to create a handle to a C library (the argument is processed +the same way as dlopen processes it).""" +) + + +class W_LibHandler(Wrappable): + def __init__(self, space, name): + self.space = space + try: + self.rlibhandler = rjitffi._LibHandler(name) + except OSError, e: + raise OperationError(space.w_OSError, space.wrap(str(e))) + self.handler = self.rlibhandler.handler + +def W_LibHandler___new__(space, w_type, name): + try: + return space.wrap(W_LibHandler(space, name)) + except OSError, e: + raise wrap_oserror(space, e) + +W_LibHandler.typedef = TypeDef( + 'LibHandler', + __new__ = interp2app(W_LibHandler___new__, unwrap_spec=[ObjSpace, + W_Root, str]) +) + +class Cache(object): + def __init__(self, space): + self.cache = {} + +class W_Get(Wrappable): + def __init__(self, space, cpu, lib, func, args_type, res_type='v'): + self.space = space + + wrap_func = (self.wrap_int_w, self.wrap_float_w, + self.wrap_ref_w, self.wrap_void_w) + self.rget = rjitffi._Get(cpu, lib, func, args_type, res_type, + wrap_func, cache=True) + + # grab from the cache if possible + arg_classes = ''.join(args_type) + key = (res_type, arg_classes) + cache = space.fromcache(Cache).cache + try: + self.rget.looptoken = cache[key] + except KeyError: + self.rget.gen_looptaken() + cache[key] = self.rget.looptoken + + def call_w(self, space, w_args=None): + if not space.is_w(w_args, space.w_None): + i = 0 + w_iterator = space.iter(w_args) + while True: + try: + w_arg = space.next(w_iterator) + except OperationError, e: + if not e.match(space, space.w_StopIteration): + raise + break # done + + if self.rget.args_type[i] == 'i': + self.rget.push_int(space.int_w(w_arg)) + elif self.rget.args_type[i] == 'f': + self.rget.push_float(space.float_w(w_arg)) + elif self.rget.args_type[i] == 'p': + self.rget.push_ref(space.int_w(w_arg)) + else: + # should never happen (raised earlier) + raise OperationError( + space.w_ValueError, + space.wrap('Unsupported type of argument: %s' + % self.rget.args_type[0])) + i += 1 + return self.rget.call() + + def wrap_int_w(self, value): + return self.space.wrap(value) + + def wrap_float_w(self, value): + return self.space.wrap(value) + + def wrap_ref_w(self, value): + return self.space.wrap(value) + + def wrap_void_w(self, w_value): + return w_value + +#def W_Get___new__(space, w_type, cpu, lib, func, args_type, res_type): +# try: +# return space.wrap(W_Get(space, w_type, cpu, lib, func, args_type, res_type)) +# except OSError, e: +# raise wrap_oserror(space, e) + +W_Get.typedef = TypeDef( + 'Get', + #__new__ = interp2app(W_Get___new__, unwrap_spec=[ObjSpace, W_Root, W_Root, W_Root, str, W_Root, str]), + call = interp2app(W_Get.call_w, unwrap_spec=['self', ObjSpace, W_Root]), + wrap_int = interp2app(W_Get.wrap_int_w, unwrap_spec=['self', int]), + wrap_float = interp2app(W_Get.wrap_float_w, unwrap_spec=['self', float]), + wrap_ref = interp2app(W_Get.wrap_ref_w, unwrap_spec=['self', int]), + wrap_void = interp2app(W_Get.wrap_void_w, unwrap_spec=['self', W_Root]) +) Added: pypy/branch/fast-ctypes/pypy/module/_ctypes/interp_test.py ============================================================================== --- (empty file) +++ pypy/branch/fast-ctypes/pypy/module/_ctypes/interp_test.py Mon Aug 16 15:22:18 2010 @@ -0,0 +1,45 @@ +from pypy.interpreter.baseobjspace import ObjSpace, W_Root, Wrappable +from pypy.interpreter.gateway import interp2app +from pypy.interpreter.typedef import TypeDef +from pypy.rpython.lltypesystem import rffi, lltype + +# for tests only +class W_Test(Wrappable): + def __init__(self, space): + self.space = space + + def get_intp_w(self, space, n, w_values): + values_w = [ space.int_w(w_x) for w_x in space.listview(w_values) ] + intp = lltype.malloc(rffi.INTP.TO, n, flavor='raw') # XXX free it! + for i in xrange(n): + intp[i] = values_w[i] + return space.wrap(rffi.cast(lltype.Signed, intp)) + + def get_charp_w(self, space, txt): + charp = rffi.str2charp(txt) + return space.wrap(rffi.cast(lltype.Signed, charp)) # XXX free it! + + def get_str_w(self, space, addr): + charp = rffi.cast(rffi.CCHARP, addr) + return space.wrap(rffi.charp2str(charp)) # XXX free it? + + def get_int_from_addr_w(self, space, addr): + intp = rffi.cast(rffi.INTP, addr) + return space.wrap(intp[0]) # return the first element + +def W_Test___new__(space, w_x): + return space.wrap(W_Test(space)) + +W_Test.typedef = TypeDef( + 'Test', + __new__ = interp2app(W_Test___new__, unwrap_spec=[ObjSpace, W_Root]), + get_intp = interp2app(W_Test.get_intp_w, + unwrap_spec=['self', ObjSpace, int, W_Root]), + get_charp = interp2app(W_Test.get_charp_w, + unwrap_spec=['self', ObjSpace, str]), + get_str = interp2app(W_Test.get_str_w, + unwrap_spec=['self', ObjSpace, int]), + get_int_from_addr = interp2app(W_Test.get_int_from_addr_w, + unwrap_spec=['self', ObjSpace, int]) +) + Added: pypy/branch/fast-ctypes/pypy/module/_ctypes/test/test_dll.py ============================================================================== --- (empty file) +++ pypy/branch/fast-ctypes/pypy/module/_ctypes/test/test_dll.py Mon Aug 16 15:22:18 2010 @@ -0,0 +1,178 @@ +from pypy.conftest import gettestobjspace +from pypy.translator.tool.cbuild import ExternalCompilationInfo +from pypy.translator.platform import platform + +import py + +class AppTestJitffi(object): + @staticmethod + def preprare_c_example(): + from pypy.tool.udir import udir + c_file = udir.ensure("test_ctypes", dir=True).join("xlib.c") + c_file.write(py.code.Source(''' + int add_integers(int a, int b) + { + return a+b; + } + + double add_floats(double a, double b) + { + return a+b; + } + + int add_intfloat(int a, double b) + { + int rb = (int)b; + return a+rb; + } + + double return_float(int a, int b) + { + return a+b; + } + + int max3(int a, int b, int c) + { + int max = a; + if (b > max) max = b; + if (c > max) max = c; + return max; + } + + int fvoid(void) + { + return 1; + } + + void return_void(int a, int b) + { + int c; + c = a + b; + } + int return_ptrvalue(int a, int *b) + { + return a+(*b); + } + int sum_intarray(int *a) + { + int i; + int sum = 0; + for(i=0; i<5; i++) + { + sum += *a+i; + } + return sum; + } + void a2b(char *txt) + { + int i; + for(i=0; txt[i] != '\0'; i++) + { + if (txt[i] == 'a') txt[i] = 'b'; + } + } + int *return_intptr(int a) + { + int *x = malloc(sizeof(int)); + *x = a; + return x; + } + ''' + )) + + symbols = ['add_integers', 'add_floats', 'add_intfloat', + 'return_float', 'max3', 'fvoid', 'return_void', + 'return_ptrvalue', 'sum_intarray', 'a2b', 'return_intptr'] + eci = ExternalCompilationInfo(export_symbols=symbols) + + return str(platform.compile([c_file], eci, 'x', standalone=False)) + + def setup_class(cls): + space = gettestobjspace(usemodules=('_ctypes',)) + cls.space = space + cls.w_lib_name = space.wrap(cls.preprare_c_example()) + + def test_missing_lib(self): + from _ctypes import dlopen + raises(OSError, dlopen, 'xxxfoo888baryyy') + + def test_get(self): + from _ctypes import dlopen + lib = dlopen(self.lib_name) + + func = lib.get('add_integers', ['i', 'i'], 'i') + assert 3 == func.call([1,2]) + func = lib.get('add_integers', ['i', 'i'], 'i') + assert 1 == func.call([-1,2]) + func = lib.get('add_integers', ['i', 'i'], 'i') + assert 0 == func.call([0,0]) + + func = lib.get('max3', ['i', 'i', 'i'], 'i') + assert 8 == func.call([2, 8, 3]) + + func = lib.get('add_floats', ['f', 'f'], 'f') + assert 2.7 == func.call([1.2, 1.5]) + + def test_get_void(self): + from _ctypes import dlopen + lib = dlopen(self.lib_name) + + func = lib.get('fvoid', [], 'i') + assert 1 == func.call() + + func = lib.get('return_void', ['i', 'i'], 'v') + assert func.call([1, 2]) is None + func = lib.get('return_void', ['i', 'i']) + assert func.call([1, 2]) is None + + def test_various_type_args(self): + from _ctypes import dlopen + lib = dlopen(self.lib_name) + + func = lib.get('add_intfloat', ['i', 'f'], 'i') + assert func.call([1, 2.9]) == 3 + assert func.call([0, 1.3]) == 1 + + def test_ptrargs(self): + from _ctypes import dlopen, Test + t = Test() + lib = dlopen(self.lib_name) + + func = lib.get('return_ptrvalue', ['i', 'p'], 'i') + intp = t.get_intp(1, [10]) + assert func.call([20, intp]) == 30 + + func = lib.get('sum_intarray', ['p'], 'i') + intp = t.get_intp(5, [ i for i in xrange(5) ]) + assert func.call([intp]) == 10 + + func = lib.get('a2b', ['p']) + charp = t.get_charp('xaxaxa') + func.call([charp]) + assert t.get_str(charp) == 'xbxbxb' + + def test_get_ptr(self): + from _ctypes import dlopen, Test + t = Test() + lib = dlopen(self.lib_name) + + func = lib.get('return_intptr', ['i'], 'p') + addr = func.call([22]) + ret = t.get_int_from_addr(addr) + assert ret == 22 + + def test_undefined_func(self): + from _ctypes import dlopen + lib = dlopen(self.lib_name) + # xxxfoo888baryyy - not existed function + raises(ValueError, lib.get, 'xxxfoo888baryyy', []) + raises(ValueError, lib.get, 'xxxfoo888baryyy', ['i'], 'i') + + def test_unknown_types(self): + from _ctypes import dlopen + lib = dlopen(self.lib_name) + # xxxfoo888baryyy - not defined types (args_type, res_type etc.) + raises(ValueError, lib.get, 'fvoid', ['xxxfoo888baryyy']) + raises(ValueError, lib.get, 'fvoid', ['i','xxxfoo888baryyy']) + raises(ValueError, lib.get, 'fvoid', ['xxxfoo888baryyy'],'i') + raises(ValueError, lib.get, 'fvoid', [], 'xxxfoo888baryyy') From getxsick at codespeak.net Mon Aug 16 17:45:28 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Mon, 16 Aug 2010 17:45:28 +0200 (CEST) Subject: [pypy-svn] r76634 - pypy/branch/fast-ctypes/pypy/module/_ctypes Message-ID: <20100816154528.CF026282BAD@codespeak.net> Author: getxsick Date: Mon Aug 16 17:45:27 2010 New Revision: 76634 Modified: pypy/branch/fast-ctypes/pypy/module/_ctypes/__init__.py Log: set __version__ for module/_ctypes Modified: pypy/branch/fast-ctypes/pypy/module/_ctypes/__init__.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/_ctypes/__init__.py (original) +++ pypy/branch/fast-ctypes/pypy/module/_ctypes/__init__.py Mon Aug 16 17:45:27 2010 @@ -2,7 +2,8 @@ class Module(MixedModule): interpleveldefs = { + '__version__' : 'space.wrap("1.0.3")', 'dlopen' : 'interp_dll.W_CDLL', - 'Test' : 'interp_test.W_Test', + 'Test' : 'interp_test.W_Test', } appleveldefs = {} From getxsick at codespeak.net Mon Aug 16 18:00:18 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Mon, 16 Aug 2010 18:00:18 +0200 (CEST) Subject: [pypy-svn] r76635 - pypy/trunk/pypy/module/_locale Message-ID: <20100816160018.863C2282BAD@codespeak.net> Author: getxsick Date: Mon Aug 16 18:00:17 2010 New Revision: 76635 Modified: pypy/trunk/pypy/module/_locale/__init__.py Log: kill import Modified: pypy/trunk/pypy/module/_locale/__init__.py ============================================================================== --- pypy/trunk/pypy/module/_locale/__init__.py (original) +++ pypy/trunk/pypy/module/_locale/__init__.py Mon Aug 16 18:00:17 2010 @@ -1,5 +1,4 @@ from pypy.interpreter.mixedmodule import MixedModule -from pypy.module._locale import interp_locale from pypy.rlib import rlocale import sys From dan at codespeak.net Mon Aug 16 18:06:45 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Mon, 16 Aug 2010 18:06:45 +0200 (CEST) Subject: [pypy-svn] r76636 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100816160645.C3BAC282BD4@codespeak.net> Author: dan Date: Mon Aug 16 18:06:44 2010 New Revision: 76636 Modified: pypy/trunk/pypy/module/cpyext/stubs.py pypy/trunk/pypy/module/cpyext/test/test_unicodeobject.py pypy/trunk/pypy/module/cpyext/unicodeobject.py Log: Finally committing unicode changes. Modified: pypy/trunk/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/stubs.py (original) +++ pypy/trunk/pypy/module/cpyext/stubs.py Mon Aug 16 18:06:44 2010 @@ -2874,36 +2874,6 @@ """ raise NotImplementedError - at cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.INTP], PyObject) -def PyUnicode_DecodeUTF16(space, s, size, errors, byteorder): - """Decode length bytes from a UTF-16 encoded buffer string and return the - corresponding Unicode object. errors (if non-NULL) defines the error - handling. It defaults to "strict". - - If byteorder is non-NULL, the decoder starts decoding using the given byte - order: - - *byteorder == -1: little endian - *byteorder == 0: native order - *byteorder == 1: big endian - - If *byteorder is zero, and the first two bytes of the input data are a - byte order mark (BOM), the decoder switches to this byte order and the BOM is - not copied into the resulting Unicode string. If *byteorder is -1 or - 1, any byte order mark is copied to the output (where it will result in - either a \ufeff or a \ufffe character). - - After completion, *byteorder is set to the current byte order at the end - of input data. - - If byteorder is NULL, the codec starts in native order mode. - - Return NULL if an exception was raised by the codec. - - This function used an int type for size. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.INTP, Py_ssize_t], PyObject) def PyUnicode_DecodeUTF16Stateful(space, s, size, errors, byteorder, consumed): """If consumed is NULL, behave like PyUnicode_DecodeUTF16(). If Modified: pypy/trunk/pypy/module/cpyext/test/test_unicodeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_unicodeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_unicodeobject.py Mon Aug 16 18:06:44 2010 @@ -172,4 +172,37 @@ result = api.PyUnicode_AsASCIIString(w_ustr) assert result is None + def test_decode_utf16(self, space, api): + def test(encoded, endian, realendian=None): + encoded_charp = rffi.str2charp(encoded) + strict_charp = rffi.str2charp("strict") + if endian is not None: + pendian = lltype.malloc(rffi.INTP.TO, 1, flavor='raw') + if endian < 0: + pendian[0] = -1 + elif endian > 0: + pendian[0] = 1 + else: + pendian[0] = 0 + else: + pendian = None + w_ustr = api.PyUnicode_DecodeUTF16(encoded_charp, len(encoded), strict_charp, pendian) + assert space.eq_w(space.call_method(w_ustr, 'encode', space.wrap('ascii')), + space.wrap("abcd")) + + rffi.free_charp(encoded_charp) + rffi.free_charp(strict_charp) + if pendian: + if realendian is not None: + assert rffi.cast(rffi.INT, realendian) == pendian[0] + lltype.free(pendian, flavor='raw') + + test("\x61\x00\x62\x00\x63\x00\x64\x00", -1) + + test("\x61\x00\x62\x00\x63\x00\x64\x00", None) + + test("\x00\x61\x00\x62\x00\x63\x00\x64", 1) + + test("\xFE\xFF\x00\x61\x00\x62\x00\x63\x00\x64", 0, 1) + test("\xFF\xFE\x61\x00\x62\x00\x63\x00\x64\x00", 0, -1) Modified: pypy/trunk/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/unicodeobject.py Mon Aug 16 18:06:44 2010 @@ -9,6 +9,7 @@ from pypy.module.cpyext.pyobject import PyObject, from_ref, make_typedescr from pypy.module.sys.interp_encoding import setdefaultencoding from pypy.objspace.std import unicodeobject, unicodetype +from pypy.rlib import runicode import sys ## See comment in stringobject.py. PyUnicode_FromUnicode(NULL, size) is not @@ -307,6 +308,64 @@ w_errors = space.w_None return space.call_method(w_str, 'decode', space.wrap("utf-8"), w_errors) + at cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.INTP], PyObject) +def PyUnicode_DecodeUTF16(space, s, size, llerrors, pbyteorder): + """Decode length bytes from a UTF-16 encoded buffer string and return the + corresponding Unicode object. errors (if non-NULL) defines the error + handling. It defaults to "strict". + + If byteorder is non-NULL, the decoder starts decoding using the given byte + order: + + *byteorder == -1: little endian + *byteorder == 0: native order + *byteorder == 1: big endian + + If *byteorder is zero, and the first two bytes of the input data are a + byte order mark (BOM), the decoder switches to this byte order and the BOM is + not copied into the resulting Unicode string. If *byteorder is -1 or + 1, any byte order mark is copied to the output (where it will result in + either a \ufeff or a \ufffe character). + + After completion, *byteorder is set to the current byte order at the end + of input data. + + If byteorder is NULL, the codec starts in native order mode. + + Return NULL if an exception was raised by the codec. + + This function used an int type for size. This might require + changes in your code for properly supporting 64-bit systems.""" + + string = rffi.charpsize2str(s, size) + + #FIXME: I don't like these prefixes + if pbyteorder is not None: # correct NULL check? + llbyteorder = rffi.cast(lltype.Signed, pbyteorder[0]) # compatible with int? + if llbyteorder < 0: + byteorder = "little" + elif llbyteorder > 0: + byteorder = "big" + else: + byteorder = "native" + else: + byteorder = "native" + + if llerrors: + errors = rffi.charp2str(llerrors) + else: + errors = None + + result, length, byteorder = runicode.str_decode_utf_16_helper(string, size, + errors, + True, # final ? false for multiple passes? + None, # errorhandler + byteorder) + if pbyteorder is not None: + pbyteorder[0] = rffi.cast(rffi.INT, byteorder) + + return space.wrap(result) + @cpython_api([PyObject], PyObject) def PyUnicode_AsASCIIString(space, w_unicode): """Encode a Unicode object using ASCII and return the result as Python string From getxsick at codespeak.net Mon Aug 16 18:18:34 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Mon, 16 Aug 2010 18:18:34 +0200 (CEST) Subject: [pypy-svn] r76637 - in pypy/branch/fast-ctypes/pypy: module/_ctypes rlib Message-ID: <20100816161834.88275282BAD@codespeak.net> Author: getxsick Date: Mon Aug 16 18:18:33 2010 New Revision: 76637 Added: pypy/branch/fast-ctypes/pypy/module/_ctypes/constants.py (contents, props changed) Modified: pypy/branch/fast-ctypes/pypy/module/_ctypes/__init__.py pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Log: add some constants Modified: pypy/branch/fast-ctypes/pypy/module/_ctypes/__init__.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/_ctypes/__init__.py (original) +++ pypy/branch/fast-ctypes/pypy/module/_ctypes/__init__.py Mon Aug 16 18:18:33 2010 @@ -2,8 +2,24 @@ class Module(MixedModule): interpleveldefs = { - '__version__' : 'space.wrap("1.0.3")', 'dlopen' : 'interp_dll.W_CDLL', 'Test' : 'interp_test.W_Test', } appleveldefs = {} + + def buildloaders(cls): + from pypy.module._ctypes.constants import constants + for constant, value in constants.iteritems(): + Module.interpleveldefs[constant] = 'space.wrap(%r)' % value + + from pypy.rlib import rjitffi + for name in ['FUNCFLAG_STDCALL', + 'FUNCFLAG_CDECL', + 'FUNCFLAG_PYTHONAPI', + ]: + if hasattr(rjitffi, name): + Module.interpleveldefs[name] = \ + "space.wrap(%r)" % getattr(rjitffi, name) + + super(Module, cls).buildloaders() + buildloaders = classmethod(buildloaders) Added: pypy/branch/fast-ctypes/pypy/module/_ctypes/constants.py ============================================================================== --- (empty file) +++ pypy/branch/fast-ctypes/pypy/module/_ctypes/constants.py Mon Aug 16 18:18:33 2010 @@ -0,0 +1,5 @@ +constants = { + '__version__' : '1.0.3', + 'RTLD_LOCAL' : 0, + 'RTLD_GLOBAL' : 256, +} Modified: pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rjitffi.py Mon Aug 16 18:18:33 2010 @@ -12,6 +12,10 @@ GLOBAL_CPU._vtable_to_descr_dict = None GLOBAL_CPU.setup() +FUNCFLAG_STDCALL = 0 +FUNCFLAG_CDECL = 1 # for WINAPI calls +FUNCFLAG_PYTHONAPI = 4 + class CDLL(object): def __init__(self, name, load=True): if load: From getxsick at codespeak.net Mon Aug 16 19:12:28 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Mon, 16 Aug 2010 19:12:28 +0200 (CEST) Subject: [pypy-svn] r76638 - pypy/branch/fast-ctypes/pypy/module/_ctypes/test Message-ID: <20100816171228.0C088282BAD@codespeak.net> Author: getxsick Date: Mon Aug 16 19:12:27 2010 New Revision: 76638 Added: pypy/branch/fast-ctypes/pypy/module/_ctypes/test/_ctypes_test.c pypy/branch/fast-ctypes/pypy/module/_ctypes/test/conftest.py pypy/branch/fast-ctypes/pypy/module/_ctypes/test/test_loading.py Log: import and modify test_loading from lib_pypy/_ctypes tests pass but some lines have to be changed in modified/ctypes (temporary, not commited) Added: pypy/branch/fast-ctypes/pypy/module/_ctypes/test/_ctypes_test.c ============================================================================== --- (empty file) +++ pypy/branch/fast-ctypes/pypy/module/_ctypes/test/_ctypes_test.c Mon Aug 16 19:12:27 2010 @@ -0,0 +1,535 @@ +#if defined(_MSC_VER) || defined(__CYGWIN__) +#include +#define MS_WIN32 +#endif + +#if defined(MS_WIN32) +#define EXPORT(x) __declspec(dllexport) x +#else +#define EXPORT(x) x +#endif + +#include +#include +#include +#include +#include + +#define HAVE_LONG_LONG +#define LONG_LONG long long +#define HAVE_WCHAR_H + + +/* some functions handy for testing */ + +EXPORT(char *)my_strtok(char *token, const char *delim) +{ + return strtok(token, delim); +} + +EXPORT(char *)my_strchr(const char *s, int c) +{ + return strchr(s, c); +} + + +EXPORT(double) my_sqrt(double a) +{ + return sqrt(a); +} + +EXPORT(void) my_qsort(void *base, size_t num, size_t width, int(*compare)(const void*, const void*)) +{ + qsort(base, num, width, compare); +} + +EXPORT(int *) _testfunc_ai8(int a[8]) +{ + return a; +} + +EXPORT(void) _testfunc_v(int a, int b, int *presult) +{ + *presult = a + b; +} + +EXPORT(int) _testfunc_i_bhilfd(signed char b, short h, int i, long l, float f, double d) +{ +/* printf("_testfunc_i_bhilfd got %d %d %d %ld %f %f\n", + b, h, i, l, f, d); +*/ + return (int)(b + h + i + l + f + d); +} + +EXPORT(float) _testfunc_f_bhilfd(signed char b, short h, int i, long l, float f, double d) +{ +/* printf("_testfunc_f_bhilfd got %d %d %d %ld %f %f\n", + b, h, i, l, f, d); +*/ + return (float)(b + h + i + l + f + d); +} + +EXPORT(double) _testfunc_d_bhilfd(signed char b, short h, int i, long l, float f, double d) +{ +/* printf("_testfunc_d_bhilfd got %d %d %d %ld %f %f\n", + b, h, i, l, f, d); +*/ + return (double)(b + h + i + l + f + d); +} + +EXPORT(char *) _testfunc_p_p(void *s) +{ + return (char *)s; +} + +EXPORT(void *) _testfunc_c_p_p(int *argcp, char **argv) +{ + return argv[(*argcp)-1]; +} + +EXPORT(void *) get_strchr(void) +{ + return (void *)strchr; +} + +EXPORT(char *) my_strdup(char *src) +{ + char *dst = (char *)malloc(strlen(src)+1); + if (!dst) + return NULL; + strcpy(dst, src); + return dst; +} + +EXPORT(void)my_free(void *ptr) +{ + free(ptr); +} + +#ifdef HAVE_WCHAR_H +EXPORT(wchar_t *) my_wcsdup(wchar_t *src) +{ + size_t len = wcslen(src); + wchar_t *ptr = (wchar_t *)malloc((len + 1) * sizeof(wchar_t)); + if (ptr == NULL) + return NULL; + memcpy(ptr, src, (len+1) * sizeof(wchar_t)); + return ptr; +} + +EXPORT(size_t) my_wcslen(wchar_t *src) +{ + return wcslen(src); +} +#endif + +#ifndef MS_WIN32 +# ifndef __stdcall +# define __stdcall /* */ +# endif +#endif + +typedef struct { + int (*c)(int, int); + int (__stdcall *s)(int, int); +} FUNCS; + +EXPORT(int) _testfunc_callfuncp(FUNCS *fp) +{ + fp->c(1, 2); + fp->s(3, 4); + return 0; +} + +EXPORT(int) _testfunc_deref_pointer(int *pi) +{ + return *pi; +} + +#ifdef MS_WIN32 +EXPORT(int) _testfunc_piunk(IUnknown FAR *piunk) +{ + piunk->lpVtbl->AddRef(piunk); + return piunk->lpVtbl->Release(piunk); +} +#endif + +EXPORT(int) _testfunc_callback_with_pointer(int (*func)(int *)) +{ + int table[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + + return (*func)(table); +} + +EXPORT(int) _testfunc_callback_opaque(int (*func)(void*), void* arg) +{ + return (*func)(arg); +} + +#ifdef HAVE_LONG_LONG +EXPORT(LONG_LONG) _testfunc_q_bhilfdq(signed char b, short h, int i, long l, float f, + double d, LONG_LONG q) +{ + return (LONG_LONG)(b + h + i + l + f + d + q); +} + +EXPORT(LONG_LONG) _testfunc_q_bhilfd(signed char b, short h, int i, long l, float f, double d) +{ + return (LONG_LONG)(b + h + i + l + f + d); +} + +EXPORT(int) _testfunc_callback_i_if(int value, int (*func)(int)) +{ + int sum = 0; + while (value != 0) { + sum += func(value); + value /= 2; + } + return sum; +} + +EXPORT(LONG_LONG) _testfunc_callback_q_qf(LONG_LONG value, + LONG_LONG (*func)(LONG_LONG)) +{ + LONG_LONG sum = 0; + + while (value != 0) { + sum += func(value); + value /= 2; + } + return sum; +} + +#endif + +typedef struct { + char *name; + char *value; +} SPAM; + +typedef struct { + char *name; + int num_spams; + SPAM *spams; +} EGG; + +SPAM my_spams[2] = { + { "name1", "value1" }, + { "name2", "value2" }, +}; + +EGG my_eggs[1] = { + { "first egg", 1, my_spams } +}; + +EXPORT(int) getSPAMANDEGGS(EGG **eggs) +{ + *eggs = my_eggs; + return 1; +} + +typedef struct tagpoint { + int x; + int y; +} point; + +EXPORT(int) _testfunc_byval(point in, point *pout) +{ + if (pout) { + pout->x = in.x; + pout->y = in.y; + } + return in.x + in.y; +} + +EXPORT (int) an_integer = 42; + +EXPORT(int) get_an_integer(void) +{ + return an_integer; +} + +EXPORT(char) a_string[16] = "0123456789abcdef"; + +EXPORT(int) get_a_string_char(int index) +{ + return a_string[index]; +} + +EXPORT(double) +integrate(double a, double b, double (*f)(double), long nstep) +{ + double x, sum=0.0, dx=(b-a)/(double)nstep; + for(x=a+0.5*dx; (b-x)*(x-a)>0.0; x+=dx) + sum += f(x); + return sum/(double)nstep; +} + +typedef struct { + void (*initialize)(void *(*)(int), void(*)(void *)); +} xxx_library; + +static void _xxx_init(void *(*Xalloc)(int), void (*Xfree)(void *)) +{ + void *ptr; + + printf("_xxx_init got %p %p\n", Xalloc, Xfree); + printf("calling\n"); + ptr = Xalloc(32); + Xfree(ptr); + printf("calls done, ptr was %p\n", ptr); +} + +xxx_library _xxx_lib = { + _xxx_init +}; + +EXPORT(xxx_library) *library_get(void) +{ + return &_xxx_lib; +} + +#ifdef MS_WIN32 +/* See Don Box (german), pp 79ff. */ +EXPORT(void) GetString(BSTR *pbstr) +{ + *pbstr = SysAllocString(L"Goodbye!"); +} +#endif + +EXPORT(void) _py_func_si(char *s, int i) +{ +} + +EXPORT(void) _py_func(void) +{ +} + +EXPORT(LONG_LONG) last_tf_arg_s; +EXPORT(unsigned LONG_LONG) last_tf_arg_u; + +struct BITS { + int A: 1, B:2, C:3, D:4, E: 5, F: 6, G: 7, H: 8, I: 9; + short M: 1, N: 2, O: 3, P: 4, Q: 5, R: 6, S: 7; +}; + +EXPORT(void) set_bitfields(struct BITS *bits, char name, int value) +{ + switch (name) { + case 'A': bits->A = value; break; + case 'B': bits->B = value; break; + case 'C': bits->C = value; break; + case 'D': bits->D = value; break; + case 'E': bits->E = value; break; + case 'F': bits->F = value; break; + case 'G': bits->G = value; break; + case 'H': bits->H = value; break; + case 'I': bits->I = value; break; + + case 'M': bits->M = value; break; + case 'N': bits->N = value; break; + case 'O': bits->O = value; break; + case 'P': bits->P = value; break; + case 'Q': bits->Q = value; break; + case 'R': bits->R = value; break; + case 'S': bits->S = value; break; + } +} + +EXPORT(int) unpack_bitfields(struct BITS *bits, char name) +{ + switch (name) { + case 'A': return bits->A; + case 'B': return bits->B; + case 'C': return bits->C; + case 'D': return bits->D; + case 'E': return bits->E; + case 'F': return bits->F; + case 'G': return bits->G; + case 'H': return bits->H; + case 'I': return bits->I; + + case 'M': return bits->M; + case 'N': return bits->N; + case 'O': return bits->O; + case 'P': return bits->P; + case 'Q': return bits->Q; + case 'R': return bits->R; + case 'S': return bits->S; + } + return 0; +} + +#define S last_tf_arg_s = (LONG_LONG)c +#define U last_tf_arg_u = (unsigned LONG_LONG)c + +EXPORT(signed char) tf_b(signed char c) { S; return c/3; } +EXPORT(unsigned char) tf_B(unsigned char c) { U; return c/3; } +EXPORT(short) tf_h(short c) { S; return c/3; } +EXPORT(unsigned short) tf_H(unsigned short c) { U; return c/3; } +EXPORT(int) tf_i(int c) { S; return c/3; } +EXPORT(unsigned int) tf_I(unsigned int c) { U; return c/3; } +EXPORT(long) tf_l(long c) { S; return c/3; } +EXPORT(unsigned long) tf_L(unsigned long c) { U; return c/3; } +EXPORT(LONG_LONG) tf_q(LONG_LONG c) { S; return c/3; } +EXPORT(unsigned LONG_LONG) tf_Q(unsigned LONG_LONG c) { U; return c/3; } +EXPORT(float) tf_f(float c) { S; return c/3; } +EXPORT(double) tf_d(double c) { S; return c/3; } + +#ifdef MS_WIN32 +EXPORT(signed char) __stdcall s_tf_b(signed char c) { S; return c/3; } +EXPORT(unsigned char) __stdcall s_tf_B(unsigned char c) { U; return c/3; } +EXPORT(short) __stdcall s_tf_h(short c) { S; return c/3; } +EXPORT(unsigned short) __stdcall s_tf_H(unsigned short c) { U; return c/3; } +EXPORT(int) __stdcall s_tf_i(int c) { S; return c/3; } +EXPORT(unsigned int) __stdcall s_tf_I(unsigned int c) { U; return c/3; } +EXPORT(long) __stdcall s_tf_l(long c) { S; return c/3; } +EXPORT(unsigned long) __stdcall s_tf_L(unsigned long c) { U; return c/3; } +EXPORT(LONG_LONG) __stdcall s_tf_q(LONG_LONG c) { S; return c/3; } +EXPORT(unsigned LONG_LONG) __stdcall s_tf_Q(unsigned LONG_LONG c) { U; return c/3; } +EXPORT(float) __stdcall s_tf_f(float c) { S; return c/3; } +EXPORT(double) __stdcall s_tf_d(double c) { S; return c/3; } +#endif +/*******/ + +EXPORT(signed char) tf_bb(signed char x, signed char c) { S; return c/3; } +EXPORT(unsigned char) tf_bB(signed char x, unsigned char c) { U; return c/3; } +EXPORT(short) tf_bh(signed char x, short c) { S; return c/3; } +EXPORT(unsigned short) tf_bH(signed char x, unsigned short c) { U; return c/3; } +EXPORT(int) tf_bi(signed char x, int c) { S; return c/3; } +EXPORT(unsigned int) tf_bI(signed char x, unsigned int c) { U; return c/3; } +EXPORT(long) tf_bl(signed char x, long c) { S; return c/3; } +EXPORT(unsigned long) tf_bL(signed char x, unsigned long c) { U; return c/3; } +EXPORT(LONG_LONG) tf_bq(signed char x, LONG_LONG c) { S; return c/3; } +EXPORT(unsigned LONG_LONG) tf_bQ(signed char x, unsigned LONG_LONG c) { U; return c/3; } +EXPORT(float) tf_bf(signed char x, float c) { S; return c/3; } +EXPORT(double) tf_bd(signed char x, double c) { S; return c/3; } +EXPORT(void) tv_i(int c) { S; return; } + +#ifdef MS_WIN32 +EXPORT(signed char) __stdcall s_tf_bb(signed char x, signed char c) { S; return c/3; } +EXPORT(unsigned char) __stdcall s_tf_bB(signed char x, unsigned char c) { U; return c/3; } +EXPORT(short) __stdcall s_tf_bh(signed char x, short c) { S; return c/3; } +EXPORT(unsigned short) __stdcall s_tf_bH(signed char x, unsigned short c) { U; return c/3; } +EXPORT(int) __stdcall s_tf_bi(signed char x, int c) { S; return c/3; } +EXPORT(unsigned int) __stdcall s_tf_bI(signed char x, unsigned int c) { U; return c/3; } +EXPORT(long) __stdcall s_tf_bl(signed char x, long c) { S; return c/3; } +EXPORT(unsigned long) __stdcall s_tf_bL(signed char x, unsigned long c) { U; return c/3; } +EXPORT(LONG_LONG) __stdcall s_tf_bq(signed char x, LONG_LONG c) { S; return c/3; } +EXPORT(unsigned LONG_LONG) __stdcall s_tf_bQ(signed char x, unsigned LONG_LONG c) { U; return c/3; } +EXPORT(float) __stdcall s_tf_bf(signed char x, float c) { S; return c/3; } +EXPORT(double) __stdcall s_tf_bd(signed char x, double c) { S; return c/3; } +EXPORT(void) __stdcall s_tv_i(int c) { S; return; } +#endif + +/********/ + +#ifndef MS_WIN32 + +typedef struct { + long x; + long y; +} POINT; + +typedef struct { + long left; + long top; + long right; + long bottom; +} RECT; + +#endif + +EXPORT(int) PointInRect(RECT *prc, POINT pt) +{ + if (pt.x < prc->left) + return 0; + if (pt.x > prc->right) + return 0; + if (pt.y < prc->top) + return 0; + if (pt.y > prc->bottom) + return 0; + return 1; +} + +typedef struct { + short x; + short y; +} S2H; + +EXPORT(S2H) ret_2h_func(S2H inp) +{ + inp.x *= 2; + inp.y *= 3; + return inp; +} + +typedef struct { + int a, b, c, d, e, f, g, h; +} S8I; + +EXPORT(S8I) ret_8i_func(S8I inp) +{ + inp.a *= 2; + inp.b *= 3; + inp.c *= 4; + inp.d *= 5; + inp.e *= 6; + inp.f *= 7; + inp.g *= 8; + inp.h *= 9; + return inp; +} + +EXPORT(int) GetRectangle(int flag, RECT *prect) +{ + if (flag == 0) + return 0; + prect->left = (int)flag; + prect->top = (int)flag + 1; + prect->right = (int)flag + 2; + prect->bottom = (int)flag + 3; + return 1; +} + +EXPORT(void) TwoOutArgs(int a, int *pi, int b, int *pj) +{ + *pi += a; + *pj += b; +} + +#ifdef MS_WIN32 +EXPORT(S2H) __stdcall s_ret_2h_func(S2H inp) { return ret_2h_func(inp); } +EXPORT(S8I) __stdcall s_ret_8i_func(S8I inp) { return ret_8i_func(inp); } +#endif + +#ifdef MS_WIN32 +/* Should port this */ +#include +#include + +EXPORT (HRESULT) KeepObject(IUnknown *punk) +{ + static IUnknown *pobj; + if (punk) + punk->lpVtbl->AddRef(punk); + if (pobj) + pobj->lpVtbl->Release(pobj); + pobj = punk; + return S_OK; +} + +#endif + +typedef union { + short x; + long y; +} UN; + +EXPORT(UN) ret_un_func(UN inp) +{ + inp.y = inp.x * 10000; + return inp; +} + Added: pypy/branch/fast-ctypes/pypy/module/_ctypes/test/conftest.py ============================================================================== --- (empty file) +++ pypy/branch/fast-ctypes/pypy/module/_ctypes/test/conftest.py Mon Aug 16 19:12:27 2010 @@ -0,0 +1,21 @@ +import py +import sys + +def compile_so_file(): + from pypy.translator.platform import platform + from pypy.translator.tool.cbuild import ExternalCompilationInfo + udir = py.test.ensuretemp('test__ctypes') + cfile = py.path.local(__file__).dirpath().join("_ctypes_test.c") + + if sys.platform == 'win32': + libraries = ['oleaut32'] + else: + libraries = [] + eci = ExternalCompilationInfo(libraries=libraries) + + return platform.compile([cfile], eci, str(udir.join('_ctypes_test')), + standalone=False) + +def pytest_configure(config): + global sofile + sofile = compile_so_file() Added: pypy/branch/fast-ctypes/pypy/module/_ctypes/test/test_loading.py ============================================================================== --- (empty file) +++ pypy/branch/fast-ctypes/pypy/module/_ctypes/test/test_loading.py Mon Aug 16 19:12:27 2010 @@ -0,0 +1,92 @@ +from pypy.conftest import gettestobjspace + +import sys +import os + +import py + + +class AppTestLoader(object): + def setup_class(cls): + space = gettestobjspace(usemodules=('_ctypes',)) + cls.space = space + cls.w_unknowndll = space.wrap("xxrandomnamexx") + + from ctypes.util import find_library + libc_name = None + if os.name == "nt": + libc_name = "msvcrt" + elif os.name == "ce": + libc_name = "coredll" + elif sys.platform == "cygwin": + libc_name = "cygwin1.dll" + else: + libc_name = find_library("c") + cls.w_libc_name = space.wrap(libc_name) + + def test_load(self): + if self.libc_name is None: + skip("Libc not found") + + import os + from ctypes import CDLL + CDLL(self.libc_name) + CDLL(os.path.basename(self.libc_name)) + raises(OSError, CDLL, self.unknowndll) + + def test_load_version(self): + import os + if self.libc_name is None \ + or os.path.basename(self.libc_name) != "libc.so.6": + skip("Libc not found or wrong libc name") + + from ctypes import cdll + cdll.LoadLibrary("libc.so.6") + # linux uses version, libc 9 should not exist + raises(OSError, cdll.LoadLibrary, "libc.so.9") + raises(OSError, cdll.LoadLibrary, self.unknowndll) + + def test_find(self): + from ctypes import cdll, CDLL + from ctypes.util import find_library + for name in ("c", "m"): + lib = find_library(name) + if lib: + cdll.LoadLibrary(lib) + CDLL(lib) + + def test_load_library(self): + import os + if os.name not in ("nt", "ce"): + skip('test is platform dependent') + + if os.name == "nt": + windll.kernel32.GetModuleHandleW + windll["kernel32"].GetModuleHandleW + windll.LoadLibrary("kernel32").GetModuleHandleW + WinDLL("kernel32").GetModuleHandleW + elif os.name == "ce": + windll.coredll.GetModuleHandleW + windll["coredll"].GetModuleHandleW + windll.LoadLibrary("coredll").GetModuleHandleW + WinDLL("coredll").GetModuleHandleW + + def test_load_ordinal_functions(self): + import os + if os.name not in ("nt", "ce"): + skip('test is platform dependent') + + import conftest + _ctypes_test = str(conftest.sofile) + dll = CDLL(_ctypes_test) + # We load the same function both via ordinal and name + func_ord = dll[2] + func_name = dll.GetString + # addressof gets the address where the function pointer is stored + a_ord = addressof(func_ord) + a_name = addressof(func_name) + f_ord_addr = c_void_p.from_address(a_ord).value + f_name_addr = c_void_p.from_address(a_name).value + assert hex(f_ord_addr) == hex(f_name_addr) + + raises(AttributeError, dll.__getitem__, 1234) From dan at codespeak.net Tue Aug 17 00:51:25 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Tue, 17 Aug 2010 00:51:25 +0200 (CEST) Subject: [pypy-svn] r76644 - in pypy/branch/micronumpy/pypy: config jit/tl module/micronumpy tool Message-ID: <20100816225125.0679F282BD4@codespeak.net> Author: dan Date: Tue Aug 17 00:51:24 2010 New Revision: 76644 Modified: pypy/branch/micronumpy/pypy/config/pypyoption.py pypy/branch/micronumpy/pypy/jit/tl/pypyjit_demo.py pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py pypy/branch/micronumpy/pypy/tool/convolve.py Log: Final commit for GSoC... Forgot to commit earlier when I fell asleep. Modified: pypy/branch/micronumpy/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/micronumpy/pypy/config/pypyoption.py (original) +++ pypy/branch/micronumpy/pypy/config/pypyoption.py Tue Aug 17 00:51:24 2010 @@ -20,7 +20,7 @@ ["_codecs", "gc", "_weakref", "marshal", "errno", "imp", "math", "_sre", "_pickle_support", "operator", "parser", "symbol", "token", "_ast", "_random", "__pypy__", - "_testing"])) + "_testing", "micronumpy"])) # --allworkingmodules Modified: pypy/branch/micronumpy/pypy/jit/tl/pypyjit_demo.py ============================================================================== --- pypy/branch/micronumpy/pypy/jit/tl/pypyjit_demo.py (original) +++ pypy/branch/micronumpy/pypy/jit/tl/pypyjit_demo.py Tue Aug 17 00:51:24 2010 @@ -38,17 +38,17 @@ ## print t2 - t1 try: - from array import array - def f(img): - i=0 - sa=0 - while i < img.__len__(): - sa+=img[i] - i+=1 - return sa + from micronumpy import zeros - img=array('h',(1,2,3,4)) - print f(img) + size = 128 + dtype = float + + for run in range(200): + ar = zeros((size,), dtype=dtype) + + for i in range(size): + ar[i] = dtype(i) * 5 + print i except Exception, e: print "Exception: ", type(e) print e Modified: pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py Tue Aug 17 00:51:24 2010 @@ -5,6 +5,10 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem import rffi +from sys import byteorder + +byteorder = '>' if byteorder == 'big' else '<' + class TypeDescr(Wrappable): def __init__(self, dtype, name): self.dtype = dtype @@ -36,7 +40,7 @@ __repr__ = interp2app(TypeDescr.descr_repr), ) -storage_type = lltype.Ptr(lltype.Array(lltype.Char)) +storage_type = lltype.Ptr(rffi.CArray(lltype.Char)) null_data = lltype.nullptr(storage_type.TO) class DescrBase(object): pass @@ -45,7 +49,7 @@ _descriptors = [] _w_descriptors = [] def descriptor(code, name, ll_type): - arraytype = lltype.Array(ll_type) + arraytype = rffi.CArray(ll_type) class DescrImpl(DescrBase): def __init__(self): self.typeid = 0 @@ -89,6 +93,14 @@ def dump(self, data): return ', '.join([str(x) for x in self.cast(data)]) + def str(self): + if self is float_descr: + code = 'f' + else: + code = self.typecode + + return ''.join([byteorder, code, self.itemsize()]) + for type in [lltype.Signed, lltype.Float]: def get_type(self, data, index): value = self.getitem(data, index) Modified: pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py Tue Aug 17 00:51:24 2010 @@ -372,6 +372,17 @@ def descr_get_shape(space, self): return space.newtuple([space.wrap(x) for x in self.shape[self.offset:]]) +def descr_array_interface(space, self): + w_dict = space.newdict() + data_ptr = space.wrap(lltype.cast_ptr_to_int(self.data)) + data = [data_ptr, space.w_False] + content = [(space.wrap('shape'), descr_get_shape(space, self)), + (space.wrap('data'), space.newtuple(data)), + (space.wrap('typestr'), space.wrap(self.dtype.dtype.str())), + (space.wrap('version'), space.wrap(3))] + w_dict.initialize_content(content) + return w_dict + #TODO: add to typedef when ready def descr_new(space, w_cls, w_shape, w_dtype=NoneNotWrapped, w_buffer=NoneNotWrapped, w_offset=NoneNotWrapped, @@ -401,6 +412,7 @@ MicroArray.typedef = TypeDef('uarray', dtype = GetSetProperty(descr_get_dtype, cls=MicroArray), shape = GetSetProperty(descr_get_shape, cls=MicroArray), + __array_interface__ = GetSetProperty(descr_get_array_interface, cls=MicroArray), __getitem__ = interp2app(MicroArray.descr_getitem), __setitem__ = interp2app(MicroArray.descr_setitem), __len__ = interp2app(MicroArray.descr_len), Modified: pypy/branch/micronumpy/pypy/tool/convolve.py ============================================================================== --- pypy/branch/micronumpy/pypy/tool/convolve.py (original) +++ pypy/branch/micronumpy/pypy/tool/convolve.py Tue Aug 17 00:51:24 2010 @@ -38,7 +38,8 @@ for t in range(t_from, t_to): v = x - smid + s w = y - tmid + t + #print "f[%d, %d] = %s" % (v, w, f[v, w]) value += g[smid - s, tmid - t] * f[v, w] - print "h[%d, %d] = %s" % (x, y, value) + #print "h[%d, %d] = %s" % (x, y, value) h[x, y] = value return h From cfbolz at codespeak.net Tue Aug 17 11:55:25 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 17 Aug 2010 11:55:25 +0200 (CEST) Subject: [pypy-svn] r76651 - in pypy/benchmarks: . own Message-ID: <20100817095525.CE1E4282BAD@codespeak.net> Author: cfbolz Date: Tue Aug 17 11:55:24 2010 New Revision: 76651 Added: pypy/benchmarks/own/go.py (contents, props changed) Modified: pypy/benchmarks/benchmarks.py Log: Add disco go benchmark from here: http://shed-skin.blogspot.com/2009/08/disco-elegant-python-go-player-update.html Modified: pypy/benchmarks/benchmarks.py ============================================================================== --- pypy/benchmarks/benchmarks.py (original) +++ pypy/benchmarks/benchmarks.py Tue Aug 17 11:55:24 2010 @@ -44,7 +44,7 @@ } for name in ['float', 'nbody_modified', 'meteor-contest', 'fannkuch', - 'spectral-norm', 'chaos', 'telco']: + 'spectral-norm', 'chaos', 'telco', 'go']: _register_new_bm(name, name, globals(), **opts.get(name, {})) for name in ['names', 'iteration', 'tcp', 'pb']:#, 'accepts', 'web']: if name == 'web': Added: pypy/benchmarks/own/go.py ============================================================================== --- (empty file) +++ pypy/benchmarks/own/go.py Tue Aug 17 11:55:24 2010 @@ -0,0 +1,446 @@ +import random, math, sys, time + +SIZE = 9 +GAMES = 200 +KOMI = 7.5 +EMPTY, WHITE, BLACK = 0, 1, 2 +SHOW = {EMPTY: '.', WHITE: 'o', BLACK: 'x'} +PASS = -1 +MAXMOVES = SIZE*SIZE*3 +TIMESTAMP = 0 +MOVES = 0 + +def to_pos(x,y): + return y * SIZE + x + +def to_xy(pos): + y, x = divmod(pos, SIZE) + return x, y + +class Square: + def __init__(self, board, pos): + self.board = board + self.pos = pos + self.timestamp = TIMESTAMP + self.removestamp = TIMESTAMP + self.zobrist_strings = [random.randrange(sys.maxint) for i in range(3)] + + def set_neighbours(self): + x, y = self.pos % SIZE, self.pos / SIZE; + self.neighbours = [] + for dx, dy in [(-1, 0), (1, 0), (0, -1), (0, 1)]: + newx, newy = x + dx, y + dy + if 0 <= newx < SIZE and 0 <= newy < SIZE: + self.neighbours.append(self.board.squares[to_pos(newx, newy)]) + + def move(self, color): + global TIMESTAMP, MOVES + TIMESTAMP += 1 + MOVES += 1 + self.board.zobrist.update(self, color) + self.color = color + self.reference = self + self.ledges = 0 + self.used = True + for neighbour in self.neighbours: + neighcolor = neighbour.color + if neighcolor == EMPTY: + self.ledges += 1 + else: + neighbour_ref = neighbour.find(update=True) + if neighcolor == color: + if neighbour_ref.reference.pos != self.pos: + self.ledges += neighbour_ref.ledges + neighbour_ref.reference = self + self.ledges -= 1 + else: + neighbour_ref.ledges -= 1 + if neighbour_ref.ledges == 0: + neighbour.remove(neighbour_ref) + self.board.zobrist.add() + + def remove(self, reference, update=True): + self.board.zobrist.update(self, EMPTY) + self.removestamp = TIMESTAMP + if update: + self.color = EMPTY + self.board.emptyset.add(self.pos) +# if color == BLACK: +# self.board.black_dead += 1 +# else: +# self.board.white_dead += 1 + for neighbour in self.neighbours: + if neighbour.color != EMPTY and neighbour.removestamp != TIMESTAMP: + neighbour_ref = neighbour.find(update) + if neighbour_ref.pos == reference.pos: + neighbour.remove(reference, update) + else: + if update: + neighbour_ref.ledges += 1 + + def find(self, update=False): + reference = self.reference + if reference.pos != self.pos: + reference = reference.find(update) + if update: + self.reference = reference + return reference + + def __repr__(self): + return repr(to_xy(self.pos)) + +class EmptySet: + def __init__(self, board): + self.board = board + self.empties = range(SIZE*SIZE) + self.empty_pos = range(SIZE*SIZE) + + def random_choice(self): + choices = len(self.empties) + while choices: + i = int(random.random()*choices) + pos = self.empties[i] + if self.board.useful(pos): + return pos + choices -= 1 + self.set(i, self.empties[choices]) + self.set(choices, pos) + return PASS + + def add(self, pos): + self.empty_pos[pos] = len(self.empties) + self.empties.append(pos) + + def remove(self, pos): + self.set(self.empty_pos[pos], self.empties[len(self.empties)-1]) + self.empties.pop() + + def set(self, i, pos): + self.empties[i] = pos + self.empty_pos[pos] = i + +class ZobristHash: + def __init__(self, board): + self.board = board + self.hash_set = set() + self.hash = 0 + for square in self.board.squares: + self.hash ^= square.zobrist_strings[EMPTY] + self.hash_set.clear() + self.hash_set.add(self.hash) + + def update(self, square, color): + self.hash ^= square.zobrist_strings[square.color] + self.hash ^= square.zobrist_strings[color] + + def add(self): + self.hash_set.add(self.hash) + + def dupe(self): + return self.hash in self.hash_set + +class Board: + def __init__(self): + self.squares = [Square(self, pos) for pos in range(SIZE*SIZE)] + for square in self.squares: + square.set_neighbours() + self.reset() + + def reset(self): + for square in self.squares: + square.color = EMPTY + square.used = False + self.emptyset = EmptySet(self) + self.zobrist = ZobristHash(self) + self.color = BLACK + self.finished = False + self.lastmove = -2 + self.history = [] + self.white_dead = 0 + self.black_dead = 0 + + def move(self, pos): + square = self.squares[pos] + if pos != PASS: + square.move(self.color) + self.emptyset.remove(square.pos) + elif self.lastmove == PASS: + self.finished = True + if self.color == BLACK: self.color = WHITE + else: self.color = BLACK + self.lastmove = pos + self.history.append(pos) + + def random_move(self): + return self.emptyset.random_choice() + + def useful_fast(self, square): + if not square.used: + for neighbour in square.neighbours: + if neighbour.color == EMPTY: + return True + return False + + def useful(self, pos): + global TIMESTAMP + TIMESTAMP += 1 + square = self.squares[pos] + if self.useful_fast(square): + return True + old_hash = self.zobrist.hash + self.zobrist.update(square, self.color) + empties = opps = weak_opps = neighs = weak_neighs = 0 + for neighbour in square.neighbours: + neighcolor = neighbour.color + if neighcolor == EMPTY: + empties += 1 + continue + neighbour_ref = neighbour.find() + if neighbour_ref.timestamp != TIMESTAMP: + if neighcolor == self.color: + neighs += 1 + else: + opps += 1 + neighbour_ref.timestamp = TIMESTAMP + neighbour_ref.temp_ledges = neighbour_ref.ledges + neighbour_ref.temp_ledges -= 1 + if neighbour_ref.temp_ledges == 0: + if neighcolor == self.color: + weak_neighs += 1 + else: + weak_opps += 1 + neighbour_ref.remove(neighbour_ref, update=False) + dupe = self.zobrist.dupe() + self.zobrist.hash = old_hash + strong_neighs = neighs-weak_neighs + strong_opps = opps-weak_opps + return not dupe and \ + (empties or weak_opps or (strong_neighs and (strong_opps or weak_neighs))) + + def useful_moves(self): + return [pos for pos in self.emptyset.empties if self.useful(pos)] + + def replay(self, history): + for pos in history: + self.move(pos) + + def score(self, color): + if color == WHITE: + count = KOMI + self.black_dead + else: + count = self.white_dead + for square in self.squares: + squarecolor = square.color + if squarecolor == color: + count += 1 + elif squarecolor == EMPTY: + surround = 0 + for neighbour in square.neighbours: + if neighbour.color == color: + surround += 1 + if surround == len(square.neighbours): + count += 1 + return count + + def check(self): + for square in self.squares: + if square.color == EMPTY: + continue + + members1 = set([square]) + changed = True + while changed: + changed = False + for member in members1.copy(): + for neighbour in member.neighbours: + if neighbour.color == square.color and neighbour not in members1: + changed = True + members1.add(neighbour) + ledges1 = 0 + for member in members1: + for neighbour in member.neighbours: + if neighbour.color == EMPTY: + ledges1 += 1 + + root = square.find() + + #print 'members1', square, root, members1 + #print 'ledges1', square, ledges1 + + members2 = set() + for square2 in self.squares: + if square2.color != EMPTY and square2.find() == root: + members2.add(square2) + + ledges2 = root.ledges + #print 'members2', square, root, members1 + #print 'ledges2', square, ledges2 + + assert members1 == members2 + assert ledges1 == ledges2, ('ledges differ at %r: %d %d' % (square, ledges1, ledges2)) + + empties1 = set(self.emptyset.empties) + + empties2 = set() + for square in self.squares: + if square.color == EMPTY: + empties2.add(square.pos) + + def __repr__(self): + result = [] + for y in range(SIZE): + start = to_pos(0, y) + result.append(''.join([SHOW[square.color]+' ' for square in self.squares[start:start+SIZE]])) + return '\n'.join(result) + +class UCTNode: + def __init__(self): + self.bestchild = None + self.pos = -1 + self.wins = 0 + self.losses = 0 + self.pos_child = [None for x in range(SIZE*SIZE)] + self.parent = None + + def play(self, board): + """ uct tree search """ + color = board.color + node = self + path = [node] + while True: + pos = node.select(board) + if pos == PASS: + break + board.move(pos) + child = node.pos_child[pos] + if not child: + child = node.pos_child[pos] = UCTNode() + child.unexplored = board.useful_moves() + child.pos = pos + child.parent = node + path.append(child) + break + path.append(child) + node = child + self.random_playout(board) + self.update_path(board, color, path) + + def select(self, board): + """ select move; unexplored children first, then according to uct value """ + if self.unexplored: + i = random.randrange(len(self.unexplored)) + pos = self.unexplored[i] + self.unexplored[i] = self.unexplored[len(self.unexplored)-1] + self.unexplored.pop() + return pos + elif self.bestchild: + return self.bestchild.pos + else: + return PASS + + def random_playout(self, board): + """ random play until both players pass """ + for x in range(MAXMOVES): # XXX while not self.finished? + if board.finished: + break + board.move(board.random_move()) + + def update_path(self, board, color, path): + """ update win/loss count along path """ + wins = board.score(BLACK) >= board.score(WHITE) + for node in path: + if color == BLACK: color = WHITE + else: color = BLACK + if wins == (color == BLACK): + node.wins += 1 + else: + node.losses += 1 + if node.parent: + node.parent.bestchild = node.parent.best_child() + + def score(self): + winrate = self.wins/float(self.wins+self.losses) + parentvisits = self.parent.wins+self.parent.losses + if not parentvisits: + return winrate + nodevisits = self.wins+self.losses + return winrate + math.sqrt((math.log(parentvisits))/(5*nodevisits)) + + def best_child(self): + maxscore = -1 + maxchild = None + for child in self.pos_child: + if child and child.score() > maxscore: + maxchild = child + maxscore = child.score() + return maxchild + + def best_visited(self): + maxvisits = -1 + maxchild = None + for child in self.pos_child: +# if child: +# print to_xy(child.pos), child.wins, child.losses, child.score() + if child and (child.wins+child.losses) > maxvisits: + maxvisits, maxchild = (child.wins+child.losses), child + return maxchild + +def user_move(board): + while True: + text = raw_input('?').strip() + if text == 'p': + return PASS + if text == 'q': + raise EOFError + try: + x, y = [int(i) for i in text.split()] + except ValueError: + continue + if not (0 <= x < SIZE and 0 <= y < SIZE): + continue + pos = to_pos(x, y) + if board.useful(pos): + return pos + +def computer_move(board): + global MOVES + pos = board.random_move() + if pos == PASS: + return PASS + tree = UCTNode() + tree.unexplored = board.useful_moves() + nboard = Board() + for game in range(GAMES): + node = tree + nboard.reset() + nboard.replay(board.history) + node.play(nboard) +# print 'moves', MOVES + return tree.best_visited().pos + +def versus_cpu(): + random.seed(1) + board = Board() + pos = computer_move(board) + +def main(n): + times = [] + for i in range(5): + versus_cpu() # warmup + for i in range(n): + t1 = time.time() + versus_cpu() + t2 = time.time() + times.append(t2 - t1) + return times + +if __name__ == "__main__": + import util, optparse + parser = optparse.OptionParser( + usage="%prog [options]", + description="Test the performance of the Chaos benchmark") + util.add_standard_options_to(parser) + options, args = parser.parse_args() + + util.run_benchmark(options, options.num_runs, main) + From getxsick at codespeak.net Tue Aug 17 14:47:22 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Tue, 17 Aug 2010 14:47:22 +0200 (CEST) Subject: [pypy-svn] r76652 - in pypy/branch/fast-ctypes/pypy/module/_ctypes: . test Message-ID: <20100817124722.B127E282BEB@codespeak.net> Author: getxsick Date: Tue Aug 17 14:47:21 2010 New Revision: 76652 Added: pypy/branch/fast-ctypes/pypy/module/_ctypes/app_dummy.py pypy/branch/fast-ctypes/pypy/module/_ctypes/test/test_import.py Modified: pypy/branch/fast-ctypes/pypy/module/_ctypes/__init__.py Log: add mock objects to get a possibility to import ctypes Modified: pypy/branch/fast-ctypes/pypy/module/_ctypes/__init__.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/_ctypes/__init__.py (original) +++ pypy/branch/fast-ctypes/pypy/module/_ctypes/__init__.py Tue Aug 17 14:47:21 2010 @@ -5,7 +5,24 @@ 'dlopen' : 'interp_dll.W_CDLL', 'Test' : 'interp_test.W_Test', } - appleveldefs = {} + appleveldefs = { + '_SimpleCData' : 'app_dummy._SimpleCData', + '_Pointer' : 'app_dummy._Pointer', + 'CFuncPtr' : 'app_dummy.CFuncPtr', + 'Union' : 'app_dummy.Union', + 'Structure' : 'app_dummy.Structure', + 'Array' : 'app_dummy.Array', + 'ArgumentError' : 'app_dummy.ArgumentError', + 'sizeof' : 'app_dummy.sizeof', + 'byref' : 'app_dummy.byref', + 'addressof' : 'app_dummy.addressof', + 'alignment' : 'app_dummy.alignment', + 'resize' : 'app_dummy.resize', + '_memmove_addr' : 'app_dummy._memmove_addr', + '_memset_addr' : 'app_dummy._memset_addr', + '_cast_addr' : 'app_dummy._cast_addr', + '_string_at' : 'app_dummy._string_at', + } def buildloaders(cls): from pypy.module._ctypes.constants import constants Added: pypy/branch/fast-ctypes/pypy/module/_ctypes/app_dummy.py ============================================================================== --- (empty file) +++ pypy/branch/fast-ctypes/pypy/module/_ctypes/app_dummy.py Tue Aug 17 14:47:21 2010 @@ -0,0 +1,29 @@ +class DummyClass(object): + def __init__(self, *args, **kwargs): + raise NotImplementedError("not-implemented ctypes function") +_Pointer = Union = Structure = Array = ArgumentError = DummyClass + +class CFuncPtr(object): + def __init__(self, *args, **kwargs): + pass + +class Meta(object): + def __init__(self, *args, **kwargs): + self.__name__ = 'MockObject' + def __mul__(self, v): + return 10 + __rmul__ = __mul__ + _type_ = '' + from_param = None + +class _SimpleCData(object): + __metaclass__ = Meta + +def dummyfunc(*args, **kwargs): + raise NotImplementedError("not-implemented ctypes function") +byref = addressof = alignment = resize = dummyfunc +_memmove_addr = _memset_addr = _cast_addr = dummyfunc +_string_at = dummyfunc + +def sizeof(tp): + return 0 Added: pypy/branch/fast-ctypes/pypy/module/_ctypes/test/test_import.py ============================================================================== --- (empty file) +++ pypy/branch/fast-ctypes/pypy/module/_ctypes/test/test_import.py Tue Aug 17 14:47:21 2010 @@ -0,0 +1,13 @@ +from pypy.conftest import gettestobjspace + +class AppTestCtypesimport(object): + def setup_class(cls): + space = gettestobjspace(usemodules=('_ctypes', 'struct')) + cls.space = space + + def test_import(self): + import ctypes + assert ctypes + + import _ctypes + assert _ctypes From getxsick at codespeak.net Tue Aug 17 14:52:03 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Tue, 17 Aug 2010 14:52:03 +0200 (CEST) Subject: [pypy-svn] r76653 - pypy/branch/fast-ctypes/lib-python/modified-2.5.2/ctypes Message-ID: <20100817125203.E1167282BAD@codespeak.net> Author: getxsick Date: Tue Aug 17 14:52:02 2010 New Revision: 76653 Modified: pypy/branch/fast-ctypes/lib-python/modified-2.5.2/ctypes/__init__.py Log: temporary comment if statement to get possiblity to import ctypes with mock objects. once sizeof() is implemented, it should be uncommented Modified: pypy/branch/fast-ctypes/lib-python/modified-2.5.2/ctypes/__init__.py ============================================================================== --- pypy/branch/fast-ctypes/lib-python/modified-2.5.2/ctypes/__init__.py (original) +++ pypy/branch/fast-ctypes/lib-python/modified-2.5.2/ctypes/__init__.py Tue Aug 17 14:52:02 2010 @@ -143,9 +143,10 @@ # Most _type_ codes are the same as used in struct typecode = typ._type_ actual, required = sizeof(typ), calcsize(typecode) - if actual != required: - raise SystemError("sizeof(%s) wrong: %d instead of %d" % \ - (typ, actual, required)) + # XXX temporary commented (see r76653) + #if actual != required: + # raise SystemError("sizeof(%s) wrong: %d instead of %d" % \ + # (typ, actual, required)) class py_object(_SimpleCData): _type_ = "O" From getxsick at codespeak.net Tue Aug 17 15:28:11 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Tue, 17 Aug 2010 15:28:11 +0200 (CEST) Subject: [pypy-svn] r76654 - pypy/branch/fast-ctypes/pypy/module/_ctypes/test Message-ID: <20100817132811.85CB2282BAD@codespeak.net> Author: getxsick Date: Tue Aug 17 15:28:10 2010 New Revision: 76654 Modified: pypy/branch/fast-ctypes/pypy/module/_ctypes/test/test_loading.py Log: add 'struct' module to usemodules Modified: pypy/branch/fast-ctypes/pypy/module/_ctypes/test/test_loading.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/_ctypes/test/test_loading.py (original) +++ pypy/branch/fast-ctypes/pypy/module/_ctypes/test/test_loading.py Tue Aug 17 15:28:10 2010 @@ -8,7 +8,7 @@ class AppTestLoader(object): def setup_class(cls): - space = gettestobjspace(usemodules=('_ctypes',)) + space = gettestobjspace(usemodules=('_ctypes', 'struct')) cls.space = space cls.w_unknowndll = space.wrap("xxrandomnamexx") From cfbolz at codespeak.net Tue Aug 17 16:57:52 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 17 Aug 2010 16:57:52 +0200 (CEST) Subject: [pypy-svn] r76655 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20100817145752.B4C85282BAD@codespeak.net> Author: cfbolz Date: Tue Aug 17 16:57:51 2010 New Revision: 76655 Modified: pypy/trunk/pypy/jit/metainterp/executor.py pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Log: (cfbolz, arigo checking): constant-fold ovf operations Modified: pypy/trunk/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/executor.py (original) +++ pypy/trunk/pypy/jit/metainterp/executor.py Tue Aug 17 16:57:51 2010 @@ -172,34 +172,36 @@ [x1box.getref_base(), x2box.getref_base()], None) def do_int_add_ovf(cpu, metainterp, box1, box2): - assert metainterp is not None + # the overflow operations can be called without a metainterp, if an + # overflow cannot occur a = box1.getint() b = box2.getint() try: z = ovfcheck(a + b) except OverflowError: + assert metainterp is not None metainterp.execute_raised(OverflowError(), constant=True) z = 0 return BoxInt(z) def do_int_sub_ovf(cpu, metainterp, box1, box2): - assert metainterp is not None a = box1.getint() b = box2.getint() try: z = ovfcheck(a - b) except OverflowError: + assert metainterp is not None metainterp.execute_raised(OverflowError(), constant=True) z = 0 return BoxInt(z) def do_int_mul_ovf(cpu, metainterp, box1, box2): - assert metainterp is not None a = box1.getint() b = box2.getint() try: z = ovfcheck(a * b) except OverflowError: + assert metainterp is not None metainterp.execute_raised(OverflowError(), constant=True) z = 0 return BoxInt(z) Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Tue Aug 17 16:57:51 2010 @@ -526,7 +526,9 @@ def propagate_forward(self): self.exception_might_have_happened = False self.newoperations = [] - for op in self.loop.operations: + self.i = 0 + while self.i < len(self.loop.operations): + op = self.loop.operations[self.i] opnum = op.opnum for value, func in optimize_ops: if opnum == value: @@ -534,6 +536,7 @@ break else: self.optimize_default(op) + self.i += 1 self.loop.operations = self.newoperations # accumulate counters self.resumedata_memo.update_counters(self.metainterp_sd.profiler) @@ -588,7 +591,12 @@ descr.make_a_counter_per_value(op) def optimize_default(self, op): - if op.is_always_pure(): + canfold = op.is_always_pure() + is_ovf = op.is_ovf() + if is_ovf: + nextop = self.loop.operations[self.i + 1] + canfold = nextop.opnum == rop.GUARD_NO_OVERFLOW + if canfold: for arg in op.args: if self.get_constant_box(arg) is None: break @@ -598,6 +606,8 @@ resbox = execute_nonspec(self.cpu, None, op.opnum, argboxes, op.descr) self.make_constant(op.result, resbox.constbox()) + if is_ovf: + self.i += 1 # skip next operation, it is the unneeded guard return # did we do the exact same operation already? @@ -611,6 +621,8 @@ if oldop is not None and oldop.descr is op.descr: assert oldop.opnum == op.opnum self.make_equal_to(op.result, self.getvalue(oldop.result)) + if is_ovf: + self.i += 1 # skip next operation, it is the unneeded guard return elif self.find_rewritable_bool(op, args): return Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Tue Aug 17 16:57:51 2010 @@ -285,6 +285,24 @@ """ self.optimize_loop(ops, '', expected) + def test_constant_propagate_ovf(self): + ops = """ + [] + i0 = int_add_ovf(2, 3) + guard_no_overflow() [] + i1 = int_is_true(i0) + guard_true(i1) [] + i2 = int_is_zero(i1) + guard_false(i2) [] + guard_value(i0, 5) [] + jump() + """ + expected = """ + [] + jump() + """ + self.optimize_loop(ops, '', expected) + def test_constfold_all(self): from pypy.jit.backend.llgraph.llimpl import TYPES # xxx fish from pypy.jit.metainterp.executor import execute_nonspec @@ -2098,6 +2116,33 @@ """ self.optimize_loop(ops, 'Not', expected) + def test_remove_duplicate_pure_op_ovf(self): + ops = """ + [i1] + i3 = int_add_ovf(i1, 1) + guard_no_overflow() [] + i3b = int_is_true(i3) + guard_true(i3b) [] + i4 = int_add_ovf(i1, 1) + guard_no_overflow() [] + i4b = int_is_true(i4) + guard_true(i4b) [] + escape(i3) + escape(i4) + jump(i1) + """ + expected = """ + [i1] + i3 = int_add_ovf(i1, 1) + guard_no_overflow() [] + i3b = int_is_true(i3) + guard_true(i3b) [] + escape(i3) + escape(i3) + jump(i1) + """ + self.optimize_loop(ops, "Not", expected) + def test_int_and_or_with_zero(self): ops = """ [i0, i1] From cfbolz at codespeak.net Tue Aug 17 17:15:45 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 17 Aug 2010 17:15:45 +0200 (CEST) Subject: [pypy-svn] r76656 - in pypy/trunk/pypy/jit/backend/x86: . test Message-ID: <20100817151545.D6F9A282BAD@codespeak.net> Author: cfbolz Date: Tue Aug 17 17:15:44 2010 New Revision: 76656 Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/test/test_runner.py Log: (cfbolz, arigo): make the output sortable Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Tue Aug 17 17:15:44 2010 @@ -241,7 +241,7 @@ f = open_file_as_stream(output_log, "w") for i in range(len(self.loop_run_counters)): name, struct = self.loop_run_counters[i] - f.write(name + ":" + str(struct.i) + "\n") + f.write(str(struct.i) + " " * (8 - len(str(struct.i))) + name + "\n") f.close() def _build_float_constants(self): Modified: pypy/trunk/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_runner.py Tue Aug 17 17:15:44 2010 @@ -503,4 +503,4 @@ assert struct.i == 10 self.cpu.finish_once() lines = py.path.local(self.logfile + ".count").readlines() - assert lines[0] == 'xyz:10\n' + assert lines[0] == '10 xyz\n' From cfbolz at codespeak.net Tue Aug 17 17:27:19 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 17 Aug 2010 17:27:19 +0200 (CEST) Subject: [pypy-svn] r76657 - pypy/trunk/pypy/objspace/std Message-ID: <20100817152719.19AB2282BAD@codespeak.net> Author: cfbolz Date: Tue Aug 17 17:27:17 2010 New Revision: 76657 Modified: pypy/trunk/pypy/objspace/std/intobject.py Log: (cfbolz, arigo): make the common case of shifts a tiny bit faster (removes two guards) Modified: pypy/trunk/pypy/objspace/std/intobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/intobject.py (original) +++ pypy/trunk/pypy/objspace/std/intobject.py Tue Aug 17 17:27:17 2010 @@ -240,34 +240,36 @@ def lshift__Int_Int(space, w_int1, w_int2): a = w_int1.intval b = w_int2.intval + if r_uint(b) < LONG_BIT: # 0 <= b < LONG_BIT + try: + c = ovfcheck_lshift(a, b) + except OverflowError: + raise FailedToImplementArgs(space.w_OverflowError, + space.wrap("integer left shift")) + return wrapint(space, c) if b < 0: raise OperationError(space.w_ValueError, space.wrap("negative shift count")) - if a == 0 or b == 0: - return get_integer(space, w_int1) - if b >= LONG_BIT: + else: #b >= LONG_BIT + if a == 0: + return get_integer(space, w_int1) raise FailedToImplementArgs(space.w_OverflowError, space.wrap("integer left shift")) - try: - c = ovfcheck_lshift(a, b) - except OverflowError: - raise FailedToImplementArgs(space.w_OverflowError, - space.wrap("integer left shift")) - return wrapint(space, c) def rshift__Int_Int(space, w_int1, w_int2): a = w_int1.intval b = w_int2.intval - if b < 0: - raise OperationError(space.w_ValueError, - space.wrap("negative shift count")) - if a == 0 or b == 0: - return get_integer(space, w_int1) - if b >= LONG_BIT: - if a < 0: - a = -1 - else: - a = 0 + if r_uint(b) >= LONG_BIT: # not (0 <= b < LONG_BIT) + if b < 0: + raise OperationError(space.w_ValueError, + space.wrap("negative shift count")) + else: # b >= LONG_BIT + if a == 0: + return get_integer(space, w_int1) + if a < 0: + a = -1 + else: + a = 0 else: a = a >> b return wrapint(space, a) From cfbolz at codespeak.net Tue Aug 17 17:59:51 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 17 Aug 2010 17:59:51 +0200 (CEST) Subject: [pypy-svn] r76659 - in pypy/benchmarks: . own Message-ID: <20100817155951.C3B0A282BAD@codespeak.net> Author: cfbolz Date: Tue Aug 17 17:59:50 2010 New Revision: 76659 Added: pypy/benchmarks/own/interpreter.tar.bz2 (contents, props changed) pypy/benchmarks/own/pyflate-fast.py (contents, props changed) Modified: pypy/benchmarks/benchmarks.py Log: (cfbolz, arigo): yet another benchmark: a pure-Python bz2 decoder Modified: pypy/benchmarks/benchmarks.py ============================================================================== --- pypy/benchmarks/benchmarks.py (original) +++ pypy/benchmarks/benchmarks.py Tue Aug 17 17:59:50 2010 @@ -44,7 +44,7 @@ } for name in ['float', 'nbody_modified', 'meteor-contest', 'fannkuch', - 'spectral-norm', 'chaos', 'telco', 'go']: + 'spectral-norm', 'chaos', 'telco', 'go', 'pyflate-fast']: _register_new_bm(name, name, globals(), **opts.get(name, {})) for name in ['names', 'iteration', 'tcp', 'pb']:#, 'accepts', 'web']: if name == 'web': Added: pypy/benchmarks/own/interpreter.tar.bz2 ============================================================================== Binary file. No diff available. Added: pypy/benchmarks/own/pyflate-fast.py ============================================================================== --- (empty file) +++ pypy/benchmarks/own/pyflate-fast.py Tue Aug 17 17:59:50 2010 @@ -0,0 +1,684 @@ +#!/usr/bin/env python +# Copyright 2006--2007-01-21 Paul Sladen +# http://www.paul.sladen.org/projects/compression/ +# +# You may use and distribute this code under any DFSG-compatible +# license (eg. BSD, GNU GPLv2). +# +# Stand-alone pure-Python DEFLATE (gzip) and bzip2 decoder/decompressor. +# This is probably most useful for research purposes/index building; there +# is certainly some room for improvement in the Huffman bit-matcher. +# +# With the as-written implementation, there was a known bug in BWT +# decoding to do with repeated strings. This has been worked around; +# see 'bwt_reverse()'. Correct output is produced in all test cases +# but ideally the problem would be found... + +class BitfieldBase(object): + def __init__(self, x): + if isinstance(x,BitfieldBase): + self.f = x.f + self.bits = x.bits + self.bitfield = x.bitfield + self.count = x.bitfield + else: + self.f = x + self.bits = 0 + self.bitfield = 0x0 + self.count = 0 + def _read(self, n): + s = self.f.read(n) + if not s: + raise "Length Error" + self.count += len(s) + return s + def needbits(self, n): + while self.bits < n: + self._more() + def _mask(self, n): + return (1 << n) - 1 + def toskip(self): + return self.bits & 0x7 + def align(self): + self.readbits(self.toskip()) + def dropbits(self, n = 8): + while n >= self.bits and n > 7: + n -= self.bits + self.bits = 0 + n -= len(self.f._read(n >> 3)) << 3 + if n: + self.readbits(n) + # No return value + def dropbytes(self, n = 1): + self.dropbits(n << 3) + def tell(self): + return self.count - ((self.bits+7) >> 3), 7 - ((self.bits-1) & 0x7) + def tellbits(self): + bytes, bits = self.tell() + return (bytes << 3) + bits + +class Bitfield(BitfieldBase): + def _more(self): + c = self._read(1) + self.bitfield += ord(c) << self.bits + self.bits += 8 + def snoopbits(self, n = 8): + if n > self.bits: + self.needbits(n) + return self.bitfield & self._mask(n) + def readbits(self, n = 8): + if n > self.bits: + self.needbits(n) + r = self.bitfield & self._mask(n) + self.bits -= n + self.bitfield >>= n + return r + +class RBitfield(BitfieldBase): + def _more(self): + c = self._read(1) + self.bitfield <<= 8 + self.bitfield += ord(c) + self.bits += 8 + def snoopbits(self, n = 8): + if n > self.bits: + self.needbits(n) + return (self.bitfield >> (self.bits - n)) & self._mask(n) + def readbits(self, n = 8): + if n > self.bits: + self.needbits(n) + r = (self.bitfield >> (self.bits - n)) & self._mask(n) + self.bits -= n + self.bitfield &= ~(self._mask(n) << self.bits) + return r + +def printbits(v, n): + o = '' + for i in range(n): + if v & 1: + o = '1' + o + else: + o = '0' + o + v >>= 1 + return o + +class HuffmanLength(object): + def __init__(self, code, bits = 0): + self.code = code + self.bits = bits + self.symbol = None + def __repr__(self): + return `(self.code, self.bits, self.symbol, self.reverse_symbol)` + def __cmp__(self, other): + if self.bits == other.bits: + return cmp(self.code, other.code) + else: + return cmp(self.bits, other.bits) + +def reverse_bits(v, n): + a = 1 << 0 + b = 1 << (n - 1) + z = 0 + for i in range(n-1, -1, -2): + z |= (v >> i) & a + z |= (v << i) & b + a <<= 1 + b >>= 1 + return z + +def reverse_bytes(v, n): + a = 0xff << 0 + b = 0xff << (n - 8) + z = 0 + for i in range(n-8, -8, -16): + z |= (v >> i) & a + z |= (v << i) & b + a <<= 8 + b >>= 8 + return z + +class HuffmanTable(object): + def __init__(self, bootstrap): + l = [] + start, bits = bootstrap[0] + for finish, endbits in bootstrap[1:]: + if bits: + for code in range(start, finish): + l.append(HuffmanLength(code, bits)) + start, bits = finish, endbits + if endbits == -1: + break + l.sort() + self.table = l + + def populate_huffman_symbols(self): + bits, symbol = -1, -1 + for x in self.table: + symbol += 1 + if x.bits != bits: + symbol <<= (x.bits - bits) + bits = x.bits + x.symbol = symbol + x.reverse_symbol = reverse_bits(symbol, bits) + #print printbits(x.symbol, bits), printbits(x.reverse_symbol, bits) + + def tables_by_bits(self): + d = {} + for x in self.table: + try: + d[x.bits].append(x) + except: + d[x.bits] = [x] + pass + + def min_max_bits(self): + self.min_bits, self.max_bits = 16, -1 + for x in self.table: + if x.bits < self.min_bits: self.min_bits = x.bits + if x.bits > self.max_bits: self.max_bits = x.bits + + def _find_symbol(self, bits, symbol, table): + for h in table: + if h.bits == bits and h.reverse_symbol == symbol: + #print "found, processing", h.code + return h.code + return -1 + + def find_next_symbol(self, field, reversed = True): + cached_length = -1 + cached = None + for x in self.table: + if cached_length != x.bits: + cached = field.snoopbits(x.bits) + cached_length = x.bits + if (reversed and x.reverse_symbol == cached) or (not reversed and x.symbol == cached): + field.readbits(x.bits) + return x.code + raise "unfound symbol, even after end of table @ " + `field.tell()` + + for bits in range(self.min_bits, self.max_bits + 1): + #print printbits(field.snoopbits(bits),bits) + r = self._find_symbol(bits, field.snoopbits(bits), self.table) + if 0 <= r: + field.readbits(bits) + return r + elif bits == self.max_bits: + raise "unfound symbol, even after max_bits" + +class OrderedHuffmanTable(HuffmanTable): + def __init__(self, lengths): + l = len(lengths) + z = zip(range(l), lengths) + [(l, -1)] + HuffmanTable.__init__(self, z) + +def code_length_orders(i): + return (16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15)[i] + +def distance_base(i): + return (1,2,3,4,5,7,9,13,17,25,33,49,65,97,129,193,257,385,513,769,1025,1537,2049,3073,4097,6145,8193,12289,16385,24577)[i] + +def length_base(i): + return (3,4,5,6,7,8,9,10,11,13,15,17,19,23,27,31,35,43,51,59,67,83,99,115,131,163,195,227,258)[i-257] + +def extra_distance_bits(n): + if 0 <= n <= 1: + return 0 + elif 2 <= n <= 29: + return (n >> 1) - 1 + else: + raise "illegal distance code" + +def extra_length_bits(n): + if 257 <= n <= 260 or n == 285: + return 0 + elif 261 <= n <= 284: + return ((n-257) >> 2) - 1 + else: + raise "illegal length code" + +def move_to_front(l, c): + l[:] = l[c:c+1] + l[0:c] + l[c+1:] + +def bwt_transform(L): + # Semi-inefficient way to get the character counts + F = ''.join(sorted(L)) + base = [] + for i in range(256): + base.append(F.find(chr(i))) + + pointers = [-1] * len(L) + for i, char in enumerate(L): + symbol = ord(char) + pointers[base[symbol]] = i + base[symbol] += 1 + return pointers + +def bwt_reverse(L, end): + out = [] + if len(L): + T = bwt_transform(L) + + # STRAGENESS WARNING: There was a bug somewhere here in that + # if the output of the BWT resolves to a perfect copy of N + # identical strings (think exact multiples of 255 'X' here), + # then a loop is formed. When decoded, the output string would + # be cut off after the first loop, typically '\0\0\0\0\xfb'. + # The previous loop construct was: + # + # next = T[end] + # while next != end: + # out += L[next] + # next = T[next] + # out += L[next] + # + # For the moment, I've instead replaced it with a check to see + # if there has been enough output generated. I didn't figured + # out where the off-by-one-ism is yet---that actually produced + # the cyclic loop. + + for i in xrange(len(L)): + end = T[end] + out.append(L[end]) + + return "".join(out) + +def compute_used(b): + huffman_used_map = b.readbits(16) + #print 'used map', hex(huffman_used_map) + map_mask = 1 << 15 + used = [] + while map_mask > 0: + if huffman_used_map & map_mask: + huffman_used_bitmap = b.readbits(16) + bit_mask = 1 << 15 + while bit_mask > 0: + if huffman_used_bitmap & bit_mask: + #print 'hit', len(used) + pass + used += [bool(huffman_used_bitmap & bit_mask)] + bit_mask >>= 1 + else: + used += [False] * 16 + map_mask >>= 1 + return used + +def compute_selectors_list(b, huffman_groups): + selectors_used = b.readbits(15) + #print 'selectors used', selectors_used + mtf = range(huffman_groups) + selectors_list = [] + for i in range(selectors_used): + # zero-terminated bit runs (0..62) of MTF'ed huffman table + c = 0 + while b.readbits(1): + c += 1 + if c >= huffman_groups: + raise "Bzip2 chosen selector greater than number of groups (max 6)" + if c >= 0: + move_to_front(mtf, c) + #print c, mtf + selectors_list.append(mtf[0]) + return selectors_list + +def compute_tables(b, huffman_groups, symbols_in_use): + groups_lengths = [] + for j in range(huffman_groups): + length = start_huffman_length = b.readbits(5) + #print 'start_huffman_length', start_huffman_length + lengths = [] + for i in range(symbols_in_use): + if not 0 <= length <= 20: + raise "Bzip2 Huffman length code outside range 0..20" + while b.readbits(1): + length -= (b.readbits(1) * 2) - 1 + lengths += [length] + groups_lengths += [lengths] + #print groups_lengths + + tables = [] + for g in groups_lengths: + codes = OrderedHuffmanTable(g) + codes.populate_huffman_symbols() + codes.min_max_bits() + tables.append(codes) + return tables + +def decode_huffman_block(b, out): + #print 'bzip2 Huffman block' + randomised = b.readbits(1) + if randomised: + raise "Bzip2 randomised support not implemented" + pointer = b.readbits(24) + #print 'pointer', pointer, hex(pointer) + used = compute_used(b) + + huffman_groups = b.readbits(3) + #print 'huffman groups', huffman_groups + if not 2 <= huffman_groups <= 6: + raise "Bzip2: Number of Huffman groups not in range 2..6" + + selectors_list = compute_selectors_list(b, huffman_groups) + symbols_in_use = sum(used) + 2 # remember RUN[AB] RLE symbols + tables = compute_tables(b, huffman_groups, symbols_in_use) + + #favourites = map(chr,range(sum(used))) + #favourites = string.join([y for x,y in map(None,used,map(chr,range(len(used)))) if x],'') + favourites = [chr(i) for i, x in enumerate(used) if x] + + data_start = b.tellbits() + selector_pointer = 0 + decoded = 0 + # Main Huffman loop + repeat = repeat_power = 0 + buffer = [] + t = None + while True: + decoded -= 1 + if decoded <= 0: + #print 'RETABLE TIME', selectors_list[selector_pointer] + decoded = 50 # Huffman table re-evaluate/switch length + if selector_pointer <= len(selectors_list): + t = tables[selectors_list[selector_pointer]] + selector_pointer += 1 + #print 'tables changed', tables[0].table + #print b.tell() + r = t.find_next_symbol(b, False) + #print 'symbol', r + if 0 <= r <= 1: + if repeat == 0: + repeat_power = 1 + #print 'run', repeat + repeat += repeat_power << r + repeat_power <<= 1 + continue + elif repeat > 0: + # Remember kids: If there is only one repeated + # real symbol, it is encoded with *zero* Huffman + # bits and not output... so buffer[-1] doesn't work. + #print 'runfinal', repeat + buffer.append(favourites[0] * repeat) + repeat = 0 + if r == symbols_in_use - 1: + #print 'finished', `buffer[:10]`, '..', `buffer[-10:]`, 'len', len(buffer) + break + else: + o = favourites[r-1] + #print 'pre ', `favourites` + move_to_front(favourites, r-1) + #print 'post', `favourites` + #print 'output', `o` + buffer.append(o) + pass + #print 'huffman', `buffer`, pointer, len(buffer) + #nearly_there = bwt_reverse(buffer, len(buffer)-pointer-1) + nt = nearly_there = bwt_reverse("".join(buffer), pointer) + #print 'nearly there', `nearly_there` + i = 0 + # Pointless/irritating run-length encoding step + while i < len(nearly_there): + #print 'RLE decode', `nt[i:]` + if i < len(nearly_there) - 4 and nt[i] == nt[i+1] == nt[i+2] == nt[i+3]: + out.append(nearly_there[i] * (ord(nearly_there[i+4]) + 4)) + i += 5 + else: + out.append(nearly_there[i]) + i += 1 + #print 'done', `done[:10]`, '..', `done[-10:]`, 'len', len(done) + + #raise "Bip2 block support not implemented" + +# Sixteen bits of magic have been removed by the time we start decoding +def bzip2_main(input): + b = RBitfield(input) + + method = b.readbits(8) + if method != ord('h'): + raise "Unknown (not type 'h'uffman Bzip2) compression method" + + blocksize = b.readbits(8) + if ord('1') <= blocksize <= ord('9'): + blocksize = blocksize - ord('0') + else: + raise "Unknown (not size '0'-'9') Bzip2 blocksize" + + out = [] + while True: + #header_start = b.tellbits() + blocktype = b.readbits(48) + crc = b.readbits(32) + #print hex(blocktype) + #print hex(crc) + if blocktype == 0x314159265359: # (pi) + decode_huffman_block(b, out) + elif blocktype == 0x177245385090: # sqrt(pi) + #print 'bzip2 end-of-stream block' + b.align() + break + else: + raise "Illegal Bzip2 blocktype" + #print len(out), set([len(s) for s in out]) + return ''.join(out) + +# Sixteen bits of magic have been removed by the time we start decoding +def gzip_main(field): + b = Bitfield(field) + method = b.readbits(8) + if method != 8: + raise "Unknown (not type eight DEFLATE) compression method" + + # Use flags, drop modification time, extra flags and OS creator type. + flags = b.readbits(8) + #print 'flags', hex(flags) + mtime = b.readbits(32) + #print 'mtime', hex(mtime) + extra_flags = b.readbits(8) + #print 'extra_flags', hex(extra_flags) + os_type = b.readbits(8) + #print 'os_type', hex(os_type) + + if flags & 0x04: # structured GZ_FEXTRA miscellaneous data + xlen = b.readbits(16) + b.dropbytes(xlen) + while flags & 0x08: # original GZ_FNAME filename + if not b.readbits(8): + break + while flags & 0x10: # human readable GZ_FCOMMENT + if not b.readbits(8): + break + if flags & 0x02: # header-only GZ_FHCRC checksum + b.readbits(16) + + #print "gzip header skip", b.tell() + out = [] + + #print 'header 0 count 0 bits', b.tellbits() + + while True: + header_start = b.tell() + bheader_start = b.tellbits() + #print 'new block at', b.tell() + lastbit = b.readbits(1) + #print "last bit", hex(lastbit) + blocktype = b.readbits(2) + #print "deflate-blocktype", blocktype, 'beginning at', header_start + + #print 'raw block data at', b.tell() + if blocktype == 0: + b.align() + length = b.readbits(16) + if length & b.readbits(16): + raise "stored block lengths do not match each other" + #print "stored block of length", length + #print 'raw data at', b.tell(), 'bits', b.tellbits() - bheader_start + #print 'header 0 count 0 bits', b.tellbits() - bheader_start + for i in range(length): + out.append(chr(b.readbits(8))) + #print 'linear', b.tell()[0], 'count', length, 'bits', b.tellbits() - bheader_start + + elif blocktype == 1 or blocktype == 2: # Huffman + main_literals, main_distances = None, None + + if blocktype == 1: # Static Huffman + static_huffman_bootstrap = [(0, 8), (144, 9), (256, 7), (280, 8), (288, -1)] + static_huffman_lengths_bootstrap = [(0, 5), (32, -1)] + main_literals = HuffmanTable(static_huffman_bootstrap) + main_distances = HuffmanTable(static_huffman_lengths_bootstrap) + + elif blocktype == 2: # Dynamic Huffman + literals = b.readbits(5) + 257 + distances = b.readbits(5) + 1 + code_lengths_length = b.readbits(4) + 4 + + l = [0] * 19 + for i in range(code_lengths_length): + l[code_length_orders(i)] = b.readbits(3) + + dynamic_codes = OrderedHuffmanTable(l) + dynamic_codes.populate_huffman_symbols() + dynamic_codes.min_max_bits() + + # Decode the code_lengths for both tables at once, + # then split the list later + + code_lengths = [] + n = 0 + while n < (literals + distances): + r = dynamic_codes.find_next_symbol(b) + if 0 <= r <= 15: # literal bitlength for this code + count = 1 + what = r + elif r == 16: # repeat last code + count = 3 + b.readbits(2) + # Is this supposed to default to '0' if in the zeroth position? + what = code_lengths[-1] + elif r == 17: # repeat zero + count = 3 + b.readbits(3) + what = 0 + elif r == 18: # repeat zero lots + count = 11 + b.readbits(7) + what = 0 + else: + raise "next code length is outside of the range 0 <= r <= 18" + code_lengths += [what] * count + n += count + + main_literals = OrderedHuffmanTable(code_lengths[:literals]) + main_distances = OrderedHuffmanTable(code_lengths[literals:]) + + # Common path for both Static and Dynamic Huffman decode now + + data_start = b.tell() + #print 'raw data at', data_start, 'bits', b.tellbits() - bheader_start + #print 'header 0 count 0 bits', b.tellbits() - bheader_start + + main_literals.populate_huffman_symbols() + main_distances.populate_huffman_symbols() + + main_literals.min_max_bits() + main_distances.min_max_bits() + + literal_count = 0 + literal_start = 0 + + while True: + lz_start = b.tellbits() + r = main_literals.find_next_symbol(b) + if 0 <= r <= 255: + if literal_count == 0: + literal_start = lz_start + literal_count += 1 + #print 'found literal', `chr(r)` + out.append(chr(r)) + elif r == 256: + if literal_count > 0: + #print 'add 0 count', literal_count, 'bits', lz_start-literal_start, 'data', `out[-literal_count:]` + literal_count = 0 + #print 'eos 0 count 0 bits', b.tellbits() - lz_start + #print 'end of Huffman block encountered' + break + elif 257 <= r <= 285: # dictionary lookup + if literal_count > 0: + #print 'add 0 count', literal_count, 'bits', lz_start-literal_start, 'data', `out[-literal_count:]` + literal_count = 0 + length_extra = b.readbits(extra_length_bits(r)) + length = length_base(r) + length_extra + #print 'dictionary lookup: length', length, + + r1 = main_distances.find_next_symbol(b) + if 0 <= r1 <= 29: + distance = distance_base(r1) + b.readbits(extra_distance_bits(r1)) + cached_length = length + while length > distance: + out += out[-distance:] + length -= distance + if length == distance: + out += out[-distance:] + else: + out += out[-distance:length-distance] + #print 'copy', -distance, 'count', cached_length, 'bits', b.tellbits() - lz_start, 'data', `out[-cached_length:]` + elif 30 <= r1 <= 31: + raise "illegal unused distance symbol in use @" + `b.tell()` + elif 286 <= r <= 287: + raise "illegal unused literal/length symbol in use @" + `b.tell()` + elif blocktype == 3: + raise "illegal unused blocktype in use @" + `b.tell()` + + if lastbit: + #print "this was the last block, time to leave", b.tell() + break + + footer_start = b.tell() + bfooter_start = b.tellbits() + b.align() + crc = b.readbits(32) + final_length = b.readbits(32) + #print len(out) + next_unused = b.tell() + #print 'deflate-end-of-stream', 5, 'beginning at', footer_start, 'raw data at', next_unused, 'bits', b.tellbits() - bfooter_start + #print 'deflate-end-of-stream' + #print 'crc', hex(crc), 'final length', final_length + #print 'header 0 count 0 bits', b.tellbits()-bfooter_start + + return "".join(out) + +import sys, os + +def _main(): + filename = os.path.join(os.path.dirname(__file__), "interpreter.tar.bz2") + input = open(filename) + field = RBitfield(input) + + magic = field.readbits(16) + if magic == 0x1f8b: # GZip + out = gzip_main(field) + elif magic == 0x425a: # BZip2 + out = bzip2_main(field) + else: + raise "Unknown file magic "+hex(magic)+", not a gzip/bzip2 file" + + import md5 + assert md5.md5(out).hexdigest() == "afa004a630fe072901b1d9628b960974" + input.close() + +def main(n): + import time + times = [] + for i in range(5): + _main() # warmup + for i in range(n): + t1 = time.time() + _main() + t2 = time.time() + times.append(t2 - t1) + return times + +if __name__ == "__main__": + import util, optparse + parser = optparse.OptionParser( + usage="%prog [options]", + description="Test the performance of the Chaos benchmark") + util.add_standard_options_to(parser) + options, args = parser.parse_args() + + util.run_benchmark(options, options.num_runs, main) + From agaynor at codespeak.net Tue Aug 17 20:14:09 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Tue, 17 Aug 2010 20:14:09 +0200 (CEST) Subject: [pypy-svn] r76660 - pypy/benchmarks/own Message-ID: <20100817181409.DC5B1282BEB@codespeak.net> Author: agaynor Date: Tue Aug 17 20:14:08 2010 New Revision: 76660 Modified: pypy/benchmarks/own/go.py Log: Fix a description, it was a copy-paste. Modified: pypy/benchmarks/own/go.py ============================================================================== --- pypy/benchmarks/own/go.py (original) +++ pypy/benchmarks/own/go.py Tue Aug 17 20:14:08 2010 @@ -438,7 +438,7 @@ import util, optparse parser = optparse.OptionParser( usage="%prog [options]", - description="Test the performance of the Chaos benchmark") + description="Test the performance of the Go benchmark") util.add_standard_options_to(parser) options, args = parser.parse_args() From hakanardo at codespeak.net Wed Aug 18 07:33:29 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Wed, 18 Aug 2010 07:33:29 +0200 (CEST) Subject: [pypy-svn] r76661 - in pypy/branch/jit-bounds/pypy: jit/metainterp jit/metainterp/test module/pypyjit/test Message-ID: <20100818053329.14CC1282BEB@codespeak.net> Author: hakanardo Date: Wed Aug 18 07:33:27 2010 New Revision: 76661 Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/jit-bounds/pypy/module/pypyjit/test/test_pypy_c.py Log: Removing guard_no_overflow, intbound support for int_add_ovf, int_sub_ovf, int_and Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py Wed Aug 18 07:33:27 2010 @@ -19,6 +19,10 @@ from pypy.jit.metainterp.history import AbstractDescr, make_hashable_int from pypy.rlib.rarithmetic import ovfcheck +import sys +MAXINT = sys.maxint +MININT = -sys.maxint - 1 + def optimize_loop_1(metainterp_sd, loop): """Optimize loop.operations to make it match the input of loop.specnodes and to remove internal overheadish operations. Note that loop.specnodes @@ -118,25 +122,37 @@ def add_bound(self, other): res = self.copy() if other.has_upper: - res.upper += other.upper + try: + res.upper = ovfcheck(res.upper + other.upper) + except OverflowError: + res.has_upper = False else: res.has_upper = False if other.has_lower: - res.lower += other.lower + try: + res.lower = ovfcheck(res.lower + other.lower) + except OverflowError: + res.has_lower = False else: res.has_lower = False return res def sub_bound(self, other): res = self.copy() - if other.has_upper: - res.lower -= other.upper - else: - res.has_lower = False if other.has_lower: - res.upper -= other.lower + try: + res.upper = ovfcheck(res.upper - other.lower) + except OverflowError: + res.has_upper = False else: res.has_upper = False + if other.has_upper: + try: + res.lower = ovfcheck(res.lower - other.upper) + except OverflowError: + res.has_lower = False + else: + res.has_lower = False return res def contains(self, val): @@ -201,7 +217,7 @@ def __init__(self, box): self.box = box - self.intbound = IntUnbounded() + self.intbound = IntBound(MININT, MAXINT) #IntUnbounded() if isinstance(box, Const): self.make_constant(box) # invariant: box is a Const if and only if level == LEVEL_CONSTANT @@ -678,7 +694,9 @@ def propagate_forward(self): self.exception_might_have_happened = False self.newoperations = [] - for op in self.loop.operations: + self.i = 0 + while self.i < len(self.loop.operations): + op = self.loop.operations[self.i] self.producer[op.result] = op opnum = op.opnum for value, func in optimize_ops: @@ -687,6 +705,7 @@ break else: self.optimize_default(op) + self.i += 1 self.loop.operations = self.newoperations # accumulate counters self.resumedata_memo.update_counters(self.metainterp_sd.profiler) @@ -757,7 +776,7 @@ arg = args[i] if arg in self.values: args[i] = self.values[arg].get_key_box() - args.append(ConstInt(op.opnum)) + args.append(ConstInt(op.opnum)) return args def optimize_default(self, op): @@ -1199,6 +1218,15 @@ self.make_constant_int(op.result, 0) else: self.optimize_default(op) + r = self.getvalue(op.result) + if v2.is_constant(): + val = v2.box.getint() + if val >= 0: + r.intbound.intersect(IntBound(0,val)) + elif v1.is_constant(): + val = v1.box.getint() + if val >= 0: + r.intbound.intersect(IntBound(0,val)) def optimize_INT_OR(self, op): v1 = self.getvalue(op.args[0]) @@ -1209,6 +1237,10 @@ self.make_equal_to(op.result, v1) else: self.optimize_default(op) + + def pure(self, opnum, args, result): + op = ResOperation(opnum, args, result) + self.pure_operations[self.make_args_key(op)] = op def optimize_INT_SUB(self, op): v1 = self.getvalue(op.args[0]) @@ -1218,8 +1250,11 @@ else: self.optimize_default(op) r = self.getvalue(op.result) - print v1.intbound.sub_bound(v2.intbound) r.intbound.intersect(v1.intbound.sub_bound(v2.intbound)) + + # Synthesize the reverse ops for optimize_default to reuse + self.pure(rop.INT_ADD, [op.result, op.args[1]], op.args[0]) + self.pure(rop.INT_SUB, [op.args[0], op.result], op.args[1]) def optimize_INT_ADD(self, op): v1 = self.getvalue(op.args[0]) @@ -1235,13 +1270,38 @@ r.intbound.intersect(v1.intbound.add_bound(v2.intbound)) # Synthesize the reverse op for optimize_default to reuse - revop = ResOperation(rop.INT_SUB, [op.result, op.args[1]], \ - op.args[0], op.descr) - self.pure_operations[self.make_args_key(revop)] = revop - revop = ResOperation(rop.INT_SUB, [op.result, op.args[0]], \ - op.args[1], op.descr) - self.pure_operations[self.make_args_key(revop)] = revop + self.pure(rop.INT_SUB, [op.result, op.args[1]], op.args[0]) + self.pure(rop.INT_SUB, [op.result, op.args[0]], op.args[1]) + def optimize_INT_ADD_OVF(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + resbound = v1.intbound.add_bound(v2.intbound) + if resbound.has_lower and resbound.has_upper and \ + self.loop.operations[self.i+1].opnum == rop.GUARD_NO_OVERFLOW: + # Transform into INT_SUB and remove guard + op.opnum = rop.INT_ADD + self.i += 1 + self.optimize_INT_ADD(op) + else: + self.optimize_default(op) + r = self.getvalue(op.result) + r.intbound.intersect(resbound) + + def optimize_INT_SUB_OVF(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + resbound = v1.intbound.sub_bound(v2.intbound) + if resbound.has_lower and resbound.has_upper and \ + self.loop.operations[self.i+1].opnum == rop.GUARD_NO_OVERFLOW: + # Transform into INT_SUB and remove guard + op.opnum = rop.INT_SUB + self.i += 1 + self.optimize_INT_SUB(op) + else: + self.optimize_default(op) + r = self.getvalue(op.result) + r.intbound.intersect(resbound) def optimize_INT_LT(self, op): v1 = self.getvalue(op.args[0]) @@ -1360,6 +1420,9 @@ # b = r.intbound.sub_bound(v1.intbound).mul(-1) # if v2.intbound.intersect(b): + propagate_bounds_INT_ADD_OVF = propagate_bounds_INT_ADD + propagate_bounds_INT_SUB_OVF = propagate_bounds_INT_SUB + optimize_ops = _findall(Optimizer, 'optimize_') propagate_bounds_ops = _findall(Optimizer, 'propagate_bounds_') Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py Wed Aug 18 07:33:27 2010 @@ -3169,6 +3169,47 @@ """ self.optimize_loop(ops, 'Not', expected) + def test_bound_lt_add_ovf(self): + ops = """ + [i0] + i1 = int_lt(i0, 4) + guard_true(i1) [] + i2 = int_add_ovf(i0, 10) + guard_no_overflow() [] + i3 = int_lt(i2, 15) + guard_true(i3) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_lt(i0, 4) + guard_true(i1) [] + i2 = int_add(i0, 10) + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_bound_lt_add_ovf_before(self): + ops = """ + [i0] + i2 = int_add_ovf(i0, 10) + guard_no_overflow() [] + i3 = int_lt(i2, 15) + guard_true(i3) [] + i1 = int_lt(i0, 6) + guard_true(i1) [] + jump(i0) + """ + expected = """ + [i0] + i2 = int_add_ovf(i0, 10) + guard_no_overflow() [] + i3 = int_lt(i2, 15) + guard_true(i3) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + def test_bound_lt_sub(self): ops = """ [i0] @@ -3292,6 +3333,79 @@ """ self.optimize_loop(ops, 'Not', expected) + def test_bound_ovf(self): + ops = """ + [i0] + i1 = int_ge(i0, 0) + guard_true(i1) [] + i2 = int_lt(i0, 10) + guard_true(i2) [] + i3 = int_add_ovf(i0, 1) + guard_no_overflow() [] + jump(i3) + """ + expected = """ + [i0] + i1 = int_ge(i0, 0) + guard_true(i1) [] + i2 = int_lt(i0, 10) + guard_true(i2) [] + i3 = int_add(i0, 1) + jump(i3) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_addsub_const(self): + ops = """ + [i0] + i1 = int_add(i0, 1) + i2 = int_sub(i1, 1) + i3 = int_add(i2, 1) + i4 = int_mul(i2, i3) + jump(i4) + """ + expected = """ + [i0] + i1 = int_add(i0, 1) + i4 = int_mul(i0, i1) + jump(i4) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_addsub_int(self): + ops = """ + [i0, i10] + i1 = int_add(i0, i10) + i2 = int_sub(i1, i10) + i3 = int_add(i2, i10) + i4 = int_add(i2, i3) + jump(i4, i10) + """ + expected = """ + [i0, i10] + i1 = int_add(i0, i10) + i4 = int_add(i0, i1) + jump(i4, i10) + """ + self.optimize_loop(ops, 'Not, Not', expected) + + def test_addsub_int2(self): + ops = """ + [i0, i10] + i1 = int_add(i10, i0) + i2 = int_sub(i1, i10) + i3 = int_add(i10, i2) + i4 = int_add(i2, i3) + jump(i4, i10) + """ + expected = """ + [i0, i10] + i1 = int_add(i10, i0) + i4 = int_add(i0, i1) + jump(i4, i10) + """ + self.optimize_loop(ops, 'Not, Not', expected) + def test_framestackdepth_overhead(self): ops = """ [p0, i22] @@ -3321,6 +3435,115 @@ jump(p0, i22) """ self.optimize_loop(ops, 'Not, Not', expected) + + def test_addsub_ovf(self): + ops = """ + [i0] + i1 = int_add_ovf(i0, 10) + guard_no_overflow() [] + i2 = int_sub_ovf(i1, 5) + guard_no_overflow() [] + jump(i2) + """ + expected = """ + [i0] + i1 = int_add_ovf(i0, 10) + guard_no_overflow() [] + i2 = int_sub(i1, 5) + jump(i2) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_subadd_ovf(self): + ops = """ + [i0] + i1 = int_sub_ovf(i0, 10) + guard_no_overflow() [] + i2 = int_add_ovf(i1, 5) + guard_no_overflow() [] + jump(i2) + """ + expected = """ + [i0] + i1 = int_sub_ovf(i0, 10) + guard_no_overflow() [] + i2 = int_add(i1, 5) + jump(i2) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_bound_and(self): + ops = """ + [i0] + i1 = int_and(i0, 255) + i2 = int_lt(i1, 500) + guard_true(i2) [] + i3 = int_le(i1, 255) + guard_true(i3) [] + i4 = int_gt(i1, -1) + guard_true(i4) [] + i5 = int_ge(i1, 0) + guard_true(i5) [] + i6 = int_lt(i1, 0) + guard_false(i6) [] + i7 = int_le(i1, -1) + guard_false(i7) [] + i8 = int_gt(i1, 255) + guard_false(i8) [] + i9 = int_ge(i1, 500) + guard_false(i9) [] + i12 = int_lt(i1, 100) + guard_true(i12) [] + i13 = int_le(i1, 90) + guard_true(i13) [] + i14 = int_gt(i1, 10) + guard_true(i14) [] + i15 = int_ge(i1, 20) + guard_true(i15) [] + jump(i1) + """ + expected = """ + [i0] + i1 = int_and(i0, 255) + i12 = int_lt(i1, 100) + guard_true(i12) [] + i13 = int_le(i1, 90) + guard_true(i13) [] + i14 = int_gt(i1, 10) + guard_true(i14) [] + i15 = int_ge(i1, 20) + guard_true(i15) [] + jump(i1) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_subsub_ovf(self): + ops = """ + [i0] + i1 = int_sub_ovf(1, i0) + guard_no_overflow() [] + i2 = int_gt(i1, 1) + guard_true(i2) [] + i3 = int_sub_ovf(1, i0) + guard_no_overflow() [] + i4 = int_gt(i3, 1) + guard_true(i4) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_sub_ovf(1, i0) + guard_no_overflow() [] + i2 = int_gt(i1, 1) + guard_true(i2) [] + i3 = int_sub_ovf(1, i0) + guard_no_overflow() [] + i4 = int_gt(i3, 1) + guard_true(i4) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + Modified: pypy/branch/jit-bounds/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/jit-bounds/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/jit-bounds/pypy/module/pypyjit/test/test_pypy_c.py Wed Aug 18 07:33:27 2010 @@ -110,6 +110,7 @@ if sys.platform.startswith('win'): py.test.skip("XXX this is not Windows-friendly") + print logfilepath child_stdout = os.popen('PYPYLOG=":%s" "%s" "%s"' % ( logfilepath, self.pypy_c, filepath), 'r') result = child_stdout.read() @@ -847,7 +848,7 @@ return intimg[i - 1] ''', maxops, ([tc], res)) - def test_intbound(self): + def test_intbound_simple(self): ops = ('<', '>', '<=', '>=') nbr = (3, 7) for o1 in ops: @@ -881,11 +882,12 @@ res[f(i)] += 1500 self.run_source(src, 220, ([], res)) - def test_intbound_addsub(self): + def test_intbound_addsub_mix(self): tests = ('i > 4', 'i > 2', 'i + 1 > 2', '1 + i > 4', 'i - 1 > 1', '1 - i > 1', '1 - i < -3') for t1 in tests: for t2 in tests: + print t1, t2 src = ''' def f(i): a, b = 3, 3 @@ -926,6 +928,16 @@ return (a, b) ''', 48, ([], (2000, 2000))) + def test_intbound_sub_lt(self): + self.run_source(''' + def main(): + i, a, b = 0, 0, 0 + while i < 2000: + if i - 10 < 1995: + a += 1 + i += 1 + return (a, b) + ''', 38, ([], (2000, 0))) def test_intbound_addsub_ge(self): self.run_source(''' @@ -934,24 +946,38 @@ while i < 2000: if i + 5 >= 5: a += 1 - #if i - 1 >= -1: if i - 1 >= -1: b += 1 i += 1 return (a, b) - ''', 0, ([], (2000, 2000))) + ''', 56, ([], (2000, 2000))) - def test_intbound_sub_lt(self): + def test_zeropadded(self): self.run_source(''' + from array import array + class ZeroPadded(array): + def __new__(cls, l): + self = array.__new__(cls, 'd', range(l)) + return self + + def __getitem__(self, i): + if i < 0 or i >= self.__len__(): + return 0 + return array.__getitem__(self, i) + + def main(): - i, a, b = 0, 0, 0 - while i < 2000: - if i - 10 < 1995: - a += 1 + buf = ZeroPadded(2000) + i = 10 + sa = 0 + while i < 2000 - 10: + sa += buf[i-2] + buf[i-1] + buf[i] + buf[i+1] + buf[i+2] i += 1 - return (a, b) - ''', 0, ([], (2000, 0))) + return sa + + ''', 232, ([], 9895050.0)) + # test_circular class AppTestJIT(PyPyCJITTests): def setup_class(cls): From cfbolz at codespeak.net Wed Aug 18 11:14:21 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 18 Aug 2010 11:14:21 +0200 (CEST) Subject: [pypy-svn] r76662 - in pypy/benchmarks: . own Message-ID: <20100818091421.32FB0282BEB@codespeak.net> Author: cfbolz Date: Wed Aug 18 11:14:19 2010 New Revision: 76662 Added: pypy/benchmarks/own/raytrace-simple.py (contents, props changed) Modified: pypy/benchmarks/benchmarks.py Log: add a simple pure-python ray tracer Modified: pypy/benchmarks/benchmarks.py ============================================================================== --- pypy/benchmarks/benchmarks.py (original) +++ pypy/benchmarks/benchmarks.py Wed Aug 18 11:14:19 2010 @@ -44,7 +44,7 @@ } for name in ['float', 'nbody_modified', 'meteor-contest', 'fannkuch', - 'spectral-norm', 'chaos', 'telco', 'go', 'pyflate-fast']: + 'spectral-norm', 'chaos', 'telco', 'go', 'pyflate-fast', 'raytrace-simple']: _register_new_bm(name, name, globals(), **opts.get(name, {})) for name in ['names', 'iteration', 'tcp', 'pb']:#, 'accepts', 'web']: if name == 'web': Added: pypy/benchmarks/own/raytrace-simple.py ============================================================================== --- (empty file) +++ pypy/benchmarks/own/raytrace-simple.py Wed Aug 18 11:14:19 2010 @@ -0,0 +1,377 @@ +# This file contains definitions for a simple raytracer. +# Copyright Callum and Tony Garnock-Jones, 2008. +# This file may be freely redistributed under the MIT license, +# http://www.opensource.org/licenses/mit-license.php + +# from http://www.lshift.net/blog/2008/10/29/toy-raytracer-in-python +from __future__ import with_statement + +import math + +EPSILON = 0.00001 + +class Vector(object): + def __init__(self, initx, inity, initz): + self.x = initx + self.y = inity + self.z = initz + + def __str__(self): + return '(%s,%s,%s)' % (self.x, self.y, self.z) + + def __repr__(self): + return 'Vector(%s,%s,%s)' % (self.x, self.y, self.z) + + def magnitude(self): + return math.sqrt(self.dot(self)) + + def __add__(self, other): + if other.isPoint(): + return Point(self.x + other.x, self.y + other.y, self.z + other.z) + else: + return Vector(self.x + other.x, self.y + other.y, self.z + other.z) + + def __sub__(self, other): + other.mustBeVector() + return Vector(self.x - other.x, self.y - other.y, self.z - other.z) + + def scale(self, factor): + return Vector(factor * self.x, factor * self.y, factor * self.z) + + def dot(self, other): + other.mustBeVector() + return (self.x * other.x) + (self.y * other.y) + (self.z * other.z) + + def cross(self, other): + other.mustBeVector() + return Vector(self.y * other.z - self.z * other.y, + self.z * other.x - self.x * other.z, + self.x * other.y - self.y * other.x) + + def normalized(self): + return self.scale(1.0 / self.magnitude()) + + def negated(self): + return self.scale(-1) + + def __eq__(self, other): + return (self.x == other.x) and (self.y == other.y) and (self.z == other.z) + + def isVector(self): + return True + + def isPoint(self): + return False + + def mustBeVector(self): + return self + + def mustBePoint(self): + raise 'Vectors are not points!' + + def reflectThrough(self, normal): + d = normal.scale(self.dot(normal)) + return self - d.scale(2) + +Vector.ZERO = Vector(0,0,0) +Vector.RIGHT = Vector(1,0,0) +Vector.UP = Vector(0,1,0) +Vector.OUT = Vector(0,0,1) + +assert Vector.RIGHT.reflectThrough(Vector.UP) == Vector.RIGHT +assert Vector(-1,-1,0).reflectThrough(Vector.UP) == Vector(-1,1,0) + +class Point(object): + def __init__(self, initx, inity, initz): + self.x = initx + self.y = inity + self.z = initz + + def __str__(self): + return '(%s,%s,%s)' % (self.x, self.y, self.z) + + def __repr__(self): + return 'Point(%s,%s,%s)' % (self.x, self.y, self.z) + + def __add__(self, other): + other.mustBeVector() + return Point(self.x + other.x, self.y + other.y, self.z + other.z) + + def __sub__(self, other): + if other.isPoint(): + return Vector(self.x - other.x, self.y - other.y, self.z - other.z) + else: + return Point(self.x - other.x, self.y - other.y, self.z - other.z) + + def isVector(self): + return False + + def isPoint(self): + return True + + def mustBeVector(self): + raise 'Points are not vectors!' + + def mustBePoint(self): + return self + +class Sphere(object): + def __init__(self, centre, radius): + centre.mustBePoint() + self.centre = centre + self.radius = radius + + def __repr__(self): + return 'Sphere(%s,%s)' % (repr(self.centre), self.radius) + + def intersectionTime(self, ray): + cp = self.centre - ray.point + v = cp.dot(ray.vector) + discriminant = (self.radius * self.radius) - (cp.dot(cp) - v*v) + if discriminant < 0: + return None + else: + return v - math.sqrt(discriminant) + + def normalAt(self, p): + return (p - self.centre).normalized() + +class Halfspace(object): + def __init__(self, point, normal): + self.point = point + self.normal = normal.normalized() + + def __repr__(self): + return 'Halfspace(%s,%s)' % (repr(self.point), repr(self.normal)) + + def intersectionTime(self, ray): + v = ray.vector.dot(self.normal) + if v: + return 1 / -v + else: + return None + + def normalAt(self, p): + return self.normal + +class Ray(object): + def __init__(self, point, vector): + self.point = point + self.vector = vector.normalized() + + def __repr__(self): + return 'Ray(%s,%s)' % (repr(self.point), repr(self.vector)) + + def pointAtTime(self, t): + return self.point + self.vector.scale(t) + +Point.ZERO = Point(0,0,0) + +a = Vector(3,4,12) +b = Vector(1,1,1) + +class PpmCanvas(object): + def __init__(self, width, height, filenameBase): + import array + self.bytes = array.array('B', [0] * (width * height * 3)) + for i in range(width * height): + self.bytes[i * 3 + 2] = 255 + self.width = width + self.height = height + self.filenameBase = filenameBase + + def plot(self, x, y, r, g, b): + i = ((self.height - y - 1) * self.width + x) * 3 + self.bytes[i ] = max(0, min(255, int(r * 255))) + self.bytes[i+1] = max(0, min(255, int(g * 255))) + self.bytes[i+2] = max(0, min(255, int(b * 255))) + + def save(self): + with open(self.filenameBase + '.ppm', 'wb') as f: + f.write('P6 %d %d 255\n' % (self.width, self.height)) + f.write(self.bytes.tostring()) + +def firstIntersection(intersections): + result = None + for i in intersections: + candidateT = i[1] + if candidateT is not None and candidateT > -EPSILON: + if result is None or candidateT < result[1]: + result = i + return result + +class Scene(object): + def __init__(self): + self.objects = [] + self.lightPoints = [] + self.position = Point(0, 1.8, 10) + self.lookingAt = Point.ZERO + self.fieldOfView = 45 + self.recursionDepth = 0 + + def moveTo(self, p): + self.position = p + + def lookAt(self, p): + self.lookingAt = p + + def addObject(self, object, surface): + self.objects.append((object, surface)) + + def addLight(self, p): + self.lightPoints.append(p) + + def render(self, canvas): + #print 'Computing field of view' + fovRadians = math.pi * (self.fieldOfView / 2.0) / 180.0 + halfWidth = math.tan(fovRadians) + halfHeight = 0.75 * halfWidth + width = halfWidth * 2 + height = halfHeight * 2 + pixelWidth = width / (canvas.width - 1) + pixelHeight = height / (canvas.height - 1) + + eye = Ray(self.position, self.lookingAt - self.position) + vpRight = eye.vector.cross(Vector.UP).normalized() + vpUp = vpRight.cross(eye.vector).normalized() + + #print 'Looping over pixels' + previousfraction = 0 + for y in range(canvas.height): + currentfraction = float(y) / canvas.height + if currentfraction - previousfraction > 0.05: + canvas.save() + #print '%d%% complete' % (currentfraction * 100) + previousfraction = currentfraction + for x in range(canvas.width): + xcomp = vpRight.scale(x * pixelWidth - halfWidth) + ycomp = vpUp.scale(y * pixelHeight - halfHeight) + ray = Ray(eye.point, eye.vector + xcomp + ycomp) + colour = self.rayColour(ray) + canvas.plot(x,y,*colour) + + #print 'Complete.' + + def rayColour(self, ray): + if self.recursionDepth > 3: + return (0,0,0) + try: + self.recursionDepth = self.recursionDepth + 1 + intersections = [(o, o.intersectionTime(ray), s) for (o, s) in self.objects] + i = firstIntersection(intersections) + if i is None: + return (0,0,0) ## the background colour + else: + (o, t, s) = i + p = ray.pointAtTime(t) + return s.colourAt(self, ray, p, o.normalAt(p)) + finally: + self.recursionDepth = self.recursionDepth - 1 + + def _lightIsVisible(self, l, p): + for (o, s) in self.objects: + t = o.intersectionTime(Ray(p,l - p)) + if t is not None and t > EPSILON: + return False + return True + + def visibleLights(self, p): + result = [] + for l in self.lightPoints: + if self._lightIsVisible(l, p): + result.append(l) + return result + +def addColours(a, scale, b): + return (a[0] + scale * b[0], + a[1] + scale * b[1], + a[2] + scale * b[2]) + +class SimpleSurface(object): + def __init__(self, **kwargs): + self.baseColour = kwargs.get('baseColour', (1,1,1)) + self.specularCoefficient = kwargs.get('specularCoefficient', 0.2) + self.lambertCoefficient = kwargs.get('lambertCoefficient', 0.6) + self.ambientCoefficient = 1.0 - self.specularCoefficient - self.lambertCoefficient + + def baseColourAt(self, p): + return self.baseColour + + def colourAt(self, scene, ray, p, normal): + b = self.baseColourAt(p) + + c = (0,0,0) + if self.specularCoefficient > 0: + reflectedRay = Ray(p, ray.vector.reflectThrough(normal)) + #print p, normal, ray.vector, reflectedRay.vector + reflectedColour = scene.rayColour(reflectedRay) + c = addColours(c, self.specularCoefficient, reflectedColour) + + if self.lambertCoefficient > 0: + lambertAmount = 0 + for lightPoint in scene.visibleLights(p): + contribution = (lightPoint - p).normalized().dot(normal) + if contribution > 0: + lambertAmount = lambertAmount + contribution + lambertAmount = min(1,lambertAmount) + c = addColours(c, self.lambertCoefficient * lambertAmount, b) + + if self.ambientCoefficient > 0: + c = addColours(c, self.ambientCoefficient, b) + + return c + +class CheckerboardSurface(SimpleSurface): + def __init__(self, **kwargs): + SimpleSurface.__init__(self, **kwargs) + self.otherColour = kwargs.get('otherColour', (0,0,0)) + self.checkSize = kwargs.get('checkSize', 1) + + def baseColourAt(self, p): + v = p - Point.ZERO + v.scale(1.0 / self.checkSize) + if (int(abs(v.x) + 0.5) + \ + int(abs(v.y) + 0.5) + \ + int(abs(v.z) + 0.5)) \ + % 2: + return self.otherColour + else: + return self.baseColour + +def _main(): + Canvas = PpmCanvas + c = Canvas(100,100,'test_raytrace') + #c = Canvas(640,480,'test_raytrace_big') + s = Scene() + s.addLight(Point(30, 30, 10)) + s.addLight(Point(-10, 100, 30)) + s.lookAt(Point(0, 3, 0)) + s.addObject(Sphere(Point(1,3,-10), 2), SimpleSurface(baseColour = (1,1,0))) + for y in range(6): + s.addObject(Sphere(Point(-3 - y * 0.4, 2.3, -5), 0.4), + SimpleSurface(baseColour = (y / 6.0, 1 - y / 6.0, 0.5))) + s.addObject(Halfspace(Point(0,0,0), Vector.UP), CheckerboardSurface()) + s.render(c) + +def main(n): + import time + times = [] + for i in range(5): + _main() # warmup + for i in range(n): + t1 = time.time() + _main() + t2 = time.time() + times.append(t2 - t1) + return times + +if __name__ == "__main__": + import util, optparse + parser = optparse.OptionParser( + usage="%prog [options]", + description="Test the performance of the raytrace benchmark") + util.add_standard_options_to(parser) + options, args = parser.parse_args() + + util.run_benchmark(options, options.num_runs, main) + From hakanardo at codespeak.net Thu Aug 19 08:28:49 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Thu, 19 Aug 2010 08:28:49 +0200 (CEST) Subject: [pypy-svn] r76668 - in pypy/trunk/pypy/module/array: . test Message-ID: <20100819062849.2E726282B9E@codespeak.net> Author: hakanardo Date: Thu Aug 19 08:28:47 2010 New Revision: 76668 Modified: pypy/trunk/pypy/module/array/interp_array.py pypy/trunk/pypy/module/array/test/test_array.py Log: potential fix to the array apptest failures? Modified: pypy/trunk/pypy/module/array/interp_array.py ============================================================================== --- pypy/trunk/pypy/module/array/interp_array.py (original) +++ pypy/trunk/pypy/module/array/interp_array.py Thu Aug 19 08:28:47 2010 @@ -15,6 +15,7 @@ from pypy.interpreter.argument import Arguments, Signature from pypy.module._file.interp_file import W_File from pypy.interpreter.buffer import RWBuffer +from pypy.objspace.std.multimethod import FailedToImplement def w_array(space, w_cls, typecode, w_args=None): if len(w_args.arguments_w) > 1: @@ -476,9 +477,11 @@ def mul__Array_ANY(space, self, w_repeat): try: - repeat = space.int_w(w_repeat) - except OperationError: - return space.w_NotImplemented + repeat = space.getindex_w(w_repeat, space.w_OverflowError) + except OperationError, e: + if e.match(space, space.w_TypeError): + raise FailedToImplement + raise a = mytype.w_class(space) repeat = max(repeat, 0) a.setlen(self.len * repeat) @@ -492,9 +495,11 @@ def inplace_mul__Array_ANY(space, self, w_repeat): try: - repeat = space.int_w(w_repeat) - except OperationError: - return space.w_NotImplemented + repeat = space.getindex_w(w_repeat, space.w_OverflowError) + except OperationError, e: + if e.match(space, space.w_TypeError): + raise FailedToImplement + raise oldlen = self.len repeat = max(repeat, 0) self.setlen(self.len * repeat) Modified: pypy/trunk/pypy/module/array/test/test_array.py ============================================================================== --- pypy/trunk/pypy/module/array/test/test_array.py (original) +++ pypy/trunk/pypy/module/array/test/test_array.py Thu Aug 19 08:28:47 2010 @@ -628,7 +628,8 @@ a = self.array('i') raises(TypeError, "a * 'hi'") raises(TypeError, "'hi' * a") - + raises(TypeError, "a *= 'hi'") + class mulable(object): def __mul__(self, other): return "mul" From getxsick at codespeak.net Thu Aug 19 13:35:16 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Thu, 19 Aug 2010 13:35:16 +0200 (CEST) Subject: [pypy-svn] r76669 - pypy/branch/fast-ctypes/pypy/module/_ctypes Message-ID: <20100819113516.1BC2F282B9E@codespeak.net> Author: getxsick Date: Thu Aug 19 13:35:15 2010 New Revision: 76669 Added: pypy/branch/fast-ctypes/pypy/module/_ctypes/app_basics.py Modified: pypy/branch/fast-ctypes/pypy/module/_ctypes/__init__.py pypy/branch/fast-ctypes/pypy/module/_ctypes/app_dummy.py Log: add ArgumentError exception. Modified: pypy/branch/fast-ctypes/pypy/module/_ctypes/__init__.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/_ctypes/__init__.py (original) +++ pypy/branch/fast-ctypes/pypy/module/_ctypes/__init__.py Thu Aug 19 13:35:15 2010 @@ -6,13 +6,13 @@ 'Test' : 'interp_test.W_Test', } appleveldefs = { + 'ArgumentError' : 'app_basics.ArgumentError', '_SimpleCData' : 'app_dummy._SimpleCData', '_Pointer' : 'app_dummy._Pointer', 'CFuncPtr' : 'app_dummy.CFuncPtr', 'Union' : 'app_dummy.Union', 'Structure' : 'app_dummy.Structure', 'Array' : 'app_dummy.Array', - 'ArgumentError' : 'app_dummy.ArgumentError', 'sizeof' : 'app_dummy.sizeof', 'byref' : 'app_dummy.byref', 'addressof' : 'app_dummy.addressof', Added: pypy/branch/fast-ctypes/pypy/module/_ctypes/app_basics.py ============================================================================== --- (empty file) +++ pypy/branch/fast-ctypes/pypy/module/_ctypes/app_basics.py Thu Aug 19 13:35:15 2010 @@ -0,0 +1,2 @@ +class ArgumentError(Exception): + pass Modified: pypy/branch/fast-ctypes/pypy/module/_ctypes/app_dummy.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/_ctypes/app_dummy.py (original) +++ pypy/branch/fast-ctypes/pypy/module/_ctypes/app_dummy.py Thu Aug 19 13:35:15 2010 @@ -1,7 +1,7 @@ class DummyClass(object): def __init__(self, *args, **kwargs): raise NotImplementedError("not-implemented ctypes function") -_Pointer = Union = Structure = Array = ArgumentError = DummyClass +_Pointer = Union = Structure = Array = DummyClass class CFuncPtr(object): def __init__(self, *args, **kwargs): From cfbolz at codespeak.net Thu Aug 19 14:48:02 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 19 Aug 2010 14:48:02 +0200 (CEST) Subject: [pypy-svn] r76670 - pypy/branch/remove-ri386-multimethod-2 Message-ID: <20100819124802.247A3282B9E@codespeak.net> Author: cfbolz Date: Thu Aug 19 14:48:01 2010 New Revision: 76670 Removed: pypy/branch/remove-ri386-multimethod-2/ Log: (cfbolz, arigo): remove branch with code that is already used somewhere From cfbolz at codespeak.net Thu Aug 19 14:52:42 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 19 Aug 2010 14:52:42 +0200 (CEST) Subject: [pypy-svn] r76671 - pypy/branch/kill-can-inline Message-ID: <20100819125242.7DC65282B9E@codespeak.net> Author: cfbolz Date: Thu Aug 19 14:52:41 2010 New Revision: 76671 Removed: pypy/branch/kill-can-inline/ Log: remove confusingly named redundant branch From cfbolz at codespeak.net Thu Aug 19 14:54:50 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 19 Aug 2010 14:54:50 +0200 (CEST) Subject: [pypy-svn] r76672 - pypy/branch/pypycpp Message-ID: <20100819125450.0AE35282B9E@codespeak.net> Author: cfbolz Date: Thu Aug 19 14:54:49 2010 New Revision: 76672 Removed: pypy/branch/pypycpp/ Log: this is made redundant by cppyy (and the new branch needs less characters too) From wlav at codespeak.net Fri Aug 20 02:19:50 2010 From: wlav at codespeak.net (wlav at codespeak.net) Date: Fri, 20 Aug 2010 02:19:50 +0200 (CEST) Subject: [pypy-svn] r76674 - in pypy/branch/reflex-support/pypy/module/cppyy: . test Message-ID: <20100820001950.CBDB0282B9E@codespeak.net> Author: wlav Date: Fri Aug 20 02:19:47 2010 New Revision: 76674 Modified: pypy/branch/reflex-support/pypy/module/cppyy/converter.py pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py Log: Allow setting of boolean data members. Modified: pypy/branch/reflex-support/pypy/module/cppyy/converter.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/converter.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/converter.py Fri Aug 20 02:19:47 2010 @@ -48,6 +48,23 @@ x[0] = arg return rffi.cast(rffi.VOIDP, x) + def from_memory(self, space, w_obj, offset): + fieldptr = self._get_fieldptr(space, w_obj, offset) + if fieldptr[0] == '\x01': + return space.wrap(True) + return space.wrap(False) + + def to_memory(self, space, w_obj, w_value, offset): + fieldptr = self._get_fieldptr(space, w_obj, offset) + arg = space.c_int_w(w_value) + if arg != False and arg != True: + raise OperationError(space.w_TypeError, + space.wrap("boolean value should be bool, or integer 1 or 0")) + if arg: + fieldptr[0] = '\x01' + else: + fieldptr[0] = '\x00' + class CharConverter(TypeConverter): def _from_space(self, space, w_value): # allow int to pass to char and make sure that str is of length 1 Modified: pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py ============================================================================== --- pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py (original) +++ pypy/branch/reflex-support/pypy/module/cppyy/test/test_datatypes.py Fri Aug 20 02:19:47 2010 @@ -41,16 +41,17 @@ c = cppyy_test_data() assert isinstance(c, cppyy_test_data) - # boolean types + # boolean types through functions c.set_bool(True); assert c.get_bool() == True c.set_bool(0); assert c.get_bool() == False - """ + + # boolean types through data members c.m_bool = True; assert c.get_bool() == True c.set_bool(True); assert c.m_bool == True c.m_bool = 0; assert c.get_bool() == False c.set_bool(0); assert c.m_bool == False - """ + raises(TypeError, 'c.set_bool(10)') # char types through functions From arigo at codespeak.net Fri Aug 20 14:04:23 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 20 Aug 2010 14:04:23 +0200 (CEST) Subject: [pypy-svn] r76676 - pypy/trunk/pypy/objspace/std Message-ID: <20100820120423.71C57282B90@codespeak.net> Author: arigo Date: Fri Aug 20 14:04:21 2010 New Revision: 76676 Modified: pypy/trunk/pypy/objspace/std/model.py Log: Only import the array module if the config says we need it. Modified: pypy/trunk/pypy/objspace/std/model.py ============================================================================== --- pypy/trunk/pypy/objspace/std/model.py (original) +++ pypy/trunk/pypy/objspace/std/model.py Fri Aug 20 14:04:21 2010 @@ -88,7 +88,8 @@ import pypy.objspace.std.default # register a few catch-all multimethods import pypy.objspace.std.marshal_impl # install marshal multimethods - import pypy.module.array + if config.objspace.usemodules.array: + import pypy.module.array # the set of implementation types self.typeorder = { From arigo at codespeak.net Fri Aug 20 14:34:13 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 20 Aug 2010 14:34:13 +0200 (CEST) Subject: [pypy-svn] r76677 - pypy/trunk/pypy/jit/backend/test Message-ID: <20100820123413.679B2282B9D@codespeak.net> Author: arigo Date: Fri Aug 20 14:34:11 2010 New Revision: 76677 Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py Log: Fix the test. Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/runner_test.py (original) +++ pypy/trunk/pypy/jit/backend/test/runner_test.py Fri Aug 20 14:34:11 2010 @@ -1852,7 +1852,7 @@ abox = BoxInt(heaptracker.adr2int(addr)) self.execute_operation(rop.SETARRAYITEM_RAW, [abox, BoxInt(5), BoxInt(12345)], - 'int', descr=descr) + 'void', descr=descr) assert a[5] == 12345 lltype.free(a, flavor='raw') From arigo at codespeak.net Fri Aug 20 14:34:31 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 20 Aug 2010 14:34:31 +0200 (CEST) Subject: [pypy-svn] r76678 - pypy/trunk/pypy/jit/backend/llgraph Message-ID: <20100820123431.93F39282B9D@codespeak.net> Author: arigo Date: Fri Aug 20 14:34:30 2010 New Revision: 76678 Modified: pypy/trunk/pypy/jit/backend/llgraph/llimpl.py Log: Add {set,get}arrayitem_raw to the llgraph backend. Modified: pypy/trunk/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/trunk/pypy/jit/backend/llgraph/llimpl.py Fri Aug 20 14:34:30 2010 @@ -123,6 +123,9 @@ 'setarrayitem_gc' : (('ref', 'int', 'intorptr'), None), 'getarrayitem_gc' : (('ref', 'int'), 'intorptr'), 'getarrayitem_gc_pure' : (('ref', 'int'), 'intorptr'), + 'setarrayitem_raw' : (('ref', 'int', 'intorptr'), None), + 'getarrayitem_raw' : (('ref', 'int'), 'intorptr'), + 'getarrayitem_raw_pure' : (('ref', 'int'), 'intorptr'), 'arraylen_gc' : (('ref',), 'int'), 'call' : (('ref', 'varargs'), 'intorptr'), 'call_assembler' : (('ref', 'varargs'), 'intorptr'), @@ -689,6 +692,18 @@ op_getarrayitem_gc_pure = op_getarrayitem_gc + def op_getarrayitem_raw(self, arraydescr, array, index): + if arraydescr.typeinfo == REF: + raise NotImplementedError("getarrayitem_raw -> gcref") + elif arraydescr.typeinfo == INT: + return do_getarrayitem_raw_int(array, index) + elif arraydescr.typeinfo == FLOAT: + return do_getarrayitem_raw_float(array, index) + else: + raise NotImplementedError + + op_getarrayitem_raw_pure = op_getarrayitem_raw + def op_getfield_gc(self, fielddescr, struct): if fielddescr.typeinfo == REF: return do_getfield_gc_ptr(struct, fielddescr.ofs) @@ -734,6 +749,16 @@ else: raise NotImplementedError + def op_setarrayitem_raw(self, arraydescr, array, index, newvalue): + if arraydescr.typeinfo == REF: + raise NotImplementedError("setarrayitem_raw <- gcref") + elif arraydescr.typeinfo == INT: + do_setarrayitem_raw_int(array, index, newvalue) + elif arraydescr.typeinfo == FLOAT: + do_setarrayitem_raw_float(array, index, newvalue) + else: + raise NotImplementedError + def op_setfield_gc(self, fielddescr, struct, newvalue): if fielddescr.typeinfo == REF: do_setfield_gc_ptr(struct, fielddescr.ofs, newvalue) From arigo at codespeak.net Fri Aug 20 17:03:11 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 20 Aug 2010 17:03:11 +0200 (CEST) Subject: [pypy-svn] r76679 - pypy/branch/rsre2/pypy/rlib/rsre Message-ID: <20100820150311.18719282B90@codespeak.net> Author: arigo Date: Fri Aug 20 17:03:09 2010 New Revision: 76679 Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre_core.py Log: A version with seriously less @specializectx around. Only part of the functions are now specialized; the calls from non-specialized to specialized code are an indirect call (so only a bit slower than direct calls), and they occur at points where the overhead is not really noticeable (as opposed to e.g. every time we call ctx.str()). Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre_core.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre_core.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre_core.py Fri Aug 20 17:03:09 2010 @@ -1,7 +1,7 @@ import sys from pypy.rlib.debug import check_nonneg +from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rsre import rsre_char -from pypy.rlib import jit from pypy.tool.sourcetools import func_with_new_name @@ -41,19 +41,29 @@ # ____________________________________________________________ def specializectx(func): - """A decorator that specializes 'func' for each concrete subclass - XyzMatchContext. It must then be called as func(ctx,...) where - ctx is known to be of a specific subclass. + """A decorator that specializes 'func(ctx,...)' for each concrete subclass + of AbstractMatchContext. During annotation, if 'ctx' is known to be a + specific subclass, calling 'func' is a direct call; if 'ctx' is only known + to be of class AbstractMatchContext, calling 'func' is an indirect call. """ - i = list(func.func_code.co_varnames).index('ctx') - func._annspecialcase_ = 'specialize:argtype(%d)' % i - return func + assert func.func_code.co_varnames[0] == 'ctx' + specname = '_spec_' + func.func_name + # Install a copy of the function under the name '_spec_funcname' in each + # concrete subclass + for prefix, concreteclass in [('str', StrMatchContext), + ('uni', UnicodeMatchContext)]: + newfunc = func_with_new_name(func, prefix + specname) + setattr(concreteclass, specname, newfunc) + # Return a dispatcher function, specialized on the exact type of 'ctx' + def dispatch(ctx, *args): + return getattr(ctx, specname)(*args) + dispatch._annspecialcase_ = 'specialize:argtype(0)' + return dispatch # ____________________________________________________________ class AbstractMatchContext(object): """Abstract base class""" - _must_specialize_ = True match_start = 0 match_end = 0 match_marks = None @@ -74,16 +84,15 @@ def str(self, index): """NOT_RPYTHON: Must be overridden in a concrete subclass. - The line below is used to generate a translation-time crash + The tag ^^^ here is used to generate a translation-time crash if there is a call to str() that is indirect. All calls must be direct for performance reasons; you need to specialize the caller with @specializectx.""" raise NotImplementedError - @specializectx - def lowstr(ctx, index): - c = ctx.str(index) - return rsre_char.getlower(c, ctx.flags) + def lowstr(self, index): + """NOT_RPYTHON: Similar to str().""" + raise NotImplementedError def get_mark(self, gid): return find_mark(self.match_marks, gid) @@ -136,6 +145,10 @@ check_nonneg(index) return ord(self._string[index]) + def lowstr(self, index): + c = self.str(index) + return rsre_char.getlower(c, self.flags) + class UnicodeMatchContext(AbstractMatchContext): """Concrete subclass for matching in a unicode string.""" @@ -149,6 +162,10 @@ check_nonneg(index) return ord(self._unicodestr[index]) + def lowstr(self, index): + c = self.str(index) + return rsre_char.getlower(c, self.flags) + # ____________________________________________________________ class Mark(object): @@ -171,7 +188,6 @@ class MatchResult(object): subresult = None - @specializectx def move_to_next_result(self, ctx): result = self.subresult if result is None: @@ -192,8 +208,6 @@ self.start_ptr = ptr self.start_marks = marks - @specializectx - @jit.unroll_safe # there are only a few branch alternatives def find_first_result(self, ctx): ppos = self.ppos while ctx.pat(ppos): @@ -213,7 +227,6 @@ self.start_ptr = ptr self.start_marks = marks - @specializectx def find_first_result(self, ctx): ptr = self.start_ptr while ptr >= self.minptr: @@ -235,7 +248,6 @@ self.start_ptr = ptr self.start_marks = marks - @specializectx def find_first_result(self, ctx): ptr = self.start_ptr while ptr <= self.maxptr: @@ -244,20 +256,27 @@ self.subresult = result self.start_ptr = ptr return self - ptr1 = find_repetition_end(ctx, self.ppos3, ptr, 1) - if ptr1 == ptr: + if not self.next_char_ok(ctx, ptr): break - ptr = ptr1 + ptr += 1 - @specializectx def find_next_result(self, ctx): ptr = self.start_ptr - ptr1 = find_repetition_end(ctx, self.ppos3, ptr, 1) - if ptr1 == ptr: + if not self.next_char_ok(ctx, ptr): return - self.start_ptr = ptr1 + self.start_ptr = ptr + 1 return self.find_first_result(ctx) + def next_char_ok(self, ctx, ptr): + if ptr == ctx.end: + return False + ppos = self.ppos3 + op = ctx.pat(ppos) + for op1, (checkerfn, _) in unroll_char_checker: + if op1 == op: + return checkerfn(ctx, ptr, ppos) + raise NotImplementedError("next_char_ok[%d]" % op) + class AbstractUntilMatchResult(MatchResult): def __init__(self, ppos, tailppos, ptr, marks): @@ -277,16 +296,13 @@ class MaxUntilMatchResult(AbstractUntilMatchResult): - @specializectx def find_first_result(self, ctx): enum = sre_match(ctx, self.ppos + 3, self.cur_ptr, self.cur_marks) return self.search_next(ctx, enum, resume=False) - @specializectx def find_next_result(self, ctx): return self.search_next(ctx, None, resume=True) - @specializectx def search_next(self, ctx, enum, resume): ppos = self.ppos min = ctx.pat(ppos+1) @@ -330,15 +346,12 @@ class MinUntilMatchResult(AbstractUntilMatchResult): - @specializectx def find_first_result(self, ctx): return self.search_next(ctx, resume=False) - @specializectx def find_next_result(self, ctx): return self.search_next(ctx, resume=True) - @specializectx def search_next(self, ctx, resume): ppos = self.ppos min = ctx.pat(ppos+1) @@ -383,8 +396,6 @@ # ____________________________________________________________ @specializectx - at jit.unroll_safe # it's safe to unroll the main 'while' loop: - # 'ppos' is only ever incremented in this function def sre_match(ctx, ppos, ptr, marks): """Returns either None or a MatchResult object. Usually we only need the first result, but there is the case of REPEAT...UNTIL where we @@ -666,69 +677,57 @@ end1 = ptr + maxcount if end1 <= end: end = end1 - op = ctx.pat(ppos) - if op == OPCODE_ANY: return fre_ANY(ctx, ptr, end) - if op == OPCODE_ANY_ALL: return end - if op == OPCODE_IN: return fre_IN(ctx, ptr, end, ppos) - if op == OPCODE_IN_IGNORE: return fre_IN_IGNORE(ctx, ptr, end, ppos) - if op == OPCODE_LITERAL: return fre_LITERAL(ctx, ptr, end, ppos) - if op == OPCODE_LITERAL_IGNORE: return fre_LITERAL_IGNORE(ctx,ptr,end,ppos) - if op == OPCODE_NOT_LITERAL: return fre_NOT_LITERAL(ctx, ptr, end, ppos) - if op == OPCODE_NOT_LITERAL_IGNORE: return fre_NOT_LITERAL_IGNORE(ctx,ptr, - end,ppos) + for op1, (_, fre) in unroll_char_checker: + if op1 == op: + return fre(ctx, ptr, end, ppos) raise NotImplementedError("rsre.find_repetition_end[%d]" % op) @specializectx -def fre_ANY(ctx, ptr, end): - # repeated dot wildcard. - while ptr < end and not rsre_char.is_linebreak(ctx.str(ptr)): - ptr += 1 - return ptr - - at specializectx -def fre_IN(ctx, ptr, end, ppos): - # repeated set - while ptr < end and rsre_char.check_charset(ctx.pattern, ppos+2, - ctx.str(ptr)): - ptr += 1 - return ptr - - at specializectx -def fre_IN_IGNORE(ctx, ptr, end, ppos): - # repeated set - while ptr < end and rsre_char.check_charset(ctx.pattern, ppos+2, - ctx.lowstr(ptr)): - ptr += 1 - return ptr - - at specializectx -def fre_LITERAL(ctx, ptr, end, ppos): - chr = ctx.pat(ppos+1) - while ptr < end and ctx.str(ptr) == chr: - ptr += 1 - return ptr - - at specializectx -def fre_LITERAL_IGNORE(ctx, ptr, end, ppos): - chr = ctx.pat(ppos+1) - while ptr < end and ctx.lowstr(ptr) == chr: - ptr += 1 - return ptr - - at specializectx -def fre_NOT_LITERAL(ctx, ptr, end, ppos): - chr = ctx.pat(ppos+1) - while ptr < end and ctx.str(ptr) != chr: - ptr += 1 - return ptr - - at specializectx -def fre_NOT_LITERAL_IGNORE(ctx, ptr, end, ppos): - chr = ctx.pat(ppos+1) - while ptr < end and ctx.lowstr(ptr) != chr: - ptr += 1 - return ptr +def match_ANY(ctx, ptr, ppos): # dot wildcard. + return not rsre_char.is_linebreak(ctx.str(ptr)) +def match_ANY_ALL(ctx, ptr, ppos): + return True # match anything (including a newline) + at specializectx +def match_IN(ctx, ptr, ppos): + return rsre_char.check_charset(ctx.pattern, ppos+2, ctx.str(ptr)) + at specializectx +def match_IN_IGNORE(ctx, ptr, ppos): + return rsre_char.check_charset(ctx.pattern, ppos+2, ctx.lowstr(ptr)) + at specializectx +def match_LITERAL(ctx, ptr, ppos): + return ctx.str(ptr) == ctx.pat(ppos+1) + at specializectx +def match_LITERAL_IGNORE(ctx, ptr, ppos): + return ctx.lowstr(ptr) == ctx.pat(ppos+1) + at specializectx +def match_NOT_LITERAL(ctx, ptr, ppos): + return ctx.str(ptr) != ctx.pat(ppos+1) + at specializectx +def match_NOT_LITERAL_IGNORE(ctx, ptr, ppos): + return ctx.lowstr(ptr) != ctx.pat(ppos+1) + +def _make_fre(checkerfn): + if checkerfn == match_ANY_ALL: + def fre(ctx, ptr, end, ppos): + return end + else: + def fre(ctx, ptr, end, ppos): + while ptr < end and checkerfn(ctx, ptr, ppos): + ptr += 1 + return ptr + return checkerfn, fre + +unroll_char_checker = unrolling_iterable([ + (OPCODE_ANY, _make_fre(match_ANY)), + (OPCODE_ANY_ALL, _make_fre(match_ANY_ALL)), + (OPCODE_IN, _make_fre(match_IN)), + (OPCODE_IN_IGNORE, _make_fre(match_IN_IGNORE)), + (OPCODE_LITERAL, _make_fre(match_LITERAL)), + (OPCODE_LITERAL_IGNORE, _make_fre(match_LITERAL_IGNORE)), + (OPCODE_NOT_LITERAL, _make_fre(match_NOT_LITERAL)), + (OPCODE_NOT_LITERAL_IGNORE, _make_fre(match_NOT_LITERAL_IGNORE)), + ]) ##### At dispatch @@ -813,17 +812,12 @@ def match(pattern, string, start=0, end=sys.maxint, flags=0): ctx = StrMatchContext(pattern, string, start, end, flags) - if match_context(ctx) is not None: - return ctx - return None + return match_context(ctx) def search(pattern, string, start=0, end=sys.maxint, flags=0): ctx = StrMatchContext(pattern, string, start, end, flags) - if search_context(ctx) is not None: - return ctx - return None + return search_context(ctx) - at specializectx def match_context(ctx): if ctx.end < ctx.match_start: return None @@ -831,7 +825,6 @@ return ctx return None - at specializectx def search_context(ctx): if ctx.end < ctx.match_start: return None @@ -840,7 +833,6 @@ return fast_search(ctx) return regular_search(ctx) - at specializectx def regular_search(ctx): start = ctx.match_start while start <= ctx.end: @@ -864,7 +856,8 @@ overlap_offset = 7 + prefix_len - 1 assert overlap_offset >= 0 pattern_offset = ctx.pat(1) + 1 - assert pattern_offset >= 0 + ppos_start = pattern_offset + 2 * prefix_skip + assert ppos_start >= 0 i = 0 string_position = ctx.match_start end = ctx.end @@ -889,8 +882,7 @@ ctx.match_end = ptr ctx.match_marks = None return ctx - ppos = pattern_offset + 2 * prefix_skip - if sre_match(ctx, ppos, ptr, None) is not None: + if sre_match(ctx, ppos_start, ptr, None) is not None: ctx.match_start = start return ctx i = ctx.pat(overlap_offset + i) From cfbolz at codespeak.net Fri Aug 20 17:22:14 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 20 Aug 2010 17:22:14 +0200 (CEST) Subject: [pypy-svn] r76680 - pypy/branch/better-map-instances Message-ID: <20100820152214.34203282B90@codespeak.net> Author: cfbolz Date: Fri Aug 20 17:22:12 2010 New Revision: 76680 Added: pypy/branch/better-map-instances/ (props changed) - copied from r76679, pypy/trunk/ Log: a branch to play with a different approach to implement maps From arigo at codespeak.net Fri Aug 20 17:31:16 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 20 Aug 2010 17:31:16 +0200 (CEST) Subject: [pypy-svn] r76681 - pypy/branch/rsre2/pypy/module/_sre/test Message-ID: <20100820153116.C4FEE36C536@codespeak.net> Author: arigo Date: Fri Aug 20 17:31:15 2010 New Revision: 76681 Modified: pypy/branch/rsre2/pypy/module/_sre/test/test_app_sre.py Log: Add a passing test about a complete undocumented detail. Modified: pypy/branch/rsre2/pypy/module/_sre/test/test_app_sre.py ============================================================================== --- pypy/branch/rsre2/pypy/module/_sre/test/test_app_sre.py (original) +++ pypy/branch/rsre2/pypy/module/_sre/test/test_app_sre.py Fri Aug 20 17:31:15 2010 @@ -266,6 +266,18 @@ p.match().group(0), p.match().group(0)) assert None == p.match() + def test_scanner_match_detail(self): + import re + p = re.compile("a").scanner("aaXaa") + assert "a" == p.match().group(0) + assert "a" == p.match().group(0) + assert None == p.match() + assert "a" == p.match().group(0) + assert "a" == p.match().group(0) + assert None == p.match() + assert None == p.match() + assert None == p.match() + def test_scanner_search(self): import re p = re.compile("\d").scanner("bla23c5a") From arigo at codespeak.net Fri Aug 20 17:45:18 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 20 Aug 2010 17:45:18 +0200 (CEST) Subject: [pypy-svn] r76683 - pypy/branch/kill-caninline/pypy/module/pypyjit Message-ID: <20100820154518.70EDA282B90@codespeak.net> Author: arigo Date: Fri Aug 20 17:45:16 2010 New Revision: 76683 Modified: pypy/branch/kill-caninline/pypy/module/pypyjit/interp_jit.py pypy/branch/kill-caninline/pypy/module/pypyjit/policy.py Log: Disable JITting generators entirely for now. Modified: pypy/branch/kill-caninline/pypy/module/pypyjit/interp_jit.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/pypyjit/interp_jit.py (original) +++ pypy/branch/kill-caninline/pypy/module/pypyjit/interp_jit.py Fri Aug 20 17:45:16 2010 @@ -9,7 +9,7 @@ import pypy.interpreter.pyopcode # for side-effects from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import ObjSpace, Arguments -from pypy.interpreter.pycode import PyCode +from pypy.interpreter.pycode import PyCode, CO_GENERATOR from pypy.interpreter.pyframe import PyFrame from pypy.interpreter.pyopcode import ExitFrame from opcode import opmap @@ -36,7 +36,8 @@ bytecode.jit_cells[next_instr] = newcell def confirm_enter_jit(next_instr, bytecode, frame, ec): - return (frame.w_f_trace is None and + return (not (bytecode.co_flags & CO_GENERATOR) and + frame.w_f_trace is None and ec.profilefunc is None and ec.w_tracefunc is None) Modified: pypy/branch/kill-caninline/pypy/module/pypyjit/policy.py ============================================================================== --- pypy/branch/kill-caninline/pypy/module/pypyjit/policy.py (original) +++ pypy/branch/kill-caninline/pypy/module/pypyjit/policy.py Fri Aug 20 17:45:16 2010 @@ -32,6 +32,8 @@ return False if mod.startswith('pypy.interpreter.pyparser.'): return False + if mod == 'pypy.interpreter.generator': + return False if mod.startswith('pypy.module.'): modname = mod[len('pypy.module.'):] if not self.look_inside_pypy_module(modname): From arigo at codespeak.net Fri Aug 20 18:41:57 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 20 Aug 2010 18:41:57 +0200 (CEST) Subject: [pypy-svn] r76685 - in pypy/trunk/pypy: interpreter interpreter/astcompiler interpreter/test jit/backend/x86/test jit/codewriter jit/codewriter/test jit/metainterp jit/metainterp/test module/__builtin__ module/pypyjit module/pypyjit/test rlib rlib/test Message-ID: <20100820164157.605EA282B90@codespeak.net> Author: arigo Date: Fri Aug 20 18:41:54 2010 New Revision: 76685 Removed: pypy/trunk/pypy/module/pypyjit/test/test_can_inline.py Modified: pypy/trunk/pypy/interpreter/astcompiler/consts.py pypy/trunk/pypy/interpreter/pycode.py pypy/trunk/pypy/interpreter/test/test_code.py pypy/trunk/pypy/jit/backend/x86/test/test_ztranslation.py pypy/trunk/pypy/jit/codewriter/assembler.py pypy/trunk/pypy/jit/codewriter/jitcode.py pypy/trunk/pypy/jit/codewriter/jtransform.py pypy/trunk/pypy/jit/codewriter/test/test_flatten.py pypy/trunk/pypy/jit/metainterp/blackhole.py pypy/trunk/pypy/jit/metainterp/history.py pypy/trunk/pypy/jit/metainterp/jitdriver.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/test/test_basic.py pypy/trunk/pypy/jit/metainterp/test/test_jitdriver.py pypy/trunk/pypy/jit/metainterp/test/test_recursive.py pypy/trunk/pypy/jit/metainterp/test/test_virtualizable.py pypy/trunk/pypy/jit/metainterp/test/test_warmspot.py pypy/trunk/pypy/jit/metainterp/test/test_warmstate.py pypy/trunk/pypy/jit/metainterp/test/test_ztranslation.py pypy/trunk/pypy/jit/metainterp/warmspot.py pypy/trunk/pypy/jit/metainterp/warmstate.py pypy/trunk/pypy/module/__builtin__/functional.py pypy/trunk/pypy/module/pypyjit/interp_jit.py pypy/trunk/pypy/module/pypyjit/policy.py pypy/trunk/pypy/rlib/jit.py pypy/trunk/pypy/rlib/test/test_jit.py Log: Merge the branch/kill-caninline, whose purpose is to kill the "can_inline" hack in the JIT. This makes some benchmarks faster because a Python function can be inlined into another one even if it contains a loop, and inlining stops only when (and if) the inner loop is reached. Left open: there is still the code to sometimes start JIT tracing from the start of a Python function; this is now only useful in case a previous tracing gave up because the trace was too long. Should ideally be handled differently. Also, JITting Python generators is fully disabled for now. Modified: pypy/trunk/pypy/interpreter/astcompiler/consts.py ============================================================================== --- pypy/trunk/pypy/interpreter/astcompiler/consts.py (original) +++ pypy/trunk/pypy/interpreter/astcompiler/consts.py Fri Aug 20 18:41:54 2010 @@ -9,7 +9,6 @@ CO_NESTED = 0x0010 CO_GENERATOR = 0x0020 CO_NOFREE = 0x0040 -CO_CONTAINSLOOP = 0x0080 CO_CONTAINSGLOBALS = 0x0800 CO_GENERATOR_ALLOWED = 0x1000 CO_FUTURE_DIVISION = 0x2000 Modified: pypy/trunk/pypy/interpreter/pycode.py ============================================================================== --- pypy/trunk/pypy/interpreter/pycode.py (original) +++ pypy/trunk/pypy/interpreter/pycode.py Fri Aug 20 18:41:54 2010 @@ -13,7 +13,7 @@ from pypy.interpreter.baseobjspace import ObjSpace, W_Root from pypy.interpreter.astcompiler.consts import (CO_OPTIMIZED, CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED, - CO_GENERATOR, CO_CONTAINSLOOP, CO_CONTAINSGLOBALS) + CO_GENERATOR, CO_CONTAINSGLOBALS) from pypy.rlib.rarithmetic import intmask from pypy.rlib.debug import make_sure_not_resized, make_sure_not_modified from pypy.rlib import jit @@ -133,9 +133,7 @@ while opcode == opcodedesc.EXTENDED_ARG.index: opcode = ord(co_code[next_instr]) next_instr += 3 - if opcode == opcodedesc.JUMP_ABSOLUTE.index: - self.co_flags |= CO_CONTAINSLOOP - elif opcode == opcodedesc.LOAD_GLOBAL.index: + if opcode == opcodedesc.LOAD_GLOBAL.index: self.co_flags |= CO_CONTAINSGLOBALS elif opcode == opcodedesc.LOAD_NAME.index: self.co_flags |= CO_CONTAINSGLOBALS Modified: pypy/trunk/pypy/interpreter/test/test_code.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_code.py (original) +++ pypy/trunk/pypy/interpreter/test/test_code.py Fri Aug 20 18:41:54 2010 @@ -184,8 +184,6 @@ # CO_NESTED assert f(4).func_code.co_flags & 0x10 assert f.func_code.co_flags & 0x10 == 0 - # check for CO_CONTAINSLOOP - assert not f.func_code.co_flags & 0x0080 # check for CO_CONTAINSGLOBALS assert not f.func_code.co_flags & 0x0800 @@ -198,9 +196,6 @@ return [l for l in [1, 2, 3, 4]] """ - # check for CO_CONTAINSLOOP - assert f.func_code.co_flags & 0x0080 - assert g.func_code.co_flags & 0x0080 # check for CO_CONTAINSGLOBALS assert f.func_code.co_flags & 0x0800 assert not g.func_code.co_flags & 0x0800 Modified: pypy/trunk/pypy/jit/backend/x86/test/test_ztranslation.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_ztranslation.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_ztranslation.py Fri Aug 20 18:41:54 2010 @@ -75,8 +75,7 @@ driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'], virtualizables = ['frame'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno: str(codeno)) class SomewhereElse(object): pass Modified: pypy/trunk/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/codewriter/assembler.py (original) +++ pypy/trunk/pypy/jit/codewriter/assembler.py Fri Aug 20 18:41:54 2010 @@ -53,6 +53,7 @@ self.liveness = {} self.startpoints = set() self.alllabels = set() + self.resulttypes = {} def emit_reg(self, reg): if reg.index >= self.count_regs[reg.kind]: @@ -165,7 +166,9 @@ raise NotImplementedError(x) # opname = insn[0] - assert '>' not in argcodes or argcodes.index('>') == len(argcodes) - 2 + if '>' in argcodes: + assert argcodes.index('>') == len(argcodes) - 2 + self.resulttypes[len(self.code)] = argcodes[-1] key = opname + '/' + ''.join(argcodes) num = self.insns.setdefault(key, len(self.insns)) self.code[startposition] = chr(num) @@ -212,7 +215,8 @@ self.count_regs['float'], liveness=self.liveness, startpoints=self.startpoints, - alllabels=self.alllabels) + alllabels=self.alllabels, + resulttypes=self.resulttypes) def see_raw_object(self, value): if value._obj not in self._seen_raw_objects: Modified: pypy/trunk/pypy/jit/codewriter/jitcode.py ============================================================================== --- pypy/trunk/pypy/jit/codewriter/jitcode.py (original) +++ pypy/trunk/pypy/jit/codewriter/jitcode.py Fri Aug 20 18:41:54 2010 @@ -19,7 +19,8 @@ def setup(self, code='', constants_i=[], constants_r=[], constants_f=[], num_regs_i=255, num_regs_r=255, num_regs_f=255, - liveness=None, startpoints=None, alllabels=None): + liveness=None, startpoints=None, alllabels=None, + resulttypes=None): self.code = code # if the following lists are empty, use a single shared empty list self.constants_i = constants_i or self._empty_i @@ -33,6 +34,7 @@ self.liveness = make_liveness_cache(liveness) self._startpoints = startpoints # debugging self._alllabels = alllabels # debugging + self._resulttypes = resulttypes # debugging def get_fnaddr_as_int(self): return heaptracker.adr2int(self.fnaddr) Modified: pypy/trunk/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/trunk/pypy/jit/codewriter/jtransform.py (original) +++ pypy/trunk/pypy/jit/codewriter/jtransform.py Fri Aug 20 18:41:54 2010 @@ -829,13 +829,20 @@ self.make_three_lists(op.args[2:2+num_green_args]) + self.make_three_lists(op.args[2+num_green_args:])) op1 = SpaceOperation('jit_merge_point', args, None) - return ops + [op1] + op2 = SpaceOperation('-live-', [], None) + # ^^^ we need a -live- for the case of do_recursive_call() + return ops + [op1, op2] - def handle_jit_marker__can_enter_jit(self, op, jitdriver): + def handle_jit_marker__loop_header(self, op, jitdriver): jd = self.callcontrol.jitdriver_sd_from_jitdriver(jitdriver) assert jd is not None c_index = Constant(jd.index, lltype.Signed) - return SpaceOperation('can_enter_jit', [c_index], None) + return SpaceOperation('loop_header', [c_index], None) + + # a 'can_enter_jit' in the source graph becomes a 'loop_header' + # operation in the transformed graph, as its only purpose in + # the transformed graph is to detect loops. + handle_jit_marker__can_enter_jit = handle_jit_marker__loop_header def rewrite_op_debug_assert(self, op): log.WARNING("found debug_assert in %r; should have be removed" % Modified: pypy/trunk/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/trunk/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/trunk/pypy/jit/codewriter/test/test_flatten.py Fri Aug 20 18:41:54 2010 @@ -593,7 +593,8 @@ -live- %i0, %i1 int_guard_value %i0 jit_merge_point $27, I[%i0], R[], F[], I[%i1], R[], F[] - can_enter_jit $27 + -live- + loop_header $27 void_return """, transform=True, liveness=True, cc=MyFakeCallControl(), jd=jd) Modified: pypy/trunk/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/blackhole.py (original) +++ pypy/trunk/pypy/jit/metainterp/blackhole.py Fri Aug 20 18:41:54 2010 @@ -2,7 +2,7 @@ from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.debug import debug_start, debug_stop -from pypy.rlib.debug import make_sure_not_resized +from pypy.rlib.debug import make_sure_not_resized, fatalerror from pypy.rpython.lltypesystem import lltype, llmemory, rclass from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.llinterp import LLException @@ -756,11 +756,15 @@ assert e reraise(e) + @arguments("r") + def bhimpl_debug_fatalerror(msg): + llop.debug_fatalerror(lltype.Void, msg) + # ---------- # the main hints and recursive calls @arguments("i") - def bhimpl_can_enter_jit(jdindex): + def bhimpl_loop_header(jdindex): pass @arguments("self", "i", "I", "R", "F", "I", "R", "F") @@ -1164,7 +1168,7 @@ # we now proceed to interpret the bytecode in this frame self.run() # - except JitException: + except JitException, e: raise # go through except Exception, e: # if we get an exception, return it to the caller frame @@ -1266,6 +1270,33 @@ e = lltype.cast_opaque_ptr(llmemory.GCREF, e) raise sd.ExitFrameWithExceptionRef(self.cpu, e) + def _handle_jitexception_in_portal(self, e): + # This case is really rare, but can occur if + # convert_and_run_from_pyjitpl() gets called in this situation: + # + # [function 1] <---- top BlackholeInterpreter() + # [recursive portal jit code] + # ... + # [bottom portal jit code] <---- bottom BlackholeInterpreter() + # + # and then "function 1" contains a call to "function 2", which + # calls "can_enter_jit". The latter can terminate by raising a + # JitException. In that case, the JitException is not supposed + # to fall through the whole chain of BlackholeInterpreters, but + # be caught and handled just below the level "recursive portal + # jit code". The present function is called to handle the case + # of recursive portal jit codes. + for jd in self.builder.metainterp_sd.jitdrivers_sd: + if jd.mainjitcode is self.jitcode: + break + else: + assert 0, "portal jitcode not found??" + # call the helper in warmspot.py. It might either raise a + # regular exception (which should then be propagated outside + # of 'self', not caught inside), or return (the return value + # gets stored in nextblackholeinterp). + jd.handle_jitexc_from_bh(self.nextblackholeinterp, e) + def _copy_data_from_miframe(self, miframe): self.setposition(miframe.jitcode, miframe.pc) for i in range(self.jitcode.num_regs_i()): @@ -1287,9 +1318,31 @@ while True: try: current_exc = blackholeinterp._resume_mainloop(current_exc) - finally: - blackholeinterp.builder.release_interp(blackholeinterp) + except JitException, e: + blackholeinterp, current_exc = _handle_jitexception( + blackholeinterp, e) + blackholeinterp.builder.release_interp(blackholeinterp) + blackholeinterp = blackholeinterp.nextblackholeinterp + +def _handle_jitexception(blackholeinterp, jitexc): + # See comments in _handle_jitexception_in_portal(). + while not blackholeinterp.jitcode.is_portal: + blackholeinterp.builder.release_interp(blackholeinterp) blackholeinterp = blackholeinterp.nextblackholeinterp + if blackholeinterp.nextblackholeinterp is None: + blackholeinterp.builder.release_interp(blackholeinterp) + raise jitexc # bottommost entry: go through + # We have reached a recursive portal level. + try: + blackholeinterp._handle_jitexception_in_portal(jitexc) + except Exception, e: + # It raised a general exception (it should not be a JitException here). + lle = get_llexception(blackholeinterp.cpu, e) + else: + # It set up the nextblackholeinterp to contain the return value. + lle = lltype.nullptr(rclass.OBJECTPTR.TO) + # We will continue to loop in _run_forever() from the parent level. + return blackholeinterp, lle def resume_in_blackhole(metainterp_sd, jitdriver_sd, resumedescr, all_virtuals=None): Modified: pypy/trunk/pypy/jit/metainterp/history.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/history.py (original) +++ pypy/trunk/pypy/jit/metainterp/history.py Fri Aug 20 18:41:54 2010 @@ -919,11 +919,12 @@ "found %d %r, expected %d" % (found, insn, expected_count)) return insns - def check_loops(self, expected=None, **check): + def check_loops(self, expected=None, everywhere=False, **check): insns = {} for loop in self.loops: - if getattr(loop, '_ignore_during_counting', False): - continue + if not everywhere: + if getattr(loop, '_ignore_during_counting', False): + continue insns = loop.summary(adding_insns=insns) if expected is not None: insns.pop('debug_merge_point', None) Modified: pypy/trunk/pypy/jit/metainterp/jitdriver.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/jitdriver.py (original) +++ pypy/trunk/pypy/jit/metainterp/jitdriver.py Fri Aug 20 18:41:54 2010 @@ -12,6 +12,7 @@ # self.result_type ... pypy.jit.metainterp.warmspot # self.virtualizable_info... pypy.jit.metainterp.warmspot # self.warmstate ... pypy.jit.metainterp.warmspot + # self.handle_jitexc_from_bh pypy.jit.metainterp.warmspot # self.index ... pypy.jit.codewriter.call # self.mainjitcode ... pypy.jit.codewriter.call Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Fri Aug 20 18:41:54 2010 @@ -149,16 +149,20 @@ assert oldbox not in registers[count:] def make_result_of_lastop(self, resultbox): - if resultbox is None: - return + got_type = resultbox.type + if not we_are_translated(): + typeof = {'i': history.INT, + 'r': history.REF, + 'f': history.FLOAT} + assert typeof[self.jitcode._resulttypes[self.pc]] == got_type target_index = ord(self.bytecode[self.pc-1]) - if resultbox.type == history.INT: + if got_type == history.INT: self.registers_i[target_index] = resultbox - elif resultbox.type == history.REF: + elif got_type == history.REF: #debug_print(' ->', # llmemory.cast_ptr_to_adr(resultbox.getref_base())) self.registers_r[target_index] = resultbox - elif resultbox.type == history.FLOAT: + elif got_type == history.FLOAT: self.registers_f[target_index] = resultbox else: raise AssertionError("bad result box type") @@ -685,11 +689,11 @@ def _opimpl_recursive_call(self, jdindex, greenboxes, redboxes): targetjitdriver_sd = self.metainterp.staticdata.jitdrivers_sd[jdindex] allboxes = greenboxes + redboxes - portal_code = targetjitdriver_sd.mainjitcode warmrunnerstate = targetjitdriver_sd.warmstate token = None if warmrunnerstate.inlining: if warmrunnerstate.can_inline_callable(greenboxes): + portal_code = targetjitdriver_sd.mainjitcode return self.metainterp.perform_call(portal_code, allboxes, greenkey=greenboxes) token = warmrunnerstate.get_assembler_token(greenboxes) @@ -697,6 +701,10 @@ # that assembler that we call is still correct self.verify_green_args(targetjitdriver_sd, greenboxes) # + return self.do_recursive_call(targetjitdriver_sd, allboxes, token) + + def do_recursive_call(self, targetjitdriver_sd, allboxes, token=None): + portal_code = targetjitdriver_sd.mainjitcode k = targetjitdriver_sd.portal_runner_adr funcbox = ConstInt(heaptracker.adr2int(k)) return self.do_residual_call(funcbox, portal_code.calldescr, @@ -786,13 +794,8 @@ return clsbox @arguments("int") - def opimpl_can_enter_jit(self, jdindex): - if self.metainterp.in_recursion: - from pypy.jit.metainterp.warmspot import CannotInlineCanEnterJit - raise CannotInlineCanEnterJit() - assert jdindex == self.metainterp.jitdriver_sd.index, ( - "found a can_enter_jit that does not match the current jitdriver") - self.metainterp.seen_can_enter_jit = True + def opimpl_loop_header(self, jdindex): + self.metainterp.seen_loop_header_for_jdindex = jdindex def verify_green_args(self, jitdriver_sd, varargs): num_green_args = jitdriver_sd.num_green_args @@ -806,22 +809,42 @@ self.verify_green_args(jitdriver_sd, greenboxes) # xxx we may disable the following line in some context later self.debug_merge_point(jitdriver_sd, greenboxes) - if self.metainterp.seen_can_enter_jit: - self.metainterp.seen_can_enter_jit = False - # Assert that it's impossible to arrive here with in_recursion - # set to a non-zero value: seen_can_enter_jit can only be set - # to True by opimpl_can_enter_jit, which should be executed - # just before opimpl_jit_merge_point (no recursion inbetween). - assert not self.metainterp.in_recursion + if self.metainterp.seen_loop_header_for_jdindex < 0: + return + # + assert self.metainterp.seen_loop_header_for_jdindex == jdindex, ( + "found a loop_header for a JitDriver that does not match " + "the following jit_merge_point's") + self.metainterp.seen_loop_header_for_jdindex = -1 + # + if not self.metainterp.in_recursion: assert jitdriver_sd is self.metainterp.jitdriver_sd # Set self.pc to point to jit_merge_point instead of just after: - # if reached_can_enter_jit() raises SwitchToBlackhole, then the + # if reached_loop_header() raises SwitchToBlackhole, then the # pc is still at the jit_merge_point, which is a point that is # much less expensive to blackhole out of. saved_pc = self.pc self.pc = orgpc - self.metainterp.reached_can_enter_jit(greenboxes, redboxes) + self.metainterp.reached_loop_header(greenboxes, redboxes) self.pc = saved_pc + else: + warmrunnerstate = jitdriver_sd.warmstate + token = warmrunnerstate.get_assembler_token(greenboxes) + # warning! careful here. We have to return from the current + # frame containing the jit_merge_point, and then use + # do_recursive_call() to follow the recursive call. This is + # needed because do_recursive_call() will write its result + # with make_result_of_lastop(), so the lastop must be right: + # it must be the call to 'self', and not the jit_merge_point + # itself, which has no result at all. + assert len(self.metainterp.framestack) >= 2 + try: + self.metainterp.finishframe(None) + except ChangeFrame: + pass + frame = self.metainterp.framestack[-1] + frame.do_recursive_call(jitdriver_sd, greenboxes + redboxes, token) + raise ChangeFrame def debug_merge_point(self, jitdriver_sd, greenkey): # debugging: produce a DEBUG_MERGE_POINT operation @@ -872,6 +895,12 @@ return exc_value_box @arguments("box") + def opimpl_debug_fatalerror(self, box): + from pypy.rpython.lltypesystem import rstr, lloperation + msg = box.getref(lltype.Ptr(rstr.STR)) + lloperation.llop.debug_fatalerror(msg) + + @arguments("box") def opimpl_virtual_ref(self, box): # Details on the content of metainterp.virtualref_boxes: # @@ -1018,9 +1047,10 @@ self.metainterp.clear_exception() resbox = self.metainterp.execute_and_record_varargs(opnum, argboxes, descr=descr) - self.make_result_of_lastop(resbox) - # ^^^ this is done before handle_possible_exception() because we need - # the box to show up in get_list_of_active_boxes() + if resbox is not None: + self.make_result_of_lastop(resbox) + # ^^^ this is done before handle_possible_exception() because we + # need the box to show up in get_list_of_active_boxes() if exc: self.metainterp.handle_possible_exception() else: @@ -1323,7 +1353,8 @@ self.last_exc_value_box = None self.popframe() if self.framestack: - self.framestack[-1].make_result_of_lastop(resultbox) + if resultbox is not None: + self.framestack[-1].make_result_of_lastop(resultbox) raise ChangeFrame else: try: @@ -1552,7 +1583,7 @@ redkey = original_boxes[num_green_args:] self.resumekey = compile.ResumeFromInterpDescr(original_greenkey, redkey) - self.seen_can_enter_jit = False + self.seen_loop_header_for_jdindex = -1 try: self.interpret() except GenerateMergePoint, gmp: @@ -1579,7 +1610,7 @@ # because we cannot reconstruct the beginning of the proper loop self.current_merge_points = [(original_greenkey, -1)] self.resumekey = key - self.seen_can_enter_jit = False + self.seen_loop_header_for_jdindex = -1 try: self.prepare_resume_from_failure(key.guard_opnum) self.interpret() @@ -1609,7 +1640,7 @@ else: duplicates[box] = None - def reached_can_enter_jit(self, greenboxes, redboxes): + def reached_loop_header(self, greenboxes, redboxes): duplicates = {} self.remove_consts_and_duplicates(redboxes, len(redboxes), duplicates) @@ -1623,7 +1654,7 @@ live_arg_boxes += self.virtualizable_boxes live_arg_boxes.pop() assert len(self.virtualref_boxes) == 0, "missing virtual_ref_finish()?" - # Called whenever we reach the 'can_enter_jit' hint. + # Called whenever we reach the 'loop_header' hint. # First, attempt to make a bridge: # - if self.resumekey is a ResumeGuardDescr, it starts from a guard # that failed; @@ -2232,7 +2263,10 @@ else: resultbox = unboundmethod(self, *args) # - self.make_result_of_lastop(resultbox) + if resultbox is not None: + self.make_result_of_lastop(resultbox) + elif not we_are_translated(): + assert self._result_argcode in 'v?' # unboundmethod = getattr(MIFrame, 'opimpl_' + name).im_func argtypes = unrolling_iterable(unboundmethod.argtypes) Modified: pypy/trunk/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_basic.py Fri Aug 20 18:41:54 2010 @@ -116,8 +116,9 @@ class JitMixin: basic = True - def check_loops(self, expected=None, **check): - get_stats().check_loops(expected=expected, **check) + def check_loops(self, expected=None, everywhere=False, **check): + get_stats().check_loops(expected=expected, everywhere=everywhere, + **check) def check_loop_count(self, count): """NB. This is a hack; use check_tree_loop_count() or check_enter_count() for the real thing. Modified: pypy/trunk/pypy/jit/metainterp/test/test_jitdriver.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_jitdriver.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_jitdriver.py Fri Aug 20 18:41:54 2010 @@ -14,7 +14,6 @@ def test_simple(self): myjitdriver1 = JitDriver(greens=[], reds=['n', 'm'], - can_inline = lambda *args: False, get_printable_location = getloc1) myjitdriver2 = JitDriver(greens=['g'], reds=['r'], get_printable_location = getloc2) @@ -30,11 +29,14 @@ while r > 0: myjitdriver2.can_enter_jit(g=g, r=r) myjitdriver2.jit_merge_point(g=g, r=r) - r += loop1(r, g) - 1 + r += loop1(r, g) + (-1) return r # res = self.meta_interp(loop2, [4, 40], repeat=7, inline=True) assert res == loop2(4, 40) + # we expect only one int_sub, corresponding to the single + # compiled instance of loop1() + self.check_loops(int_sub=1) # the following numbers are not really expectations of the test # itself, but just the numbers that we got after looking carefully # at the generated machine code @@ -42,11 +44,10 @@ self.check_tree_loop_count(4) # 2 x loop, 2 x enter bridge self.check_enter_count(7) - def test_simple_inline(self): + def test_inline(self): # this is not an example of reasonable code: loop1() is unrolled # 'n/m' times, where n and m are given as red arguments. myjitdriver1 = JitDriver(greens=[], reds=['n', 'm'], - can_inline = lambda *args: True, get_printable_location = getloc1) myjitdriver2 = JitDriver(greens=['g'], reds=['r'], get_printable_location = getloc2) Modified: pypy/trunk/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_recursive.py Fri Aug 20 18:41:54 2010 @@ -2,10 +2,11 @@ from pypy.rlib.jit import JitDriver, we_are_jitted, OPTIMIZER_SIMPLE, hint from pypy.rlib.jit import unroll_safe, dont_look_inside from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.debug import fatalerror from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin from pypy.jit.codewriter.policy import StopAtXPolicy from pypy.rpython.annlowlevel import hlstr -from pypy.jit.metainterp.warmspot import CannotInlineCanEnterJit, get_stats +from pypy.jit.metainterp.warmspot import get_stats class RecursiveTests: @@ -98,23 +99,18 @@ policy=StopAtXPolicy(opaque)) assert res == 1 - def get_interpreter(self, codes, always_inline=False): + def get_interpreter(self, codes): ADD = "0" JUMP_BACK = "1" CALL = "2" EXIT = "3" - if always_inline: - def can_inline(*args): - return True - else: - def can_inline(i, code): - code = hlstr(code) - return not JUMP_BACK in code + def getloc(i, code): + return 'code="%s", i=%d' % (code, i) jitdriver = JitDriver(greens = ['i', 'code'], reds = ['n'], - can_inline = can_inline) - + get_printable_location = getloc) + def interpret(codenum, n, i): code = codes[codenum] while i < len(code): @@ -162,31 +158,16 @@ assert self.meta_interp(f, [0, 0, 0], optimizer=OPTIMIZER_SIMPLE, inline=True) == 42 - self.check_loops(call_may_force = 1, call = 0) - - def test_inline_faulty_can_inline(self): - code = "021" - subcode = "301" - codes = [code, subcode] - - f = self.get_interpreter(codes, always_inline=True) - - try: - self.meta_interp(f, [0, 0, 0], optimizer=OPTIMIZER_SIMPLE, - inline=True) - except CannotInlineCanEnterJit: - pass - else: - py.test.fail("DID NOT RAISE") + # the call is fully inlined, because we jump to subcode[1], thus + # skipping completely the JUMP_BACK in subcode[0] + self.check_loops(call_may_force = 0, call_assembler = 0, call = 0) def test_guard_failure_in_inlined_function(self): def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(pc, code): - return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], - get_printable_location=p, can_inline=c) + get_printable_location=p) def f(code, n): pc = 0 while pc < len(code): @@ -219,10 +200,8 @@ def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(pc, code): - return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n', 'flag'], - get_printable_location=p, can_inline=c) + get_printable_location=p) def f(code, n): pc = 0 flag = False @@ -262,10 +241,8 @@ def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(pc, code): - return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], - get_printable_location=p, can_inline=c) + get_printable_location=p) class Exc(Exception): pass @@ -307,10 +284,8 @@ def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(pc, code): - return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], - get_printable_location=p, can_inline=c) + get_printable_location=p) def f(code, n): pc = 0 @@ -524,10 +499,8 @@ def p(pc, code): code = hlstr(code) return "'%s' at %d: %s" % (code, pc, code[pc]) - def c(pc, code): - return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], - get_printable_location=p, can_inline=c) + get_printable_location=p) def f(code, n): pc = 0 @@ -558,6 +531,8 @@ result += f('+-cl--', i) g(50) self.meta_interp(g, [50], backendopt=True) + py.test.skip("tracing from start is by now only longer enabled " + "if a trace gets too big") self.check_tree_loop_count(3) self.check_history(int_add=1) @@ -565,10 +540,8 @@ def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(pc, code): - return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], - get_printable_location=p, can_inline=c) + get_printable_location=p) def f(code, n): pc = 0 @@ -607,8 +580,7 @@ def test_directly_call_assembler(self): driver = JitDriver(greens = ['codeno'], reds = ['i'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) def portal(codeno): i = 0 @@ -624,28 +596,29 @@ def test_recursion_cant_call_assembler_directly(self): driver = JitDriver(greens = ['codeno'], reds = ['i', 'j'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) def portal(codeno, j): i = 0 - while i < 1: - driver.can_enter_jit(codeno=codeno, i=i, j=j) + while 1: driver.jit_merge_point(codeno=codeno, i=i, j=j) - i += 1 - if j == 0: + if i == 1: + if j == 0: + return + portal(2, j - 1) + elif i == 3: return - portal(2, j - 1) + i += 1 + driver.can_enter_jit(codeno=codeno, i=i, j=j) portal(2, 50) self.meta_interp(portal, [2, 20], inline=True) - self.check_history(call_assembler=0, call_may_force=1) - self.check_enter_count_at_most(1) + self.check_loops(call_assembler=0, call_may_force=1, + everywhere=True) def test_directly_call_assembler_return(self): driver = JitDriver(greens = ['codeno'], reds = ['i', 'k'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) def portal(codeno): i = 0 @@ -668,8 +641,7 @@ self.x = x driver = JitDriver(greens = ['codeno'], reds = ['i'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) def portal(codeno): i = 0 @@ -690,8 +662,7 @@ def test_directly_call_assembler_fail_guard(self): driver = JitDriver(greens = ['codeno'], reds = ['i', 'k'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) def portal(codeno, k): i = 0 @@ -722,8 +693,7 @@ driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'], virtualizables = ['frame'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) def main(codeno): frame = Frame() @@ -761,8 +731,7 @@ driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'], virtualizables = ['frame'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) @dont_look_inside def check_frame(subframe): @@ -802,7 +771,7 @@ res = self.meta_interp(main, [0], inline=True) assert res == main(0) - def test_directly_call_assembler_virtualizable_force(self): + def test_directly_call_assembler_virtualizable_force1(self): class Thing(object): def __init__(self, val): self.val = val @@ -812,8 +781,7 @@ driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'], virtualizables = ['frame'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) class SomewhereElse(object): pass @@ -830,6 +798,7 @@ return frame.thing.val def portal(codeno, frame): + print 'ENTER:', codeno, frame.thing.val i = 0 while i < 10: driver.can_enter_jit(frame=frame, codeno=codeno, i=i) @@ -839,11 +808,15 @@ subframe = Frame() subframe.thing = Thing(nextval) nextval = portal(1, subframe) - elif frame.thing.val > 40: - change(Thing(13)) - nextval = 13 + elif codeno == 1: + if frame.thing.val > 40: + change(Thing(13)) + nextval = 13 + else: + fatalerror("bad codeno = " + str(codeno)) frame.thing = Thing(nextval + 1) i += 1 + print 'LEAVE:', codeno, frame.thing.val return frame.thing.val res = self.meta_interp(main, [0], inline=True, @@ -852,8 +825,7 @@ def test_directly_call_assembler_virtualizable_with_array(self): myjitdriver = JitDriver(greens = ['codeno'], reds = ['n', 'x', 'frame'], - virtualizables = ['frame'], - can_inline = lambda codeno : False) + virtualizables = ['frame']) class Frame(object): _virtualizable2_ = ['l[*]', 's'] @@ -899,8 +871,7 @@ driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'], virtualizables = ['frame'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) class SomewhereElse(object): pass @@ -942,17 +913,16 @@ def test_assembler_call_red_args(self): driver = JitDriver(greens = ['codeno'], reds = ['i', 'k'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) def residual(k): - if k > 40: + if k > 150: return 0 return 1 def portal(codeno, k): i = 0 - while i < 10: + while i < 15: driver.can_enter_jit(codeno=codeno, i=i, k=k) driver.jit_merge_point(codeno=codeno, i=i, k=k) if codeno == 2: @@ -969,10 +939,130 @@ assert res == portal(2, 0) self.check_loops(call_assembler=2) - # There is a test which I fail to write. - # * what happens if we call recursive_call while blackholing - # this seems to be completely corner case and not really happening - # in the wild + def test_inline_without_hitting_the_loop(self): + driver = JitDriver(greens = ['codeno'], reds = ['i'], + get_printable_location = lambda codeno : str(codeno)) + + def portal(codeno): + i = 0 + while True: + driver.jit_merge_point(codeno=codeno, i=i) + if codeno < 10: + i += portal(20) + codeno += 1 + elif codeno == 10: + if i > 63: + return i + codeno = 0 + driver.can_enter_jit(codeno=codeno, i=i) + else: + return 1 + + assert portal(0) == 70 + res = self.meta_interp(portal, [0], inline=True) + assert res == 70 + self.check_loops(call_assembler=0) + + def test_inline_with_hitting_the_loop_sometimes(self): + driver = JitDriver(greens = ['codeno'], reds = ['i', 'k'], + get_printable_location = lambda codeno : str(codeno)) + + def portal(codeno, k): + if k > 2: + return 1 + i = 0 + while True: + driver.jit_merge_point(codeno=codeno, i=i, k=k) + if codeno < 10: + i += portal(codeno + 5, k+1) + codeno += 1 + elif codeno == 10: + if i > [-1, 2000, 63][k]: + return i + codeno = 0 + driver.can_enter_jit(codeno=codeno, i=i, k=k) + else: + return 1 + + assert portal(0, 1) == 2095 + res = self.meta_interp(portal, [0, 1], inline=True) + assert res == 2095 + self.check_loops(call_assembler=6, everywhere=True) + + def test_inline_with_hitting_the_loop_sometimes_exc(self): + driver = JitDriver(greens = ['codeno'], reds = ['i', 'k'], + get_printable_location = lambda codeno : str(codeno)) + class GotValue(Exception): + def __init__(self, result): + self.result = result + + def portal(codeno, k): + if k > 2: + raise GotValue(1) + i = 0 + while True: + driver.jit_merge_point(codeno=codeno, i=i, k=k) + if codeno < 10: + try: + portal(codeno + 5, k+1) + except GotValue, e: + i += e.result + codeno += 1 + elif codeno == 10: + if i > [-1, 2000, 63][k]: + raise GotValue(i) + codeno = 0 + driver.can_enter_jit(codeno=codeno, i=i, k=k) + else: + raise GotValue(1) + + def main(codeno, k): + try: + portal(codeno, k) + except GotValue, e: + return e.result + + assert main(0, 1) == 2095 + res = self.meta_interp(main, [0, 1], inline=True) + assert res == 2095 + self.check_loops(call_assembler=6, everywhere=True) + + def test_handle_jitexception_in_portal(self): + # a test for _handle_jitexception_in_portal in blackhole.py + driver = JitDriver(greens = ['codeno'], reds = ['i', 'str'], + get_printable_location = lambda codeno: str(codeno)) + def do_can_enter_jit(codeno, i, str): + i = (i+1)-1 # some operations + driver.can_enter_jit(codeno=codeno, i=i, str=str) + def intermediate(codeno, i, str): + if i == 9: + do_can_enter_jit(codeno, i, str) + def portal(codeno, str): + i = value.initial + while i < 10: + intermediate(codeno, i, str) + driver.jit_merge_point(codeno=codeno, i=i, str=str) + i += 1 + if codeno == 64 and i == 10: + str = portal(96, str) + str += chr(codeno+i) + return str + class Value: + initial = -1 + value = Value() + def main(): + value.initial = 0 + return (portal(64, '') + + portal(64, '') + + portal(64, '') + + portal(64, '') + + portal(64, '')) + assert main() == 'ABCDEFGHIabcdefghijJ' * 5 + for tlimit in [95, 90, 102]: + print 'tlimit =', tlimit + res = self.meta_interp(main, [], inline=True, trace_limit=tlimit) + assert ''.join(res.chars) == 'ABCDEFGHIabcdefghijJ' * 5 + class TestLLtype(RecursiveTests, LLJitMixin): pass Modified: pypy/trunk/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_virtualizable.py Fri Aug 20 18:41:54 2010 @@ -1330,11 +1330,9 @@ def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(pc, code): - return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['frame'], virtualizables=["frame"], - get_printable_location=p, can_inline=c) + get_printable_location=p) def f(code, frame): pc = 0 while pc < len(code): Modified: pypy/trunk/pypy/jit/metainterp/test/test_warmspot.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_warmspot.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_warmspot.py Fri Aug 20 18:41:54 2010 @@ -272,6 +272,30 @@ self.check_enter_count_at_most(2) self.check_loops(call=0) + def test_loop_header(self): + # artificial test: we enter into the JIT only when can_enter_jit() + # is seen, but we close a loop in the JIT much more quickly + # because of loop_header(). + mydriver = JitDriver(reds = ['n', 'm'], greens = []) + + def f(m): + n = 0 + while True: + mydriver.jit_merge_point(n=n, m=m) + if n > m: + m -= 1 + if m < 0: + return n + n = 0 + mydriver.can_enter_jit(n=n, m=m) + else: + n += 1 + mydriver.loop_header() + assert f(15) == 1 + res = self.meta_interp(f, [15], backendopt=True) + assert res == 1 + self.check_loops(int_add=1) # I get 13 without the loop_header() + class TestLLWarmspot(WarmspotTests, LLJitMixin): CPUClass = runner.LLtypeCPU Modified: pypy/trunk/pypy/jit/metainterp/test/test_warmstate.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_warmstate.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_warmstate.py Fri Aug 20 18:41:54 2010 @@ -61,12 +61,14 @@ _green_args_spec = [lltype.Signed, lltype.Float] state = WarmEnterState(None, FakeJitDriverSD()) get_jitcell = state._make_jitcell_getter_default() - cell1 = get_jitcell(42, 42.5) + cell1 = get_jitcell(True, 42, 42.5) assert isinstance(cell1, JitCell) - cell2 = get_jitcell(42, 42.5) + cell2 = get_jitcell(True, 42, 42.5) assert cell1 is cell2 - cell3 = get_jitcell(41, 42.5) - cell4 = get_jitcell(42, 0.25) + cell3 = get_jitcell(True, 41, 42.5) + assert get_jitcell(False, 42, 0.25) is None + cell4 = get_jitcell(True, 42, 0.25) + assert get_jitcell(False, 42, 0.25) is cell4 assert cell1 is not cell3 is not cell4 is not cell1 def test_make_jitcell_getter(): @@ -75,8 +77,8 @@ _get_jitcell_at_ptr = None state = WarmEnterState(None, FakeJitDriverSD()) get_jitcell = state.make_jitcell_getter() - cell1 = get_jitcell(1.75) - cell2 = get_jitcell(1.75) + cell1 = get_jitcell(True, 1.75) + cell2 = get_jitcell(True, 1.75) assert cell1 is cell2 assert get_jitcell is state.make_jitcell_getter() @@ -103,14 +105,16 @@ # state = WarmEnterState(FakeWarmRunnerDesc(), FakeJitDriverSD()) get_jitcell = state._make_jitcell_getter_custom() - cell1 = get_jitcell(5, 42.5) + cell1 = get_jitcell(True, 5, 42.5) assert isinstance(cell1, JitCell) assert cell1.x == 5 assert cell1.y == 42.5 - cell2 = get_jitcell(5, 42.5) + cell2 = get_jitcell(True, 5, 42.5) assert cell2 is cell1 - cell3 = get_jitcell(41, 42.5) - cell4 = get_jitcell(42, 0.25) + cell3 = get_jitcell(True, 41, 42.5) + assert get_jitcell(False, 42, 0.25) is None + cell4 = get_jitcell(True, 42, 0.25) + assert get_jitcell(False, 42, 0.25) is cell4 assert cell1 is not cell3 is not cell4 is not cell1 def test_make_set_future_values(): @@ -153,52 +157,25 @@ state.attach_unoptimized_bridge_from_interp([ConstInt(5), ConstFloat(2.25)], "entry loop token") - cell1 = get_jitcell(5, 2.25) + cell1 = get_jitcell(True, 5, 2.25) assert cell1.counter < 0 assert cell1.entry_loop_token == "entry loop token" def test_make_jitdriver_callbacks_1(): class FakeJitDriverSD: _green_args_spec = [lltype.Signed, lltype.Float] - _can_inline_ptr = None _get_printable_location_ptr = None _confirm_enter_jit_ptr = None class FakeCell: dont_trace_here = False state = WarmEnterState(None, FakeJitDriverSD()) - def jit_getter(*args): + def jit_getter(build, *args): return FakeCell() state.jit_getter = jit_getter state.make_jitdriver_callbacks() - res = state.can_inline_callable([ConstInt(5), ConstFloat(42.5)]) - assert res is True res = state.get_location_str([ConstInt(5), ConstFloat(42.5)]) assert res == '(no jitdriver.get_printable_location!)' -def test_make_jitdriver_callbacks_2(): - def can_inline(x, y): - assert x == 5 - assert y == 42.5 - return False - CAN_INLINE = lltype.Ptr(lltype.FuncType([lltype.Signed, lltype.Float], - lltype.Bool)) - class FakeCell: - dont_trace_here = False - class FakeWarmRunnerDesc: - rtyper = None - class FakeJitDriverSD: - _green_args_spec = [lltype.Signed, lltype.Float] - _can_inline_ptr = llhelper(CAN_INLINE, can_inline) - _get_printable_location_ptr = None - _confirm_enter_jit_ptr = None - state = WarmEnterState(FakeWarmRunnerDesc(), FakeJitDriverSD()) - def jit_getter(*args): - return FakeCell() - state.jit_getter = jit_getter - state.make_jitdriver_callbacks() - res = state.can_inline_callable([ConstInt(5), ConstFloat(42.5)]) - assert res is False - def test_make_jitdriver_callbacks_3(): def get_location(x, y): assert x == 5 @@ -210,7 +187,6 @@ rtyper = None class FakeJitDriverSD: _green_args_spec = [lltype.Signed, lltype.Float] - _can_inline_ptr = None _get_printable_location_ptr = llhelper(GET_LOCATION, get_location) _confirm_enter_jit_ptr = None _get_jitcell_at_ptr = None @@ -231,7 +207,6 @@ rtyper = None class FakeJitDriverSD: _green_args_spec = [lltype.Signed, lltype.Float] - _can_inline_ptr = None _get_printable_location_ptr = None _confirm_enter_jit_ptr = llhelper(ENTER_JIT, confirm_enter_jit) _get_jitcell_at_ptr = None Modified: pypy/trunk/pypy/jit/metainterp/test/test_ztranslation.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_ztranslation.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_ztranslation.py Fri Aug 20 18:41:54 2010 @@ -37,15 +37,12 @@ return jitcellcache.entry def get_printable_location(): return '(hello world)' - def can_inline(): - return False jitdriver = JitDriver(greens = [], reds = ['total', 'frame'], virtualizables = ['frame'], get_jitcell_at=get_jitcell_at, set_jitcell_at=set_jitcell_at, - get_printable_location=get_printable_location, - can_inline=can_inline) + get_printable_location=get_printable_location) def f(i): for param in unroll_parameters: defl = PARAMETERS[param] @@ -63,8 +60,7 @@ frame.i -= 1 return total * 10 # - myjitdriver2 = JitDriver(greens = ['g'], reds = ['m', 'x'], - can_inline = lambda *args: False) + myjitdriver2 = JitDriver(greens = ['g'], reds = ['m', 'x']) def f2(g, m, x): while m > 0: myjitdriver2.can_enter_jit(g=g, m=m, x=x) Modified: pypy/trunk/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/warmspot.py (original) +++ pypy/trunk/pypy/jit/metainterp/warmspot.py Fri Aug 20 18:41:54 2010 @@ -136,9 +136,6 @@ class ContinueRunningNormallyBase(JitException): pass -class CannotInlineCanEnterJit(JitException): - pass - # ____________________________________________________________ class WarmRunnerDesc(object): @@ -402,7 +399,7 @@ can_inline = state.can_inline_greenargs num_green_args = jd.num_green_args def maybe_enter_from_start(*args): - if can_inline is not None and not can_inline(*args[:num_green_args]): + if not can_inline(*args[:num_green_args]): maybe_compile_and_run(*args) maybe_enter_from_start._always_inline_ = True jd._maybe_enter_from_start_fn = maybe_enter_from_start @@ -423,8 +420,6 @@ s_BaseJitCell_not_None) jd._get_jitcell_at_ptr = self._make_hook_graph(jd, annhelper, jd.jitdriver.get_jitcell_at, s_BaseJitCell_or_None) - jd._can_inline_ptr = self._make_hook_graph(jd, - annhelper, jd.jitdriver.can_inline, annmodel.s_Bool) jd._get_printable_location_ptr = self._make_hook_graph(jd, annhelper, jd.jitdriver.get_printable_location, s_Str) jd._confirm_enter_jit_ptr = self._make_hook_graph(jd, @@ -478,7 +473,13 @@ jitdriver = op.args[1].value assert jitdriver in sublists, \ "can_enter_jit with no matching jit_merge_point" - sublists[jitdriver].append((graph, block, index)) + origportalgraph = jd._jit_merge_point_pos[0] + if graph is not origportalgraph: + sublists[jitdriver].append((graph, block, index)) + else: + pass # a 'can_enter_jit' before the 'jit-merge_point', but + # originally in the same function: we ignore it here + # see e.g. test_jitdriver.test_simple for jd in self.jitdrivers_sd: sublist = sublists[jd.jitdriver] assert len(sublist) > 0, \ @@ -557,6 +558,7 @@ # Prepare the portal_runner() helper # from pypy.jit.metainterp.warmstate import specialize_value + from pypy.jit.metainterp.warmstate import unspecialize_value portal_ptr = self.cpu.ts.functionptr(PORTALFUNC, 'portal', graph = portalgraph) jd._portal_ptr = portal_ptr @@ -611,6 +613,37 @@ value = cast_base_ptr_to_instance(Exception, value) raise Exception, value + def handle_jitexception(e): + # XXX the bulk of this function is a copy-paste from above :-( + try: + raise e + except self.ContinueRunningNormally, e: + args = () + for ARGTYPE, attrname, count in portalfunc_ARGS: + x = getattr(e, attrname)[count] + x = specialize_value(ARGTYPE, x) + args = args + (x,) + return ll_portal_runner(*args) + except self.DoneWithThisFrameVoid: + assert result_kind == 'void' + return + except self.DoneWithThisFrameInt, e: + assert result_kind == 'int' + return specialize_value(RESULT, e.result) + except self.DoneWithThisFrameRef, e: + assert result_kind == 'ref' + return specialize_value(RESULT, e.result) + except self.DoneWithThisFrameFloat, e: + assert result_kind == 'float' + return specialize_value(RESULT, e.result) + except self.ExitFrameWithExceptionRef, e: + value = ts.cast_to_baseclass(e.value) + if not we_are_translated(): + raise LLException(ts.get_typeptr(value), value) + else: + value = cast_base_ptr_to_instance(Exception, value) + raise Exception, value + jd._ll_portal_runner = ll_portal_runner # for debugging jd.portal_runner_ptr = self.helper_func(jd._PTR_PORTAL_FUNCTYPE, ll_portal_runner) @@ -631,32 +664,8 @@ vinfo.reset_vable_token(virtualizable) try: loop_token = fail_descr.handle_fail(self.metainterp_sd, jd) - except self.ContinueRunningNormally, e: - args = () - for ARGTYPE, attrname, count in portalfunc_ARGS: - x = getattr(e, attrname)[count] - x = specialize_value(ARGTYPE, x) - args = args + (x,) - return ll_portal_runner(*args) - except self.DoneWithThisFrameVoid: - assert result_kind == 'void' - return - except self.DoneWithThisFrameInt, e: - assert result_kind == 'int' - return specialize_value(RESULT, e.result) - except self.DoneWithThisFrameRef, e: - assert result_kind == 'ref' - return specialize_value(RESULT, e.result) - except self.DoneWithThisFrameFloat, e: - assert result_kind == 'float' - return specialize_value(RESULT, e.result) - except self.ExitFrameWithExceptionRef, e: - value = ts.cast_to_baseclass(e.value) - if not we_are_translated(): - raise LLException(ts.get_typeptr(value), value) - else: - value = cast_base_ptr_to_instance(Exception, value) - raise Exception, value + except JitException, e: + return handle_jitexception(e) fail_descr = self.cpu.execute_token(loop_token) jd._assembler_call_helper = assembler_call_helper # for debugging @@ -668,6 +677,21 @@ if vinfo is not None: jd.vable_token_descr = vinfo.vable_token_descr + def handle_jitexception_from_blackhole(bhcaller, e): + result = handle_jitexception(e) + # + if result_kind != 'void': + result = unspecialize_value(result) + if result_kind == 'int': + bhcaller._setup_return_value_i(result) + elif result_kind == 'ref': + bhcaller._setup_return_value_r(result) + elif result_kind == 'float': + bhcaller._setup_return_value_f(result) + else: + assert False + jd.handle_jitexc_from_bh = handle_jitexception_from_blackhole + # ____________________________________________________________ # Now mutate origportalgraph to end with a call to portal_runner_ptr # @@ -687,17 +711,6 @@ origblock.exitswitch = None origblock.recloseblock(Link([v_result], origportalgraph.returnblock)) # - # Also kill any can_enter_jit left behind (example: see - # test_jitdriver.test_simple, which has a can_enter_jit in - # loop1's origportalgraph) - can_enter_jits = _find_jit_marker([origportalgraph], 'can_enter_jit') - for _, block, i in can_enter_jits: - op = block.operations[i] - assert op.opname == 'jit_marker' - block.operations[i] = SpaceOperation('same_as', - [Constant(None, lltype.Void)], - op.result) - # checkgraph(origportalgraph) def add_finish(self): Modified: pypy/trunk/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/warmstate.py (original) +++ pypy/trunk/pypy/jit/metainterp/warmstate.py Fri Aug 20 18:41:54 2010 @@ -30,6 +30,22 @@ else: return lltype.cast_opaque_ptr(TYPE, x) + at specialize.ll() +def unspecialize_value(value): + """Casts 'value' to a Signed, a GCREF or a Float.""" + if isinstance(lltype.typeOf(value), lltype.Ptr): + if lltype.typeOf(value).TO._gckind == 'gc': + return lltype.cast_opaque_ptr(llmemory.GCREF, value) + else: + adr = llmemory.cast_ptr_to_adr(value) + return heaptracker.adr2int(adr) + elif isinstance(lltype.typeOf(value), ootype.OOType): + return ootype.cast_to_object(value) + elif isinstance(value, float): + return value + else: + return intmask(value) + @specialize.arg(0) def unwrap(TYPE, box): if TYPE is lltype.Void: @@ -232,7 +248,7 @@ # look for the cell corresponding to the current greenargs greenargs = args[:num_green_args] - cell = get_jitcell(*greenargs) + cell = get_jitcell(True, *greenargs) if cell.counter >= 0: # update the profiling counter @@ -324,7 +340,7 @@ # def jit_cell_at_key(greenkey): greenargs = unwrap_greenkey(greenkey) - return jit_getter(*greenargs) + return jit_getter(True, *greenargs) self.jit_cell_at_key = jit_cell_at_key self.jit_getter = jit_getter # @@ -355,10 +371,12 @@ # jitcell_dict = r_dict(comparekey, hashkey) # - def get_jitcell(*greenargs): + def get_jitcell(build, *greenargs): try: cell = jitcell_dict[greenargs] except KeyError: + if not build: + return None cell = JitCell() jitcell_dict[greenargs] = cell return cell @@ -371,38 +389,41 @@ set_jitcell_at_ptr = self.jitdriver_sd._set_jitcell_at_ptr lltohlhack = {} # - def get_jitcell(*greenargs): + def get_jitcell(build, *greenargs): fn = support.maybe_on_top_of_llinterp(rtyper, get_jitcell_at_ptr) cellref = fn(*greenargs) # if we_are_translated(): BASEJITCELL = lltype.typeOf(cellref) cell = cast_base_ptr_to_instance(JitCell, cellref) - elif isinstance(cellref, (BaseJitCell, type(None))): - BASEJITCELL = None - cell = cellref else: - BASEJITCELL = lltype.typeOf(cellref) - if cellref: - cell = lltohlhack[rtyper.type_system.deref(cellref)] + if isinstance(cellref, (BaseJitCell, type(None))): + BASEJITCELL = None + cell = cellref else: - cell = None - # + BASEJITCELL = lltype.typeOf(cellref) + if cellref: + cell = lltohlhack[rtyper.type_system.deref(cellref)] + else: + cell = None + if not build: + return cell if cell is None: cell = JitCell() # if we_are_translated(): cellref = cast_object_to_ptr(BASEJITCELL, cell) - elif BASEJITCELL is None: - cellref = cell else: - if isinstance(BASEJITCELL, lltype.Ptr): - cellref = lltype.malloc(BASEJITCELL.TO) - elif isinstance(BASEJITCELL, ootype.Instance): - cellref = ootype.new(BASEJITCELL) + if BASEJITCELL is None: + cellref = cell else: - assert False, "no clue" - lltohlhack[rtyper.type_system.deref(cellref)] = cell + if isinstance(BASEJITCELL, lltype.Ptr): + cellref = lltype.malloc(BASEJITCELL.TO) + elif isinstance(BASEJITCELL, ootype.Instance): + cellref = ootype.new(BASEJITCELL) + else: + assert False, "no clue" + lltohlhack[rtyper.type_system.deref(cellref)] = cell # fn = support.maybe_on_top_of_llinterp(rtyper, set_jitcell_at_ptr) @@ -468,34 +489,24 @@ if hasattr(self, 'get_location_str'): return # - can_inline_ptr = self.jitdriver_sd._can_inline_ptr unwrap_greenkey = self.make_unwrap_greenkey() jit_getter = self.make_jitcell_getter() - if can_inline_ptr is None: - def can_inline_callable(*greenargs): - # XXX shouldn't it be False by default? - return True - else: - rtyper = self.warmrunnerdesc.rtyper - # - def can_inline_callable(*greenargs): - fn = support.maybe_on_top_of_llinterp(rtyper, can_inline_ptr) - return fn(*greenargs) - def can_inline(*greenargs): - cell = jit_getter(*greenargs) - if cell.dont_trace_here: + + def can_inline_greenargs(*greenargs): + cell = jit_getter(False, *greenargs) + if cell is not None and cell.dont_trace_here: return False - return can_inline_callable(*greenargs) - self.can_inline_greenargs = can_inline - def can_inline_greenkey(greenkey): + return True + def can_inline_callable(greenkey): greenargs = unwrap_greenkey(greenkey) - return can_inline(*greenargs) - self.can_inline_callable = can_inline_greenkey + return can_inline_greenargs(*greenargs) + self.can_inline_greenargs = can_inline_greenargs + self.can_inline_callable = can_inline_callable def get_assembler_token(greenkey): greenargs = unwrap_greenkey(greenkey) - cell = jit_getter(*greenargs) - if cell.counter >= 0: + cell = jit_getter(False, *greenargs) + if cell is None or cell.counter >= 0: return None return cell.entry_loop_token self.get_assembler_token = get_assembler_token Modified: pypy/trunk/pypy/module/__builtin__/functional.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/functional.py (original) +++ pypy/trunk/pypy/module/__builtin__/functional.py Fri Aug 20 18:41:54 2010 @@ -221,8 +221,7 @@ from pypy.rlib.jit import JitDriver mapjitdriver = JitDriver(greens = ['code'], - reds = ['w_func', 'w_iter', 'result_w'], - can_inline = lambda *args: False) + reds = ['w_func', 'w_iter', 'result_w']) def map_single_user_function(code, w_func, w_iter): result_w = [] while True: Modified: pypy/trunk/pypy/module/pypyjit/interp_jit.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/interp_jit.py (original) +++ pypy/trunk/pypy/module/pypyjit/interp_jit.py Fri Aug 20 18:41:54 2010 @@ -9,7 +9,7 @@ import pypy.interpreter.pyopcode # for side-effects from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import ObjSpace, Arguments -from pypy.interpreter.pycode import PyCode, CO_CONTAINSLOOP +from pypy.interpreter.pycode import PyCode, CO_GENERATOR from pypy.interpreter.pyframe import PyFrame from pypy.interpreter.pyopcode import ExitFrame from opcode import opmap @@ -24,9 +24,6 @@ JUMP_ABSOLUTE = opmap['JUMP_ABSOLUTE'] -def can_inline(next_instr, bytecode): - return not bool(bytecode.co_flags & CO_CONTAINSLOOP) - def get_printable_location(next_instr, bytecode): from pypy.tool.stdlib_opcode import opcode_method_names name = opcode_method_names[ord(bytecode.co_code[next_instr])] @@ -39,7 +36,8 @@ bytecode.jit_cells[next_instr] = newcell def confirm_enter_jit(next_instr, bytecode, frame, ec): - return (frame.w_f_trace is None and + return (not (bytecode.co_flags & CO_GENERATOR) and + frame.w_f_trace is None and ec.profilefunc is None and ec.w_tracefunc is None) @@ -57,8 +55,7 @@ ## blockstack = frame.blockstack ## return (valuestackdepth, blockstack) -pypyjitdriver = PyPyJitDriver(can_inline = can_inline, - get_printable_location = get_printable_location, +pypyjitdriver = PyPyJitDriver(get_printable_location = get_printable_location, get_jitcell_at = get_jitcell_at, set_jitcell_at = set_jitcell_at, confirm_enter_jit = confirm_enter_jit) Modified: pypy/trunk/pypy/module/pypyjit/policy.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/policy.py (original) +++ pypy/trunk/pypy/module/pypyjit/policy.py Fri Aug 20 18:41:54 2010 @@ -32,6 +32,8 @@ return False if mod.startswith('pypy.interpreter.pyparser.'): return False + if mod == 'pypy.interpreter.generator': + return False if mod.startswith('pypy.module.'): modname = mod[len('pypy.module.'):] if not self.look_inside_pypy_module(modname): Modified: pypy/trunk/pypy/rlib/jit.py ============================================================================== --- pypy/trunk/pypy/rlib/jit.py (original) +++ pypy/trunk/pypy/rlib/jit.py Fri Aug 20 18:41:54 2010 @@ -253,8 +253,7 @@ def __init__(self, greens=None, reds=None, virtualizables=None, get_jitcell_at=None, set_jitcell_at=None, - can_inline=None, get_printable_location=None, - confirm_enter_jit=None): + get_printable_location=None, confirm_enter_jit=None): if greens is not None: self.greens = greens if reds is not None: @@ -270,7 +269,6 @@ self.get_jitcell_at = get_jitcell_at self.set_jitcell_at = set_jitcell_at self.get_printable_location = get_printable_location - self.can_inline = can_inline self.confirm_enter_jit = confirm_enter_jit def _freeze_(self): @@ -284,6 +282,10 @@ # special-cased by ExtRegistryEntry assert dict.fromkeys(livevars) == _self._alllivevars + def loop_header(self): + # special-cased by ExtRegistryEntry + pass + def _set_param(self, name, value): # special-cased by ExtRegistryEntry # (internal, must receive a constant 'name') @@ -323,11 +325,15 @@ # specifically for them. self.jit_merge_point = self.jit_merge_point self.can_enter_jit = self.can_enter_jit + self.loop_header = self.loop_header self._set_param = self._set_param class Entry(ExtEnterLeaveMarker): _about_ = (self.jit_merge_point, self.can_enter_jit) + class Entry(ExtLoopHeader): + _about_ = self.loop_header + class Entry(ExtSetParam): _about_ = self._set_param @@ -384,7 +390,6 @@ self.annotate_hook(driver.get_jitcell_at, driver.greens, **kwds_s) self.annotate_hook(driver.set_jitcell_at, driver.greens, [s_jitcell], **kwds_s) - self.annotate_hook(driver.can_inline, driver.greens, **kwds_s) self.annotate_hook(driver.get_printable_location, driver.greens, **kwds_s) def annotate_hook(self, func, variables, args_s=[], **kwds_s): @@ -425,6 +430,23 @@ return hop.genop('jit_marker', vlist, resulttype=lltype.Void) +class ExtLoopHeader(ExtRegistryEntry): + # Replace a call to myjitdriver.loop_header() + # with an operation jit_marker('loop_header', myjitdriver). + + def compute_result_annotation(self, **kwds_s): + from pypy.annotation import model as annmodel + return annmodel.s_None + + def specialize_call(self, hop): + from pypy.rpython.lltypesystem import lltype + driver = self.instance.im_self + hop.exception_cannot_occur() + vlist = [hop.inputconst(lltype.Void, 'loop_header'), + hop.inputconst(lltype.Void, driver)] + return hop.genop('jit_marker', vlist, + resulttype=lltype.Void) + class ExtSetParam(ExtRegistryEntry): def compute_result_annotation(self, s_name, s_value): Modified: pypy/trunk/pypy/rlib/test/test_jit.py ============================================================================== --- pypy/trunk/pypy/rlib/test/test_jit.py (original) +++ pypy/trunk/pypy/rlib/test/test_jit.py Fri Aug 20 18:41:54 2010 @@ -59,11 +59,9 @@ def test_annotate_hooks(self): - def can_inline(m): pass def get_printable_location(m): pass myjitdriver = JitDriver(greens=['m'], reds=['n'], - can_inline=can_inline, get_printable_location=get_printable_location) def fn(n): m = 42.5 @@ -81,9 +79,8 @@ return [v.concretetype for v in graph.getargs()] raise Exception, 'function %r has not been annotated' % func - can_inline_args = getargs(can_inline) get_printable_location_args = getargs(get_printable_location) - assert can_inline_args == get_printable_location_args == [lltype.Float] + assert get_printable_location_args == [lltype.Float] def test_annotate_argumenterror(self): myjitdriver = JitDriver(greens=['m'], reds=['n']) From arigo at codespeak.net Fri Aug 20 18:42:21 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 20 Aug 2010 18:42:21 +0200 (CEST) Subject: [pypy-svn] r76686 - pypy/branch/kill-caninline Message-ID: <20100820164221.28AA5282BDE@codespeak.net> Author: arigo Date: Fri Aug 20 18:42:19 2010 New Revision: 76686 Removed: pypy/branch/kill-caninline/ Log: Remove merged branch. From hakanardo at codespeak.net Fri Aug 20 18:50:29 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Fri, 20 Aug 2010 18:50:29 +0200 (CEST) Subject: [pypy-svn] r76687 - in pypy/branch/jit-bounds/pypy: jit/metainterp jit/metainterp/test module/pypyjit/test Message-ID: <20100820165029.EEFB9282B90@codespeak.net> Author: hakanardo Date: Fri Aug 20 18:50:28 2010 New Revision: 76687 Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/jit-bounds/pypy/module/pypyjit/test/test_pypy_c.py Log: int_eq support Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py Fri Aug 20 18:50:28 2010 @@ -711,6 +711,11 @@ self.resumedata_memo.update_counters(self.metainterp_sd.profiler) def propagate_bounds_backward(self, box): + v = self.getvalue(box) + b = v.intbound + if b.has_lower and b.has_upper and b.lower == b.upper: + v.make_constant(ConstInt(b.lower)) + try: op = self.producer[box] except KeyError: @@ -812,13 +817,15 @@ oldop = self.pure_operations.get(targs, None) if oldop is not None and oldop.descr is op.descr: value = self.getvalue(oldop.result) + print value if value.is_constant(): - if value.box is CONST_1: + if value.box.same_constant(CONST_1): self.make_constant(op.result, CONST_0) return True - elif value.box is CONST_0: + elif value.box.same_constant(CONST_0): self.make_constant(op.result, CONST_1) return True + return False @@ -1343,6 +1350,16 @@ else: self.optimize_default(op) + def optimize_INT_EQ(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + if v1.intbound.known_gt(v2.intbound): + self.make_constant_int(op.result, 0) + elif v1.intbound.known_lt(v2.intbound): + self.make_constant_int(op.result, 0) + else: + self.optimize_default(op) + def make_int_lt(self, args): v1 = self.getvalue(args[0]) v2 = self.getvalue(args[1]) @@ -1398,6 +1415,17 @@ else: self.make_int_lt(op.args) + def propagate_bounds_INT_EQ(self, op): + r = self.getvalue(op.result) + if r.is_constant(): + if r.box.same_constant(CONST_1): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + if v1.intbound.intersect(v2.intbound): + self.propagate_bounds_backward(op.args[0]) + if v2.intbound.intersect(v1.intbound): + self.propagate_bounds_backward(op.args[1]) + def propagate_bounds_INT_ADD(self, op): v1 = self.getvalue(op.args[0]) v2 = self.getvalue(op.args[1]) Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py Fri Aug 20 18:50:28 2010 @@ -266,7 +266,13 @@ guard_value(i0, 0) [i0] jump(i) """ - self.optimize_loop(ops, 'Not', ops) + expected = """ + [i] + i0 = int_sub(i, 1) + guard_value(i0, 0) [i0] + jump(1) + """ + self.optimize_loop(ops, 'Not', expected) def test_constant_propagate(self): ops = """ @@ -662,7 +668,13 @@ guard_value(i1, 0) [i] jump(i) """ - self.optimize_loop(ops, 'Not', ops) + expected = """ + [i] + i1 = int_add(i, 3) + guard_value(i1, 0) [i] + jump(-3) + """ + self.optimize_loop(ops, 'Not', expected) def test_int_is_true_of_bool(self): ops = """ @@ -3091,7 +3103,7 @@ guard_false(i1) [] i2 = int_lt(i0, 5) guard_true(i2) [] - jump(i0) + jump(4) """ self.optimize_loop(ops, 'Not', expected) @@ -3544,6 +3556,65 @@ """ self.optimize_loop(ops, 'Not', expected) + def test_bound_eq(self): + ops = """ + [i0, i1] + i2 = int_le(i0, 4) + guard_true(i2) [] + i3 = int_eq(i0, i1) + guard_true(i3) [] + i4 = int_lt(i1, 5) + guard_true(i4) [] + jump(i0, i1) + """ + expected = """ + [i0, i1] + i2 = int_le(i0, 4) + guard_true(i2) [] + i3 = int_eq(i0, i1) + guard_true(i3) [] + jump(i0, i1) + """ + self.optimize_loop(ops, 'Not, Not', expected) + + def test_bound_eq_const(self): + ops = """ + [i0] + i1 = int_eq(i0, 7) + guard_true(i1) [] + i2 = int_add(i0, 3) + jump(i2) + """ + expected = """ + [i0] + i1 = int_eq(i0, 7) + guard_true(i1) [] + jump(10) + + """ + self.optimize_loop(ops, 'Not', expected) + + def test_bound_lege_const(self): + ops = """ + [i0] + i1 = int_ge(i0, 7) + guard_true(i1) [] + i2 = int_le(i0, 7) + guard_true(i2) [] + i3 = int_add(i0, 3) + jump(i3) + """ + expected = """ + [i0] + i1 = int_ge(i0, 7) + guard_true(i1) [] + i2 = int_le(i0, 7) + guard_true(i2) [] + jump(10) + + """ + self.optimize_loop(ops, 'Not', expected) + Modified: pypy/branch/jit-bounds/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/jit-bounds/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/jit-bounds/pypy/module/pypyjit/test/test_pypy_c.py Fri Aug 20 18:50:28 2010 @@ -884,7 +884,8 @@ def test_intbound_addsub_mix(self): tests = ('i > 4', 'i > 2', 'i + 1 > 2', '1 + i > 4', - 'i - 1 > 1', '1 - i > 1', '1 - i < -3') + 'i - 1 > 1', '1 - i > 1', '1 - i < -3', + 'i == 1', 'i == 5') for t1 in tests: for t2 in tests: print t1, t2 @@ -952,6 +953,32 @@ return (a, b) ''', 56, ([], (2000, 2000))) + def test_intbound_eq(self): + self.run_source(''' + def main(a): + i, s = 0, 0 + while i < 1500: + if a == 7: + s += a + 1 + elif i == 10: + s += i + else: + s += 1 + i += 1 + return s + ''', 69, ([7], 12000), ([42], 1509), ([10], 1509)) + + def test_assert(self): + self.run_source(''' + def main(a): + i, s = 0, 0 + while i < 1500: + assert a == 7 + s += a + 1 + i += 1 + return s + ''', 38, ([7], 8*1500)) + def test_zeropadded(self): self.run_source(''' from array import array @@ -977,6 +1004,29 @@ ''', 232, ([], 9895050.0)) + def test_circular(self): + self.run_source(''' + from array import array + class Circular(array): + def __new__(cls): + self = array.__new__(cls, 'd', range(256)) + return self + def __getitem__(self, i): + return array.__getitem__(self, i & 255) + + def main(): + buf = Circular() + i = 10 + sa = 0 + while i < 2000 - 10: + sa += buf[i-2] + buf[i-1] + buf[i] + buf[i+1] + buf[i+2] + i += 1 + return sa + + ''', 170, ([], 1239690.0)) + + + # test_circular class AppTestJIT(PyPyCJITTests): From cfbolz at codespeak.net Fri Aug 20 23:29:24 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 20 Aug 2010 23:29:24 +0200 (CEST) Subject: [pypy-svn] r76691 - pypy/branch/better-map-instances/pypy/objspace/std Message-ID: <20100820212924.23AFF282B90@codespeak.net> Author: cfbolz Date: Fri Aug 20 23:29:22 2010 New Revision: 76691 Modified: pypy/branch/better-map-instances/pypy/objspace/std/dictmultiobject.py Log: cleanup whitespace Modified: pypy/branch/better-map-instances/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/dictmultiobject.py Fri Aug 20 23:29:22 2010 @@ -102,17 +102,17 @@ else: return None - # _________________________________________________________________ + # _________________________________________________________________ # implementation methods def impl_getitem(self, w_key): #return w_value or None raise NotImplementedError("abstract base class") - def impl_getitem_str(self, w_key): + def impl_getitem_str(self, key): #return w_value or None raise NotImplementedError("abstract base class") - def impl_setitem_str(self, key, w_value, shadows_type=True): + def impl_setitem_str(self, key, w_value, shadows_type=True): raise NotImplementedError("abstract base class") def impl_setitem(self, w_key, w_value): @@ -120,7 +120,7 @@ def impl_delitem(self, w_key): raise NotImplementedError("abstract base class") - + def impl_length(self): raise NotImplementedError("abstract base class") From cfbolz at codespeak.net Fri Aug 20 23:30:01 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 20 Aug 2010 23:30:01 +0200 (CEST) Subject: [pypy-svn] r76692 - pypy/branch/better-map-instances/pypy/interpreter Message-ID: <20100820213001.134F2282B90@codespeak.net> Author: cfbolz Date: Fri Aug 20 23:29:59 2010 New Revision: 76692 Modified: pypy/branch/better-map-instances/pypy/interpreter/baseobjspace.py pypy/branch/better-map-instances/pypy/interpreter/typedef.py Log: pass in the member object into get/setslotvalue for reasons that will become clear soon. Modified: pypy/branch/better-map-instances/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/better-map-instances/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/better-map-instances/pypy/interpreter/baseobjspace.py Fri Aug 20 23:29:59 2010 @@ -105,10 +105,10 @@ return space.wrap("<%s at 0x%s%s>" % (info, addrstring, moreinfo)) - def getslotvalue(self, index): + def getslotvalue(self, member): raise NotImplementedError - def setslotvalue(self, index, w_val): + def setslotvalue(self, member, w_val): raise NotImplementedError def descr_call_mismatch(self, space, opname, RequiredClass, args): Modified: pypy/branch/better-map-instances/pypy/interpreter/typedef.py ============================================================================== --- pypy/branch/better-map-instances/pypy/interpreter/typedef.py (original) +++ pypy/branch/better-map-instances/pypy/interpreter/typedef.py Fri Aug 20 23:29:59 2010 @@ -239,10 +239,10 @@ def user_setup_slots(self, nslots): if nslots > 0: self.slots_w = [None] * nslots - def setslotvalue(self, index, w_value): - self.slots_w[index] = w_value - def getslotvalue(self, index): - return self.slots_w[index] + def setslotvalue(self, member, w_value): + self.slots_w[member.index] = w_value + def getslotvalue(self, member): + return self.slots_w[member.index] add(Proto) wantdict = "dict" in features @@ -521,7 +521,7 @@ else: self = member self.typecheck(space, w_obj) - w_result = w_obj.getslotvalue(self.index) + w_result = w_obj.getslotvalue(self) if w_result is None: raise OperationError(space.w_AttributeError, space.wrap(self.name)) # XXX better message @@ -532,14 +532,14 @@ Write into the slot 'member' of the given 'obj'.""" self = member self.typecheck(space, w_obj) - w_obj.setslotvalue(self.index, w_value) + w_obj.setslotvalue(self, w_value) def descr_member_del(space, member, w_obj): """member.__delete__(obj) Delete the value of the slot 'member' from the given 'obj'.""" self = member self.typecheck(space, w_obj) - w_obj.setslotvalue(self.index, None) + w_obj.setslotvalue(self, None) Member.typedef = TypeDef( "member_descriptor", From cfbolz at codespeak.net Fri Aug 20 23:44:57 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 20 Aug 2010 23:44:57 +0200 (CEST) Subject: [pypy-svn] r76693 - in pypy/branch/better-map-instances/pypy/objspace/std: . test Message-ID: <20100820214457.11A74282B90@codespeak.net> Author: cfbolz Date: Fri Aug 20 23:44:55 2010 New Revision: 76693 Added: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py (contents, props changed) pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py (contents, props changed) Log: (cfbolz, arigo there for parts of the work): A new approach to sharing dicts. Decompose the shared structure into a linked list of attribute descriptors. It has some nice properties: - it unifies slots with the shared structure - it unifies the weakref lifeline with the shared structure - the size of an instance will be 5 + n words, where n is the number of attributes - code is cleaner and more extensible - it makes you coffee disadvantage: it's likely that it will be slow withouth the JIT This is work in progress. It's not integrated and not all corner cases are implemented. Added: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py ============================================================================== --- (empty file) +++ pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py Fri Aug 20 23:44:55 2010 @@ -0,0 +1,310 @@ + + +# ____________________________________________________________ +# attribute shapes + +NUM_DIGITS = 4 + +class AbstractAttribute(object): + cache_attrs = None + _size_estimate = 0 + + def read(self, obj, selector): + raise NotImplementedError("abstract base class") + + def write(self, obj, selector, w_value): + raise NotImplementedError("abstract base class") + + def delete(self, obj, selector): + raise NotImplementedError("abstract base class") + + def length(self): + raise NotImplementedError("abstract base class") + + def get_terminator(self): + raise NotImplementedError("abstract base class") + + def set_terminator(self, obj, terminator): + raise NotImplementedError("abstract base class") + + def size_estimate(self): + return self._size_estimate >> NUM_DIGITS + + def search(self, attrtype): + return None + + def add_attr(self, obj, selector, w_value): + cache = self.cache_attrs + if cache is None: + cache = self.cache_attrs = {} + attr = cache.get(selector, None) + if attr is None: + attr = PlainAttribute(selector, self) + cache[selector] = attr + oldattr = obj.map + oldattr._size_estimate += attr.size_estimate() - oldattr.size_estimate() + if attr.length() > len(obj.storage): + new_storage = [None] * attr.size_estimate() + for i in range(len(obj.storage)): + new_storage[i] = obj.storage[i] + obj.storage = new_storage + + obj.storage[attr.position] = w_value + obj.map = attr + + +class Terminator(AbstractAttribute): + def __init__(self, w_cls=None): + self.w_cls = w_cls + + def read(self, obj, selector): + return None + + def write(self, obj, selector, w_value): + obj.map.add_attr(obj, selector, w_value) + return True + + def delete(self, obj, selector): + result = Object() + result._init_empty(self) + return result + + def length(self): + return 0 + + def get_terminator(self): + return self + + def set_terminator(self, obj, terminator): + result = Object() + result._init_empty(terminator) + return result + +class NoDictTerminator(Terminator): + def write(self, obj, selector, w_value): + if selector[1] == DICT: + return False + return Terminator.write(self, obj, selector, w_value) + +class PlainAttribute(AbstractAttribute): + def __init__(self, selector, back): + self.selector = selector + self.position = back.length() + self.back = back + self._size_estimate = self.length() << NUM_DIGITS + + def _copy_attr(self, obj, new_obj): + w_value = self.read(obj, self.selector) + new_obj.map.add_attr(new_obj, self.selector, w_value) + + def read(self, obj, selector): + if selector == self.selector: + return obj.storage[self.position] + return self.back.read(obj, selector) + + def write(self, obj, selector, w_value): + if selector == self.selector: + obj.storage[self.position] = w_value + return True + return self.back.write(obj, selector, w_value) + + def delete(self, obj, selector): + new_obj = self.back.delete(obj, selector) + if self.selector != selector: + self._copy_attr(obj, new_obj) + return new_obj + + def length(self): + return self.position + 1 + + def get_terminator(self): + return self.back.get_terminator() + + def set_terminator(self, obj, terminator): + new_obj = self.back.set_terminator(obj, terminator) + self._copy_attr(obj, new_obj) + return new_obj + + def search(self, attrtype): + if self.selector[1] == attrtype: + return self + return self.back.search(attrtype) + + +# ____________________________________________________________ +# object implementation + +DICT = 6 +SLOT = 1 +SPECIAL = 2 + +class Object(object): + def _init_empty(self, map): + self.map = map + self.storage = [None] * map.size_estimate() + def _become(self, new_obj): + self.map = new_obj.map + self.storage = new_obj.storage + + # _____________________________________________ + # objspace interface + + def getdictvalue(self, space, attrname): + return self.map.read(self, (attrname, DICT)) + + def setdictvalue(self, space, attrname, w_value, shadows_type=True): + return self.map.write(self, (attrname, DICT), w_value) + + def deldictvalue(self, space, w_name): + attrname = space.str_w(w_name) + new_obj = self.map.delete(self, (attrname, DICT)) + # XXX too slow? + if new_obj.map is self.map and new_obj.storage == self.storage: + return False + self._become(new_obj) + return True + + def getdict(self): + w_dict = self.map.read(self, ("dict", SPECIAL)) + if w_dict is not None: + return w_dict + w_dict = MapDictImplementation(self.space, self) + self.map.write(self, ("dict", SPECIAL), w_dict) + return w_dict + + def setdict(self, space, w_dict): + XXX_here_be_monster + typename = space.type(self).getname(space, '?') + raise operationerrfmt(space.w_TypeError, + "attribute '__dict__' of %s objects " + "is not writable", typename) + + def getclass(self, space): + return self.map.get_terminator().w_cls + + def setclass(self, space, w_cls): + new_obj = self.map.set_terminator(self, w_cls.terminator) + self._become(new_obj) + + def user_setup(self, space, w_subtype): + self.space = space + self._init_empty(w_subtype.terminator) + + def getslotvalue(self, member): + return self.map.read(self, (member.name, SLOT)) + + def setslotvalue(self, member, w_value): + self.map.write(self, (member.name, SLOT), w_value) + + # used by _weakref implemenation + + def getweakref(self): + return self.map.read(self, ("weakref", SPECIAL)) + + def setweakref(self, space, weakreflifeline): + self.map.write(self, ("weakref", SPECIAL), weakreflifeline) + +# ____________________________________________________________ +# dict implementation + +from pypy.objspace.std.dictmultiobject import W_DictMultiObject +from pypy.objspace.std.dictmultiobject import IteratorImplementation + +class MapDictImplementation(W_DictMultiObject): + def __init__(self, space, w_obj): + self.space = space + self.w_obj = w_obj + + def impl_getitem(self, w_lookup): + space = self.space + w_lookup_type = space.type(w_lookup) + if space.is_w(w_lookup_type, space.w_str): + return self.impl_getitem_str(space.str_w(w_lookup)) + elif _is_sane_hash(space, w_lookup_type): + return None + else: + return self._as_rdict().getitem(w_lookup) + + def impl_getitem_str(self, key): + return self.w_obj.getdictvalue(self.space, key) + + def impl_setitem_str(self, key, w_value, shadows_type=True): + flag = self.w_obj.setdictvalue(self.space, key, w_value) + assert flag + + def impl_setitem(self, w_key, w_value): + space = self.space + if space.is_w(space.type(w_key), space.w_str): + self.impl_setitem_str(self.space.str_w(w_key), w_value) + else: + self._as_rdict().setitem(w_key, w_value) + + def impl_delitem(self, w_key): + space = self.space + w_key_type = space.type(w_key) + if space.is_w(w_key_type, space.w_str): + flag = self.w_obj.deldictvalue(space, w_key) + if not flag: + raise KeyError + elif _is_sane_hash(space, w_key_type): + raise KeyError + else: + self._as_rdict().delitem(w_key) + + def impl_length(self): + res = 0 + # XXX not RPython yet + curr = self.w_obj.map.search(DICT) + while curr is not None: + curr = curr.back + curr = curr.search(DICT) + res += 1 + return res + + def impl_iter(self): + return MapDictIteratorImplementation(self.space, self) + + def impl_clear(self): + raise NotImplementedError("abstract base class") + + def _clear_fields(self): + self.w_obj = None + + def _as_rdict(self): + r_dict_content = self.initialize_as_rdict() + space = self.space + w_obj = self.w_obj + curr = w_obj.map.search(DICT) + while curr is not None: + attr = curr.selector[0] + r_dict_content[space.wrap(attr)] = w_obj.getdictvalue(space, attr) + curr = curr.back + curr = curr.search(DICT) + self._clear_fields() + return self + + +def _materialize_r_dict(space, obj, w_d): + assert isinstance(w_d, MapDictImplementation) + #XXX + +class MapDictIteratorImplementation(IteratorImplementation): + def __init__(self, space, dictimplementation): + IteratorImplementation.__init__(self, space, dictimplementation) + w_obj = dictimplementation.w_obj + self.w_obj = w_obj + # XXX + self.orig_map = self.curr_map = w_obj.map + + def next_entry(self): + implementation = self.dictimplementation + assert isinstance(implementation, MapDictImplementation) + if self.orig_map is not self.w_obj.map: + return None, None + if self.curr_map: + curr_map = self.curr_map.search(DICT) + if curr_map: + self.curr_map = curr_map.back + attr = curr_map.selector[0] + return attr, self.w_obj.getdictvalue(self.space, attr) + return None, None Added: pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py ============================================================================== --- (empty file) +++ pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py Fri Aug 20 23:44:55 2010 @@ -0,0 +1,249 @@ +from pypy.objspace.std.mapdict import * + +class FakeSpace(object): + def str_w(self, s): + return s + def wrap(self, x): + return x +space = FakeSpace() + +class FakeMember(object): + def __init__(self, name): + self.name = name + +class Class(object): + def __init__(self, hasdict=True): + self.hasdict = True + if hasdict: + self.terminator = Terminator(self) + else: + self.terminator = NoDictTerminator(self) + + def instantiate(self, sp=None): + if sp is None: + sp = space + result = Object() + result.user_setup(sp, self) + return result + +def test_plain_attribute(): + aa = PlainAttribute(("b", DICT), PlainAttribute(("a", DICT), Terminator())) + obj = Object() + obj.map, obj.storage = aa, [10, 20] + assert obj.getdictvalue(space, "a") == 10 + assert obj.getdictvalue(space, "b") == 20 + assert obj.getdictvalue(space, "c") is None + + obj = Object() + obj.map, obj.storage = aa, [30, 40] + obj.setdictvalue(space, "a", 50) + assert obj.storage == [50, 40] + assert obj.getdictvalue(space, "a") == 50 + obj.setdictvalue(space, "b", 60) + assert obj.storage == [50, 60] + assert obj.getdictvalue(space, "b") == 60 + + assert aa.length() == 2 + + assert aa.get_terminator() is aa.back.back + +def test_search(): + aa = PlainAttribute(("b", DICT), PlainAttribute(("a", DICT), Terminator())) + assert aa.search(DICT) is aa + assert aa.search(SLOT) is None + assert aa.search(SPECIAL) is None + bb = PlainAttribute(("C", SPECIAL), PlainAttribute(("A", SLOT), aa)) + assert bb.search(DICT) is aa + assert bb.search(SLOT) is bb.back + assert bb.search(SPECIAL) is bb + +def test_add_attribute(): + cls = Class() + obj = cls.instantiate() + obj.setdictvalue(space, "a", 10) + assert obj.storage == [10] + assert obj.getdictvalue(space, "a") == 10 + assert obj.getdictvalue(space, "b") is None + assert obj.getdictvalue(space, "c") is None + obj.setdictvalue(space, "a", 20) + assert obj.getdictvalue(space, "a") == 20 + assert obj.getdictvalue(space, "b") is None + assert obj.getdictvalue(space, "c") is None + + obj.setdictvalue(space, "b", 30) + assert obj.storage == [20, 30] + assert obj.getdictvalue(space, "a") == 20 + assert obj.getdictvalue(space, "b") == 30 + assert obj.getdictvalue(space, "c") is None + obj.setdictvalue(space, "b", 40) + assert obj.getdictvalue(space, "a") == 20 + assert obj.getdictvalue(space, "b") == 40 + assert obj.getdictvalue(space, "c") is None + + obj2 = cls.instantiate() + obj2.setdictvalue(space, "a", 50) + obj2.setdictvalue(space, "b", 60) + assert obj2.getdictvalue(space, "a") == 50 + assert obj2.getdictvalue(space, "b") == 60 + assert obj2.map is obj.map + +def test_delete(): + for i, dattr in enumerate(["a", "b", "c"]): + c = Class() + obj = c.instantiate() + obj.setdictvalue(space, "a", 50) + obj.setdictvalue(space, "b", 60) + obj.setdictvalue(space, "c", 70) + assert obj.storage == [50, 60, 70] + res = obj.deldictvalue(space, dattr) + assert res + s = [50, 60, 70] + del s[i] + assert obj.storage == s + + obj = c.instantiate() + obj.setdictvalue(space, "a", 50) + obj.setdictvalue(space, "b", 60) + obj.setdictvalue(space, "c", 70) + assert not obj.deldictvalue(space, "d") + + +def test_class(): + c = Class() + obj = c.instantiate() + assert obj.getclass(space) is c + obj.setdictvalue(space, "a", 50) + assert obj.getclass(space) is c + obj.setdictvalue(space, "b", 60) + assert obj.getclass(space) is c + obj.setdictvalue(space, "c", 70) + assert obj.getclass(space) is c + + c2 = Class() + obj.setclass(space, c2) + assert obj.getclass(space) is c2 + assert obj.storage == [50, 60, 70] + +def test_special(): + c = Class() + obj = c.instantiate() + obj.setdictvalue(space, "a", 50) + obj.setdictvalue(space, "b", 60) + obj.setdictvalue(space, "c", 70) + obj.setweakref(space, 100) + assert obj.getdictvalue(space, "a") == 50 + assert obj.getdictvalue(space, "b") == 60 + assert obj.getdictvalue(space, "c") == 70 + assert obj.storage == [50, 60, 70, 100] + assert obj.getweakref() == 100 + + obj2 = c.instantiate() + obj2.setdictvalue(space, "a", 150) + obj2.setdictvalue(space, "b", 160) + obj2.setdictvalue(space, "c", 170) + obj2.setweakref(space, 1100) + assert obj2.storage == [150, 160, 170, 1100] + assert obj2.getweakref() == 1100 + + assert obj2.map is obj.map + + assert obj.getdictvalue(space, "weakref") is None + obj.setdictvalue(space, "weakref", 41) + assert obj.getweakref() == 100 + assert obj.getdictvalue(space, "weakref") == 41 + + +def test_slots(): + cls = Class() + obj = cls.instantiate() + a = FakeMember("a") + b = FakeMember("b") + c = FakeMember("c") + obj.setslotvalue(a, 50) + obj.setslotvalue(b, 60) + obj.setslotvalue(c, 70) + assert obj.getslotvalue(a) == 50 + assert obj.getslotvalue(b) == 60 + assert obj.getslotvalue(c) == 70 + assert obj.storage == [50, 60, 70] + + obj.setdictvalue(space, "a", 5) + obj.setdictvalue(space, "b", 6) + obj.setdictvalue(space, "c", 7) + assert obj.getdictvalue(space, "a") == 5 + assert obj.getdictvalue(space, "b") == 6 + assert obj.getdictvalue(space, "c") == 7 + assert obj.getslotvalue(a) == 50 + assert obj.getslotvalue(b) == 60 + assert obj.getslotvalue(c) == 70 + assert obj.storage == [50, 60, 70, 5, 6, 7] + + obj2 = cls.instantiate() + obj2.setslotvalue(a, 501) + obj2.setslotvalue(b, 601) + obj2.setslotvalue(c, 701) + obj2.setdictvalue(space, "a", 51) + obj2.setdictvalue(space, "b", 61) + obj2.setdictvalue(space, "c", 71) + assert obj2.storage == [501, 601, 701, 51, 61, 71] + assert obj.map is obj2.map + +def test_slots_no_dict(): + cls = Class(hasdict=False) + obj = cls.instantiate() + a = FakeMember("a") + b = FakeMember("b") + c = FakeMember("c") + obj.setslotvalue(a, 50) + obj.setslotvalue(b, 60) + assert obj.getslotvalue(a) == 50 + assert obj.getslotvalue(b) == 60 + assert obj.storage == [50, 60] + assert not obj.setdictvalue(space, "a", 70) + +def test_getdict(): + cls = Class() + obj = cls.instantiate() + obj.setdictvalue(space, "a", 51) + obj.setdictvalue(space, "b", 61) + obj.setdictvalue(space, "c", 71) + assert obj.getdict() is obj.getdict() + assert obj.getdict().length() == 3 + + + +def test_size_prediction(): + for i in range(10): + c = Class() + assert c.terminator.size_estimate() == 0 + for j in range(1000): + obj = c.instantiate() + for a in "abcdefghij"[:i]: + obj.setdictvalue(space, a, 50) + assert c.terminator.size_estimate() == i + for i in range(1, 10): + c = Class() + assert c.terminator.size_estimate() == 0 + for j in range(1000): + obj = c.instantiate() + for a in "abcdefghij"[:i]: + obj.setdictvalue(space, a, 50) + obj = c.instantiate() + for a in "klmnopqars": + obj.setdictvalue(space, a, 50) + assert c.terminator.size_estimate() in [(i + 10) // 2, (i + 11) // 2] + +# ___________________________________________________________ +# dict tests + +from pypy.objspace.std.test.test_dictmultiobject import BaseTestRDictImplementation, BaseTestDevolvedDictImplementation +def get_impl(self): + cls = Class() + w_obj = cls.instantiate(self.fakespace) + return w_obj.getdict() +class TestMapDictImplementation(BaseTestRDictImplementation): + ImplementionClass = MapDictImplementation + get_impl = get_impl +class TestDevolvedMapDictImplementation(BaseTestDevolvedDictImplementation): + get_impl = get_impl + ImplementionClass = MapDictImplementation From cfbolz at codespeak.net Sat Aug 21 00:33:18 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 21 Aug 2010 00:33:18 +0200 (CEST) Subject: [pypy-svn] r76694 - in pypy/branch/better-map-instances/pypy/objspace/std: . test Message-ID: <20100820223318.4363E282B90@codespeak.net> Author: cfbolz Date: Sat Aug 21 00:33:14 2010 New Revision: 76694 Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py Log: another piece of the puzzle: turn map-instances into devolved multi-dicts Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py Sat Aug 21 00:33:14 2010 @@ -52,6 +52,9 @@ obj.storage[attr.position] = w_value obj.map = attr + def materialize_r_dict(self, space, obj, w_d): + raise NotImplementedError("abstract base class") + class Terminator(AbstractAttribute): def __init__(self, w_cls=None): @@ -80,6 +83,11 @@ result._init_empty(terminator) return result + def materialize_r_dict(self, space, obj, w_d): + result = Object() + result._init_empty(self) # XXX + return result + class NoDictTerminator(Terminator): def write(self, obj, selector, w_value): if selector[1] == DICT: @@ -130,6 +138,15 @@ return self return self.back.search(attrtype) + def materialize_r_dict(self, space, obj, w_d): + new_obj = self.back.materialize_r_dict(space, obj, w_d) + if self.selector[1] == DICT: + w_attr = space.wrap(self.selector[0]) + w_d.r_dict_content[w_attr] = obj.storage[self.position] + else: + self._copy_attr(obj, new_obj) + return new_obj + # ____________________________________________________________ # object implementation @@ -169,7 +186,8 @@ if w_dict is not None: return w_dict w_dict = MapDictImplementation(self.space, self) - self.map.write(self, ("dict", SPECIAL), w_dict) + flag = self.map.write(self, ("dict", SPECIAL), w_dict) + assert flag return w_dict def setdict(self, space, w_dict): @@ -274,19 +292,16 @@ r_dict_content = self.initialize_as_rdict() space = self.space w_obj = self.w_obj - curr = w_obj.map.search(DICT) - while curr is not None: - attr = curr.selector[0] - r_dict_content[space.wrap(attr)] = w_obj.getdictvalue(space, attr) - curr = curr.back - curr = curr.search(DICT) + materialize_r_dict(space, w_obj, self) self._clear_fields() return self -def _materialize_r_dict(space, obj, w_d): - assert isinstance(w_d, MapDictImplementation) - #XXX +def materialize_r_dict(space, obj, w_d): + map = obj.map # XXX + assert obj.getdict() is w_d + new_obj = map.materialize_r_dict(space, obj, w_d) + obj._become(new_obj) class MapDictIteratorImplementation(IteratorImplementation): def __init__(self, space, dictimplementation): Modified: pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py Sat Aug 21 00:33:14 2010 @@ -211,6 +211,32 @@ assert obj.getdict().length() == 3 +def test_materialize_r_dict(): + cls = Class() + obj = cls.instantiate() + a = FakeMember("a") + b = FakeMember("b") + c = FakeMember("c") + obj.setslotvalue(a, 50) + obj.setslotvalue(b, 60) + obj.setslotvalue(c, 70) + obj.setdictvalue(space, "a", 5) + obj.setdictvalue(space, "b", 6) + obj.setdictvalue(space, "c", 7) + assert obj.storage == [50, 60, 70, 5, 6, 7] + + class FakeDict(object): + def __init__(self, d): + self.r_dict_content = d + + d = {} + w_d = FakeDict(d) + flag = obj.map.write(obj, ("dict", SPECIAL), w_d) + assert flag + materialize_r_dict(space, obj, w_d) + assert d == {"a": 5, "b": 6, "c": 7} + assert obj.storage == [50, 60, 70, w_d] + def test_size_prediction(): for i in range(10): From cfbolz at codespeak.net Sat Aug 21 00:44:13 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 21 Aug 2010 00:44:13 +0200 (CEST) Subject: [pypy-svn] r76695 - pypy/branch/better-map-instances/pypy/objspace/std/test Message-ID: <20100820224413.EBC67282B9D@codespeak.net> Author: cfbolz Date: Sat Aug 21 00:44:10 2010 New Revision: 76695 Modified: pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py Log: use the existing fake space Modified: pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py Sat Aug 21 00:44:10 2010 @@ -1,10 +1,6 @@ +from pypy.objspace.std.test.test_dictmultiobject import FakeSpace from pypy.objspace.std.mapdict import * -class FakeSpace(object): - def str_w(self, s): - return s - def wrap(self, x): - return x space = FakeSpace() class FakeMember(object): From cfbolz at codespeak.net Sat Aug 21 10:34:12 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 21 Aug 2010 10:34:12 +0200 (CEST) Subject: [pypy-svn] r76696 - in pypy/branch/better-map-instances/pypy/objspace/std: . test Message-ID: <20100821083412.8B9EF282B90@codespeak.net> Author: cfbolz Date: Sat Aug 21 10:34:10 2010 New Revision: 76696 Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py pypy/branch/better-map-instances/pypy/objspace/std/test/test_dictmultiobject.py pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py Log: better support for devolding dicts Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py Sat Aug 21 10:34:10 2010 @@ -83,17 +83,51 @@ result._init_empty(terminator) return result + +class DictTerminator(Terminator): + def __init__(self, w_cls=None): + Terminator.__init__(self, w_cls) + self.devolved_dict_terminator = DevolvedDictTerminator(w_cls) + def materialize_r_dict(self, space, obj, w_d): result = Object() - result._init_empty(self) # XXX + result._init_empty(self.devolved_dict_terminator) return result + class NoDictTerminator(Terminator): def write(self, obj, selector, w_value): if selector[1] == DICT: return False return Terminator.write(self, obj, selector, w_value) + +class DevolvedDictTerminator(Terminator): + def read(self, obj, selector): + w_dict = obj.getdict() + space = obj.space # XXX + return space.finditem_str(w_dict, selector[0]) + + def write(self, obj, selector, w_value): + if selector[1] == DICT: + w_dict = obj.getdict() + space = obj.space # XXX + space.setitem_str(w_dict, selector[0], w_value) + return True + Terminator.write(self, obj, selector, w_value) + + def delete(self, obj, selector): + if selector[1] == DICT: + w_dict = obj.getdict() + space = obj.space # XXX + try: + space.delitem(w_dict, w_name) + except OperationError, ex: + if not ex.match(space, space.w_KeyError): + raise + return Terminator.delete(self, obj, selector) + + class PlainAttribute(AbstractAttribute): def __init__(self, selector, back): self.selector = selector @@ -175,7 +209,7 @@ def deldictvalue(self, space, w_name): attrname = space.str_w(w_name) new_obj = self.map.delete(self, (attrname, DICT)) - # XXX too slow? + # XXX too slow? XXX wrong for devolved dicts if new_obj.map is self.map and new_obj.storage == self.storage: return False self._become(new_obj) Modified: pypy/branch/better-map-instances/pypy/objspace/std/test/test_dictmultiobject.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/test/test_dictmultiobject.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/test/test_dictmultiobject.py Sat Aug 21 10:34:10 2010 @@ -602,6 +602,12 @@ classofinstance=classofinstance, from_strdict_shared=from_strdict_shared) + def finditem_str(self, w_dict, s): + return w_dict.getitem_str(s) # assume it's a multidict + + def setitem_str(self, w_dict, s, w_value): + return w_dict.setitem_str(s, w_value) # assume it's a multidict + def allocate_instance(self, cls, type): return object.__new__(cls) Modified: pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py Sat Aug 21 10:34:10 2010 @@ -11,7 +11,7 @@ def __init__(self, hasdict=True): self.hasdict = True if hasdict: - self.terminator = Terminator(self) + self.terminator = DictTerminator(self) else: self.terminator = NoDictTerminator(self) @@ -269,3 +269,44 @@ class TestDevolvedMapDictImplementation(BaseTestDevolvedDictImplementation): get_impl = get_impl ImplementionClass = MapDictImplementation + +# ___________________________________________________________ +# tests that check the obj interface after the dict has devolved + +def devolve_dict(obj): + w_d = obj.getdict() + w_d._as_rdict() + +def test_get_setdictvalue_after_devolve(): + cls = Class() + obj = cls.instantiate() + a = FakeMember("a") + b = FakeMember("b") + c = FakeMember("c") + obj.setslotvalue(a, 50) + obj.setslotvalue(b, 60) + obj.setslotvalue(c, 70) + obj.setdictvalue(space, "a", 5) + obj.setdictvalue(space, "b", 6) + obj.setdictvalue(space, "c", 7) + devolve_dict(obj) + assert obj.getdictvalue(space, "a") == 5 + assert obj.getdictvalue(space, "b") == 6 + assert obj.getdictvalue(space, "c") == 7 + assert obj.getslotvalue(a) == 50 + assert obj.getslotvalue(b) == 60 + assert obj.getslotvalue(c) == 70 + + obj.setslotvalue(a, 501) + obj.setslotvalue(b, 601) + obj.setslotvalue(c, 701) + obj.setdictvalue(space, "a", 51) + obj.setdictvalue(space, "b", 61) + obj.setdictvalue(space, "c", 71) + assert obj.getdictvalue(space, "a") == 51 + assert obj.getdictvalue(space, "b") == 61 + assert obj.getdictvalue(space, "c") == 71 + assert obj.getslotvalue(a) == 501 + assert obj.getslotvalue(b) == 601 + assert obj.getslotvalue(c) == 701 + From cfbolz at codespeak.net Sat Aug 21 12:04:04 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 21 Aug 2010 12:04:04 +0200 (CEST) Subject: [pypy-svn] r76697 - in pypy/branch/better-map-instances/pypy/objspace/std: . test Message-ID: <20100821100404.E4DE3282B90@codespeak.net> Author: cfbolz Date: Sat Aug 21 12:04:03 2010 New Revision: 76697 Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py pypy/branch/better-map-instances/pypy/objspace/std/test/test_dictmultiobject.py pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py Log: fix deletion in devolved dicts Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py Sat Aug 21 12:04:03 2010 @@ -1,5 +1,3 @@ - - # ____________________________________________________________ # attribute shapes @@ -16,6 +14,9 @@ raise NotImplementedError("abstract base class") def delete(self, obj, selector): + return None + + def copy(self, obj): raise NotImplementedError("abstract base class") def length(self): @@ -67,7 +68,7 @@ obj.map.add_attr(obj, selector, w_value) return True - def delete(self, obj, selector): + def copy(self, obj): result = Object() result._init_empty(self) return result @@ -117,14 +118,16 @@ Terminator.write(self, obj, selector, w_value) def delete(self, obj, selector): + from pypy.interpreter.error import OperationError if selector[1] == DICT: w_dict = obj.getdict() space = obj.space # XXX try: - space.delitem(w_dict, w_name) + space.delitem(w_dict, space.wrap(selector[0])) except OperationError, ex: if not ex.match(space, space.w_KeyError): raise + return Terminator.copy(self, obj) return Terminator.delete(self, obj, selector) @@ -151,11 +154,19 @@ return self.back.write(obj, selector, w_value) def delete(self, obj, selector): + if self.selector == selector: + # ok, attribute is deleted + return self.back.copy(obj) new_obj = self.back.delete(obj, selector) - if self.selector != selector: + if new_obj is not None: self._copy_attr(obj, new_obj) return new_obj + def copy(self, obj): + new_obj = self.back.copy(obj) + self._copy_attr(obj, new_obj) + return new_obj + def length(self): return self.position + 1 @@ -209,8 +220,7 @@ def deldictvalue(self, space, w_name): attrname = space.str_w(w_name) new_obj = self.map.delete(self, (attrname, DICT)) - # XXX too slow? XXX wrong for devolved dicts - if new_obj.map is self.map and new_obj.storage == self.storage: + if new_obj is None: return False self._become(new_obj) return True Modified: pypy/branch/better-map-instances/pypy/objspace/std/test/test_dictmultiobject.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/test/test_dictmultiobject.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/test/test_dictmultiobject.py Sat Aug 21 12:04:03 2010 @@ -608,6 +608,9 @@ def setitem_str(self, w_dict, s, w_value): return w_dict.setitem_str(s, w_value) # assume it's a multidict + def delitem(self, w_dict, w_s): + return w_dict.delitem(w_s) # assume it's a multidict + def allocate_instance(self, cls, type): return object.__new__(cls) Modified: pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py Sat Aug 21 12:04:03 2010 @@ -309,4 +309,6 @@ assert obj.getslotvalue(a) == 501 assert obj.getslotvalue(b) == 601 assert obj.getslotvalue(c) == 701 - + res = obj.deldictvalue(space, "a") + assert res + assert obj.getdictvalue(space, "a") is None From cfbolz at codespeak.net Sat Aug 21 12:24:39 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 21 Aug 2010 12:24:39 +0200 (CEST) Subject: [pypy-svn] r76698 - in pypy/branch/better-map-instances/pypy/objspace/std: . test Message-ID: <20100821102439.E9930282B90@codespeak.net> Author: cfbolz Date: Sat Aug 21 12:24:38 2010 New Revision: 76698 Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py pypy/branch/better-map-instances/pypy/objspace/std/test/test_dictmultiobject.py pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py Log: implement setdict Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py Sat Aug 21 12:24:38 2010 @@ -232,14 +232,17 @@ w_dict = MapDictImplementation(self.space, self) flag = self.map.write(self, ("dict", SPECIAL), w_dict) assert flag + assert isinstance(w_dict, W_DictMultiObject) return w_dict def setdict(self, space, w_dict): - XXX_here_be_monster - typename = space.type(self).getname(space, '?') - raise operationerrfmt(space.w_TypeError, - "attribute '__dict__' of %s objects " - "is not writable", typename) + from pypy.interpreter.typedef import check_new_dictionary + w_dict = check_new_dictionary(space, w_dict) + w_olddict = self.getdict() + assert isinstance(w_dict, W_DictMultiObject) + w_olddict._as_rdict() + flag = self.map.write(self, ("dict", SPECIAL), w_dict) + assert flag def getclass(self, space): return self.map.get_terminator().w_cls Modified: pypy/branch/better-map-instances/pypy/objspace/std/test/test_dictmultiobject.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/test/test_dictmultiobject.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/test/test_dictmultiobject.py Sat Aug 21 12:24:38 2010 @@ -620,7 +620,7 @@ w_StopIteration = StopIteration w_None = None StringObjectCls = FakeString - w_dict = None + w_dict = W_DictMultiObject iter = iter fixedview = list listview = list Modified: pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py Sat Aug 21 12:24:38 2010 @@ -312,3 +312,26 @@ res = obj.deldictvalue(space, "a") assert res assert obj.getdictvalue(space, "a") is None + assert obj.getdictvalue(space, "b") == 61 + assert obj.getdictvalue(space, "c") == 71 + assert obj.getslotvalue(a) == 501 + assert obj.getslotvalue(b) == 601 + assert obj.getslotvalue(c) == 701 + +def test_setdict(): + cls = Class() + obj = cls.instantiate() + obj.setdictvalue(space, "a", 5) + obj.setdictvalue(space, "b", 6) + obj.setdictvalue(space, "c", 7) + w_d = obj.getdict() + obj2 = cls.instantiate() + obj2.setdictvalue(space, "d", 8) + obj.setdict(space, obj2.getdict()) + assert obj.getdictvalue(space, "a") is None + assert obj.getdictvalue(space, "b") is None + assert obj.getdictvalue(space, "c") is None + assert obj.getdictvalue(space, "d") == 8 + assert w_d.getitem_str("a") == 5 + assert w_d.getitem_str("b") == 6 + assert w_d.getitem_str("c") == 7 From cfbolz at codespeak.net Sat Aug 21 19:24:29 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 21 Aug 2010 19:24:29 +0200 (CEST) Subject: [pypy-svn] r76700 - pypy/branch/better-map-instances/pypy/module/_weakref Message-ID: <20100821172429.80180282B90@codespeak.net> Author: cfbolz Date: Sat Aug 21 19:24:27 2010 New Revision: 76700 Modified: pypy/branch/better-map-instances/pypy/module/_weakref/interp__weakref.py Log: the mapdict implementation would really like the weakref lifeline to be a subclass of W_Root. Should have no implications for anybody else Modified: pypy/branch/better-map-instances/pypy/module/_weakref/interp__weakref.py ============================================================================== --- pypy/branch/better-map-instances/pypy/module/_weakref/interp__weakref.py (original) +++ pypy/branch/better-map-instances/pypy/module/_weakref/interp__weakref.py Sat Aug 21 19:24:27 2010 @@ -7,7 +7,7 @@ import weakref -class WeakrefLifeline(object): +class WeakrefLifeline(W_Root): def __init__(self, space): self.space = space # this is here for W_Root.clear_all_weakrefs() self.refs_weak = [] From cfbolz at codespeak.net Tue Aug 24 10:39:51 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 24 Aug 2010 10:39:51 +0200 (CEST) Subject: [pypy-svn] r76701 - in pypy/branch/better-map-instances/pypy: config interpreter objspace/std objspace/std/test Message-ID: <20100824083951.C6A4D282B90@codespeak.net> Author: cfbolz Date: Tue Aug 24 10:39:48 2010 New Revision: 76701 Modified: pypy/branch/better-map-instances/pypy/config/pypyoption.py pypy/branch/better-map-instances/pypy/interpreter/baseobjspace.py pypy/branch/better-map-instances/pypy/interpreter/typedef.py pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py pypy/branch/better-map-instances/pypy/objspace/std/test/test_shadowtracking.py pypy/branch/better-map-instances/pypy/objspace/std/typeobject.py Log: Work on making things RPython. In progress: Annotation works, but call normalization doesn't. Modified: pypy/branch/better-map-instances/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/better-map-instances/pypy/config/pypyoption.py (original) +++ pypy/branch/better-map-instances/pypy/config/pypyoption.py Tue Aug 24 10:39:48 2010 @@ -238,6 +238,14 @@ default=False, requires=[("objspace.std.withshadowtracking", False)]), + BoolOption("withmapdict", + "make instances really small but slow without the JIT", + default=True, + requires=[("objspace.std.withshadowtracking", False), + ("objspace.std.withinlineddict", False), + ("objspace.std.withsharingdict", False), + ]), + BoolOption("withrangelist", "enable special range list implementation that does not " "actually create the full list until the resulting " Modified: pypy/branch/better-map-instances/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/better-map-instances/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/better-map-instances/pypy/interpreter/baseobjspace.py Tue Aug 24 10:39:48 2010 @@ -166,6 +166,16 @@ def _call_builtin_destructor(self): pass # method overridden in typedef.py + # hooks that the mapdict implementations needs: + def _get_mapdict_map(self): + return None + def _get_mapdict_storage(self): + return None + def _set_mapdict_map(self, map): + raise NotImplementedError + def _set_mapdict_storage(self, storage): + raise NotImplementedError + class Wrappable(W_Root): """A subclass of Wrappable is an internal, interpreter-level class Modified: pypy/branch/better-map-instances/pypy/interpreter/typedef.py ============================================================================== --- pypy/branch/better-map-instances/pypy/interpreter/typedef.py (original) +++ pypy/branch/better-map-instances/pypy/interpreter/typedef.py Tue Aug 24 10:39:48 2010 @@ -133,6 +133,13 @@ typedef = cls.typedef if wants_dict and typedef.hasdict: wants_dict = False + if config.objspace.std.withmapdict and not typedef.hasdict: + # mapdict only works if the type does not already have a dict + if wants_del: + parentcls = get_unique_interplevel_subclass(config, cls, True, True, + False, True) + return _usersubclswithfeature(config, parentcls, "del") + return _usersubclswithfeature(config, cls, "user", "dict", "weakref", "slots") # Forest of if's - see the comment above. if wants_del: if wants_dict: @@ -187,9 +194,18 @@ def add(Proto): for key, value in Proto.__dict__.items(): if not key.startswith('__') or key == '__del__': + if hasattr(value, "func_name"): + value = func_with_new_name(value, value.func_name) body[key] = value + if (features == ("user", "dict", "weakref", "slots") and + config.objspace.std.withmapdict): + from pypy.objspace.std.mapdict import Object + add(Object) + features = () + if "user" in features: # generic feature needed by all subcls + class Proto(object): user_overridden_class = True Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py Tue Aug 24 10:39:48 2010 @@ -42,34 +42,36 @@ if attr is None: attr = PlainAttribute(selector, self) cache[selector] = attr - oldattr = obj.map + oldattr = obj._get_mapdict_map() oldattr._size_estimate += attr.size_estimate() - oldattr.size_estimate() - if attr.length() > len(obj.storage): + if attr.length() > len(obj._get_mapdict_storage()): new_storage = [None] * attr.size_estimate() - for i in range(len(obj.storage)): - new_storage[i] = obj.storage[i] - obj.storage = new_storage + for i in range(len(obj._get_mapdict_storage())): + new_storage[i] = obj._get_mapdict_storage()[i] + obj._set_mapdict_storage(new_storage) - obj.storage[attr.position] = w_value - obj.map = attr + obj._get_mapdict_storage()[attr.position] = w_value + obj._set_mapdict_map(attr) def materialize_r_dict(self, space, obj, w_d): raise NotImplementedError("abstract base class") class Terminator(AbstractAttribute): - def __init__(self, w_cls=None): + def __init__(self, w_cls, space): self.w_cls = w_cls + self.space = space def read(self, obj, selector): return None def write(self, obj, selector, w_value): - obj.map.add_attr(obj, selector, w_value) + obj._get_mapdict_map().add_attr(obj, selector, w_value) return True def copy(self, obj): result = Object() + result.space = self.space result._init_empty(self) return result @@ -81,17 +83,19 @@ def set_terminator(self, obj, terminator): result = Object() + result.space = self.space result._init_empty(terminator) return result class DictTerminator(Terminator): - def __init__(self, w_cls=None): - Terminator.__init__(self, w_cls) - self.devolved_dict_terminator = DevolvedDictTerminator(w_cls) + def __init__(self, w_cls, space): + Terminator.__init__(self, w_cls, space) + self.devolved_dict_terminator = DevolvedDictTerminator(w_cls, space) def materialize_r_dict(self, space, obj, w_d): result = Object() + result.space = space result._init_empty(self.devolved_dict_terminator) return result @@ -106,22 +110,22 @@ class DevolvedDictTerminator(Terminator): def read(self, obj, selector): w_dict = obj.getdict() - space = obj.space # XXX + space = self.space return space.finditem_str(w_dict, selector[0]) def write(self, obj, selector, w_value): if selector[1] == DICT: w_dict = obj.getdict() - space = obj.space # XXX + space = self.space space.setitem_str(w_dict, selector[0], w_value) return True - Terminator.write(self, obj, selector, w_value) + return Terminator.write(self, obj, selector, w_value) def delete(self, obj, selector): from pypy.interpreter.error import OperationError if selector[1] == DICT: w_dict = obj.getdict() - space = obj.space # XXX + space = self.space try: space.delitem(w_dict, space.wrap(selector[0])) except OperationError, ex: @@ -140,16 +144,16 @@ def _copy_attr(self, obj, new_obj): w_value = self.read(obj, self.selector) - new_obj.map.add_attr(new_obj, self.selector, w_value) + new_obj._get_mapdict_map().add_attr(new_obj, self.selector, w_value) def read(self, obj, selector): if selector == self.selector: - return obj.storage[self.position] + return obj._get_mapdict_storage()[self.position] return self.back.read(obj, selector) def write(self, obj, selector, w_value): if selector == self.selector: - obj.storage[self.position] = w_value + obj._get_mapdict_storage()[self.position] = w_value return True return self.back.write(obj, selector, w_value) @@ -187,7 +191,7 @@ new_obj = self.back.materialize_r_dict(space, obj, w_d) if self.selector[1] == DICT: w_attr = space.wrap(self.selector[0]) - w_d.r_dict_content[w_attr] = obj.storage[self.position] + w_d.r_dict_content[w_attr] = obj._get_mapdict_storage()[self.position] else: self._copy_attr(obj, new_obj) return new_obj @@ -200,7 +204,9 @@ SLOT = 1 SPECIAL = 2 -class Object(object): +from pypy.interpreter.baseobjspace import W_Root + +class Object(W_Root): # slightly evil to make it inherit from W_Root def _init_empty(self, map): self.map = map self.storage = [None] * map.size_estimate() @@ -208,6 +214,15 @@ self.map = new_obj.map self.storage = new_obj.storage + def _get_mapdict_map(self): + return self.map + def _get_mapdict_storage(self): + return self.storage + def _set_mapdict_map(self, map): + self.map = map + def _set_mapdict_storage(self, storage): + self.storage = storage + # _____________________________________________ # objspace interface @@ -228,11 +243,11 @@ def getdict(self): w_dict = self.map.read(self, ("dict", SPECIAL)) if w_dict is not None: + assert isinstance(w_dict, W_DictMultiObject) return w_dict w_dict = MapDictImplementation(self.space, self) flag = self.map.write(self, ("dict", SPECIAL), w_dict) assert flag - assert isinstance(w_dict, W_DictMultiObject) return w_dict def setdict(self, space, w_dict): @@ -253,6 +268,7 @@ def user_setup(self, space, w_subtype): self.space = space + assert not self.typedef.hasdict self._init_empty(w_subtype.terminator) def getslotvalue(self, member): @@ -264,16 +280,25 @@ # used by _weakref implemenation def getweakref(self): - return self.map.read(self, ("weakref", SPECIAL)) + from pypy.module._weakref.interp__weakref import WeakrefLifeline + lifeline = self.map.read(self, ("weakref", SPECIAL)) + if lifeline is None: + return None + assert isinstance(lifeline, WeakrefLifeline) + return lifeline def setweakref(self, space, weakreflifeline): + from pypy.module._weakref.interp__weakref import WeakrefLifeline + assert isinstance(weakreflifeline, WeakrefLifeline) self.map.write(self, ("weakref", SPECIAL), weakreflifeline) + # ____________________________________________________________ # dict implementation from pypy.objspace.std.dictmultiobject import W_DictMultiObject from pypy.objspace.std.dictmultiobject import IteratorImplementation +from pypy.objspace.std.dictmultiobject import _is_sane_hash class MapDictImplementation(W_DictMultiObject): def __init__(self, space, w_obj): @@ -318,8 +343,7 @@ def impl_length(self): res = 0 - # XXX not RPython yet - curr = self.w_obj.map.search(DICT) + curr = self.w_obj._get_mapdict_map().search(DICT) while curr is not None: curr = curr.back curr = curr.search(DICT) @@ -345,28 +369,31 @@ def materialize_r_dict(space, obj, w_d): - map = obj.map # XXX + map = obj._get_mapdict_map() assert obj.getdict() is w_d new_obj = map.materialize_r_dict(space, obj, w_d) - obj._become(new_obj) + # XXX this is like _become, really, but we cannot use that due to RPython + # reasons + obj._set_mapdict_map(new_obj.map) + obj._set_mapdict_storage(new_obj.storage) class MapDictIteratorImplementation(IteratorImplementation): def __init__(self, space, dictimplementation): IteratorImplementation.__init__(self, space, dictimplementation) w_obj = dictimplementation.w_obj self.w_obj = w_obj - # XXX - self.orig_map = self.curr_map = w_obj.map + self.orig_map = self.curr_map = w_obj._get_mapdict_map() def next_entry(self): implementation = self.dictimplementation assert isinstance(implementation, MapDictImplementation) - if self.orig_map is not self.w_obj.map: + if self.orig_map is not self.w_obj._get_mapdict_map(): return None, None if self.curr_map: curr_map = self.curr_map.search(DICT) if curr_map: self.curr_map = curr_map.back attr = curr_map.selector[0] - return attr, self.w_obj.getdictvalue(self.space, attr) + w_attr = self.space.wrap(attr) + return w_attr, self.w_obj.getdictvalue(self.space, attr) return None, None Modified: pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py Tue Aug 24 10:39:48 2010 @@ -1,4 +1,4 @@ -from pypy.objspace.std.test.test_dictmultiobject import FakeSpace +from pypy.objspace.std.test.test_dictmultiobject import FakeSpace, W_DictMultiObject from pypy.objspace.std.mapdict import * space = FakeSpace() @@ -11,9 +11,9 @@ def __init__(self, hasdict=True): self.hasdict = True if hasdict: - self.terminator = DictTerminator(self) + self.terminator = DictTerminator(self, space) else: - self.terminator = NoDictTerminator(self) + self.terminator = NoDictTerminator(self, space) def instantiate(self, sp=None): if sp is None: @@ -22,8 +22,12 @@ result.user_setup(sp, self) return result +class Object(Object): + class typedef: + hasdict = False + def test_plain_attribute(): - aa = PlainAttribute(("b", DICT), PlainAttribute(("a", DICT), Terminator())) + aa = PlainAttribute(("b", DICT), PlainAttribute(("a", DICT), Terminator(None, None))) obj = Object() obj.map, obj.storage = aa, [10, 20] assert obj.getdictvalue(space, "a") == 10 @@ -44,7 +48,7 @@ assert aa.get_terminator() is aa.back.back def test_search(): - aa = PlainAttribute(("b", DICT), PlainAttribute(("a", DICT), Terminator())) + aa = PlainAttribute(("b", DICT), PlainAttribute(("a", DICT), Terminator(None, None))) assert aa.search(DICT) is aa assert aa.search(SLOT) is None assert aa.search(SPECIAL) is None @@ -121,31 +125,35 @@ assert obj.storage == [50, 60, 70] def test_special(): + from pypy.module._weakref.interp__weakref import WeakrefLifeline + lifeline1 = WeakrefLifeline(space) + lifeline2 = WeakrefLifeline(space) c = Class() obj = c.instantiate() + assert obj.getweakref() is None obj.setdictvalue(space, "a", 50) obj.setdictvalue(space, "b", 60) obj.setdictvalue(space, "c", 70) - obj.setweakref(space, 100) + obj.setweakref(space, lifeline1) assert obj.getdictvalue(space, "a") == 50 assert obj.getdictvalue(space, "b") == 60 assert obj.getdictvalue(space, "c") == 70 - assert obj.storage == [50, 60, 70, 100] - assert obj.getweakref() == 100 + assert obj.storage == [50, 60, 70, lifeline1] + assert obj.getweakref() is lifeline1 obj2 = c.instantiate() obj2.setdictvalue(space, "a", 150) obj2.setdictvalue(space, "b", 160) obj2.setdictvalue(space, "c", 170) - obj2.setweakref(space, 1100) - assert obj2.storage == [150, 160, 170, 1100] - assert obj2.getweakref() == 1100 + obj2.setweakref(space, lifeline2) + assert obj2.storage == [150, 160, 170, lifeline2] + assert obj2.getweakref() is lifeline2 assert obj2.map is obj.map assert obj.getdictvalue(space, "weakref") is None obj.setdictvalue(space, "weakref", 41) - assert obj.getweakref() == 100 + assert obj.getweakref() is lifeline1 assert obj.getdictvalue(space, "weakref") == 41 @@ -221,7 +229,7 @@ obj.setdictvalue(space, "c", 7) assert obj.storage == [50, 60, 70, 5, 6, 7] - class FakeDict(object): + class FakeDict(W_DictMultiObject): def __init__(self, d): self.r_dict_content = d @@ -300,7 +308,8 @@ obj.setslotvalue(a, 501) obj.setslotvalue(b, 601) obj.setslotvalue(c, 701) - obj.setdictvalue(space, "a", 51) + res = obj.setdictvalue(space, "a", 51) + assert res obj.setdictvalue(space, "b", 61) obj.setdictvalue(space, "c", 71) assert obj.getdictvalue(space, "a") == 51 Modified: pypy/branch/better-map-instances/pypy/objspace/std/test/test_shadowtracking.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/test/test_shadowtracking.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/test/test_shadowtracking.py Tue Aug 24 10:39:48 2010 @@ -3,7 +3,8 @@ class TestShadowTracking(object): def setup_class(cls): - cls.space = gettestobjspace(**{"objspace.std.withshadowtracking": True}) + cls.space = gettestobjspace(**{"objspace.std.withshadowtracking": True, + "objspace.std.withmapdict": False}) def test_simple_shadowing(self): space = self.space Modified: pypy/branch/better-map-instances/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/typeobject.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/typeobject.py Tue Aug 24 10:39:48 2010 @@ -116,6 +116,12 @@ # dict_w of any of the types in the mro changes, or if the mro # itself changes w_self._version_tag = VersionTag() + if space.config.objspace.std.withmapdict: + from pypy.objspace.std.mapdict import DictTerminator, NoDictTerminator + if w_self.hasdict: + w_self.terminator = DictTerminator(w_self, space) + else: + w_self.terminator = NoDictTerminator(w_self, space) def mutated(w_self): space = w_self.space From arigo at codespeak.net Tue Aug 24 11:53:03 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 24 Aug 2010 11:53:03 +0200 (CEST) Subject: [pypy-svn] r76702 - in pypy/branch/rsre2/pypy: module/_sre module/_sre/test rlib rlib/rsre rlib/rsre/test Message-ID: <20100824095303.0D479282B90@codespeak.net> Author: arigo Date: Tue Aug 24 11:52:33 2010 New Revision: 76702 Removed: pypy/branch/rsre2/pypy/module/_sre/app_sre.py pypy/branch/rsre2/pypy/rlib/rsre/test/test_more.py Modified: pypy/branch/rsre2/pypy/module/_sre/__init__.py pypy/branch/rsre2/pypy/module/_sre/interp_sre.py pypy/branch/rsre2/pypy/module/_sre/test/test_app_sre.py pypy/branch/rsre2/pypy/rlib/debug.py pypy/branch/rsre2/pypy/rlib/rsre/rsre_char.py pypy/branch/rsre2/pypy/rlib/rsre/rsre_core.py pypy/branch/rsre2/pypy/rlib/rsre/rsre_re.py pypy/branch/rsre2/pypy/rlib/rsre/test/test_re.py pypy/branch/rsre2/pypy/rlib/rsre/test/test_search.py Log: Expose rlib/rsre to app-level via module/_sre. Modified: pypy/branch/rsre2/pypy/module/_sre/__init__.py ============================================================================== --- pypy/branch/rsre2/pypy/module/_sre/__init__.py (original) +++ pypy/branch/rsre2/pypy/module/_sre/__init__.py Tue Aug 24 11:52:33 2010 @@ -1,24 +1,14 @@ from pypy.interpreter.mixedmodule import MixedModule class Module(MixedModule): - """A pure Python reimplementation of the _sre module from CPython 2.4 -Copyright 2005 Nik Haldimann, licensed under the MIT license -This code is based on material licensed under CNRI's Python 1.6 license and -copyrighted by: Copyright (c) 1997-2001 by Secret Labs AB -""" - appleveldefs = { - 'compile': 'app_sre.compile', } interpleveldefs = { 'CODESIZE': 'space.wrap(interp_sre.CODESIZE)', 'MAGIC': 'space.wrap(interp_sre.MAGIC)', - 'copyright': 'space.wrap(interp_sre.copyright)', + 'compile': 'interp_sre.W_SRE_Pattern', 'getlower': 'interp_sre.w_getlower', 'getcodesize': 'interp_sre.w_getcodesize', - '_State': 'interp_sre.make_state', - '_match': 'interp_sre.w_match', - '_search': 'interp_sre.w_search', } Modified: pypy/branch/rsre2/pypy/module/_sre/interp_sre.py ============================================================================== --- pypy/branch/rsre2/pypy/module/_sre/interp_sre.py (original) +++ pypy/branch/rsre2/pypy/module/_sre/interp_sre.py Tue Aug 24 11:52:33 2010 @@ -1,27 +1,20 @@ +import sys from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.typedef import GetSetProperty, TypeDef from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w +from pypy.interpreter.typedef import make_weakref_descr from pypy.interpreter.gateway import interp2app, ObjSpace, W_Root from pypy.interpreter.error import OperationError from pypy.rlib.rarithmetic import intmask +from pypy.tool.pairtype import extendabletype -# This can be compiled in two ways: -# -# * THREE_VERSIONS_OF_CORE=True: you get three copies of the whole -# regexp searching and matching code: for strings, for unicode strings, -# and for generic buffer objects (like mmap.mmap or array.array). -# -# * THREE_VERSIONS_OF_CORE=False: there is only one copy of the code, -# at the cost of an indirect method call to fetch each character. - -THREE_VERSIONS_OF_CORE = True +# ____________________________________________________________ +# +# Constants and exposed functions -#### Constants and exposed functions - -from pypy.rlib.rsre import rsre -from pypy.rlib.rsre.rsre_char import MAGIC, CODESIZE, getlower -copyright = "_sre.py 2.4 Copyright 2005 by Nik Haldimann" +from pypy.rlib.rsre import rsre_core +from pypy.rlib.rsre.rsre_char import MAGIC, CODESIZE, getlower, set_unicode_db def w_getlower(space, char_ord, flags): return space.wrap(getlower(char_ord, flags)) @@ -31,166 +24,529 @@ return space.wrap(CODESIZE) # use the same version of unicodedb as the standard objspace -from pypy.objspace.std.unicodeobject import unicodedb -rsre.set_unicode_db(unicodedb) +import pypy.objspace.std.unicodeobject +set_unicode_db(pypy.objspace.std.unicodeobject.unicodedb) -#### State classes +# ____________________________________________________________ +# +# Additional methods on the classes XxxMatchContext + +class __extend__(rsre_core.AbstractMatchContext): + __metaclass__ = extendabletype + def _w_slice(self, space, start, end): + raise NotImplementedError + def _w_string(self, space): + raise NotImplementedError -def make_state(space, w_string, start, end, flags): - # XXX maybe turn this into a __new__ method of W_State - if space.is_true(space.isinstance(w_string, space.w_str)): - cls = W_StringState - elif space.is_true(space.isinstance(w_string, space.w_unicode)): - cls = W_UnicodeState - else: - cls = W_GenericState - return space.wrap(cls(space, w_string, start, end, flags)) -make_state.unwrap_spec = [ObjSpace, W_Root, int, int, int] - - -class W_State(Wrappable): - if not THREE_VERSIONS_OF_CORE: - rsre.insert_sre_methods(locals(), 'all') - - def __init__(self, space, w_string, start, end, flags): - self.space = space - self.w_string = w_string - length = self.unwrap_object() - if start < 0: - start = 0 - if end > length: - end = length - self.start = start - self.pos = start # records the original start position - self.end = end - self.flags = flags - self.reset() +class __extend__(rsre_core.StrMatchContext): + __metaclass__ = extendabletype + def _w_slice(self, space, start, end): + return space.wrap(self._string[start:end]) + def _w_string(self, space): + return space.wrap(self._string) + +class __extend__(rsre_core.UnicodeMatchContext): + __metaclass__ = extendabletype + def _w_slice(self, space, start, end): + return space.wrap(self._unicodestr[start:end]) + def _w_string(self, space): + return space.wrap(self._unicodestr) + +def slice_w(space, ctx, start, end, w_default): + if 0 <= start <= end: + return ctx._w_slice(space, start, end) + return w_default + +def do_flatten_marks(ctx, num_groups): + # Returns a list of RPython-level integers. + # Unlike the app-level groups() method, groups are numbered from 0 + # and the returned list does not start with the whole match range. + if num_groups == 0: + return None + result = [-1] * (2*num_groups) + mark = ctx.match_marks + while mark is not None: + index = mark.gid + if result[index] == -1: + result[index] = mark.position + mark = mark.prev + return result + +def allgroups_w(space, ctx, fmarks, num_groups, w_default): + grps = [slice_w(space, ctx, fmarks[i*2], fmarks[i*2+1], w_default) + for i in range(num_groups)] + return space.newtuple(grps) + +def import_re(space): + w_builtin = space.getbuiltinmodule('__builtin__') + w_import = space.getattr(w_builtin, space.wrap("__import__")) + return space.call_function(w_import, space.wrap("re")) - def lower(self, char_ord): - return getlower(char_ord, self.flags) +def matchcontext(space, ctx): + try: + return rsre_core.match_context(ctx) + except rsre_core.Error, e: + raise OperationError(space.w_RuntimeError, space.wrap(e.msg)) - # methods overridden by subclasses +def searchcontext(space, ctx): + try: + return rsre_core.search_context(ctx) + except rsre_core.Error, e: + raise OperationError(space.w_RuntimeError, space.wrap(e.msg)) - def unwrap_object(self): - raise NotImplementedError +# ____________________________________________________________ +# +# SRE_Pattern class - if 'reset' not in locals(): - def reset(self): - raise NotImplementedError - - if 'search' not in locals(): - def search(self, pattern_codes): - raise NotImplementedError - - if 'match' not in locals(): - def match(self, pattern_codes): - raise NotImplementedError - - # Accessors for the typedef - - def w_reset(self): - self.reset() - - def create_regs(self, group_count): - """ Purely abstract method - """ - raise NotImplementedError +class W_SRE_Pattern(Wrappable): - def w_create_regs(self, group_count): - """Creates a tuple of index pairs representing matched groups, a format - that's convenient for SRE_Match.""" + def cannot_copy_w(self): space = self.space - return space.newtuple([ - space.newtuple([space.wrap(value1), - space.wrap(value2)]) - for value1, value2 in self.create_regs(group_count)]) - w_create_regs.unwrap_spec = ['self', int] + raise OperationError(space.w_TypeError, + space.wrap("cannot copy this pattern object")) - def fget_start(space, self): - return space.wrap(self.start) - - def fset_start(space, self, w_value): - self.start = space.int_w(w_value) + def make_ctx(self, w_string, pos=0, endpos=sys.maxint): + """Make a StrMatchContext or a UnicodeMatchContext for searching + in the given w_string object.""" + space = self.space + if pos < 0: pos = 0 + if endpos < pos: endpos = pos + if space.is_true(space.isinstance(w_string, space.w_unicode)): + unicodestr = space.unicode_w(w_string) + if pos > len(unicodestr): pos = len(unicodestr) + if endpos > len(unicodestr): endpos = len(unicodestr) + return rsre_core.UnicodeMatchContext(self.code, unicodestr, + pos, endpos, self.flags) + else: + str = space.bufferstr_w(w_string) + if pos > len(str): pos = len(str) + if endpos > len(str): endpos = len(str) + return rsre_core.StrMatchContext(self.code, str, + pos, endpos, self.flags) + + def getmatch(self, ctx, found): + if found: + return W_SRE_Match(self, ctx) + else: + return self.space.w_None + + def match_w(self, w_string, pos=0, endpos=sys.maxint): + ctx = self.make_ctx(w_string, pos, endpos) + return self.getmatch(ctx, matchcontext(self.space, ctx)) + match_w.unwrap_spec = ['self', W_Root, int, int] + + def search_w(self, w_string, pos=0, endpos=sys.maxint): + ctx = self.make_ctx(w_string, pos, endpos) + return self.getmatch(ctx, searchcontext(self.space, ctx)) + search_w.unwrap_spec = ['self', W_Root, int, int] - def fget_string_position(space, self): - return space.wrap(self.string_position) + def findall_w(self, w_string, pos=0, endpos=sys.maxint): + space = self.space + matchlist_w = [] + ctx = self.make_ctx(w_string, pos, endpos) + while ctx.match_start <= ctx.end: + if not searchcontext(space, ctx): + break + num_groups = self.num_groups + w_emptystr = space.wrap("") + if num_groups == 0: + w_item = slice_w(space, ctx, ctx.match_start, ctx.match_end, + w_emptystr) + else: + fmarks = do_flatten_marks(ctx, num_groups) + if num_groups == 1: + w_item = slice_w(space, ctx, fmarks[0], fmarks[1], + w_emptystr) + else: + w_item = allgroups_w(space, ctx, fmarks, num_groups, + w_emptystr) + matchlist_w.append(w_item) + no_progress = (ctx.match_start == ctx.match_end) + ctx.reset(ctx.match_end + no_progress) + return space.newlist(matchlist_w) + findall_w.unwrap_spec = ['self', W_Root, int, int] + + def finditer_w(self, w_string, pos=0, endpos=sys.maxint): + # this also works as the implementation of the undocumented + # scanner() method. + ctx = self.make_ctx(w_string, pos, endpos) + scanner = W_SRE_Scanner(self, ctx) + return self.space.wrap(scanner) + finditer_w.unwrap_spec = ['self', W_Root, int, int] - def fset_string_position(space, self, w_value): - self.start = space.int_w(w_value) + def split_w(self, w_string, maxsplit=0): + space = self.space + splitlist = [] + n = 0 + last = 0 + ctx = self.make_ctx(w_string) + while not maxsplit or n < maxsplit: + if not searchcontext(space, ctx): + break + if ctx.match_start == ctx.match_end: # zero-width match + if ctx.match_start == ctx.end: # or end of string + break + ctx.reset(ctx.match_end + 1) + continue + splitlist.append(slice_w(space, ctx, last, ctx.match_start, + space.w_None)) + # add groups (if any) + fmarks = do_flatten_marks(ctx, self.num_groups) + for groupnum in range(self.num_groups): + groupstart, groupend = fmarks[groupnum*2], fmarks[groupnum*2+1] + splitlist.append(slice_w(space, ctx, groupstart, groupend, + space.w_None)) + n += 1 + last = ctx.match_end + ctx.reset(last) + splitlist.append(slice_w(space, ctx, last, ctx.end, space.w_None)) + return space.newlist(splitlist) + split_w.unwrap_spec = ['self', W_Root, int] + + def sub_w(self, w_repl, w_string, count=0): + w_item, n = self.subx(w_repl, w_string, count) + return w_item + sub_w.unwrap_spec = ['self', W_Root, W_Root, int] - def get_char_ord(self, p): - raise NotImplementedError + def subn_w(self, w_repl, w_string, count=0): + w_item, n = self.subx(w_repl, w_string, count) + space = self.space + return space.newtuple([w_item, space.wrap(n)]) + subn_w.unwrap_spec = ['self', W_Root, W_Root, int] -getset_start = GetSetProperty(W_State.fget_start, W_State.fset_start, cls=W_State) -getset_string_position = GetSetProperty(W_State.fget_string_position, - W_State.fset_string_position, cls=W_State) - -W_State.typedef = TypeDef("W_State", - string = interp_attrproperty_w("w_string", W_State), - start = getset_start, - end = interp_attrproperty("end", W_State), - string_position = getset_string_position, - pos = interp_attrproperty("pos", W_State), - lastindex = interp_attrproperty("lastindex", W_State), - reset = interp2app(W_State.w_reset), - create_regs = interp2app(W_State.w_create_regs), + def subx(self, w_ptemplate, w_string, count): + space = self.space + if space.is_true(space.callable(w_ptemplate)): + w_filter = w_ptemplate + filter_is_callable = True + else: + if space.is_true(space.isinstance(w_ptemplate, space.w_unicode)): + filter_as_unicode = space.unicode_w(w_ptemplate) + literal = u'\\' not in filter_as_unicode + else: + try: + filter_as_string = space.str_w(w_ptemplate) + except OperationError, e: + if e.async(space): + raise + literal = False + else: + literal = '\\' not in filter_as_string + if literal: + w_filter = w_ptemplate + filter_is_callable = False + else: + # not a literal; hand it over to the template compiler + w_re = import_re(space) + w_filter = space.call_method(w_re, '_subx', + space.wrap(self), w_ptemplate) + filter_is_callable = space.is_true(space.callable(w_filter)) + # + ctx = self.make_ctx(w_string) + sublist_w = [] + n = last_pos = 0 + while not count or n < count: + if not searchcontext(space, ctx): + break + if last_pos < ctx.match_start: + sublist_w.append(slice_w(space, ctx, last_pos, + ctx.match_start, space.w_None)) + start = ctx.match_end + if start == ctx.match_start: + start += 1 + nextctx = ctx.fresh_copy(start) + if not (last_pos == ctx.match_start + == ctx.match_end and n > 0): + # the above ignores empty matches on latest position + if filter_is_callable: + w_match = self.getmatch(ctx, True) + w_piece = space.call_function(w_filter, w_match) + if not space.is_w(w_piece, space.w_None): + sublist_w.append(w_piece) + else: + sublist_w.append(w_filter) + last_pos = ctx.match_end + n += 1 + elif last_pos >= ctx.end: + break # empty match at the end: finished + ctx = nextctx + + if last_pos < ctx.end: + sublist_w.append(slice_w(space, ctx, last_pos, ctx.end, + space.w_None)) + if n == 0: + # not just an optimization -- see test_sub_unicode + return w_string, n + + if space.is_true(space.isinstance(w_string, space.w_unicode)): + w_emptystr = space.wrap(u'') + else: + w_emptystr = space.wrap('') + w_item = space.call_method(w_emptystr, 'join', + space.newlist(sublist_w)) + return w_item, n + + +def SRE_Pattern__new__(space, w_subtype, w_pattern, flags, w_code, + groups=0, w_groupindex=None, w_indexgroup=None): + n = space.int_w(space.len(w_code)) + code = [0] * n + for i in range(n): + x = space.uint_w(space.getitem(w_code, space.wrap(i))) + code[i] = intmask(x) + # + w_srepat = space.allocate_instance(W_SRE_Pattern, w_subtype) + srepat = space.interp_w(W_SRE_Pattern, w_srepat) + srepat.space = space + srepat.w_pattern = w_pattern # the original uncompiled pattern + srepat.flags = flags + srepat.code = code + srepat.num_groups = groups + srepat.w_groupindex = w_groupindex + srepat.w_indexgroup = w_indexgroup + return w_srepat +SRE_Pattern__new__.unwrap_spec = [ObjSpace, W_Root, W_Root, int, W_Root, + int, W_Root, W_Root] + + +W_SRE_Pattern.typedef = TypeDef( + 'SRE_Pattern', + __new__ = interp2app(SRE_Pattern__new__), + __copy__ = interp2app(W_SRE_Pattern.cannot_copy_w), + __deepcopy__ = interp2app(W_SRE_Pattern.cannot_copy_w), + __weakref__ = make_weakref_descr(W_SRE_Pattern), + findall = interp2app(W_SRE_Pattern.findall_w), + finditer = interp2app(W_SRE_Pattern.finditer_w), + match = interp2app(W_SRE_Pattern.match_w), + scanner = interp2app(W_SRE_Pattern.finditer_w), # reuse finditer() + search = interp2app(W_SRE_Pattern.search_w), + split = interp2app(W_SRE_Pattern.split_w), + sub = interp2app(W_SRE_Pattern.sub_w), + subn = interp2app(W_SRE_Pattern.subn_w), + flags = interp_attrproperty('flags', W_SRE_Pattern), + groupindex = interp_attrproperty_w('w_groupindex', W_SRE_Pattern), + groups = interp_attrproperty('num_groups', W_SRE_Pattern), + pattern = interp_attrproperty_w('w_pattern', W_SRE_Pattern), ) +# ____________________________________________________________ +# +# SRE_Match class -class W_StringState(W_State): - if THREE_VERSIONS_OF_CORE: - rsre.insert_sre_methods(locals(), 'str') - - def unwrap_object(self): - self.string = self.space.str_w(self.w_string) - return len(self.string) - - def get_char_ord(self, p): - return ord(self.string[p]) +class W_SRE_Match(Wrappable): + flatten_cache = None + def __init__(self, srepat, ctx): + self.space = srepat.space + self.srepat = srepat + self.ctx = ctx -class W_UnicodeState(W_State): - if THREE_VERSIONS_OF_CORE: - rsre.insert_sre_methods(locals(), 'unicode') + def cannot_copy_w(self): + space = self.space + raise OperationError(space.w_TypeError, + space.wrap("cannot copy this match object")) - def unwrap_object(self): - self.unicode = self.space.unicode_w(self.w_string) - return len(self.unicode) + def group_w(self, args_w): + space = self.space + ctx = self.ctx + if len(args_w) <= 1: + if len(args_w) == 0: + start, end = ctx.match_start, ctx.match_end + else: + start, end = self.do_span(args_w[0]) + return slice_w(space, ctx, start, end, space.w_None) + else: + results = [None] * len(args_w) + for i in range(len(args_w)): + start, end = self.do_span(args_w[i]) + results[i] = slice_w(space, ctx, start, end, space.w_None) + return space.newtuple(results) + group_w.unwrap_spec = ['self', 'args_w'] + + def groups_w(self, w_default=None): + fmarks = self.flatten_marks() + num_groups = self.srepat.num_groups + return allgroups_w(self.space, self.ctx, fmarks, num_groups, w_default) - def get_char_ord(self, p): - return ord(self.unicode[p]) + def groupdict_w(self, w_default=None): + space = self.space + w_dict = space.newdict() + w_groupindex = self.srepat.w_groupindex + w_iterator = space.iter(w_groupindex) + while True: + try: + w_key = space.next(w_iterator) + except OperationError, e: + if not e.match(space, space.w_StopIteration): + raise + break # done + w_value = space.getitem(w_groupindex, w_key) + start, end = self.do_span(w_value) + w_grp = slice_w(space, self.ctx, start, end, w_default) + space.setitem(w_dict, w_key, w_grp) + return w_dict + def expand_w(self, w_template): + space = self.space + w_re = import_re(space) + return space.call_method(w_re, '_expand', space.wrap(self.srepat), + space.wrap(self), w_template) + + def start_w(self, w_groupnum=0): + return self.space.wrap(self.do_span(w_groupnum)[0]) + + def end_w(self, w_groupnum=0): + return self.space.wrap(self.do_span(w_groupnum)[1]) + + def span_w(self, w_groupnum=0): + start, end = self.do_span(w_groupnum) + return self.space.newtuple([self.space.wrap(start), + self.space.wrap(end)]) + + def flatten_marks(self): + if self.flatten_cache is None: + num_groups = self.srepat.num_groups + self.flatten_cache = do_flatten_marks(self.ctx, num_groups) + return self.flatten_cache -class W_GenericState(W_State): - if THREE_VERSIONS_OF_CORE: - rsre.insert_sre_methods(locals(), 'generic') + def do_span(self, w_arg): + space = self.space + try: + groupnum = space.int_w(w_arg) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + w_groupnum = space.getitem(self.srepat.w_groupindex, w_arg) + groupnum = space.int_w(w_groupnum) + if groupnum == 0: + return self.ctx.match_start, self.ctx.match_end + elif 1 <= groupnum <= self.srepat.num_groups: + fmarks = self.flatten_marks() + idx = 2*(groupnum-1) + assert idx >= 0 + return fmarks[idx], fmarks[idx+1] + else: + raise OperationError(space.w_IndexError, + space.wrap("group index out of range")) + + def _last_index(self): + mark = self.ctx.match_marks + if mark is not None: + return mark.gid // 2 + 1 + return -1 + + def fget_lastgroup(space, self): + lastindex = self._last_index() + if lastindex < 0: + return space.w_None + w_result = space.finditem(self.srepat.w_indexgroup, + space.wrap(lastindex)) + if w_result is None: + return space.w_None + return w_result + + def fget_lastindex(space, self): + lastindex = self._last_index() + if lastindex >= 0: + return space.wrap(lastindex) + return space.w_None - def unwrap_object(self): - self.buffer = self.space.buffer_w(self.w_string) - return self.buffer.getlength() + def fget_pos(space, self): + return space.wrap(self.ctx.original_pos) - def get_char_ord(self, p): - return ord(self.buffer.getitem(p)) + def fget_endpos(space, self): + return space.wrap(self.ctx.end) + def fget_regs(space, self): + space = self.space + fmarks = self.flatten_marks() + num_groups = self.srepat.num_groups + result_w = [None] * (num_groups + 1) + ctx = self.ctx + result_w[0] = space.newtuple([space.wrap(ctx.match_start), + space.wrap(ctx.match_end)]) + for i in range(num_groups): + result_w[i + 1] = space.newtuple([space.wrap(fmarks[i*2]), + space.wrap(fmarks[i*2+1])]) + return space.newtuple(result_w) + + def fget_string(space, self): + return self.ctx._w_string(space) + + +W_SRE_Match.typedef = TypeDef( + 'SRE_Match', + __copy__ = interp2app(W_SRE_Match.cannot_copy_w), + __deepcopy__ = interp2app(W_SRE_Match.cannot_copy_w), + group = interp2app(W_SRE_Match.group_w), + groups = interp2app(W_SRE_Match.groups_w), + groupdict = interp2app(W_SRE_Match.groupdict_w), + start = interp2app(W_SRE_Match.start_w), + end = interp2app(W_SRE_Match.end_w), + span = interp2app(W_SRE_Match.span_w), + expand = interp2app(W_SRE_Match.expand_w), + # + re = interp_attrproperty('srepat', W_SRE_Match), + string = GetSetProperty(W_SRE_Match.fget_string), + pos = GetSetProperty(W_SRE_Match.fget_pos), + endpos = GetSetProperty(W_SRE_Match.fget_endpos), + lastgroup = GetSetProperty(W_SRE_Match.fget_lastgroup), + lastindex = GetSetProperty(W_SRE_Match.fget_lastindex), + regs = GetSetProperty(W_SRE_Match.fget_regs), +) -def w_search(space, w_state, w_pattern_codes): - state = space.interp_w(W_State, w_state) - pattern_codes = [intmask(space.uint_w(code)) for code - in space.unpackiterable(w_pattern_codes)] - try: - res = state.search(pattern_codes) - except RuntimeError: - raise OperationError(space.w_RuntimeError, - space.wrap("Internal re error")) - return space.newbool(res) - -def w_match(space, w_state, w_pattern_codes): - state = space.interp_w(W_State, w_state) - pattern_codes = [intmask(space.uint_w(code)) for code - in space.unpackiterable(w_pattern_codes)] - try: - res = state.match(pattern_codes) - except RuntimeError: - raise OperationError(space.w_RuntimeError, - space.wrap("Internal re error")) - return space.newbool(res) +# ____________________________________________________________ +# +# SRE_Scanner class +# This is mostly an internal class in CPython. +# Our version is also directly iterable, to make finditer() easier. + +class W_SRE_Scanner(Wrappable): + + def __init__(self, pattern, ctx): + self.space = pattern.space + self.srepat = pattern + self.ctx = ctx + # 'self.ctx' is always a fresh context in which no searching + # or matching succeeded so far. + + def iter_w(self): + return self.space.wrap(self) + + def next_w(self): + if self.ctx.match_start > self.ctx.end: + raise OperationError(self.space.w_StopIteration, self.space.w_None) + if not searchcontext(self.space, self.ctx): + raise OperationError(self.space.w_StopIteration, self.space.w_None) + return self.getmatch(True) + + def match_w(self): + if self.ctx.match_start > self.ctx.end: + return self.space.w_None + return self.getmatch(matchcontext(self.space, self.ctx)) + + def search_w(self): + if self.ctx.match_start > self.ctx.end: + return self.space.w_None + return self.getmatch(searchcontext(self.space, self.ctx)) + + def getmatch(self, found): + if found: + ctx = self.ctx + nextstart = ctx.match_end + nextstart += (ctx.match_start == nextstart) + self.ctx = ctx.fresh_copy(nextstart) + match = W_SRE_Match(self.srepat, ctx) + return self.space.wrap(match) + else: + self.ctx.match_start += 1 # obscure corner case + return None + +W_SRE_Scanner.typedef = TypeDef( + 'SRE_Scanner', + __iter__ = interp2app(W_SRE_Scanner.iter_w, unwrap_spec=['self']), + next = interp2app(W_SRE_Scanner.next_w, unwrap_spec=['self']), + match = interp2app(W_SRE_Scanner.match_w, unwrap_spec=['self']), + search = interp2app(W_SRE_Scanner.search_w, unwrap_spec=['self']), + pattern = interp_attrproperty('srepat', W_SRE_Scanner), +) Modified: pypy/branch/rsre2/pypy/module/_sre/test/test_app_sre.py ============================================================================== --- pypy/branch/rsre2/pypy/module/_sre/test/test_app_sre.py (original) +++ pypy/branch/rsre2/pypy/module/_sre/test/test_app_sre.py Tue Aug 24 11:52:33 2010 @@ -33,8 +33,8 @@ # copy support is disabled by default in _sre.c import re p = re.compile("b") - raises(TypeError, p.__copy__) - raises(TypeError, p.__deepcopy__) + raises(TypeError, p.__copy__) # p.__copy__() should raise + raises(TypeError, p.__deepcopy__) # p.__deepcopy__() should raise def test_creation_attributes(self): import re @@ -85,6 +85,10 @@ assert ['', 'a', None, 'l', 'u', None, 'lla'] == ( re.split("b([ua]|(s))", "balbulla")) + def test_weakref(self): + import re, _weakref + _weakref.ref(re.compile(r"")) + class AppTestSreMatch: @@ -200,6 +204,20 @@ return ret assert ("bbbbb", 3) == re.subn("a", call_me, "ababa") + def test_sub_callable_returns_none(self): + import re + def call_me(match): + return None + assert "acd" == re.sub("b", call_me, "abcd") + + def test_sub_callable_suddenly_unicode(self): + import re + def call_me(match): + if match.group() == 'A': + return unichr(0x3039) + return '' + assert (u"bb\u3039b", 2) == re.subn("[aA]", call_me, "babAb") + def test_match_array(self): import re, array a = array.array('c', 'hello') @@ -663,69 +681,6 @@ s.ATCODES["at_uni_non_boundary"], s.OPCODES["success"]] s.assert_match(opcodes, ["blaha", u"bl%sja" % UPPER_PI]) - def test_category_digit(self): - INDIAN_DIGIT = u"\u0966" - opcodes = [s.OPCODES["category"], s.CHCODES["category_digit"]] \ - + s.encode_literal("b") + [s.OPCODES["success"]] - s.assert_match(opcodes, ["1b", "a1b"]) - s.assert_no_match(opcodes, ["bb", "b1", u"%sb" % INDIAN_DIGIT]) - - def test_category_not_digit(self): - INDIAN_DIGIT = u"\u0966" - opcodes = [s.OPCODES["category"], s.CHCODES["category_not_digit"]] \ - + s.encode_literal("b") + [s.OPCODES["success"]] - s.assert_match(opcodes, ["bb", "1ab", u"%sb" % INDIAN_DIGIT]) - s.assert_no_match(opcodes, ["1b", "a1b"]) - - def test_category_space(self): - EM_SPACE = u"\u2001" - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_space"], s.OPCODES["success"]] - s.assert_match(opcodes, ["b ", "b\n", "b\t", "b\r", "b\v", "b\f"]) - s.assert_no_match(opcodes, ["bb", "b1", u"b%s" % EM_SPACE]) - - def test_category_not_space(self): - EM_SPACE = u"\u2001" - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_not_space"], s.OPCODES["success"]] - s.assert_match(opcodes, ["bb", "b1", u"b%s" % EM_SPACE]) - s.assert_no_match(opcodes, ["b ", "b\n", "b\t", "b\r", "b\v", "b\f"]) - - def test_category_word(self): - LOWER_PI = u"\u03c0" - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_word"], s.OPCODES["success"]] - s.assert_match(opcodes, ["bl", "b4", "b_"]) - s.assert_no_match(opcodes, ["b ", "b\n", u"b%s" % LOWER_PI]) - - def test_category_not_word(self): - LOWER_PI = u"\u03c0" - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_not_word"], s.OPCODES["success"]] - s.assert_match(opcodes, ["b ", "b\n", u"b%s" % LOWER_PI]) - s.assert_no_match(opcodes, ["bl", "b4", "b_"]) - - def test_category_linebreak(self): - LINE_SEP = u"\u2028" - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_linebreak"], s.OPCODES["success"]] - s.assert_match(opcodes, ["b\n"]) - s.assert_no_match(opcodes, ["b ", "bs", "b\r", u"b%s" % LINE_SEP]) - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_uni_linebreak"], s.OPCODES["success"]] - s.assert_match(opcodes, ["b\n", u"b%s" % LINE_SEP]) - - def test_category_not_linebreak(self): - LINE_SEP = u"\u2028" - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_not_linebreak"], s.OPCODES["success"]] - s.assert_match(opcodes, ["b ", "bs", u"b%s" % LINE_SEP]) - s.assert_no_match(opcodes, ["b\n"]) - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_uni_not_linebreak"], s.OPCODES["success"]] - s.assert_match(opcodes, ["b ", "bs"]) - s.assert_no_match(opcodes, ["b\n", u"b%s" % LINE_SEP, "b\r"]) - def test_category_loc_word(self): import locale try: @@ -883,10 +838,6 @@ s.assert_match(opcodes, ["ab", "aaaab", "baabb"]) s.assert_no_match(opcodes, ["aaa", "", "ac"]) - def test_max_until_error(self): - opcodes = [s.OPCODES["max_until"], s.OPCODES["success"]] - raises(RuntimeError, s.search, opcodes, "a") - def test_max_until_zero_width_match(self): # re.compile won't compile prospective zero-with matches (all of them?), # so we can only produce an example by directly constructing bytecodes. @@ -906,10 +857,6 @@ s.assert_no_match(opcodes, ["b"]) assert "aab" == s.search(opcodes, "aabb").group(0) - def test_min_until_error(self): - opcodes = [s.OPCODES["min_until"], s.OPCODES["success"]] - raises(RuntimeError, s.search, opcodes, "a") - def test_groupref(self): opcodes = [s.OPCODES["mark"], 0, s.OPCODES["any"], s.OPCODES["mark"], 1] \ + s.encode_literal("a") + [s.OPCODES["groupref"], 0, s.OPCODES["success"]] Modified: pypy/branch/rsre2/pypy/rlib/debug.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/debug.py (original) +++ pypy/branch/rsre2/pypy/rlib/debug.py Tue Aug 24 11:52:33 2010 @@ -255,7 +255,16 @@ class IntegerCanBeNegative(Exception): pass +class UnexpectedRUInt(Exception): + pass + def check_nonneg(x): + """Give a translation-time error if 'x' is not known to be non-negative. + To help debugging, this also gives a translation-time error if 'x' is + actually typed as an r_uint (in which case the call to check_nonneg() + is a bit strange and probably unexpected). + """ + assert type(x)(-1) < 0 # otherwise, 'x' is a r_uint or similar assert x >= 0 return x @@ -264,6 +273,9 @@ def compute_result_annotation(self, s_arg): from pypy.annotation.model import SomeInteger + if isinstance(s_arg, SomeInteger) and s_arg.unsigned: + raise UnexpectedRUInt("check_nonneg() arg is a %s" % ( + s_arg.knowntype,)) s_nonneg = SomeInteger(nonneg=True) if not s_nonneg.contains(s_arg): raise IntegerCanBeNegative Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre_char.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre_char.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre_char.py Tue Aug 24 11:52:33 2010 @@ -6,18 +6,22 @@ from pypy.rlib.unroll import unrolling_iterable # Note: the unicode parts of this module require you to call -# rsre.set_unicode_db() first, to select one of the modules +# rsre_char.set_unicode_db() first, to select one of the modules # pypy.module.unicodedata.unicodedb_x_y_z. This allows PyPy to use sre # with the same version of the unicodedb as it uses for # unicodeobject.py. If unset, the RPython program cannot use unicode # matching. -unicodedb = None # possibly patched by rsre.set_unicode_db() +unicodedb = None # possibly patched by set_unicode_db() + +def set_unicode_db(newunicodedb): + global unicodedb + unicodedb = newunicodedb #### Constants -# Identifying as _sre from Python 2.3 or 2.4 +# Identifying as _sre from Python 2.3 and onwards (at least up to 2.7) MAGIC = 20031017 # In _sre.c this is bytesize of the code word type of the C implementation. @@ -58,22 +62,6 @@ return char_ord -class MatchContextBase(object): - - UNDECIDED = 0 - MATCHED = 1 - NOT_MATCHED = 2 - - def peek_code(self, peek=0): - return self.pattern_codes[self.code_position + peek] - - def skip_code(self, skip_count): - self.code_position = self.code_position + skip_count - - def has_remaining_codes(self): - return len(self.pattern_codes) != self.code_position - - #### Category helpers ascii_char_info = [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 2, Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre_core.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre_core.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre_core.py Tue Aug 24 11:52:33 2010 @@ -3,6 +3,7 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rsre import rsre_char from pypy.tool.sourcetools import func_with_new_name +from pypy.rlib.objectmodel import we_are_translated OPCODE_FAILURE = 0 @@ -14,7 +15,7 @@ OPCODE_AT = 6 OPCODE_BRANCH = 7 #OPCODE_CALL = 8 -#OPCODE_CATEGORY = 9 +OPCODE_CATEGORY = 9 #OPCODE_CHARSET = 10 #OPCODE_BIGCHARSET = 11 OPCODE_GROUPREF = 12 @@ -40,6 +41,8 @@ # ____________________________________________________________ +_seen_specname = {} + def specializectx(func): """A decorator that specializes 'func(ctx,...)' for each concrete subclass of AbstractMatchContext. During annotation, if 'ctx' is known to be a @@ -48,11 +51,15 @@ """ assert func.func_code.co_varnames[0] == 'ctx' specname = '_spec_' + func.func_name + while specname in _seen_specname: + specname += '_' + _seen_specname[specname] = True # Install a copy of the function under the name '_spec_funcname' in each # concrete subclass for prefix, concreteclass in [('str', StrMatchContext), ('uni', UnicodeMatchContext)]: newfunc = func_with_new_name(func, prefix + specname) + assert not hasattr(concreteclass, specname) setattr(concreteclass, specname, newfunc) # Return a dispatcher function, specialized on the exact type of 'ctx' def dispatch(ctx, *args): @@ -62,6 +69,10 @@ # ____________________________________________________________ +class Error(Exception): + def __init__(self, msg): + self.msg = msg + class AbstractMatchContext(object): """Abstract base class""" match_start = 0 @@ -70,17 +81,29 @@ match_marks_flat = None def __init__(self, pattern, match_start, end, flags): - # here, 'end' must be at most len(string) + # 'match_start' and 'end' must be known to be non-negative + # and they must not be more than len(string). + check_nonneg(match_start) + check_nonneg(end) self.pattern = pattern - if match_start < 0: - match_start = 0 self.match_start = match_start self.end = end self.flags = flags + def reset(self, start): + self.match_start = start + self.match_marks = None + self.match_marks_flat = None + def pat(self, index): check_nonneg(index) - return self.pattern[index] + result = self.pattern[index] + # Check that we only return non-negative integers from this helper. + # It is possible that self.pattern contains negative integers + # (see set_charset() and set_bigcharset() in rsre_char.py) + # but they should not be fetched via this helper here. + assert result >= 0 + return result def str(self, index): """NOT_RPYTHON: Must be overridden in a concrete subclass. @@ -132,12 +155,13 @@ else: return None + def fresh_copy(self, start): + raise NotImplementedError + class StrMatchContext(AbstractMatchContext): """Concrete subclass for matching in a plain string.""" def __init__(self, pattern, string, match_start, end, flags): - if end > len(string): - end = len(string) AbstractMatchContext.__init__(self, pattern, match_start, end, flags) self._string = string @@ -149,12 +173,14 @@ c = self.str(index) return rsre_char.getlower(c, self.flags) + def fresh_copy(self, start): + return StrMatchContext(self.pattern, self._string, start, + self.end, self.flags) + class UnicodeMatchContext(AbstractMatchContext): """Concrete subclass for matching in a unicode string.""" def __init__(self, pattern, unicodestr, match_start, end, flags): - if end > len(unicodestr): - end = len(unicodestr) AbstractMatchContext.__init__(self, pattern, match_start, end, flags) self._unicodestr = unicodestr @@ -166,6 +192,10 @@ c = self.str(index) return rsre_char.getlower(c, self.flags) + def fresh_copy(self, start): + return UnicodeMatchContext(self.pattern, self._unicodestr, start, + self.end, self.flags) + # ____________________________________________________________ class Mark(object): @@ -275,7 +305,7 @@ for op1, (checkerfn, _) in unroll_char_checker: if op1 == op: return checkerfn(ctx, ptr, ppos) - raise NotImplementedError("next_char_ok[%d]" % op) + raise Error("next_char_ok[%d]" % op) class AbstractUntilMatchResult(MatchResult): @@ -311,32 +341,33 @@ marks = self.cur_marks while True: while True: - if enum is not None: - # matched one more 'item'. record it and continue + if (enum is not None and + (ptr != ctx.match_end or self.num_pending < min)): + # ^^^^^^^^^^ zero-width match protection + # matched one more 'item'. record it and continue. self.pending = Pending(ptr, marks, enum, self.pending) self.num_pending += 1 ptr = ctx.match_end marks = ctx.match_marks break - else: - # 'item' no longer matches. - if not resume and self.num_pending >= min: - # try to match 'tail' if we have enough 'item' - result = sre_match(ctx, self.tailppos, ptr, marks) - if result is not None: - self.subresult = result - self.cur_ptr = ptr - self.cur_marks = marks - return self - resume = False - p = self.pending - if p is None: - return - self.pending = p.next - self.num_pending -= 1 - ptr = p.ptr - marks = p.marks - enum = p.enum.move_to_next_result(ctx) + # 'item' no longer matches. + if not resume and self.num_pending >= min: + # try to match 'tail' if we have enough 'item' + result = sre_match(ctx, self.tailppos, ptr, marks) + if result is not None: + self.subresult = result + self.cur_ptr = ptr + self.cur_marks = marks + return self + resume = False + p = self.pending + if p is None: + return + self.pending = p.next + self.num_pending -= 1 + ptr = p.ptr + marks = p.marks + enum = p.enum.move_to_next_result(ctx) # if max == 65535 or self.num_pending < max: # try to match one more 'item' @@ -375,7 +406,9 @@ else: enum = None # 'max' reached, no more matches - while enum is None: + while (enum is None or + (ptr == ctx.match_end and self.num_pending >= min)): + # ^^^^^^^^^^ zero-width match protection # 'item' does not match; try to get further results from # the 'pending' list. p = self.pending @@ -459,8 +492,15 @@ result = BranchMatchResult(ppos, ptr, marks) return result.find_first_result(ctx) - #elif op == OPCODE_CATEGORY: - # seems to be never produced + elif op == OPCODE_CATEGORY: + # seems to be never produced, but used by some tests from + # pypy/module/_sre/test + # + if (ptr == ctx.end or + not rsre_char.category_dispatch(ctx.pat(ppos), ctx.str(ptr))): + return + ptr += 1 + ppos += 1 elif op == OPCODE_GROUPREF: # match backreference @@ -588,7 +628,7 @@ return result.find_first_result(ctx) else: - raise AssertionError("missing UNTIL after REPEAT") + raise Error("missing UNTIL after REPEAT") elif op == OPCODE_REPEAT_ONE: # match repeated sequence (maximizing regexp). @@ -639,7 +679,7 @@ return result.find_first_result(ctx) else: - assert 0, "bad pattern code %d" % op + raise Error("bad pattern code %d" % op) def get_group_ref(marks, groupnum): @@ -681,7 +721,7 @@ for op1, (_, fre) in unroll_char_checker: if op1 == op: return fre(ctx, ptr, end, ppos) - raise NotImplementedError("rsre.find_repetition_end[%d]" % op) + raise Error("rsre.find_repetition_end[%d]" % op) @specializectx def match_ANY(ctx, ptr, ppos): # dot wildcard. @@ -810,24 +850,39 @@ # ____________________________________________________________ +def _adjust(start, end, length): + if start < 0: start = 0 + elif start > length: start = length + if end < 0: end = 0 + elif end > length: end = length + return start, end + def match(pattern, string, start=0, end=sys.maxint, flags=0): + start, end = _adjust(start, end, len(string)) ctx = StrMatchContext(pattern, string, start, end, flags) - return match_context(ctx) + if match_context(ctx): + return ctx + else: + return None def search(pattern, string, start=0, end=sys.maxint, flags=0): + start, end = _adjust(start, end, len(string)) ctx = StrMatchContext(pattern, string, start, end, flags) - return search_context(ctx) + if search_context(ctx): + return ctx + else: + return None def match_context(ctx): + ctx.original_pos = ctx.match_start if ctx.end < ctx.match_start: - return None - if sre_match(ctx, 0, ctx.match_start, None) is not None: - return ctx - return None + return False + return sre_match(ctx, 0, ctx.match_start, None) is not None def search_context(ctx): + ctx.original_pos = ctx.match_start if ctx.end < ctx.match_start: - return None + return False if ctx.pat(0) == OPCODE_INFO: if ctx.pat(2) & rsre_char.SRE_INFO_PREFIX and ctx.pat(5) > 1: return fast_search(ctx) @@ -838,9 +893,9 @@ while start <= ctx.end: if sre_match(ctx, 0, start, None) is not None: ctx.match_start = start - return ctx + return True start += 1 - return None + return False @specializectx def fast_search(ctx): @@ -881,11 +936,11 @@ ctx.match_start = start ctx.match_end = ptr ctx.match_marks = None - return ctx + return True if sre_match(ctx, ppos_start, ptr, None) is not None: ctx.match_start = start - return ctx + return True i = ctx.pat(overlap_offset + i) break string_position += 1 - return None + return False Modified: pypy/branch/rsre2/pypy/rlib/rsre/rsre_re.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/rsre_re.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/rsre_re.py Tue Aug 24 11:52:33 2010 @@ -1,3 +1,7 @@ +""" +Testing code. This is not used in a PyPy translation. +It exports the same interface as the Python 're' module. +""" import re, sys from pypy.rlib.rsre import rsre_core, rsre_char from pypy.rlib.rsre.test.test_match import get_code as _get_code @@ -5,12 +9,12 @@ rsre_char.unicodedb = unicodedb_3_2_0 -I = IGNORECASE = 2 # ignore case -L = LOCALE = 4 # assume current 8-bit locale -U = UNICODE = 32 # assume unicode locale -M = MULTILINE = 8 # make anchors look for newline -S = DOTALL = 16 # make dot match newline -X = VERBOSE = 64 # ignore whitespace and comments +I = IGNORECASE = re.I # ignore case +L = LOCALE = re.L # assume current 8-bit locale +U = UNICODE = re.U # assume unicode locale +M = MULTILINE = re.M # make anchors look for newline +S = DOTALL = re.S # make dot match newline +X = VERBOSE = re.X # ignore whitespace and comments def match(pattern, string, flags=0): @@ -25,6 +29,15 @@ def finditer(pattern, string, flags=0): return compile(pattern, flags).finditer(string) +def sub(pattern, repl, string, count=0): + return compile(pattern).sub(repl, string, count) + +def subn(pattern, repl, string, count=0): + return compile(pattern).subn(repl, string, count) + +def split(pattern, string, maxsplit=0): + return compile(pattern).split(string, maxsplit) + def compile(pattern, flags=0): code, flags, args = _get_code(pattern, flags, allargs=True) return RSREPattern(pattern, code, flags, *args) @@ -54,9 +67,9 @@ pos, endpos, flags=self.flags)) - def findall(self, string): + def findall(self, string, pos=0, endpos=sys.maxint): matchlist = [] - for match in self.finditer(string): + for match in self.finditer(string, pos, endpos): if self.groups == 0 or self.groups == 1: item = match.group(self.groups) else: @@ -64,22 +77,87 @@ matchlist.append(item) return matchlist - def finditer(self, string): - matchlist = [] + def finditer(self, string, pos=0, endpos=sys.maxint): + return iter(self.scanner(string, pos, endpos).search, None) + + def subn(self, repl, string, count=0): + filter = repl + if not callable(repl) and "\\" in repl: + # handle non-literal strings; hand it over to the template compiler + filter = re._subx(self, repl) start = 0 - while True: + sublist = [] + force_unicode = (isinstance(string, unicode) or + isinstance(repl, unicode)) + n = last_pos = 0 + while not count or n < count: match = rsre_core.search(self._code, string, start, flags=self.flags) if match is None: break - end = match.match_end - yield RSREMatch(self, match) - if start == end: + if last_pos < match.match_start: + sublist.append(string[last_pos:match.match_start]) + if not (last_pos == match.match_start + == match.match_end and n > 0): + # the above ignores empty matches on latest position + if callable(filter): + piece = filter(self._make_match(match)) + else: + piece = filter + sublist.append(piece) + last_pos = match.match_end + n += 1 + elif last_pos >= len(string): + break # empty match at the end: finished + # + start = match.match_end + if start == match.match_start: start += 1 - if start > len(string): + + if last_pos < len(string): + sublist.append(string[last_pos:]) + + if n == 0: + # not just an optimization -- see test_sub_unicode + return string, n + + if force_unicode: + item = u"".join(sublist) + else: + item = "".join(sublist) + return item, n + + def sub(self, repl, string, count=0): + item, n = self.subn(repl, string, count) + return item + + def split(self, string, maxsplit=0): + splitlist = [] + start = 0 + n = 0 + last = 0 + while not maxsplit or n < maxsplit: + match = rsre_core.search(self._code, string, start, + flags=self.flags) + if match is None: + break + if match.match_start == match.match_end: # zero-width match + if match.match_start == len(string): # at end of string break - else: - start = end + start = match.match_end + 1 + continue + splitlist.append(string[last:match.match_start]) + # add groups (if any) + if self.groups: + match1 = self._make_match(match) + splitlist.extend(match1.groups(None)) + n += 1 + last = start = match.match_end + splitlist.append(string[last:]) + return splitlist + + def scanner(self, string, start=0, end=sys.maxint): + return SREScanner(self, string, start, end) def _make_match(self, res): if res is None: @@ -169,3 +247,67 @@ @property def endpos(self): return self._ctx.end + + +class SREScanner(object): + def __init__(self, pattern, string, start, end): + self.pattern = pattern + self._string = string + self._start = start + self._end = end + + def _match_search(self, matcher): + if self._start > len(self._string): + return None + match = matcher(self._string, self._start, self._end) + if match is None: + self._start += 1 # obscure corner case + else: + self._start = match.end() + if match.start() == self._start: + self._start += 1 + return match + + def match(self): + return self._match_search(self.pattern.match) + + def search(self): + return self._match_search(self.pattern.search) + +class Scanner: + # This class is copied directly from re.py. + def __init__(self, lexicon, flags=0): + from sre_constants import BRANCH, SUBPATTERN + import sre_parse + self.lexicon = lexicon + # combine phrases into a compound pattern + p = [] + s = sre_parse.Pattern() + s.flags = flags + for phrase, action in lexicon: + p.append(sre_parse.SubPattern(s, [ + (SUBPATTERN, (len(p)+1, sre_parse.parse(phrase, flags))), + ])) + s.groups = len(p)+1 + p = sre_parse.SubPattern(s, [(BRANCH, (None, p))]) + self.scanner = compile(p) + def scan(self, string): + result = [] + append = result.append + match = self.scanner.scanner(string).match + i = 0 + while 1: + m = match() + if not m: + break + j = m.end() + if i == j: + break + action = self.lexicon[m.lastindex-1][1] + if callable(action): + self.match = m + action = action(self, m.group()) + if action is not None: + append(action) + i = j + return result, string[i:] Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_re.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/test_re.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_re.py Tue Aug 24 11:52:33 2010 @@ -21,7 +21,7 @@ int_value = int(matchobj.group(0)) return str(int_value + 1) - def XXXtest_basic_re_sub(self): + def test_basic_re_sub(self): assert re.sub("(?i)b+", "x", "bbbb BBBB") == 'x x' assert re.sub(r'\d+', self.bump_num, '08.2 -2 23x99y') == ( '9.3 -3 24x100y') @@ -49,12 +49,12 @@ assert re.sub('^\s*', 'X', 'test') == 'Xtest' - def XXXtest_bug_449964(self): + def test_bug_449964(self): # fails for group followed by other escape assert re.sub(r'(?Px)', '\g<1>\g<1>\\b', 'xx') == ( 'xx\bxx\b') - def XXXtest_bug_449000(self): + def test_bug_449000(self): # Test for sub() on escaped characters assert re.sub(r'\r\n', r'\n', 'abc\r\ndef\r\n') == ( 'abc\ndef\n') @@ -65,7 +65,7 @@ assert re.sub('\r\n', '\n', 'abc\r\ndef\r\n') == ( 'abc\ndef\n') - def XXXtest_bug_1140(self): + def test_bug_1140(self): # re.sub(x, y, u'') should return u'', not '', and # re.sub(x, y, '') should return '', not u''. # Also: @@ -90,7 +90,7 @@ assert z == y assert type(z) == type(y) - def XXXtest_sub_template_numeric_escape(self): + def test_sub_template_numeric_escape(self): # bug 776311 and friends assert re.sub('x', r'\0', 'x') == '\0' assert re.sub('x', r'\000', 'x') == '\000' @@ -132,20 +132,20 @@ assert re.sub('((((((((((y))))))))))(.)', r'\11a', 'xyz') == ( 'xza') - def XXXtest_qualified_re_sub(self): + def test_qualified_re_sub(self): assert re.sub('a', 'b', 'aaaaa') == 'bbbbb' assert re.sub('a', 'b', 'aaaaa', 1) == 'baaaa' - def XXXtest_bug_114660(self): + def test_bug_114660(self): assert re.sub(r'(\S)\s+(\S)', r'\1 \2', 'hello there') == ( 'hello there') - def XXXtest_bug_462270(self): + def test_bug_462270(self): # Test for empty sub() behaviour, see SF bug #462270 assert re.sub('x*', '-', 'abxd') == '-a-b-d-' assert re.sub('x+', '-', 'abxd') == 'ab-d' - def XXXtest_symbolic_refs(self): + def test_symbolic_refs(self): raises(re.error, re.sub, '(?Px)', '\gx)', '\g<', 'xx') raises(re.error, re.sub, '(?Px)', '\g', 'xx') @@ -156,14 +156,14 @@ raises(re.error, re.sub, '(?Px)|(?Py)', '\\2', 'xx') raises(re.error, re.sub, '(?Px)', '\g<-1>', 'xx') - def XXXtest_re_subn(self): + def test_re_subn(self): assert re.subn("(?i)b+", "x", "bbbb BBBB") == ('x x', 2) assert re.subn("b+", "x", "bbbb BBBB") == ('x BBBB', 1) assert re.subn("b+", "x", "xyz") == ('xyz', 0) assert re.subn("b*", "x", "xyz") == ('xxxyxzx', 4) assert re.subn("b*", "x", "xyz", 2) == ('xxxyz', 2) - def XXXtest_re_split(self): + def test_re_split(self): assert re.split(":", ":a:b::c") == ['', 'a', 'b', '', 'c'] assert re.split(":*", ":a:b::c") == ['', 'a', 'b', 'c'] assert re.split("(:*)", ":a:b::c") == ( @@ -179,7 +179,7 @@ assert re.split("(?:b)|(?::+)", ":a:b::c") == ( ['', 'a', '', '', 'c']) - def XXXtest_qualified_re_split(self): + def test_qualified_re_split(self): assert re.split(":", ":a:b::c", 2) == ['', 'a', 'b::c'] assert re.split(':', 'a:b:c:d', 2) == ['a', 'b', 'c:d'] assert re.split("(:)", ":a:b::c", 2) == ( @@ -512,13 +512,13 @@ assert re.match('(x)*y', 50000*'x'+'y').group(1) == 'x' assert re.match('(x)*?y', 50000*'x'+'y').group(1) == 'x' - def XXXtest_scanner(self): + def test_scanner(self): def s_ident(scanner, token): return token def s_operator(scanner, token): return "op%s" % token def s_float(scanner, token): return float(token) def s_int(scanner, token): return int(token) - scanner = Scanner([ + scanner = re.Scanner([ (r"[a-zA-Z_]\w*", s_ident), (r"\d+\.\d*", s_float), (r"\d+", s_int), @@ -590,7 +590,7 @@ assert (re.compile('bug_926075') is not re.compile(eval("u'bug_926075'"))) - def XXXtest_bug_931848(self): + def test_bug_931848(self): try: unicode except NameError: Modified: pypy/branch/rsre2/pypy/rlib/rsre/test/test_search.py ============================================================================== --- pypy/branch/rsre2/pypy/rlib/rsre/test/test_search.py (original) +++ pypy/branch/rsre2/pypy/rlib/rsre/test/test_search.py Tue Aug 24 11:52:33 2010 @@ -1,5 +1,5 @@ from pypy.rlib.rsre import rsre_core -from pypy.rlib.rsre.test.test_match import get_code +from pypy.rlib.rsre.test.test_match import get_code, get_code_and_re class TestSearch: @@ -23,3 +23,174 @@ res = rsre_core.search(r_code3, "foo bar foobar baz") assert res is not None assert res.span() == (8, 14) + + def test_code3(self): + r_code1 = get_code(r'\s*(.*?)') + res = rsre_core.match(r_code1, " abcdef") + assert res is not None + + def test_max_until_0_65535(self): + r_code2 = get_code(r'(?:xy)*xy') + #res = rsre_core.match(r_code2, 'def') + #assert res is None + #res = rsre_core.match(r_code2, 'xydef') + #assert res is not None + res = rsre_core.match(r_code2, 'xyxyxydef') + assert res is not None + res = rsre_core.match(r_code2, '' + 'xy'*1000 + 'def') + assert res is not None + + def test_max_until_3_5(self): + r_code2, r = get_code_and_re(r'(?:xy){3,5}xy') + for i in range(8): + s = '' + 'xy'*i + 'defdefdefdefdef' + assert (r.match(s) is not None) is (3 <= i-1 <= 5) + res = rsre_core.match(r_code2, s) + assert (res is not None) is (3 <= i-1 <= 5) + + def test_min_until_0_65535(self): + r_code2 = get_code(r'(?:xy)*?xy') + res = rsre_core.match(r_code2, 'def') + assert res is None + res = rsre_core.match(r_code2, 'xydef') + assert res is not None + res = rsre_core.match(r_code2, 'xyxyxydef') + assert res is not None + res = rsre_core.match(r_code2, '' + 'xy'*1000 + 'def') + assert res is not None + + def test_min_until_3_5(self): + r_code2, r = get_code_and_re(r'(?:xy){3,5}?xy') + for i in range(8): + s = '' + 'xy'*i + 'defdefdefdefdef' + assert (r.match(s) is not None) is (3 <= i-1 <= 5) + res = rsre_core.match(r_code2, s) + assert (res is not None) is (3 <= i-1 <= 5) + + def test_min_repeat_one(self): + r_code3 = get_code(r'.{3,5}?y') + for i in range(8): + res = rsre_core.match(r_code3, '' + 'x'*i + 'y') + assert (res is not None) is (3 <= i <= 5) + + def test_simple_group(self): + r_code4 = get_code(r'(x.)') + res = rsre_core.match(r_code4, 'xadef') + assert res is not None + assert res.get_mark(0) == 5 + assert res.get_mark(1) == 7 + + def test_max_until_groups(self): + r_code4 = get_code(r'(x.)*xy') + res = rsre_core.match(r_code4, 'xaxbxydef') + assert res is not None + assert res.get_mark(0) == 7 + assert res.get_mark(1) == 9 + + def test_group_branch(self): + r_code5 = get_code(r'(ab|c)') + res = rsre_core.match(r_code5, 'abdef') + assert (res.get_mark(0), res.get_mark(1)) == (5, 7) + res = rsre_core.match(r_code5, 'cdef') + assert (res.get_mark(0), res.get_mark(1)) == (5, 6) + res = rsre_core.match(r_code5, 'dedef') + assert res is None + + def test_group_branch_max_until(self): + r_code6 = get_code(r'(ab|c)*a') + res = rsre_core.match(r_code6, 'ccabcccabadef') + assert (res.get_mark(0), res.get_mark(1)) == (12, 14) + r_code7 = get_code(r'((ab)|(c))*a') + res = rsre_core.match(r_code7, 'ccabcccabadef') + assert (res.get_mark(0), res.get_mark(1)) == (12, 14) + assert (res.get_mark(2), res.get_mark(3)) == (12, 14) + assert (res.get_mark(4), res.get_mark(5)) == (11, 12) + + def test_group_7(self): + r_code7, r7 = get_code_and_re(r'((a)?(b))*') + match = r7.match('bbbabbbb') + assert match.span(1) == (12, 13) + assert match.span(3) == (12, 13) + assert match.span(2) == (8, 9) + res = rsre_core.match(r_code7, 'bbbabbbb') + assert (res.get_mark(0), res.get_mark(1)) == (12, 13) + assert (res.get_mark(4), res.get_mark(5)) == (12, 13) + assert (res.get_mark(2), res.get_mark(3)) == (8, 9) + + def test_group_branch_repeat_complex_case(self): + r_code8, r8 = get_code_and_re(r'((a)|(b))*') + match = r8.match('ab') + assert match.span(1) == (6, 7) + assert match.span(3) == (6, 7) + assert match.span(2) == (5, 6) + res = rsre_core.match(r_code8, 'ab') + assert (res.get_mark(0), res.get_mark(1)) == (6, 7) + assert (res.get_mark(4), res.get_mark(5)) == (6, 7) + assert (res.get_mark(2), res.get_mark(3)) == (5, 6) + + def test_minuntil_lastmark_restore(self): + r_code9, r9 = get_code_and_re(r'(x|yz)+?(y)??c') + match = r9.match('xyzxc') + assert match.span(1) == (3, 4) + assert match.span(2) == (-1, -1) + res = rsre_core.match(r_code9, 'xyzxc') + assert (res.get_mark(0), res.get_mark(1)) == (3, 4) + assert (res.get_mark(2), res.get_mark(3)) == (-1, -1) + + def test_minuntil_bug(self): + r_code9, r9 = get_code_and_re(r'((x|yz)+?(y)??c)*') + match = r9.match('xycxyzxc') + assert match.span(2) == (6, 7) + #assert match.span(3) == (1, 2) --- bug of CPython + res = rsre_core.match(r_code9, 'xycxyzxc') + assert (res.get_mark(2), res.get_mark(3)) == (6, 7) + assert (res.get_mark(4), res.get_mark(5)) == (1, 2) + + def test_empty_maxuntil(self): + r_code, r = get_code_and_re(r'(a?)+y') + assert r.match('y') + res = rsre_core.match(r_code, 'y') + assert res + # + r_code, r = get_code_and_re(r'(a?){4,6}y') + assert r.match('y') + res = rsre_core.match(r_code, 'y') + assert res + # + r_code, r = get_code_and_re(r'(a?)*y') + assert r.match('y') + res = rsre_core.match(r_code, 'y') + assert res + + def test_empty_minuntil(self): + r_code, r = get_code_and_re(r'(a?)+?y') + #assert not r.match('z') -- CPython bug (at least 2.5) eats all memory + res = rsre_core.match(r_code, 'z') + assert not res + # + r_code, r = get_code_and_re(r'(a?){4,6}?y') + assert not r.match('z') + res = rsre_core.match(r_code, 'z') + assert not res + # + r_code, r = get_code_and_re(r'(a?)*?y') + #assert not r.match('z') -- CPython bug (at least 2.5) eats all memory + res = rsre_core.match(r_code, 'z') + assert not res + + def test_empty_search(self): + r_code, r = get_code_and_re(r'') + for j in range(-2, 6): + for i in range(-2, 6): + match = r.search('abc', i, j) + res = rsre_core.search(r_code, 'abc', i, j) + jk = min(max(j, 0), 3) + ik = min(max(i, 0), 3) + if ik <= jk: + assert match is not None + assert match.span() == (ik, ik) + assert res is not None + assert res.match_start == ik and res.match_end == ik + else: + assert match is None + assert res is None From arigo at codespeak.net Tue Aug 24 13:31:24 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 24 Aug 2010 13:31:24 +0200 (CEST) Subject: [pypy-svn] r76708 - in pypy/branch/rsre2/pypy/rpython: . test Message-ID: <20100824113124.C0B8E282B90@codespeak.net> Author: arigo Date: Tue Aug 24 13:31:23 2010 New Revision: 76708 Modified: pypy/branch/rsre2/pypy/rpython/rclass.py pypy/branch/rsre2/pypy/rpython/test/test_rclass.py Log: Test and fix for classes that have _immutable_fields_=[...] but not their parent. Modified: pypy/branch/rsre2/pypy/rpython/rclass.py ============================================================================== --- pypy/branch/rsre2/pypy/rpython/rclass.py (original) +++ pypy/branch/rsre2/pypy/rpython/rclass.py Tue Aug 24 13:31:23 2010 @@ -156,14 +156,13 @@ if '_immutable_' in self.classdef.classdesc.classdict: hints = hints.copy() hints['immutable'] = True + self.immutable_field_list = [] # unless overwritten below if self.classdef.classdesc.lookup('_immutable_fields_') is not None: hints = hints.copy() immutable_fields = self.classdef.classdesc.classdict.get( '_immutable_fields_') if immutable_fields is not None: self.immutable_field_list = immutable_fields.value - else: - self.immutable_field_list = [] accessor = FieldListAccessor() hints['immutable_fields'] = accessor return hints Modified: pypy/branch/rsre2/pypy/rpython/test/test_rclass.py ============================================================================== --- pypy/branch/rsre2/pypy/rpython/test/test_rclass.py (original) +++ pypy/branch/rsre2/pypy/rpython/test/test_rclass.py Tue Aug 24 13:31:23 2010 @@ -777,6 +777,25 @@ assert accessor.fields == {"inst_x" : "", "inst_y" : ""} or \ accessor.fields == {"ox" : "", "oy" : ""} # for ootype + def test_immutable_fields_only_in_subclass(self): + from pypy.jit.metainterp.typesystem import deref + class A(object): + def __init__(self, x): + self.x = x + class B(A): + _immutable_fields_ = ["y"] + def __init__(self, x, y): + A.__init__(self, x) + self.y = y + + def f(): + return B(3, 5) + t, typer, graph = self.gengraph(f, []) + B_TYPE = deref(graph.getreturnvar().concretetype) + accessor = B_TYPE._hints["immutable_fields"] + assert accessor.fields == {"inst_y" : ""} or \ + accessor.fields == {"oy" : ""} # for ootype + def test_immutable_inheritance(self): class I(object): def __init__(self, v): From afa at codespeak.net Tue Aug 24 16:17:10 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 24 Aug 2010 16:17:10 +0200 (CEST) Subject: [pypy-svn] r76709 - in pypy/trunk: lib_pypy pypy/module/test_lib_pypy Message-ID: <20100824141710.C741B282BD4@codespeak.net> Author: afa Date: Tue Aug 24 16:17:08 2010 New Revision: 76709 Added: pypy/trunk/pypy/module/test_lib_pypy/test_msvcrt.py (contents, props changed) Modified: pypy/trunk/lib_pypy/msvcrt.py Log: issue556: add msvcrt.locking(). Implementation provided by pclinch, thanks! Modified: pypy/trunk/lib_pypy/msvcrt.py ============================================================================== --- pypy/trunk/lib_pypy/msvcrt.py (original) +++ pypy/trunk/lib_pypy/msvcrt.py Tue Aug 24 16:17:08 2010 @@ -5,9 +5,12 @@ """ # XXX incomplete: implemented only functions needed by subprocess.py +# PAC: 2010/08 added MS locking for Whoosh import ctypes from ctypes_support import standard_c_lib as _c +from ctypes_support import get_errno +import errno try: open_osfhandle = _c._open_osfhandle @@ -25,4 +28,17 @@ setmode.argtypes = [ctypes.c_int, ctypes.c_int] setmode.restype = ctypes.c_int +LK_UNLCK, LK_LOCK, LK_NBLCK, LK_RLCK, LK_NBRLCK = range(5) + +_locking = _c._locking +_locking.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int] +_locking.restype = ctypes.c_int + +def locking(fd, mode, nbytes): + '''lock or unlock a number of bytes in a file.''' + rv = _locking(fd, mode, nbytes) + if rv != 0: + e = get_errno() + raise IOError(e, errno.errorcode[e]) + del ctypes Added: pypy/trunk/pypy/module/test_lib_pypy/test_msvcrt.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/test_lib_pypy/test_msvcrt.py Tue Aug 24 16:17:08 2010 @@ -0,0 +1,30 @@ +from pypy.conftest import gettestobjspace +from pypy.tool.udir import udir +import py +import sys + +# not an applevel test: errno is not preserved +class TestMsvcrt: + def test_locking(self): + if sys.platform != 'win32': + py.test.skip("only on Windows") + + filename = udir.join('locking_test') + filename.ensure() + + import os, msvcrt, errno + msvcrt.locking + + fd = os.open(str(filename), 0) + try: + msvcrt.locking(fd, 1, 1) + + # lock again: it fails + e = raises(IOError, msvcrt.locking, fd, 1, 1) + assert e.value.errno == errno.EDEADLOCK + + # unlock and relock sucessfully + msvcrt.locking(fd, 0, 1) + msvcrt.locking(fd, 1, 1) + finally: + os.close(fd) From arigo at codespeak.net Tue Aug 24 16:30:38 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 24 Aug 2010 16:30:38 +0200 (CEST) Subject: [pypy-svn] r76710 - pypy/trunk/pypy/rlib/rsre Message-ID: <20100824143038.26DCF282B90@codespeak.net> Author: arigo Date: Tue Aug 24 16:30:36 2010 New Revision: 76710 Removed: pypy/trunk/pypy/rlib/rsre/ Log: Remove the old rsre engine. (See next checkin.) From arigo at codespeak.net Tue Aug 24 16:30:52 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 24 Aug 2010 16:30:52 +0200 (CEST) Subject: [pypy-svn] r76711 - in pypy/trunk/pypy: annotation jit/metainterp/test module/_sre module/_sre/test rlib rlib/rsre rpython rpython/test Message-ID: <20100824143052.B6BC9282B90@codespeak.net> Author: arigo Date: Tue Aug 24 16:30:49 2010 New Revision: 76711 Added: pypy/trunk/pypy/rlib/rsre/ - copied from r76709, pypy/branch/rsre2/pypy/rlib/rsre/ Removed: pypy/trunk/pypy/module/_sre/app_sre.py Modified: pypy/trunk/pypy/annotation/specialize.py pypy/trunk/pypy/jit/metainterp/test/test_immutable.py pypy/trunk/pypy/module/_sre/__init__.py pypy/trunk/pypy/module/_sre/interp_sre.py pypy/trunk/pypy/module/_sre/test/test_app_sre.py pypy/trunk/pypy/rlib/debug.py pypy/trunk/pypy/rpython/rclass.py pypy/trunk/pypy/rpython/test/test_rclass.py Log: Merge branch/rsre2. This new version of the regexp engine, rewritten from scratch to be nicer and more (R)Pythonic, gives huuuuuge speed-ups to the benchmarks. Modified: pypy/trunk/pypy/annotation/specialize.py ============================================================================== --- pypy/trunk/pypy/annotation/specialize.py (original) +++ pypy/trunk/pypy/annotation/specialize.py Tue Aug 24 16:30:49 2010 @@ -354,6 +354,12 @@ def specialize_argtype(funcdesc, args_s, *argindices): key = tuple([args_s[i].knowntype for i in argindices]) + for cls in key: + try: + assert '_must_specialize_' not in cls.classdesc.pyobj.__dict__, ( + "%s has the tag _must_specialize_" % (cls,)) + except AttributeError: + pass return maybe_star_args(funcdesc, key, args_s) def specialize_arglistitemtype(funcdesc, args_s, i): Modified: pypy/trunk/pypy/jit/metainterp/test/test_immutable.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_immutable.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_immutable.py Tue Aug 24 16:30:49 2010 @@ -17,6 +17,38 @@ assert res == 28 self.check_operations_history(getfield_gc=0, getfield_gc_pure=1, int_add=1) + def test_fields_subclass(self): + class X(object): + _immutable_fields_ = ["x"] + + def __init__(self, x): + self.x = x + + class Y(X): + _immutable_fields_ = ["y"] + + def __init__(self, x, y): + X.__init__(self, x) + self.y = y + + def f(x, y): + X(x) # force the field 'x' to be on class 'X' + z = Y(x, y) + return z.x + z.y + 5 + res = self.interp_operations(f, [23, 11]) + assert res == 39 + self.check_operations_history(getfield_gc=0, getfield_gc_pure=2, + int_add=2) + + def f(x, y): + # this time, the field 'x' only shows up on subclass 'Y' + z = Y(x, y) + return z.x + z.y + 5 + res = self.interp_operations(f, [23, 11]) + assert res == 39 + self.check_operations_history(getfield_gc=0, getfield_gc_pure=2, + int_add=2) + def test_array(self): class X(object): _immutable_fields_ = ["y[*]"] Modified: pypy/trunk/pypy/module/_sre/__init__.py ============================================================================== --- pypy/trunk/pypy/module/_sre/__init__.py (original) +++ pypy/trunk/pypy/module/_sre/__init__.py Tue Aug 24 16:30:49 2010 @@ -1,24 +1,14 @@ from pypy.interpreter.mixedmodule import MixedModule class Module(MixedModule): - """A pure Python reimplementation of the _sre module from CPython 2.4 -Copyright 2005 Nik Haldimann, licensed under the MIT license -This code is based on material licensed under CNRI's Python 1.6 license and -copyrighted by: Copyright (c) 1997-2001 by Secret Labs AB -""" - appleveldefs = { - 'compile': 'app_sre.compile', } interpleveldefs = { 'CODESIZE': 'space.wrap(interp_sre.CODESIZE)', 'MAGIC': 'space.wrap(interp_sre.MAGIC)', - 'copyright': 'space.wrap(interp_sre.copyright)', + 'compile': 'interp_sre.W_SRE_Pattern', 'getlower': 'interp_sre.w_getlower', 'getcodesize': 'interp_sre.w_getcodesize', - '_State': 'interp_sre.make_state', - '_match': 'interp_sre.w_match', - '_search': 'interp_sre.w_search', } Modified: pypy/trunk/pypy/module/_sre/interp_sre.py ============================================================================== --- pypy/trunk/pypy/module/_sre/interp_sre.py (original) +++ pypy/trunk/pypy/module/_sre/interp_sre.py Tue Aug 24 16:30:49 2010 @@ -1,27 +1,20 @@ +import sys from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.typedef import GetSetProperty, TypeDef from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w +from pypy.interpreter.typedef import make_weakref_descr from pypy.interpreter.gateway import interp2app, ObjSpace, W_Root from pypy.interpreter.error import OperationError from pypy.rlib.rarithmetic import intmask +from pypy.tool.pairtype import extendabletype -# This can be compiled in two ways: -# -# * THREE_VERSIONS_OF_CORE=True: you get three copies of the whole -# regexp searching and matching code: for strings, for unicode strings, -# and for generic buffer objects (like mmap.mmap or array.array). -# -# * THREE_VERSIONS_OF_CORE=False: there is only one copy of the code, -# at the cost of an indirect method call to fetch each character. - -THREE_VERSIONS_OF_CORE = True +# ____________________________________________________________ +# +# Constants and exposed functions -#### Constants and exposed functions - -from pypy.rlib.rsre import rsre -from pypy.rlib.rsre.rsre_char import MAGIC, CODESIZE, getlower -copyright = "_sre.py 2.4 Copyright 2005 by Nik Haldimann" +from pypy.rlib.rsre import rsre_core +from pypy.rlib.rsre.rsre_char import MAGIC, CODESIZE, getlower, set_unicode_db def w_getlower(space, char_ord, flags): return space.wrap(getlower(char_ord, flags)) @@ -31,166 +24,529 @@ return space.wrap(CODESIZE) # use the same version of unicodedb as the standard objspace -from pypy.objspace.std.unicodeobject import unicodedb -rsre.set_unicode_db(unicodedb) +import pypy.objspace.std.unicodeobject +set_unicode_db(pypy.objspace.std.unicodeobject.unicodedb) -#### State classes +# ____________________________________________________________ +# +# Additional methods on the classes XxxMatchContext + +class __extend__(rsre_core.AbstractMatchContext): + __metaclass__ = extendabletype + def _w_slice(self, space, start, end): + raise NotImplementedError + def _w_string(self, space): + raise NotImplementedError -def make_state(space, w_string, start, end, flags): - # XXX maybe turn this into a __new__ method of W_State - if space.is_true(space.isinstance(w_string, space.w_str)): - cls = W_StringState - elif space.is_true(space.isinstance(w_string, space.w_unicode)): - cls = W_UnicodeState - else: - cls = W_GenericState - return space.wrap(cls(space, w_string, start, end, flags)) -make_state.unwrap_spec = [ObjSpace, W_Root, int, int, int] - - -class W_State(Wrappable): - if not THREE_VERSIONS_OF_CORE: - rsre.insert_sre_methods(locals(), 'all') - - def __init__(self, space, w_string, start, end, flags): - self.space = space - self.w_string = w_string - length = self.unwrap_object() - if start < 0: - start = 0 - if end > length: - end = length - self.start = start - self.pos = start # records the original start position - self.end = end - self.flags = flags - self.reset() +class __extend__(rsre_core.StrMatchContext): + __metaclass__ = extendabletype + def _w_slice(self, space, start, end): + return space.wrap(self._string[start:end]) + def _w_string(self, space): + return space.wrap(self._string) + +class __extend__(rsre_core.UnicodeMatchContext): + __metaclass__ = extendabletype + def _w_slice(self, space, start, end): + return space.wrap(self._unicodestr[start:end]) + def _w_string(self, space): + return space.wrap(self._unicodestr) + +def slice_w(space, ctx, start, end, w_default): + if 0 <= start <= end: + return ctx._w_slice(space, start, end) + return w_default + +def do_flatten_marks(ctx, num_groups): + # Returns a list of RPython-level integers. + # Unlike the app-level groups() method, groups are numbered from 0 + # and the returned list does not start with the whole match range. + if num_groups == 0: + return None + result = [-1] * (2*num_groups) + mark = ctx.match_marks + while mark is not None: + index = mark.gid + if result[index] == -1: + result[index] = mark.position + mark = mark.prev + return result + +def allgroups_w(space, ctx, fmarks, num_groups, w_default): + grps = [slice_w(space, ctx, fmarks[i*2], fmarks[i*2+1], w_default) + for i in range(num_groups)] + return space.newtuple(grps) + +def import_re(space): + w_builtin = space.getbuiltinmodule('__builtin__') + w_import = space.getattr(w_builtin, space.wrap("__import__")) + return space.call_function(w_import, space.wrap("re")) - def lower(self, char_ord): - return getlower(char_ord, self.flags) +def matchcontext(space, ctx): + try: + return rsre_core.match_context(ctx) + except rsre_core.Error, e: + raise OperationError(space.w_RuntimeError, space.wrap(e.msg)) - # methods overridden by subclasses +def searchcontext(space, ctx): + try: + return rsre_core.search_context(ctx) + except rsre_core.Error, e: + raise OperationError(space.w_RuntimeError, space.wrap(e.msg)) - def unwrap_object(self): - raise NotImplementedError +# ____________________________________________________________ +# +# SRE_Pattern class - if 'reset' not in locals(): - def reset(self): - raise NotImplementedError - - if 'search' not in locals(): - def search(self, pattern_codes): - raise NotImplementedError - - if 'match' not in locals(): - def match(self, pattern_codes): - raise NotImplementedError - - # Accessors for the typedef - - def w_reset(self): - self.reset() - - def create_regs(self, group_count): - """ Purely abstract method - """ - raise NotImplementedError +class W_SRE_Pattern(Wrappable): - def w_create_regs(self, group_count): - """Creates a tuple of index pairs representing matched groups, a format - that's convenient for SRE_Match.""" + def cannot_copy_w(self): space = self.space - return space.newtuple([ - space.newtuple([space.wrap(value1), - space.wrap(value2)]) - for value1, value2 in self.create_regs(group_count)]) - w_create_regs.unwrap_spec = ['self', int] + raise OperationError(space.w_TypeError, + space.wrap("cannot copy this pattern object")) - def fget_start(space, self): - return space.wrap(self.start) - - def fset_start(space, self, w_value): - self.start = space.int_w(w_value) + def make_ctx(self, w_string, pos=0, endpos=sys.maxint): + """Make a StrMatchContext or a UnicodeMatchContext for searching + in the given w_string object.""" + space = self.space + if pos < 0: pos = 0 + if endpos < pos: endpos = pos + if space.is_true(space.isinstance(w_string, space.w_unicode)): + unicodestr = space.unicode_w(w_string) + if pos > len(unicodestr): pos = len(unicodestr) + if endpos > len(unicodestr): endpos = len(unicodestr) + return rsre_core.UnicodeMatchContext(self.code, unicodestr, + pos, endpos, self.flags) + else: + str = space.bufferstr_w(w_string) + if pos > len(str): pos = len(str) + if endpos > len(str): endpos = len(str) + return rsre_core.StrMatchContext(self.code, str, + pos, endpos, self.flags) + + def getmatch(self, ctx, found): + if found: + return W_SRE_Match(self, ctx) + else: + return self.space.w_None + + def match_w(self, w_string, pos=0, endpos=sys.maxint): + ctx = self.make_ctx(w_string, pos, endpos) + return self.getmatch(ctx, matchcontext(self.space, ctx)) + match_w.unwrap_spec = ['self', W_Root, int, int] + + def search_w(self, w_string, pos=0, endpos=sys.maxint): + ctx = self.make_ctx(w_string, pos, endpos) + return self.getmatch(ctx, searchcontext(self.space, ctx)) + search_w.unwrap_spec = ['self', W_Root, int, int] - def fget_string_position(space, self): - return space.wrap(self.string_position) + def findall_w(self, w_string, pos=0, endpos=sys.maxint): + space = self.space + matchlist_w = [] + ctx = self.make_ctx(w_string, pos, endpos) + while ctx.match_start <= ctx.end: + if not searchcontext(space, ctx): + break + num_groups = self.num_groups + w_emptystr = space.wrap("") + if num_groups == 0: + w_item = slice_w(space, ctx, ctx.match_start, ctx.match_end, + w_emptystr) + else: + fmarks = do_flatten_marks(ctx, num_groups) + if num_groups == 1: + w_item = slice_w(space, ctx, fmarks[0], fmarks[1], + w_emptystr) + else: + w_item = allgroups_w(space, ctx, fmarks, num_groups, + w_emptystr) + matchlist_w.append(w_item) + no_progress = (ctx.match_start == ctx.match_end) + ctx.reset(ctx.match_end + no_progress) + return space.newlist(matchlist_w) + findall_w.unwrap_spec = ['self', W_Root, int, int] + + def finditer_w(self, w_string, pos=0, endpos=sys.maxint): + # this also works as the implementation of the undocumented + # scanner() method. + ctx = self.make_ctx(w_string, pos, endpos) + scanner = W_SRE_Scanner(self, ctx) + return self.space.wrap(scanner) + finditer_w.unwrap_spec = ['self', W_Root, int, int] - def fset_string_position(space, self, w_value): - self.start = space.int_w(w_value) + def split_w(self, w_string, maxsplit=0): + space = self.space + splitlist = [] + n = 0 + last = 0 + ctx = self.make_ctx(w_string) + while not maxsplit or n < maxsplit: + if not searchcontext(space, ctx): + break + if ctx.match_start == ctx.match_end: # zero-width match + if ctx.match_start == ctx.end: # or end of string + break + ctx.reset(ctx.match_end + 1) + continue + splitlist.append(slice_w(space, ctx, last, ctx.match_start, + space.w_None)) + # add groups (if any) + fmarks = do_flatten_marks(ctx, self.num_groups) + for groupnum in range(self.num_groups): + groupstart, groupend = fmarks[groupnum*2], fmarks[groupnum*2+1] + splitlist.append(slice_w(space, ctx, groupstart, groupend, + space.w_None)) + n += 1 + last = ctx.match_end + ctx.reset(last) + splitlist.append(slice_w(space, ctx, last, ctx.end, space.w_None)) + return space.newlist(splitlist) + split_w.unwrap_spec = ['self', W_Root, int] + + def sub_w(self, w_repl, w_string, count=0): + w_item, n = self.subx(w_repl, w_string, count) + return w_item + sub_w.unwrap_spec = ['self', W_Root, W_Root, int] - def get_char_ord(self, p): - raise NotImplementedError + def subn_w(self, w_repl, w_string, count=0): + w_item, n = self.subx(w_repl, w_string, count) + space = self.space + return space.newtuple([w_item, space.wrap(n)]) + subn_w.unwrap_spec = ['self', W_Root, W_Root, int] -getset_start = GetSetProperty(W_State.fget_start, W_State.fset_start, cls=W_State) -getset_string_position = GetSetProperty(W_State.fget_string_position, - W_State.fset_string_position, cls=W_State) - -W_State.typedef = TypeDef("W_State", - string = interp_attrproperty_w("w_string", W_State), - start = getset_start, - end = interp_attrproperty("end", W_State), - string_position = getset_string_position, - pos = interp_attrproperty("pos", W_State), - lastindex = interp_attrproperty("lastindex", W_State), - reset = interp2app(W_State.w_reset), - create_regs = interp2app(W_State.w_create_regs), + def subx(self, w_ptemplate, w_string, count): + space = self.space + if space.is_true(space.callable(w_ptemplate)): + w_filter = w_ptemplate + filter_is_callable = True + else: + if space.is_true(space.isinstance(w_ptemplate, space.w_unicode)): + filter_as_unicode = space.unicode_w(w_ptemplate) + literal = u'\\' not in filter_as_unicode + else: + try: + filter_as_string = space.str_w(w_ptemplate) + except OperationError, e: + if e.async(space): + raise + literal = False + else: + literal = '\\' not in filter_as_string + if literal: + w_filter = w_ptemplate + filter_is_callable = False + else: + # not a literal; hand it over to the template compiler + w_re = import_re(space) + w_filter = space.call_method(w_re, '_subx', + space.wrap(self), w_ptemplate) + filter_is_callable = space.is_true(space.callable(w_filter)) + # + ctx = self.make_ctx(w_string) + sublist_w = [] + n = last_pos = 0 + while not count or n < count: + if not searchcontext(space, ctx): + break + if last_pos < ctx.match_start: + sublist_w.append(slice_w(space, ctx, last_pos, + ctx.match_start, space.w_None)) + start = ctx.match_end + if start == ctx.match_start: + start += 1 + nextctx = ctx.fresh_copy(start) + if not (last_pos == ctx.match_start + == ctx.match_end and n > 0): + # the above ignores empty matches on latest position + if filter_is_callable: + w_match = self.getmatch(ctx, True) + w_piece = space.call_function(w_filter, w_match) + if not space.is_w(w_piece, space.w_None): + sublist_w.append(w_piece) + else: + sublist_w.append(w_filter) + last_pos = ctx.match_end + n += 1 + elif last_pos >= ctx.end: + break # empty match at the end: finished + ctx = nextctx + + if last_pos < ctx.end: + sublist_w.append(slice_w(space, ctx, last_pos, ctx.end, + space.w_None)) + if n == 0: + # not just an optimization -- see test_sub_unicode + return w_string, n + + if space.is_true(space.isinstance(w_string, space.w_unicode)): + w_emptystr = space.wrap(u'') + else: + w_emptystr = space.wrap('') + w_item = space.call_method(w_emptystr, 'join', + space.newlist(sublist_w)) + return w_item, n + + +def SRE_Pattern__new__(space, w_subtype, w_pattern, flags, w_code, + groups=0, w_groupindex=None, w_indexgroup=None): + n = space.int_w(space.len(w_code)) + code = [0] * n + for i in range(n): + x = space.uint_w(space.getitem(w_code, space.wrap(i))) + code[i] = intmask(x) + # + w_srepat = space.allocate_instance(W_SRE_Pattern, w_subtype) + srepat = space.interp_w(W_SRE_Pattern, w_srepat) + srepat.space = space + srepat.w_pattern = w_pattern # the original uncompiled pattern + srepat.flags = flags + srepat.code = code + srepat.num_groups = groups + srepat.w_groupindex = w_groupindex + srepat.w_indexgroup = w_indexgroup + return w_srepat +SRE_Pattern__new__.unwrap_spec = [ObjSpace, W_Root, W_Root, int, W_Root, + int, W_Root, W_Root] + + +W_SRE_Pattern.typedef = TypeDef( + 'SRE_Pattern', + __new__ = interp2app(SRE_Pattern__new__), + __copy__ = interp2app(W_SRE_Pattern.cannot_copy_w), + __deepcopy__ = interp2app(W_SRE_Pattern.cannot_copy_w), + __weakref__ = make_weakref_descr(W_SRE_Pattern), + findall = interp2app(W_SRE_Pattern.findall_w), + finditer = interp2app(W_SRE_Pattern.finditer_w), + match = interp2app(W_SRE_Pattern.match_w), + scanner = interp2app(W_SRE_Pattern.finditer_w), # reuse finditer() + search = interp2app(W_SRE_Pattern.search_w), + split = interp2app(W_SRE_Pattern.split_w), + sub = interp2app(W_SRE_Pattern.sub_w), + subn = interp2app(W_SRE_Pattern.subn_w), + flags = interp_attrproperty('flags', W_SRE_Pattern), + groupindex = interp_attrproperty_w('w_groupindex', W_SRE_Pattern), + groups = interp_attrproperty('num_groups', W_SRE_Pattern), + pattern = interp_attrproperty_w('w_pattern', W_SRE_Pattern), ) +# ____________________________________________________________ +# +# SRE_Match class -class W_StringState(W_State): - if THREE_VERSIONS_OF_CORE: - rsre.insert_sre_methods(locals(), 'str') - - def unwrap_object(self): - self.string = self.space.str_w(self.w_string) - return len(self.string) - - def get_char_ord(self, p): - return ord(self.string[p]) +class W_SRE_Match(Wrappable): + flatten_cache = None + def __init__(self, srepat, ctx): + self.space = srepat.space + self.srepat = srepat + self.ctx = ctx -class W_UnicodeState(W_State): - if THREE_VERSIONS_OF_CORE: - rsre.insert_sre_methods(locals(), 'unicode') + def cannot_copy_w(self): + space = self.space + raise OperationError(space.w_TypeError, + space.wrap("cannot copy this match object")) - def unwrap_object(self): - self.unicode = self.space.unicode_w(self.w_string) - return len(self.unicode) + def group_w(self, args_w): + space = self.space + ctx = self.ctx + if len(args_w) <= 1: + if len(args_w) == 0: + start, end = ctx.match_start, ctx.match_end + else: + start, end = self.do_span(args_w[0]) + return slice_w(space, ctx, start, end, space.w_None) + else: + results = [None] * len(args_w) + for i in range(len(args_w)): + start, end = self.do_span(args_w[i]) + results[i] = slice_w(space, ctx, start, end, space.w_None) + return space.newtuple(results) + group_w.unwrap_spec = ['self', 'args_w'] + + def groups_w(self, w_default=None): + fmarks = self.flatten_marks() + num_groups = self.srepat.num_groups + return allgroups_w(self.space, self.ctx, fmarks, num_groups, w_default) - def get_char_ord(self, p): - return ord(self.unicode[p]) + def groupdict_w(self, w_default=None): + space = self.space + w_dict = space.newdict() + w_groupindex = self.srepat.w_groupindex + w_iterator = space.iter(w_groupindex) + while True: + try: + w_key = space.next(w_iterator) + except OperationError, e: + if not e.match(space, space.w_StopIteration): + raise + break # done + w_value = space.getitem(w_groupindex, w_key) + start, end = self.do_span(w_value) + w_grp = slice_w(space, self.ctx, start, end, w_default) + space.setitem(w_dict, w_key, w_grp) + return w_dict + def expand_w(self, w_template): + space = self.space + w_re = import_re(space) + return space.call_method(w_re, '_expand', space.wrap(self.srepat), + space.wrap(self), w_template) + + def start_w(self, w_groupnum=0): + return self.space.wrap(self.do_span(w_groupnum)[0]) + + def end_w(self, w_groupnum=0): + return self.space.wrap(self.do_span(w_groupnum)[1]) + + def span_w(self, w_groupnum=0): + start, end = self.do_span(w_groupnum) + return self.space.newtuple([self.space.wrap(start), + self.space.wrap(end)]) + + def flatten_marks(self): + if self.flatten_cache is None: + num_groups = self.srepat.num_groups + self.flatten_cache = do_flatten_marks(self.ctx, num_groups) + return self.flatten_cache -class W_GenericState(W_State): - if THREE_VERSIONS_OF_CORE: - rsre.insert_sre_methods(locals(), 'generic') + def do_span(self, w_arg): + space = self.space + try: + groupnum = space.int_w(w_arg) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + w_groupnum = space.getitem(self.srepat.w_groupindex, w_arg) + groupnum = space.int_w(w_groupnum) + if groupnum == 0: + return self.ctx.match_start, self.ctx.match_end + elif 1 <= groupnum <= self.srepat.num_groups: + fmarks = self.flatten_marks() + idx = 2*(groupnum-1) + assert idx >= 0 + return fmarks[idx], fmarks[idx+1] + else: + raise OperationError(space.w_IndexError, + space.wrap("group index out of range")) + + def _last_index(self): + mark = self.ctx.match_marks + if mark is not None: + return mark.gid // 2 + 1 + return -1 + + def fget_lastgroup(space, self): + lastindex = self._last_index() + if lastindex < 0: + return space.w_None + w_result = space.finditem(self.srepat.w_indexgroup, + space.wrap(lastindex)) + if w_result is None: + return space.w_None + return w_result + + def fget_lastindex(space, self): + lastindex = self._last_index() + if lastindex >= 0: + return space.wrap(lastindex) + return space.w_None - def unwrap_object(self): - self.buffer = self.space.buffer_w(self.w_string) - return self.buffer.getlength() + def fget_pos(space, self): + return space.wrap(self.ctx.original_pos) - def get_char_ord(self, p): - return ord(self.buffer.getitem(p)) + def fget_endpos(space, self): + return space.wrap(self.ctx.end) + def fget_regs(space, self): + space = self.space + fmarks = self.flatten_marks() + num_groups = self.srepat.num_groups + result_w = [None] * (num_groups + 1) + ctx = self.ctx + result_w[0] = space.newtuple([space.wrap(ctx.match_start), + space.wrap(ctx.match_end)]) + for i in range(num_groups): + result_w[i + 1] = space.newtuple([space.wrap(fmarks[i*2]), + space.wrap(fmarks[i*2+1])]) + return space.newtuple(result_w) + + def fget_string(space, self): + return self.ctx._w_string(space) + + +W_SRE_Match.typedef = TypeDef( + 'SRE_Match', + __copy__ = interp2app(W_SRE_Match.cannot_copy_w), + __deepcopy__ = interp2app(W_SRE_Match.cannot_copy_w), + group = interp2app(W_SRE_Match.group_w), + groups = interp2app(W_SRE_Match.groups_w), + groupdict = interp2app(W_SRE_Match.groupdict_w), + start = interp2app(W_SRE_Match.start_w), + end = interp2app(W_SRE_Match.end_w), + span = interp2app(W_SRE_Match.span_w), + expand = interp2app(W_SRE_Match.expand_w), + # + re = interp_attrproperty('srepat', W_SRE_Match), + string = GetSetProperty(W_SRE_Match.fget_string), + pos = GetSetProperty(W_SRE_Match.fget_pos), + endpos = GetSetProperty(W_SRE_Match.fget_endpos), + lastgroup = GetSetProperty(W_SRE_Match.fget_lastgroup), + lastindex = GetSetProperty(W_SRE_Match.fget_lastindex), + regs = GetSetProperty(W_SRE_Match.fget_regs), +) -def w_search(space, w_state, w_pattern_codes): - state = space.interp_w(W_State, w_state) - pattern_codes = [intmask(space.uint_w(code)) for code - in space.unpackiterable(w_pattern_codes)] - try: - res = state.search(pattern_codes) - except RuntimeError: - raise OperationError(space.w_RuntimeError, - space.wrap("Internal re error")) - return space.newbool(res) - -def w_match(space, w_state, w_pattern_codes): - state = space.interp_w(W_State, w_state) - pattern_codes = [intmask(space.uint_w(code)) for code - in space.unpackiterable(w_pattern_codes)] - try: - res = state.match(pattern_codes) - except RuntimeError: - raise OperationError(space.w_RuntimeError, - space.wrap("Internal re error")) - return space.newbool(res) +# ____________________________________________________________ +# +# SRE_Scanner class +# This is mostly an internal class in CPython. +# Our version is also directly iterable, to make finditer() easier. + +class W_SRE_Scanner(Wrappable): + + def __init__(self, pattern, ctx): + self.space = pattern.space + self.srepat = pattern + self.ctx = ctx + # 'self.ctx' is always a fresh context in which no searching + # or matching succeeded so far. + + def iter_w(self): + return self.space.wrap(self) + + def next_w(self): + if self.ctx.match_start > self.ctx.end: + raise OperationError(self.space.w_StopIteration, self.space.w_None) + if not searchcontext(self.space, self.ctx): + raise OperationError(self.space.w_StopIteration, self.space.w_None) + return self.getmatch(True) + + def match_w(self): + if self.ctx.match_start > self.ctx.end: + return self.space.w_None + return self.getmatch(matchcontext(self.space, self.ctx)) + + def search_w(self): + if self.ctx.match_start > self.ctx.end: + return self.space.w_None + return self.getmatch(searchcontext(self.space, self.ctx)) + + def getmatch(self, found): + if found: + ctx = self.ctx + nextstart = ctx.match_end + nextstart += (ctx.match_start == nextstart) + self.ctx = ctx.fresh_copy(nextstart) + match = W_SRE_Match(self.srepat, ctx) + return self.space.wrap(match) + else: + self.ctx.match_start += 1 # obscure corner case + return None + +W_SRE_Scanner.typedef = TypeDef( + 'SRE_Scanner', + __iter__ = interp2app(W_SRE_Scanner.iter_w, unwrap_spec=['self']), + next = interp2app(W_SRE_Scanner.next_w, unwrap_spec=['self']), + match = interp2app(W_SRE_Scanner.match_w, unwrap_spec=['self']), + search = interp2app(W_SRE_Scanner.search_w, unwrap_spec=['self']), + pattern = interp_attrproperty('srepat', W_SRE_Scanner), +) Modified: pypy/trunk/pypy/module/_sre/test/test_app_sre.py ============================================================================== --- pypy/trunk/pypy/module/_sre/test/test_app_sre.py (original) +++ pypy/trunk/pypy/module/_sre/test/test_app_sre.py Tue Aug 24 16:30:49 2010 @@ -33,8 +33,8 @@ # copy support is disabled by default in _sre.c import re p = re.compile("b") - raises(TypeError, p.__copy__) - raises(TypeError, p.__deepcopy__) + raises(TypeError, p.__copy__) # p.__copy__() should raise + raises(TypeError, p.__deepcopy__) # p.__deepcopy__() should raise def test_creation_attributes(self): import re @@ -85,6 +85,10 @@ assert ['', 'a', None, 'l', 'u', None, 'lla'] == ( re.split("b([ua]|(s))", "balbulla")) + def test_weakref(self): + import re, _weakref + _weakref.ref(re.compile(r"")) + class AppTestSreMatch: def setup_class(cls): @@ -202,6 +206,20 @@ return ret assert ("bbbbb", 3) == re.subn("a", call_me, "ababa") + def test_sub_callable_returns_none(self): + import re + def call_me(match): + return None + assert "acd" == re.sub("b", call_me, "abcd") + + def test_sub_callable_suddenly_unicode(self): + import re + def call_me(match): + if match.group() == 'A': + return unichr(0x3039) + return '' + assert (u"bb\u3039b", 2) == re.subn("[aA]", call_me, "babAb") + def test_match_array(self): import re, array a = array.array('c', 'hello') @@ -268,6 +286,18 @@ p.match().group(0), p.match().group(0)) assert None == p.match() + def test_scanner_match_detail(self): + import re + p = re.compile("a").scanner("aaXaa") + assert "a" == p.match().group(0) + assert "a" == p.match().group(0) + assert None == p.match() + assert "a" == p.match().group(0) + assert "a" == p.match().group(0) + assert None == p.match() + assert None == p.match() + assert None == p.match() + def test_scanner_search(self): import re p = re.compile("\d").scanner("bla23c5a") @@ -653,69 +683,6 @@ s.ATCODES["at_uni_non_boundary"], s.OPCODES["success"]] s.assert_match(opcodes, ["blaha", u"bl%sja" % UPPER_PI]) - def test_category_digit(self): - INDIAN_DIGIT = u"\u0966" - opcodes = [s.OPCODES["category"], s.CHCODES["category_digit"]] \ - + s.encode_literal("b") + [s.OPCODES["success"]] - s.assert_match(opcodes, ["1b", "a1b"]) - s.assert_no_match(opcodes, ["bb", "b1", u"%sb" % INDIAN_DIGIT]) - - def test_category_not_digit(self): - INDIAN_DIGIT = u"\u0966" - opcodes = [s.OPCODES["category"], s.CHCODES["category_not_digit"]] \ - + s.encode_literal("b") + [s.OPCODES["success"]] - s.assert_match(opcodes, ["bb", "1ab", u"%sb" % INDIAN_DIGIT]) - s.assert_no_match(opcodes, ["1b", "a1b"]) - - def test_category_space(self): - EM_SPACE = u"\u2001" - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_space"], s.OPCODES["success"]] - s.assert_match(opcodes, ["b ", "b\n", "b\t", "b\r", "b\v", "b\f"]) - s.assert_no_match(opcodes, ["bb", "b1", u"b%s" % EM_SPACE]) - - def test_category_not_space(self): - EM_SPACE = u"\u2001" - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_not_space"], s.OPCODES["success"]] - s.assert_match(opcodes, ["bb", "b1", u"b%s" % EM_SPACE]) - s.assert_no_match(opcodes, ["b ", "b\n", "b\t", "b\r", "b\v", "b\f"]) - - def test_category_word(self): - LOWER_PI = u"\u03c0" - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_word"], s.OPCODES["success"]] - s.assert_match(opcodes, ["bl", "b4", "b_"]) - s.assert_no_match(opcodes, ["b ", "b\n", u"b%s" % LOWER_PI]) - - def test_category_not_word(self): - LOWER_PI = u"\u03c0" - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_not_word"], s.OPCODES["success"]] - s.assert_match(opcodes, ["b ", "b\n", u"b%s" % LOWER_PI]) - s.assert_no_match(opcodes, ["bl", "b4", "b_"]) - - def test_category_linebreak(self): - LINE_SEP = u"\u2028" - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_linebreak"], s.OPCODES["success"]] - s.assert_match(opcodes, ["b\n"]) - s.assert_no_match(opcodes, ["b ", "bs", "b\r", u"b%s" % LINE_SEP]) - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_uni_linebreak"], s.OPCODES["success"]] - s.assert_match(opcodes, ["b\n", u"b%s" % LINE_SEP]) - - def test_category_not_linebreak(self): - LINE_SEP = u"\u2028" - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_not_linebreak"], s.OPCODES["success"]] - s.assert_match(opcodes, ["b ", "bs", u"b%s" % LINE_SEP]) - s.assert_no_match(opcodes, ["b\n"]) - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_uni_not_linebreak"], s.OPCODES["success"]] - s.assert_match(opcodes, ["b ", "bs"]) - s.assert_no_match(opcodes, ["b\n", u"b%s" % LINE_SEP, "b\r"]) - def test_category_loc_word(self): import locale try: @@ -873,10 +840,6 @@ s.assert_match(opcodes, ["ab", "aaaab", "baabb"]) s.assert_no_match(opcodes, ["aaa", "", "ac"]) - def test_max_until_error(self): - opcodes = [s.OPCODES["max_until"], s.OPCODES["success"]] - raises(RuntimeError, s.search, opcodes, "a") - def test_max_until_zero_width_match(self): # re.compile won't compile prospective zero-with matches (all of them?), # so we can only produce an example by directly constructing bytecodes. @@ -896,10 +859,6 @@ s.assert_no_match(opcodes, ["b"]) assert "aab" == s.search(opcodes, "aabb").group(0) - def test_min_until_error(self): - opcodes = [s.OPCODES["min_until"], s.OPCODES["success"]] - raises(RuntimeError, s.search, opcodes, "a") - def test_groupref(self): opcodes = [s.OPCODES["mark"], 0, s.OPCODES["any"], s.OPCODES["mark"], 1] \ + s.encode_literal("a") + [s.OPCODES["groupref"], 0, s.OPCODES["success"]] Modified: pypy/trunk/pypy/rlib/debug.py ============================================================================== --- pypy/trunk/pypy/rlib/debug.py (original) +++ pypy/trunk/pypy/rlib/debug.py Tue Aug 24 16:30:49 2010 @@ -255,11 +255,32 @@ class IntegerCanBeNegative(Exception): pass -def _check_nonneg(ann, bk): - from pypy.annotation.model import SomeInteger - s_nonneg = SomeInteger(nonneg=True) - if not s_nonneg.contains(ann): - raise IntegerCanBeNegative +class UnexpectedRUInt(Exception): + pass def check_nonneg(x): - check_annotation(x, _check_nonneg) + """Give a translation-time error if 'x' is not known to be non-negative. + To help debugging, this also gives a translation-time error if 'x' is + actually typed as an r_uint (in which case the call to check_nonneg() + is a bit strange and probably unexpected). + """ + assert type(x)(-1) < 0 # otherwise, 'x' is a r_uint or similar + assert x >= 0 + return x + +class Entry(ExtRegistryEntry): + _about_ = check_nonneg + + def compute_result_annotation(self, s_arg): + from pypy.annotation.model import SomeInteger + if isinstance(s_arg, SomeInteger) and s_arg.unsigned: + raise UnexpectedRUInt("check_nonneg() arg is a %s" % ( + s_arg.knowntype,)) + s_nonneg = SomeInteger(nonneg=True) + if not s_nonneg.contains(s_arg): + raise IntegerCanBeNegative + return s_arg + + def specialize_call(self, hop): + hop.exception_cannot_occur() + return hop.inputarg(hop.args_r[0], arg=0) Modified: pypy/trunk/pypy/rpython/rclass.py ============================================================================== --- pypy/trunk/pypy/rpython/rclass.py (original) +++ pypy/trunk/pypy/rpython/rclass.py Tue Aug 24 16:30:49 2010 @@ -156,9 +156,13 @@ if '_immutable_' in self.classdef.classdesc.classdict: hints = hints.copy() hints['immutable'] = True - if '_immutable_fields_' in self.classdef.classdesc.classdict: + self.immutable_field_list = [] # unless overwritten below + if self.classdef.classdesc.lookup('_immutable_fields_') is not None: hints = hints.copy() - self.immutable_field_list = self.classdef.classdesc.classdict['_immutable_fields_'].value + immutable_fields = self.classdef.classdesc.classdict.get( + '_immutable_fields_') + if immutable_fields is not None: + self.immutable_field_list = immutable_fields.value accessor = FieldListAccessor() hints['immutable_fields'] = accessor return hints @@ -181,7 +185,13 @@ hints = self.object_type._hints if "immutable_fields" in hints: accessor = hints["immutable_fields"] - self._parse_field_list(self.immutable_field_list, accessor) + immutable_fields = {} + rbase = self + while rbase.classdef is not None: + immutable_fields.update( + dict.fromkeys(rbase.immutable_field_list)) + rbase = rbase.rbase + self._parse_field_list(immutable_fields, accessor) def _parse_field_list(self, fields, accessor): with_suffix = {} @@ -191,7 +201,10 @@ suffix = '[*]' else: suffix = '' - mangled_name, r = self._get_field(name) + try: + mangled_name, r = self._get_field(name) + except KeyError: + continue with_suffix[mangled_name] = suffix accessor.initialize(self.object_type, with_suffix) return with_suffix Modified: pypy/trunk/pypy/rpython/test/test_rclass.py ============================================================================== --- pypy/trunk/pypy/rpython/test/test_rclass.py (original) +++ pypy/trunk/pypy/rpython/test/test_rclass.py Tue Aug 24 16:30:49 2010 @@ -738,6 +738,64 @@ assert accessor.fields == {"inst_x" : "", "inst_y" : "[*]"} or \ accessor.fields == {"ox" : "", "oy" : "[*]"} # for ootype + def test_immutable_fields_subclass_1(self): + from pypy.jit.metainterp.typesystem import deref + class A(object): + _immutable_fields_ = ["x"] + def __init__(self, x): + self.x = x + class B(A): + def __init__(self, x, y): + A.__init__(self, x) + self.y = y + + def f(): + return B(3, 5) + t, typer, graph = self.gengraph(f, []) + B_TYPE = deref(graph.getreturnvar().concretetype) + accessor = B_TYPE._hints["immutable_fields"] + assert accessor.fields == {"inst_x" : ""} or \ + accessor.fields == {"ox" : ""} # for ootype + + def test_immutable_fields_subclass_2(self): + from pypy.jit.metainterp.typesystem import deref + class A(object): + _immutable_fields_ = ["x"] + def __init__(self, x): + self.x = x + class B(A): + _immutable_fields_ = ["y"] + def __init__(self, x, y): + A.__init__(self, x) + self.y = y + + def f(): + return B(3, 5) + t, typer, graph = self.gengraph(f, []) + B_TYPE = deref(graph.getreturnvar().concretetype) + accessor = B_TYPE._hints["immutable_fields"] + assert accessor.fields == {"inst_x" : "", "inst_y" : ""} or \ + accessor.fields == {"ox" : "", "oy" : ""} # for ootype + + def test_immutable_fields_only_in_subclass(self): + from pypy.jit.metainterp.typesystem import deref + class A(object): + def __init__(self, x): + self.x = x + class B(A): + _immutable_fields_ = ["y"] + def __init__(self, x, y): + A.__init__(self, x) + self.y = y + + def f(): + return B(3, 5) + t, typer, graph = self.gengraph(f, []) + B_TYPE = deref(graph.getreturnvar().concretetype) + accessor = B_TYPE._hints["immutable_fields"] + assert accessor.fields == {"inst_y" : ""} or \ + accessor.fields == {"oy" : ""} # for ootype + def test_immutable_inheritance(self): class I(object): def __init__(self, v): From arigo at codespeak.net Tue Aug 24 16:34:51 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 24 Aug 2010 16:34:51 +0200 (CEST) Subject: [pypy-svn] r76712 - pypy/trunk/pypy/rlib/rsre/test Message-ID: <20100824143451.AAEDC282B90@codespeak.net> Author: arigo Date: Tue Aug 24 16:34:50 2010 New Revision: 76712 Added: pypy/trunk/pypy/rlib/rsre/test/test_char.py (contents, props changed) Log: Forgot to checkin a test file. Added: pypy/trunk/pypy/rlib/rsre/test/test_char.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/rlib/rsre/test/test_char.py Tue Aug 24 16:34:50 2010 @@ -0,0 +1,107 @@ +from pypy.rlib.rsre import rsre_char +from pypy.rlib.rsre.rsre_char import SRE_FLAG_UNICODE + +def setup_module(mod): + from pypy.module.unicodedata import unicodedb_4_1_0 as unicodedb + rsre_char.set_unicode_db(unicodedb) + +UPPER_PI = 0x3a0 +LOWER_PI = 0x3c0 +INDIAN_DIGIT = 0x966 +EM_SPACE = 0x2001 +LINE_SEP = 0x2028 + + +# XXX very incomplete test + +def test_getlower(): + assert rsre_char.getlower(ord('A'), 0) == ord('a') + assert rsre_char.getlower(ord('2'), 0) == ord('2') + assert rsre_char.getlower(10, 0) == 10 + assert rsre_char.getlower(UPPER_PI, 0) == UPPER_PI + # + assert rsre_char.getlower(ord('A'), SRE_FLAG_UNICODE) == ord('a') + assert rsre_char.getlower(ord('2'), SRE_FLAG_UNICODE) == ord('2') + assert rsre_char.getlower(10, SRE_FLAG_UNICODE) == 10 + assert rsre_char.getlower(UPPER_PI, SRE_FLAG_UNICODE) == LOWER_PI + +def test_is_word(): + assert rsre_char.is_word(ord('A')) + assert rsre_char.is_word(ord('_')) + assert not rsre_char.is_word(UPPER_PI) + assert not rsre_char.is_word(LOWER_PI) + assert not rsre_char.is_word(ord(',')) + # + assert rsre_char.is_uni_word(ord('A')) + assert rsre_char.is_uni_word(ord('_')) + assert rsre_char.is_uni_word(UPPER_PI) + assert rsre_char.is_uni_word(LOWER_PI) + assert not rsre_char.is_uni_word(ord(',')) + +def test_category(): + from sre_constants import CHCODES + cat = rsre_char.category_dispatch + # + assert cat(CHCODES["category_digit"], ord('1')) + assert not cat(CHCODES["category_digit"], ord('a')) + assert not cat(CHCODES["category_digit"], INDIAN_DIGIT) + # + assert not cat(CHCODES["category_not_digit"], ord('1')) + assert cat(CHCODES["category_not_digit"], ord('a')) + assert cat(CHCODES["category_not_digit"], INDIAN_DIGIT) + # + assert not cat(CHCODES["category_space"], ord('1')) + assert not cat(CHCODES["category_space"], ord('a')) + assert cat(CHCODES["category_space"], ord(' ')) + assert cat(CHCODES["category_space"], ord('\n')) + assert cat(CHCODES["category_space"], ord('\t')) + assert cat(CHCODES["category_space"], ord('\r')) + assert cat(CHCODES["category_space"], ord('\v')) + assert cat(CHCODES["category_space"], ord('\f')) + assert not cat(CHCODES["category_space"], EM_SPACE) + # + assert cat(CHCODES["category_not_space"], ord('1')) + assert cat(CHCODES["category_not_space"], ord('a')) + assert not cat(CHCODES["category_not_space"], ord(' ')) + assert not cat(CHCODES["category_not_space"], ord('\n')) + assert not cat(CHCODES["category_not_space"], ord('\t')) + assert not cat(CHCODES["category_not_space"], ord('\r')) + assert not cat(CHCODES["category_not_space"], ord('\v')) + assert not cat(CHCODES["category_not_space"], ord('\f')) + assert cat(CHCODES["category_not_space"], EM_SPACE) + # + assert cat(CHCODES["category_word"], ord('l')) + assert cat(CHCODES["category_word"], ord('4')) + assert cat(CHCODES["category_word"], ord('_')) + assert not cat(CHCODES["category_word"], ord(' ')) + assert not cat(CHCODES["category_word"], ord('\n')) + assert not cat(CHCODES["category_word"], LOWER_PI) + # + assert not cat(CHCODES["category_not_word"], ord('l')) + assert not cat(CHCODES["category_not_word"], ord('4')) + assert not cat(CHCODES["category_not_word"], ord('_')) + assert cat(CHCODES["category_not_word"], ord(' ')) + assert cat(CHCODES["category_not_word"], ord('\n')) + assert cat(CHCODES["category_not_word"], LOWER_PI) + # + assert cat(CHCODES["category_linebreak"], ord('\n')) + assert not cat(CHCODES["category_linebreak"], ord(' ')) + assert not cat(CHCODES["category_linebreak"], ord('s')) + assert not cat(CHCODES["category_linebreak"], ord('\r')) + assert not cat(CHCODES["category_linebreak"], LINE_SEP) + # + assert cat(CHCODES["category_uni_linebreak"], ord('\n')) + assert not cat(CHCODES["category_uni_linebreak"], ord(' ')) + assert not cat(CHCODES["category_uni_linebreak"], ord('s')) + assert cat(CHCODES["category_uni_linebreak"], LINE_SEP) + # + assert not cat(CHCODES["category_not_linebreak"], ord('\n')) + assert cat(CHCODES["category_not_linebreak"], ord(' ')) + assert cat(CHCODES["category_not_linebreak"], ord('s')) + assert cat(CHCODES["category_not_linebreak"], ord('\r')) + assert cat(CHCODES["category_not_linebreak"], LINE_SEP) + # + assert not cat(CHCODES["category_uni_not_linebreak"], ord('\n')) + assert cat(CHCODES["category_uni_not_linebreak"], ord(' ')) + assert cat(CHCODES["category_uni_not_linebreak"], ord('s')) + assert not cat(CHCODES["category_uni_not_linebreak"], LINE_SEP) From arigo at codespeak.net Tue Aug 24 16:34:55 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 24 Aug 2010 16:34:55 +0200 (CEST) Subject: [pypy-svn] r76713 - pypy/branch/rsre2 Message-ID: <20100824143455.7BBBE282BDE@codespeak.net> Author: arigo Date: Tue Aug 24 16:34:53 2010 New Revision: 76713 Removed: pypy/branch/rsre2/ Log: Remove merged branch. From arigo at codespeak.net Tue Aug 24 17:01:19 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 24 Aug 2010 17:01:19 +0200 (CEST) Subject: [pypy-svn] r76714 - pypy/trunk/pypy/interpreter/test Message-ID: <20100824150119.9A7F7282B90@codespeak.net> Author: arigo Date: Tue Aug 24 17:01:17 2010 New Revision: 76714 Modified: pypy/trunk/pypy/interpreter/test/test_zpy.py Log: Rewrite this test to not use py.process.cmdexec(), which tries to be too clever (it decodes the stdout and stderr to unicode, for some reason, and that can crash if it contains characters not supported by the default encoding). Directly subprocess.Popen() instead. Modified: pypy/trunk/pypy/interpreter/test/test_zpy.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_zpy.py (original) +++ pypy/trunk/pypy/interpreter/test/test_zpy.py Tue Aug 24 17:01:17 2010 @@ -3,27 +3,28 @@ import py import sys import pypy +import subprocess pypypath = py.path.local(pypy.__file__).dirpath("bin", "py.py") -def cmdexec(s): - if sys.platform == 'win32': - s = '"%s"' % s # double double quotes - return py.process.cmdexec(s) +def run(*args): + argslist = map(str, args) + popen = subprocess.Popen(argslist, stdout=subprocess.PIPE) + stdout, stderr = popen.communicate() + return stdout def test_executable(): """Ensures sys.executable points to the py.py script""" # TODO : watch out for spaces/special chars in pypypath - output = cmdexec( '''"%s" "%s" -c "import sys;print sys.executable" ''' % - (sys.executable, pypypath) ) + output = run(sys.executable, pypypath, + "-c", "import sys;print sys.executable") assert output.splitlines()[-1] == pypypath def test_special_names(): """Test the __name__ and __file__ special global names""" cmd = "print __name__; print '__file__' in globals()" - output = cmdexec( '''"%s" "%s" -c "%s" ''' % - (sys.executable, pypypath, cmd) ) + output = run(sys.executable, pypypath, '-c', cmd) assert output.splitlines()[-2] == '__main__' assert output.splitlines()[-1] == 'False' @@ -32,26 +33,25 @@ tmpfile.write("print __name__; print __file__\n") tmpfile.close() - output = cmdexec( '''"%s" "%s" "%s" ''' % - (sys.executable, pypypath, tmpfilepath) ) + output = run(sys.executable, pypypath, tmpfilepath) assert output.splitlines()[-2] == '__main__' assert output.splitlines()[-1] == str(tmpfilepath) def test_argv_command(): """Some tests on argv""" # test 1 : no arguments - output = cmdexec( '''"%s" "%s" -c "import sys;print sys.argv" ''' % - (sys.executable, pypypath) ) + output = run(sys.executable, pypypath, + "-c", "import sys;print sys.argv") assert output.splitlines()[-1] == str(['-c']) # test 2 : some arguments after - output = cmdexec( '''"%s" "%s" -c "import sys;print sys.argv" hello''' % - (sys.executable, pypypath) ) + output = run(sys.executable, pypypath, + "-c", "import sys;print sys.argv", "hello") assert output.splitlines()[-1] == str(['-c','hello']) # test 3 : additionnal pypy parameters - output = cmdexec( '''"%s" "%s" -O -c "import sys;print sys.argv" hello''' % - (sys.executable, pypypath) ) + output = run(sys.executable, pypypath, + "-O", "-c", "import sys;print sys.argv", "hello") assert output.splitlines()[-1] == str(['-c','hello']) SCRIPT_1 = """ @@ -65,18 +65,15 @@ tmpfile.close() # test 1 : no arguments - output = cmdexec( '''"%s" "%s" "%s" ''' % - (sys.executable, pypypath, tmpfilepath) ) + output = run(sys.executable, pypypath, tmpfilepath) assert output.splitlines()[-1] == str([tmpfilepath]) # test 2 : some arguments after - output = cmdexec( '''"%s" "%s" "%s" hello''' % - (sys.executable, pypypath, tmpfilepath) ) + output = run(sys.executable, pypypath, tmpfilepath, "hello") assert output.splitlines()[-1] == str([tmpfilepath,'hello']) # test 3 : additionnal pypy parameters - output = cmdexec( '''"%s" "%s" -O "%s" hello''' % - (sys.executable, pypypath, tmpfilepath) ) + output = run(sys.executable, pypypath, "-O", tmpfilepath, "hello") assert output.splitlines()[-1] == str([tmpfilepath,'hello']) @@ -98,11 +95,7 @@ tmpfile.write(TB_NORMALIZATION_CHK) tmpfile.close() - e = None - try: - output = cmdexec( '''"%s" "%s" "%s" ''' % - (sys.executable, pypypath, tmpfilepath) ) - except py.process.cmdexec.Error, e: - pass - assert e," expected failure" - assert e.err.splitlines()[-1] == 'KeyError: ' + popen = subprocess.Popen([sys.executable, str(pypypath), tmpfilepath], + stderr=subprocess.PIPE) + _, stderr = popen.communicate() + assert stderr.endswith('KeyError: \n') From arigo at codespeak.net Tue Aug 24 17:17:26 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 24 Aug 2010 17:17:26 +0200 (CEST) Subject: [pypy-svn] r76715 - pypy/trunk/lib_pypy Message-ID: <20100824151726.DB826282B90@codespeak.net> Author: arigo Date: Tue Aug 24 17:17:24 2010 New Revision: 76715 Removed: pypy/trunk/lib_pypy/_sre.py Log: issue543 resolved Kill the pure Python _sre.py. It may work or not work, but it has no test. Moreover it's based on the "old" algorithm, now gone. If you want a pure Python replacement for _sre, there is a rough one in pypy/rlib/rsre/rsre_re.py, used by tests only for now. From arigo at codespeak.net Tue Aug 24 17:35:52 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 24 Aug 2010 17:35:52 +0200 (CEST) Subject: [pypy-svn] r76716 - pypy/trunk/pypy/translator/platform Message-ID: <20100824153552.733F7282B90@codespeak.net> Author: arigo Date: Tue Aug 24 17:35:45 2010 New Revision: 76716 Modified: pypy/trunk/pypy/translator/platform/__init__.py pypy/trunk/pypy/translator/platform/darwin.py pypy/trunk/pypy/translator/platform/freebsd7.py pypy/trunk/pypy/translator/platform/linux.py pypy/trunk/pypy/translator/platform/maemo.py pypy/trunk/pypy/translator/platform/posix.py pypy/trunk/pypy/translator/platform/windows.py Log: issue546 testing Convert class attributes from lists to tuples, to avoid accidentally mutating them. Modified: pypy/trunk/pypy/translator/platform/__init__.py ============================================================================== --- pypy/trunk/pypy/translator/platform/__init__.py (original) +++ pypy/trunk/pypy/translator/platform/__init__.py Tue Aug 24 17:35:45 2010 @@ -38,9 +38,9 @@ name = "abstract platform" c_environ = None - relevant_environ = [] + relevant_environ = () - so_prefixes = [''] + so_prefixes = ('',) def __init__(self, cc): if self.__class__ is Platform: @@ -146,7 +146,7 @@ extra = self.standalone_only else: extra = self.shared_only - cflags = self.cflags + extra + cflags = list(self.cflags) + list(extra) return (cflags + list(eci.compile_extra) + args) def _preprocess_library_dirs(self, library_dirs): @@ -158,7 +158,7 @@ libraries = self._libs(eci.libraries) link_files = self._linkfiles(eci.link_files) export_flags = self._exportsymbols_link_flags(eci) - return (library_dirs + self.link_flags + export_flags + + return (library_dirs + list(self.link_flags) + export_flags + link_files + list(eci.link_extra) + libraries) def _exportsymbols_link_flags(self, eci, relto=None): Modified: pypy/trunk/pypy/translator/platform/darwin.py ============================================================================== --- pypy/trunk/pypy/translator/platform/darwin.py (original) +++ pypy/trunk/pypy/translator/platform/darwin.py Tue Aug 24 17:35:45 2010 @@ -5,10 +5,10 @@ class Darwin(posix.BasePosix): name = "darwin" - link_flags = ['-mmacosx-version-min=10.4'] - cflags = ['-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4'] - standalone_only = ['-mdynamic-no-pic'] - shared_only = [] + link_flags = ('-mmacosx-version-min=10.4',) + cflags = ('-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4') + standalone_only = ('-mdynamic-no-pic',) + shared_only = () so_ext = 'so' @@ -18,8 +18,9 @@ self.cc = cc def _args_for_shared(self, args): - return (self.shared_only + ['-dynamiclib', '-undefined', 'dynamic_lookup'] - + args) + return (list(self.shared_only) + + ['-dynamiclib', '-undefined', 'dynamic_lookup'] + + args) def _preprocess_include_dirs(self, include_dirs): res_incl_dirs = list(include_dirs) @@ -72,11 +73,12 @@ class Darwin_i386(Darwin): name = "darwin_i386" - link_flags = ['-arch', 'i386', '-mmacosx-version-min=10.4'] - cflags = ['-arch', 'i386', '-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4'] + link_flags = ('-arch', 'i386', '-mmacosx-version-min=10.4') + cflags = ('-arch', 'i386', '-O3', '-fomit-frame-pointer', + '-mmacosx-version-min=10.4') class Darwin_x86_64(Darwin): name = "darwin_x86_64" - link_flags = ['-arch', 'x86_64', '-mmacosx-version-min=10.4'] - cflags = ['-arch', 'x86_64', '-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4'] - + link_flags = ('-arch', 'x86_64', '-mmacosx-version-min=10.4') + cflags = ('-arch', 'x86_64', '-O3', '-fomit-frame-pointer', + '-mmacosx-version-min=10.4') Modified: pypy/trunk/pypy/translator/platform/freebsd7.py ============================================================================== --- pypy/trunk/pypy/translator/platform/freebsd7.py (original) +++ pypy/trunk/pypy/translator/platform/freebsd7.py Tue Aug 24 17:35:45 2010 @@ -5,10 +5,10 @@ class Freebsd7(posix.BasePosix): name = "freebsd7" - link_flags = ['-pthread'] - cflags = ['-O3', '-pthread', '-fomit-frame-pointer'] - standalone_only = [] - shared_only = [] + link_flags = ('-pthread',) + cflags = ('-O3', '-pthread', '-fomit-frame-pointer') + standalone_only = () + shared_only = () so_ext = 'so' make_cmd = 'gmake' @@ -32,4 +32,4 @@ return ['/usr/local/lib'] class Freebsd7_64(Freebsd7): - shared_only = ['-fPIC'] + shared_only = ('-fPIC',) Modified: pypy/trunk/pypy/translator/platform/linux.py ============================================================================== --- pypy/trunk/pypy/translator/platform/linux.py (original) +++ pypy/trunk/pypy/translator/platform/linux.py Tue Aug 24 17:35:45 2010 @@ -6,12 +6,13 @@ class Linux(BasePosix): name = "linux" - link_flags = ['-pthread', '-lrt'] - cflags = ['-O3', '-pthread', '-fomit-frame-pointer', '-Wall', '-Wno-unused'] - standalone_only = [] - shared_only = ['-fPIC'] + link_flags = ('-pthread', '-lrt') + cflags = ('-O3', '-pthread', '-fomit-frame-pointer', + '-Wall', '-Wno-unused') + standalone_only = () + shared_only = ('-fPIC',) so_ext = 'so' - so_prefixes = ['lib', ''] + so_prefixes = ('lib', '') def _args_for_shared(self, args): return ['-shared'] + args @@ -30,4 +31,4 @@ class Linux64(Linux): - shared_only = ['-fPIC'] + shared_only = ('-fPIC',) Modified: pypy/trunk/pypy/translator/platform/maemo.py ============================================================================== --- pypy/trunk/pypy/translator/platform/maemo.py (original) +++ pypy/trunk/pypy/translator/platform/maemo.py Tue Aug 24 17:35:45 2010 @@ -13,7 +13,7 @@ class Maemo(Linux): name = "maemo" - available_includedirs = ['/usr/include', '/tmp'] + available_includedirs = ('/usr/include', '/tmp') copied_cache = {} def _invent_new_name(self, basepath, base): Modified: pypy/trunk/pypy/translator/platform/posix.py ============================================================================== --- pypy/trunk/pypy/translator/platform/posix.py (original) +++ pypy/trunk/pypy/translator/platform/posix.py Tue Aug 24 17:35:45 2010 @@ -10,7 +10,7 @@ exe_ext = '' make_cmd = 'make' - relevant_environ=['CPATH', 'LIBRARY_PATH', 'C_INCLUDE_PATH'] + relevant_environ = ('CPATH', 'LIBRARY_PATH', 'C_INCLUDE_PATH') def __init__(self, cc=None): if cc is None: @@ -92,7 +92,7 @@ else: exe_name = exe_name.new(ext=self.exe_ext) - linkflags = self.link_flags[:] + linkflags = list(self.link_flags) if shared: linkflags = self._args_for_shared(linkflags) Modified: pypy/trunk/pypy/translator/platform/windows.py ============================================================================== --- pypy/trunk/pypy/translator/platform/windows.py (original) +++ pypy/trunk/pypy/translator/platform/windows.py Tue Aug 24 17:35:45 2010 @@ -72,10 +72,10 @@ cc = 'cl.exe' link = 'link.exe' - cflags = ['/MD', '/O2'] - link_flags = [] - standalone_only = [] - shared_only = [] + cflags = ('/MD', '/O2') + link_flags = () + standalone_only = () + shared_only = () environ = None def __init__(self, cc=None): @@ -218,7 +218,7 @@ m.exe_name = exe_name m.eci = eci - linkflags = self.link_flags[:] + linkflags = list(self.link_flags) if shared: linkflags = self._args_for_shared(linkflags) + [ '/EXPORT:$(PYPY_MAIN_FUNCTION)'] @@ -336,10 +336,10 @@ class MingwPlatform(posix.BasePosix): name = 'mingw32' - standalone_only = [] - shared_only = [] - cflags = ['-O3'] - link_flags = [] + standalone_only = () + shared_only = () + cflags = ('-O3',) + link_flags = () exe_ext = 'exe' so_ext = 'dll' From hakanardo at codespeak.net Tue Aug 24 17:52:16 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Tue, 24 Aug 2010 17:52:16 +0200 (CEST) Subject: [pypy-svn] r76717 - in pypy/branch/jit-bounds/pypy: jit/metainterp jit/metainterp/test module/pypyjit/test Message-ID: <20100824155216.210D4282B90@codespeak.net> Author: hakanardo Date: Tue Aug 24 17:52:13 2010 New Revision: 76717 Added: pypy/branch/jit-bounds/pypy/module/pypyjit/test/randomized.py Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_intbound.py pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/jit-bounds/pypy/module/pypyjit/test/test_pypy_c.py Log: int_mul and int_ne suppert, randomized tests, more traces Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py Tue Aug 24 17:52:13 2010 @@ -118,6 +118,9 @@ except OverflowError: res.has_upper = False return res + + def mul(self, value): + return self.mul_bound(IntBound(value, value)) def add_bound(self, other): res = self.copy() @@ -155,6 +158,35 @@ res.has_lower = False return res + def mul_bound(self, other): + if self.has_upper and self.has_lower and \ + other.has_upper and other.has_lower: + try: + vals = (ovfcheck(self.upper * other.upper), + ovfcheck(self.upper * other.lower), + ovfcheck(self.lower * other.upper), + ovfcheck(self.lower * other.lower)) + return IntBound(min4(vals), max4(vals)) + except OverflowError: + return IntUnbounded() + else: + return IntUnbounded() + + def div_bound(self, other): + if self.has_upper and self.has_lower and \ + other.has_upper and other.has_lower and \ + not other.contains(0): + try: + vals = (ovfcheck(self.upper / other.upper), + ovfcheck(self.upper / other.lower), + ovfcheck(self.lower / other.upper), + ovfcheck(self.lower / other.lower)) + return IntBound(min4(vals), max4(vals)) + except OverflowError: + return IntUnbounded() + else: + return IntUnbounded() + def contains(self, val): if self.has_lower and val < self.lower: return False @@ -711,6 +743,9 @@ self.resumedata_memo.update_counters(self.metainterp_sd.profiler) def propagate_bounds_backward(self, box): + # FIXME: This takes care of the instruction where box is the reuslt + # but the bounds produced by all instructions where box is + # an argument might also be tighten v = self.getvalue(box) b = v.intbound if b.has_lower and b.has_upper and b.lower == b.upper: @@ -817,7 +852,6 @@ oldop = self.pure_operations.get(targs, None) if oldop is not None and oldop.descr is op.descr: value = self.getvalue(oldop.result) - print value if value.is_constant(): if value.box.same_constant(CONST_1): self.make_constant(op.result, CONST_0) @@ -1280,13 +1314,29 @@ self.pure(rop.INT_SUB, [op.result, op.args[1]], op.args[0]) self.pure(rop.INT_SUB, [op.result, op.args[0]], op.args[1]) + def optimize_INT_MUL(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + # If one side of the op is 0 the result is the other side. + if v1.is_constant() and v1.box.getint() == 1: + self.make_equal_to(op.result, v2) + elif v2.is_constant() and v2.box.getint() == 1: + self.make_equal_to(op.result, v1) + elif (v1.is_constant() and v1.box.getint() == 0) or \ + (v2.is_constant() and v2.box.getint() == 0): + self.make_constant_int(op.result, 0) + else: + self.optimize_default(op) + r = self.getvalue(op.result) + r.intbound.intersect(v1.intbound.mul_bound(v2.intbound)) + def optimize_INT_ADD_OVF(self, op): v1 = self.getvalue(op.args[0]) v2 = self.getvalue(op.args[1]) resbound = v1.intbound.add_bound(v2.intbound) if resbound.has_lower and resbound.has_upper and \ self.loop.operations[self.i+1].opnum == rop.GUARD_NO_OVERFLOW: - # Transform into INT_SUB and remove guard + # Transform into INT_ADD and remove guard op.opnum = rop.INT_ADD self.i += 1 self.optimize_INT_ADD(op) @@ -1310,6 +1360,21 @@ r = self.getvalue(op.result) r.intbound.intersect(resbound) + def optimize_INT_MUL_OVF(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + resbound = v1.intbound.mul_bound(v2.intbound) + if resbound.has_lower and resbound.has_upper and \ + self.loop.operations[self.i+1].opnum == rop.GUARD_NO_OVERFLOW: + # Transform into INT_MUL and remove guard + op.opnum = rop.INT_MUL + self.i += 1 + self.optimize_INT_MUL(op) + else: + self.optimize_default(op) + r = self.getvalue(op.result) + r.intbound.intersect(resbound) + def optimize_INT_LT(self, op): v1 = self.getvalue(op.args[0]) v2 = self.getvalue(op.args[1]) @@ -1360,6 +1425,16 @@ else: self.optimize_default(op) + def optimize_INT_NE(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + if v1.intbound.known_gt(v2.intbound): + self.make_constant_int(op.result, 1) + elif v1.intbound.known_lt(v2.intbound): + self.make_constant_int(op.result, 1) + else: + self.optimize_default(op) + def make_int_lt(self, args): v1 = self.getvalue(args[0]) v2 = self.getvalue(args[1]) @@ -1426,6 +1501,17 @@ if v2.intbound.intersect(v1.intbound): self.propagate_bounds_backward(op.args[1]) + def propagate_bounds_INT_NE(self, op): + r = self.getvalue(op.result) + if r.is_constant(): + if r.box.same_constant(CONST_0): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + if v1.intbound.intersect(v2.intbound): + self.propagate_bounds_backward(op.args[0]) + if v2.intbound.intersect(v1.intbound): + self.propagate_bounds_backward(op.args[1]) + def propagate_bounds_INT_ADD(self, op): v1 = self.getvalue(op.args[0]) v2 = self.getvalue(op.args[1]) @@ -1444,12 +1530,24 @@ b = r.intbound.add_bound(v2.intbound) if v1.intbound.intersect(b): self.propagate_bounds_backward(op.args[0]) - # FIXME: - # b = r.intbound.sub_bound(v1.intbound).mul(-1) - # if v2.intbound.intersect(b): + b = r.intbound.sub_bound(v1.intbound).mul(-1) + if v2.intbound.intersect(b): + self.propagate_bounds_backward(op.args[1]) + + def propagate_bounds_INT_MUL(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + r = self.getvalue(op.result) + b = r.intbound.div_bound(v2.intbound) + if v1.intbound.intersect(b): + self.propagate_bounds_backward(op.args[0]) + b = r.intbound.div_bound(v1.intbound) + if v2.intbound.intersect(b): + self.propagate_bounds_backward(op.args[1]) propagate_bounds_INT_ADD_OVF = propagate_bounds_INT_ADD propagate_bounds_INT_SUB_OVF = propagate_bounds_INT_SUB + propagate_bounds_INT_MUL_OVF = propagate_bounds_INT_MUL optimize_ops = _findall(Optimizer, 'optimize_') propagate_bounds_ops = _findall(Optimizer, 'propagate_bounds_') @@ -1691,3 +1789,8 @@ write=True) +def min4(t): + return min(min(t[0], t[1]), min(t[2], t[3])) + +def max4(t): + return max(max(t[0], t[1]), max(t[2], t[3])) Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_intbound.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_intbound.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_intbound.py Tue Aug 24 17:52:13 2010 @@ -185,6 +185,53 @@ assert not a.contains(2) assert not a.contains(7) +def test_mul_bound(): + for _, _, b1 in some_bounds(): + for _, _, b2 in some_bounds(): + b3 = b1.mul_bound(b2) + for n1 in nbr: + for n2 in nbr: + if b1.contains(n1) and b2.contains(n2): + assert b3.contains(n1 * n2) + + a=bound(2, 4).mul_bound(bound(1, 2)) + assert not a.contains(1) + assert not a.contains(9) + + a=bound(-3, 2).mul_bound(bound(1, 2)) + assert not a.contains(-7) + assert not a.contains(5) + assert a.contains(-6) + assert a.contains(4) + + a=bound(-3, 2).mul(-1) + for i in range(-2,4): + assert a.contains(i) + assert not a.contains(4) + assert not a.contains(-3) + +def test_div_bound(): + for _, _, b1 in some_bounds(): + for _, _, b2 in some_bounds(): + b3 = b1.div_bound(b2) + for n1 in nbr: + for n2 in nbr: + if b1.contains(n1) and b2.contains(n2): + if n2 != 0: + assert b3.contains(n1 / n2) + + a=bound(2, 4).div_bound(bound(1, 2)) + assert not a.contains(0) + assert not a.contains(5) + + a=bound(-3, 2).div_bound(bound(1, 2)) + assert not a.contains(-4) + assert not a.contains(3) + assert a.contains(-3) + assert a.contains(0) + + + def test_sub_bound(): for _, _, b1 in some_bounds(): for _, _, b2 in some_bounds(): Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py Tue Aug 24 17:52:13 2010 @@ -3548,10 +3548,7 @@ guard_no_overflow() [] i2 = int_gt(i1, 1) guard_true(i2) [] - i3 = int_sub_ovf(1, i0) - guard_no_overflow() [] - i4 = int_gt(i3, 1) - guard_true(i4) [] + i3 = int_sub(1, i0) jump(i0) """ self.optimize_loop(ops, 'Not', expected) @@ -3594,6 +3591,75 @@ """ self.optimize_loop(ops, 'Not', expected) + def test_bound_eq_const_not(self): + ops = """ + [i0] + i1 = int_eq(i0, 7) + guard_false(i1) [] + i2 = int_add(i0, 3) + jump(i2) + """ + expected = """ + [i0] + i1 = int_eq(i0, 7) + guard_false(i1) [] + i2 = int_add(i0, 3) + jump(i2) + + """ + self.optimize_loop(ops, 'Not', expected) + + def test_bound_ne_const(self): + ops = """ + [i0] + i1 = int_ne(i0, 7) + guard_false(i1) [] + i2 = int_add(i0, 3) + jump(i2) + """ + expected = """ + [i0] + i1 = int_ne(i0, 7) + guard_false(i1) [] + jump(10) + + """ + self.optimize_loop(ops, 'Not', expected) + + def test_bound_ne_const_not(self): + ops = """ + [i0] + i1 = int_ne(i0, 7) + guard_true(i1) [] + i2 = int_add(i0, 3) + jump(i2) + """ + expected = """ + [i0] + i1 = int_ne(i0, 7) + guard_true(i1) [] + i2 = int_add(i0, 3) + jump(i2) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_bound_ltne(self): + ops = """ + [i0, i1] + i2 = int_lt(i0, 7) + guard_true(i2) [] + i3 = int_ne(i0, 10) + guard_true(i2) [] + jump(i0, i1) + """ + expected = """ + [i0, i1] + i2 = int_lt(i0, 7) + guard_true(i2) [] + jump(i0, i1) + """ + self.optimize_loop(ops, 'Not, Not', expected) + def test_bound_lege_const(self): ops = """ [i0] @@ -3615,6 +3681,96 @@ """ self.optimize_loop(ops, 'Not', expected) + def test_mul_ovf(self): + ops = """ + [i0, i1] + i2 = int_and(i0, 255) + i3 = int_lt(i1, 5) + guard_true(i3) [] + i4 = int_gt(i1, -10) + guard_true(i4) [] + i5 = int_mul_ovf(i2, i1) + guard_no_overflow() [] + i6 = int_lt(i5, -2550) + guard_false(i6) [] + i7 = int_ge(i5, 1276) + guard_false(i7) [] + i8 = int_gt(i5, 126) + guard_true(i8) [] + jump(i0, i1) + """ + expected = """ + [i0, i1] + i2 = int_and(i0, 255) + i3 = int_lt(i1, 5) + guard_true(i3) [] + i4 = int_gt(i1, -10) + guard_true(i4) [] + i5 = int_mul(i2, i1) + i8 = int_gt(i5, 126) + guard_true(i8) [] + jump(i0, i1) + """ + self.optimize_loop(ops, 'Not, Not', expected) + + def test_mul_ovf_before(self): + ops = """ + [i0, i1] + i2 = int_and(i0, 255) + i22 = int_add(i2, 1) + i3 = int_mul_ovf(i22, i1) + guard_no_overflow() [] + i4 = int_lt(i3, 10) + guard_true(i4) [] + i5 = int_gt(i3, 2) + guard_true(i5) [] + i6 = int_lt(i1, 0) + guard_false(i6) [] + jump(i0, i1) + """ + expected = """ + [i0, i1] + i2 = int_and(i0, 255) + i22 = int_add(i2, 1) + i3 = int_mul_ovf(i22, i1) + guard_no_overflow() [] + i4 = int_lt(i3, 10) + guard_true(i4) [] + i5 = int_gt(i3, 2) + guard_true(i5) [] + jump(i0, i1) + """ + self.optimize_loop(ops, 'Not, Not', expected) + + def test_sub_ovf_before(self): + ops = """ + [i0, i1] + i2 = int_and(i0, 255) + i3 = int_sub_ovf(i2, i1) + guard_no_overflow() [] + i4 = int_le(i3, 10) + guard_true(i4) [] + i5 = int_ge(i3, 2) + guard_true(i5) [] + i6 = int_lt(i1, -10) + guard_false(i6) [] + i7 = int_gt(i1, 253) + guard_false(i7) [] + jump(i0, i1) + """ + expected = """ + [i0, i1] + i2 = int_and(i0, 255) + i3 = int_sub_ovf(i2, i1) + guard_no_overflow() [] + i4 = int_le(i3, 10) + guard_true(i4) [] + i5 = int_ge(i3, 2) + guard_true(i5) [] + jump(i0, i1) + """ + self.optimize_loop(ops, 'Not, Not', expected) + Added: pypy/branch/jit-bounds/pypy/module/pypyjit/test/randomized.py ============================================================================== --- (empty file) +++ pypy/branch/jit-bounds/pypy/module/pypyjit/test/randomized.py Tue Aug 24 17:52:13 2010 @@ -0,0 +1,124 @@ +#!/usr/bin/python + +import random, inspect, os + +class RandomCode(object): + + maxifdepth = 10 + maxopdepth = 20 + + def __init__(self): + self.vars = set() + + def sample(self, population): + return random.sample(population, 1)[0] + + def chose(self, *args): + return self.sample(args)() + + def expression(self): + if len(self.vars) == 0: + return self.constant() + elif self.depth() > self.maxopdepth: + return self.chose(self.variable, self.constant) + else: + return self.chose(self.variable, self.opperation, self.constant) + + def variable(self): + return self.sample(self.vars) + + def opperation(self): + return self.expression() + ' ' + self.sample(self.opperators) + \ + ' ' + self.expression() + + def test(self): + return self.expression() + ' ' + self.sample(self.tests) + \ + ' ' + self.expression() + + def constant(self): + return str(self.sample(self.constants)) + + def depth(self): + return len(inspect.getouterframes(inspect.currentframe())) + + def statement(self): + if self.depth() > self.maxifdepth: + return self.assignment() + else: + return self.chose(self.assignment, self.ifstatement) + + def assignment(self): + v = self.sample(self.varnames) + s = v + ' = ' + self.expression() + '\n' + self.vars.add(v) + return s + + def indent(self, s): + lines = s.split('\n') + lines = [' ' + l for l in lines[:-1]] + return '\n'.join(lines) + '\n' + + def ifstatement(self): + return 'if ' + self.test() + ':\n' + self.indent(self.block(5)) + + def block(self, n): + s = '' + for i in range(random.randrange(1,n)): + s += self.statement() + return s + + def whileloop(self): + self.vars.add('i') + return 'i = 0\nwhile i < 10:\n' + \ + self.indent(self.block(5) + 'i += 1\n') + + def setupvars(self): + return ', '.join(self.vars) + ' = ' + \ + ', '.join('0' * len(self.vars)) + '\n' + + def return_statement(self): + return 'return (' + ', '.join(self.vars) + ')\n' + + +class IntBounds(RandomCode): + opperators = ('+', '-', '*') + tests = ('<', '>', '<=', '>=', '==', '!=') + constants = range(-3,4) + varnames = 'abcd' + + def function(self, name='f'): + body = self.block(3) + self.whileloop() + self.return_statement() + body = self.setupvars() + body + return 'def %s():\n' % name + self.indent(body) + + +def run(python, code): + (s,r,e) = os.popen3(python) + s.write(code) + s.close() + res = r.read() + err = e.read() + r.close() + return res, err + +while True: + code = ''' +try: # make the file runnable by CPython + import pypyjit + pypyjit.set_param(threshold=3) +except ImportError: + pass + +%s +print f() +''' % IntBounds().function('f') + + r1,e1 = run('/usr/bin/python', code) + r2,e2 = run('../../../translator/goal/pypy-c', code) + if r1 != r2: + print + print '******************** FAILED ******************' + print code + print 'cpython: ', r1, e1 + print 'pypy: ', r2, e2 + Modified: pypy/branch/jit-bounds/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/jit-bounds/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/jit-bounds/pypy/module/pypyjit/test/test_pypy_c.py Tue Aug 24 17:52:13 2010 @@ -849,7 +849,7 @@ ''', maxops, ([tc], res)) def test_intbound_simple(self): - ops = ('<', '>', '<=', '>=') + ops = ('<', '>', '<=', '>=', '==', '!=') nbr = (3, 7) for o1 in ops: for o2 in ops: @@ -870,7 +870,10 @@ def main(): res = [0] * 4 - for i in range(15) * 1500: + idx = [] + for i in range(15): + idx.extend([i] * 1500) + for i in idx: res[f(i)] += 1 return res @@ -880,12 +883,12 @@ res = [0] * 4 for i in range(15): res[f(i)] += 1500 - self.run_source(src, 220, ([], res)) + self.run_source(src, 268, ([], res)) def test_intbound_addsub_mix(self): tests = ('i > 4', 'i > 2', 'i + 1 > 2', '1 + i > 4', 'i - 1 > 1', '1 - i > 1', '1 - i < -3', - 'i == 1', 'i == 5') + 'i == 1', 'i == 5', 'i != 1', '-2 * i < -4') for t1 in tests: for t2 in tests: print t1, t2 @@ -904,7 +907,10 @@ def main(): res = [0] * 4 - for i in range(15) * 1500: + idx = [] + for i in range(15): + idx.extend([i] * 1500) + for i in idx: res[f(i)] += 1 return res @@ -914,7 +920,7 @@ res = [0] * 4 for i in range(15): res[f(i)] += 1500 - self.run_source(src, 232, ([], res)) + self.run_source(src, 280, ([], res)) def test_intbound_gt(self): self.run_source(''' @@ -953,6 +959,19 @@ return (a, b) ''', 56, ([], (2000, 2000))) + def test_intbound_addmul_ge(self): + self.run_source(''' + def main(): + i, a, b = 0, 0, 0 + while i < 2000: + if i + 5 >= 5: + a += 1 + if 2 * i >= 0: + b += 1 + i += 1 + return (a, b) + ''', 53, ([], (2000, 2000))) + def test_intbound_eq(self): self.run_source(''' def main(a): @@ -968,6 +987,20 @@ return s ''', 69, ([7], 12000), ([42], 1509), ([10], 1509)) + def test_intbound_mul(self): + self.run_source(''' + def main(a): + i, s = 0, 0 + while i < 1500: + assert i >= 0 + if 2 * i < 30000: + s += 1 + else: + s += a + i += 1 + return s + ''', 43, ([7], 1500)) + def test_assert(self): self.run_source(''' def main(a): @@ -1012,6 +1045,7 @@ self = array.__new__(cls, 'd', range(256)) return self def __getitem__(self, i): + # assert self.__len__() == 256 (FIXME: does not improve) return array.__getitem__(self, i & 255) def main(): From hakanardo at codespeak.net Tue Aug 24 18:54:30 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Tue, 24 Aug 2010 18:54:30 +0200 (CEST) Subject: [pypy-svn] r76718 - in pypy/branch/jit-bounds: . lib_pypy pypy/annotation pypy/interpreter pypy/interpreter/astcompiler pypy/interpreter/test pypy/jit/backend/llgraph pypy/jit/backend/llsupport pypy/jit/backend/test pypy/jit/backend/x86 pypy/jit/backend/x86/test pypy/jit/codewriter pypy/jit/codewriter/test pypy/jit/metainterp pypy/jit/metainterp/test pypy/module/__builtin__ pypy/module/_locale pypy/module/_sre pypy/module/_sre/test pypy/module/array pypy/module/array/benchmark pypy/module/array/test pypy/module/cpyext pypy/module/cpyext/test pypy/module/pypyjit pypy/module/pypyjit/test pypy/module/test_lib_pypy pypy/objspace/std pypy/rlib pypy/rlib/rsre pypy/rlib/rsre/test pypy/rlib/test pypy/rpython pypy/rpython/test pypy/tool/release pypy/tool/release/test pypy/translator/platform Message-ID: <20100824165430.C659C282B90@codespeak.net> Author: hakanardo Date: Tue Aug 24 18:54:23 2010 New Revision: 76718 Added: pypy/branch/jit-bounds/pypy/module/test_lib_pypy/test_msvcrt.py - copied unchanged from r76716, pypy/trunk/pypy/module/test_lib_pypy/test_msvcrt.py pypy/branch/jit-bounds/pypy/rlib/rsre/ - copied from r76716, pypy/trunk/pypy/rlib/rsre/ pypy/branch/jit-bounds/pypy/rlib/rsre/__init__.py - copied unchanged from r76716, pypy/trunk/pypy/rlib/rsre/__init__.py pypy/branch/jit-bounds/pypy/rlib/rsre/rsre_char.py - copied unchanged from r76716, pypy/trunk/pypy/rlib/rsre/rsre_char.py pypy/branch/jit-bounds/pypy/rlib/rsre/rsre_core.py - copied unchanged from r76716, pypy/trunk/pypy/rlib/rsre/rsre_core.py pypy/branch/jit-bounds/pypy/rlib/rsre/test/ - copied from r76716, pypy/trunk/pypy/rlib/rsre/test/ pypy/branch/jit-bounds/pypy/rlib/rsre/test/__init__.py - copied unchanged from r76716, pypy/trunk/pypy/rlib/rsre/test/__init__.py pypy/branch/jit-bounds/pypy/rlib/rsre/test/targetrsre.py - copied unchanged from r76716, pypy/trunk/pypy/rlib/rsre/test/targetrsre.py pypy/branch/jit-bounds/pypy/rlib/rsre/test/test_search.py - copied unchanged from r76716, pypy/trunk/pypy/rlib/rsre/test/test_search.py pypy/branch/jit-bounds/pypy/rlib/rsre/test/test_zinterp.py - copied unchanged from r76716, pypy/trunk/pypy/rlib/rsre/test/test_zinterp.py Removed: pypy/branch/jit-bounds/lib_pypy/_sre.py pypy/branch/jit-bounds/pypy/module/_sre/app_sre.py pypy/branch/jit-bounds/pypy/module/pypyjit/test/test_can_inline.py Modified: pypy/branch/jit-bounds/ (props changed) pypy/branch/jit-bounds/lib_pypy/msvcrt.py pypy/branch/jit-bounds/pypy/annotation/specialize.py pypy/branch/jit-bounds/pypy/interpreter/astcompiler/consts.py pypy/branch/jit-bounds/pypy/interpreter/pycode.py pypy/branch/jit-bounds/pypy/interpreter/test/test_code.py pypy/branch/jit-bounds/pypy/interpreter/test/test_zpy.py pypy/branch/jit-bounds/pypy/jit/backend/llgraph/llimpl.py pypy/branch/jit-bounds/pypy/jit/backend/llsupport/descr.py pypy/branch/jit-bounds/pypy/jit/backend/test/runner_test.py pypy/branch/jit-bounds/pypy/jit/backend/x86/assembler.py pypy/branch/jit-bounds/pypy/jit/backend/x86/test/test_runner.py pypy/branch/jit-bounds/pypy/jit/backend/x86/test/test_ztranslation.py pypy/branch/jit-bounds/pypy/jit/codewriter/assembler.py pypy/branch/jit-bounds/pypy/jit/codewriter/jitcode.py pypy/branch/jit-bounds/pypy/jit/codewriter/jtransform.py pypy/branch/jit-bounds/pypy/jit/codewriter/test/test_flatten.py pypy/branch/jit-bounds/pypy/jit/metainterp/blackhole.py pypy/branch/jit-bounds/pypy/jit/metainterp/executor.py pypy/branch/jit-bounds/pypy/jit/metainterp/history.py pypy/branch/jit-bounds/pypy/jit/metainterp/jitdriver.py pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py pypy/branch/jit-bounds/pypy/jit/metainterp/pyjitpl.py pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_basic.py pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_immutable.py pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_jitdriver.py pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_recursive.py pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_virtualizable.py pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_warmspot.py pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_warmstate.py pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_ztranslation.py pypy/branch/jit-bounds/pypy/jit/metainterp/warmspot.py pypy/branch/jit-bounds/pypy/jit/metainterp/warmstate.py pypy/branch/jit-bounds/pypy/module/__builtin__/functional.py pypy/branch/jit-bounds/pypy/module/_locale/__init__.py pypy/branch/jit-bounds/pypy/module/_sre/__init__.py pypy/branch/jit-bounds/pypy/module/_sre/interp_sre.py pypy/branch/jit-bounds/pypy/module/_sre/test/test_app_sre.py pypy/branch/jit-bounds/pypy/module/array/benchmark/Makefile (props changed) pypy/branch/jit-bounds/pypy/module/array/benchmark/intimg.c (props changed) pypy/branch/jit-bounds/pypy/module/array/benchmark/intimgtst.c (props changed) pypy/branch/jit-bounds/pypy/module/array/benchmark/intimgtst.py (props changed) pypy/branch/jit-bounds/pypy/module/array/benchmark/loop.c (props changed) pypy/branch/jit-bounds/pypy/module/array/benchmark/sum.c (props changed) pypy/branch/jit-bounds/pypy/module/array/benchmark/sumtst.c (props changed) pypy/branch/jit-bounds/pypy/module/array/benchmark/sumtst.py (props changed) pypy/branch/jit-bounds/pypy/module/array/interp_array.py pypy/branch/jit-bounds/pypy/module/array/test/test_array.py pypy/branch/jit-bounds/pypy/module/array/test/test_array_old.py (props changed) pypy/branch/jit-bounds/pypy/module/cpyext/stubs.py pypy/branch/jit-bounds/pypy/module/cpyext/test/test_unicodeobject.py pypy/branch/jit-bounds/pypy/module/cpyext/unicodeobject.py pypy/branch/jit-bounds/pypy/module/pypyjit/interp_jit.py pypy/branch/jit-bounds/pypy/module/pypyjit/policy.py pypy/branch/jit-bounds/pypy/objspace/std/intobject.py pypy/branch/jit-bounds/pypy/objspace/std/model.py pypy/branch/jit-bounds/pypy/rlib/debug.py pypy/branch/jit-bounds/pypy/rlib/jit.py pypy/branch/jit-bounds/pypy/rlib/test/test_jit.py pypy/branch/jit-bounds/pypy/rpython/rclass.py pypy/branch/jit-bounds/pypy/rpython/test/test_rclass.py pypy/branch/jit-bounds/pypy/tool/release/package.py pypy/branch/jit-bounds/pypy/tool/release/test/test_package.py pypy/branch/jit-bounds/pypy/translator/platform/__init__.py pypy/branch/jit-bounds/pypy/translator/platform/darwin.py pypy/branch/jit-bounds/pypy/translator/platform/freebsd7.py pypy/branch/jit-bounds/pypy/translator/platform/linux.py pypy/branch/jit-bounds/pypy/translator/platform/maemo.py pypy/branch/jit-bounds/pypy/translator/platform/posix.py pypy/branch/jit-bounds/pypy/translator/platform/windows.py Log: svn merge -r76604:76716 svn+ssh://hakanardo at codespeak.net/svn/pypy/trunk Modified: pypy/branch/jit-bounds/lib_pypy/msvcrt.py ============================================================================== --- pypy/branch/jit-bounds/lib_pypy/msvcrt.py (original) +++ pypy/branch/jit-bounds/lib_pypy/msvcrt.py Tue Aug 24 18:54:23 2010 @@ -5,9 +5,12 @@ """ # XXX incomplete: implemented only functions needed by subprocess.py +# PAC: 2010/08 added MS locking for Whoosh import ctypes from ctypes_support import standard_c_lib as _c +from ctypes_support import get_errno +import errno try: open_osfhandle = _c._open_osfhandle @@ -25,4 +28,17 @@ setmode.argtypes = [ctypes.c_int, ctypes.c_int] setmode.restype = ctypes.c_int +LK_UNLCK, LK_LOCK, LK_NBLCK, LK_RLCK, LK_NBRLCK = range(5) + +_locking = _c._locking +_locking.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int] +_locking.restype = ctypes.c_int + +def locking(fd, mode, nbytes): + '''lock or unlock a number of bytes in a file.''' + rv = _locking(fd, mode, nbytes) + if rv != 0: + e = get_errno() + raise IOError(e, errno.errorcode[e]) + del ctypes Modified: pypy/branch/jit-bounds/pypy/annotation/specialize.py ============================================================================== --- pypy/branch/jit-bounds/pypy/annotation/specialize.py (original) +++ pypy/branch/jit-bounds/pypy/annotation/specialize.py Tue Aug 24 18:54:23 2010 @@ -354,6 +354,12 @@ def specialize_argtype(funcdesc, args_s, *argindices): key = tuple([args_s[i].knowntype for i in argindices]) + for cls in key: + try: + assert '_must_specialize_' not in cls.classdesc.pyobj.__dict__, ( + "%s has the tag _must_specialize_" % (cls,)) + except AttributeError: + pass return maybe_star_args(funcdesc, key, args_s) def specialize_arglistitemtype(funcdesc, args_s, i): Modified: pypy/branch/jit-bounds/pypy/interpreter/astcompiler/consts.py ============================================================================== --- pypy/branch/jit-bounds/pypy/interpreter/astcompiler/consts.py (original) +++ pypy/branch/jit-bounds/pypy/interpreter/astcompiler/consts.py Tue Aug 24 18:54:23 2010 @@ -9,7 +9,6 @@ CO_NESTED = 0x0010 CO_GENERATOR = 0x0020 CO_NOFREE = 0x0040 -CO_CONTAINSLOOP = 0x0080 CO_CONTAINSGLOBALS = 0x0800 CO_GENERATOR_ALLOWED = 0x1000 CO_FUTURE_DIVISION = 0x2000 Modified: pypy/branch/jit-bounds/pypy/interpreter/pycode.py ============================================================================== --- pypy/branch/jit-bounds/pypy/interpreter/pycode.py (original) +++ pypy/branch/jit-bounds/pypy/interpreter/pycode.py Tue Aug 24 18:54:23 2010 @@ -13,7 +13,7 @@ from pypy.interpreter.baseobjspace import ObjSpace, W_Root from pypy.interpreter.astcompiler.consts import (CO_OPTIMIZED, CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED, - CO_GENERATOR, CO_CONTAINSLOOP, CO_CONTAINSGLOBALS) + CO_GENERATOR, CO_CONTAINSGLOBALS) from pypy.rlib.rarithmetic import intmask from pypy.rlib.debug import make_sure_not_resized, make_sure_not_modified from pypy.rlib import jit @@ -133,9 +133,7 @@ while opcode == opcodedesc.EXTENDED_ARG.index: opcode = ord(co_code[next_instr]) next_instr += 3 - if opcode == opcodedesc.JUMP_ABSOLUTE.index: - self.co_flags |= CO_CONTAINSLOOP - elif opcode == opcodedesc.LOAD_GLOBAL.index: + if opcode == opcodedesc.LOAD_GLOBAL.index: self.co_flags |= CO_CONTAINSGLOBALS elif opcode == opcodedesc.LOAD_NAME.index: self.co_flags |= CO_CONTAINSGLOBALS Modified: pypy/branch/jit-bounds/pypy/interpreter/test/test_code.py ============================================================================== --- pypy/branch/jit-bounds/pypy/interpreter/test/test_code.py (original) +++ pypy/branch/jit-bounds/pypy/interpreter/test/test_code.py Tue Aug 24 18:54:23 2010 @@ -184,8 +184,6 @@ # CO_NESTED assert f(4).func_code.co_flags & 0x10 assert f.func_code.co_flags & 0x10 == 0 - # check for CO_CONTAINSLOOP - assert not f.func_code.co_flags & 0x0080 # check for CO_CONTAINSGLOBALS assert not f.func_code.co_flags & 0x0800 @@ -198,9 +196,6 @@ return [l for l in [1, 2, 3, 4]] """ - # check for CO_CONTAINSLOOP - assert f.func_code.co_flags & 0x0080 - assert g.func_code.co_flags & 0x0080 # check for CO_CONTAINSGLOBALS assert f.func_code.co_flags & 0x0800 assert not g.func_code.co_flags & 0x0800 Modified: pypy/branch/jit-bounds/pypy/interpreter/test/test_zpy.py ============================================================================== --- pypy/branch/jit-bounds/pypy/interpreter/test/test_zpy.py (original) +++ pypy/branch/jit-bounds/pypy/interpreter/test/test_zpy.py Tue Aug 24 18:54:23 2010 @@ -3,27 +3,28 @@ import py import sys import pypy +import subprocess pypypath = py.path.local(pypy.__file__).dirpath("bin", "py.py") -def cmdexec(s): - if sys.platform == 'win32': - s = '"%s"' % s # double double quotes - return py.process.cmdexec(s) +def run(*args): + argslist = map(str, args) + popen = subprocess.Popen(argslist, stdout=subprocess.PIPE) + stdout, stderr = popen.communicate() + return stdout def test_executable(): """Ensures sys.executable points to the py.py script""" # TODO : watch out for spaces/special chars in pypypath - output = cmdexec( '''"%s" "%s" -c "import sys;print sys.executable" ''' % - (sys.executable, pypypath) ) + output = run(sys.executable, pypypath, + "-c", "import sys;print sys.executable") assert output.splitlines()[-1] == pypypath def test_special_names(): """Test the __name__ and __file__ special global names""" cmd = "print __name__; print '__file__' in globals()" - output = cmdexec( '''"%s" "%s" -c "%s" ''' % - (sys.executable, pypypath, cmd) ) + output = run(sys.executable, pypypath, '-c', cmd) assert output.splitlines()[-2] == '__main__' assert output.splitlines()[-1] == 'False' @@ -32,26 +33,25 @@ tmpfile.write("print __name__; print __file__\n") tmpfile.close() - output = cmdexec( '''"%s" "%s" "%s" ''' % - (sys.executable, pypypath, tmpfilepath) ) + output = run(sys.executable, pypypath, tmpfilepath) assert output.splitlines()[-2] == '__main__' assert output.splitlines()[-1] == str(tmpfilepath) def test_argv_command(): """Some tests on argv""" # test 1 : no arguments - output = cmdexec( '''"%s" "%s" -c "import sys;print sys.argv" ''' % - (sys.executable, pypypath) ) + output = run(sys.executable, pypypath, + "-c", "import sys;print sys.argv") assert output.splitlines()[-1] == str(['-c']) # test 2 : some arguments after - output = cmdexec( '''"%s" "%s" -c "import sys;print sys.argv" hello''' % - (sys.executable, pypypath) ) + output = run(sys.executable, pypypath, + "-c", "import sys;print sys.argv", "hello") assert output.splitlines()[-1] == str(['-c','hello']) # test 3 : additionnal pypy parameters - output = cmdexec( '''"%s" "%s" -O -c "import sys;print sys.argv" hello''' % - (sys.executable, pypypath) ) + output = run(sys.executable, pypypath, + "-O", "-c", "import sys;print sys.argv", "hello") assert output.splitlines()[-1] == str(['-c','hello']) SCRIPT_1 = """ @@ -65,18 +65,15 @@ tmpfile.close() # test 1 : no arguments - output = cmdexec( '''"%s" "%s" "%s" ''' % - (sys.executable, pypypath, tmpfilepath) ) + output = run(sys.executable, pypypath, tmpfilepath) assert output.splitlines()[-1] == str([tmpfilepath]) # test 2 : some arguments after - output = cmdexec( '''"%s" "%s" "%s" hello''' % - (sys.executable, pypypath, tmpfilepath) ) + output = run(sys.executable, pypypath, tmpfilepath, "hello") assert output.splitlines()[-1] == str([tmpfilepath,'hello']) # test 3 : additionnal pypy parameters - output = cmdexec( '''"%s" "%s" -O "%s" hello''' % - (sys.executable, pypypath, tmpfilepath) ) + output = run(sys.executable, pypypath, "-O", tmpfilepath, "hello") assert output.splitlines()[-1] == str([tmpfilepath,'hello']) @@ -98,11 +95,7 @@ tmpfile.write(TB_NORMALIZATION_CHK) tmpfile.close() - e = None - try: - output = cmdexec( '''"%s" "%s" "%s" ''' % - (sys.executable, pypypath, tmpfilepath) ) - except py.process.cmdexec.Error, e: - pass - assert e," expected failure" - assert e.err.splitlines()[-1] == 'KeyError: ' + popen = subprocess.Popen([sys.executable, str(pypypath), tmpfilepath], + stderr=subprocess.PIPE) + _, stderr = popen.communicate() + assert stderr.endswith('KeyError: \n') Modified: pypy/branch/jit-bounds/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/jit-bounds/pypy/jit/backend/llgraph/llimpl.py Tue Aug 24 18:54:23 2010 @@ -123,6 +123,9 @@ 'setarrayitem_gc' : (('ref', 'int', 'intorptr'), None), 'getarrayitem_gc' : (('ref', 'int'), 'intorptr'), 'getarrayitem_gc_pure' : (('ref', 'int'), 'intorptr'), + 'setarrayitem_raw' : (('ref', 'int', 'intorptr'), None), + 'getarrayitem_raw' : (('ref', 'int'), 'intorptr'), + 'getarrayitem_raw_pure' : (('ref', 'int'), 'intorptr'), 'arraylen_gc' : (('ref',), 'int'), 'call' : (('ref', 'varargs'), 'intorptr'), 'call_assembler' : (('ref', 'varargs'), 'intorptr'), @@ -689,6 +692,18 @@ op_getarrayitem_gc_pure = op_getarrayitem_gc + def op_getarrayitem_raw(self, arraydescr, array, index): + if arraydescr.typeinfo == REF: + raise NotImplementedError("getarrayitem_raw -> gcref") + elif arraydescr.typeinfo == INT: + return do_getarrayitem_raw_int(array, index) + elif arraydescr.typeinfo == FLOAT: + return do_getarrayitem_raw_float(array, index) + else: + raise NotImplementedError + + op_getarrayitem_raw_pure = op_getarrayitem_raw + def op_getfield_gc(self, fielddescr, struct): if fielddescr.typeinfo == REF: return do_getfield_gc_ptr(struct, fielddescr.ofs) @@ -734,6 +749,16 @@ else: raise NotImplementedError + def op_setarrayitem_raw(self, arraydescr, array, index, newvalue): + if arraydescr.typeinfo == REF: + raise NotImplementedError("setarrayitem_raw <- gcref") + elif arraydescr.typeinfo == INT: + do_setarrayitem_raw_int(array, index, newvalue) + elif arraydescr.typeinfo == FLOAT: + do_setarrayitem_raw_float(array, index, newvalue) + else: + raise NotImplementedError + def op_setfield_gc(self, fielddescr, struct, newvalue): if fielddescr.typeinfo == REF: do_setfield_gc_ptr(struct, fielddescr.ofs, newvalue) Modified: pypy/branch/jit-bounds/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/branch/jit-bounds/pypy/jit/backend/llsupport/descr.py Tue Aug 24 18:54:23 2010 @@ -198,9 +198,13 @@ try: return cache[ARRAY] except KeyError: + # we only support Arrays that are either GcArrays, or raw no-length + # non-gc Arrays. if ARRAY._hints.get('nolength', False): + assert not isinstance(ARRAY, lltype.GcArray) arraydescr = getArrayNoLengthDescrClass(ARRAY)() else: + assert isinstance(ARRAY, lltype.GcArray) arraydescr = getArrayDescrClass(ARRAY)() # verify basic assumption that all arrays' basesize and ofslength # are equal Modified: pypy/branch/jit-bounds/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/jit-bounds/pypy/jit/backend/test/runner_test.py Tue Aug 24 18:54:23 2010 @@ -1832,6 +1832,31 @@ assert self.cpu.get_latest_value_float(0) == 13.5 assert called + def test_raw_malloced_getarrayitem(self): + ARRAY = rffi.CArray(lltype.Signed) + descr = self.cpu.arraydescrof(ARRAY) + a = lltype.malloc(ARRAY, 10, flavor='raw') + a[7] = -4242 + addr = llmemory.cast_ptr_to_adr(a) + abox = BoxInt(heaptracker.adr2int(addr)) + r1 = self.execute_operation(rop.GETARRAYITEM_RAW, [abox, BoxInt(7)], + 'int', descr=descr) + assert r1.getint() == -4242 + lltype.free(a, flavor='raw') + + def test_raw_malloced_setarrayitem(self): + ARRAY = rffi.CArray(lltype.Signed) + descr = self.cpu.arraydescrof(ARRAY) + a = lltype.malloc(ARRAY, 10, flavor='raw') + addr = llmemory.cast_ptr_to_adr(a) + abox = BoxInt(heaptracker.adr2int(addr)) + self.execute_operation(rop.SETARRAYITEM_RAW, [abox, BoxInt(5), + BoxInt(12345)], + 'void', descr=descr) + assert a[5] == 12345 + lltype.free(a, flavor='raw') + + class OOtypeBackendTest(BaseBackendTest): type_system = 'ootype' Modified: pypy/branch/jit-bounds/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/jit-bounds/pypy/jit/backend/x86/assembler.py Tue Aug 24 18:54:23 2010 @@ -241,7 +241,7 @@ f = open_file_as_stream(output_log, "w") for i in range(len(self.loop_run_counters)): name, struct = self.loop_run_counters[i] - f.write(name + ":" + str(struct.i) + "\n") + f.write(str(struct.i) + " " * (8 - len(str(struct.i))) + name + "\n") f.close() def _build_float_constants(self): Modified: pypy/branch/jit-bounds/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/branch/jit-bounds/pypy/jit/backend/x86/test/test_runner.py Tue Aug 24 18:54:23 2010 @@ -503,4 +503,4 @@ assert struct.i == 10 self.cpu.finish_once() lines = py.path.local(self.logfile + ".count").readlines() - assert lines[0] == 'xyz:10\n' + assert lines[0] == '10 xyz\n' Modified: pypy/branch/jit-bounds/pypy/jit/backend/x86/test/test_ztranslation.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/backend/x86/test/test_ztranslation.py (original) +++ pypy/branch/jit-bounds/pypy/jit/backend/x86/test/test_ztranslation.py Tue Aug 24 18:54:23 2010 @@ -75,8 +75,7 @@ driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'], virtualizables = ['frame'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno: str(codeno)) class SomewhereElse(object): pass Modified: pypy/branch/jit-bounds/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/jit-bounds/pypy/jit/codewriter/assembler.py Tue Aug 24 18:54:23 2010 @@ -53,6 +53,7 @@ self.liveness = {} self.startpoints = set() self.alllabels = set() + self.resulttypes = {} def emit_reg(self, reg): if reg.index >= self.count_regs[reg.kind]: @@ -165,7 +166,9 @@ raise NotImplementedError(x) # opname = insn[0] - assert '>' not in argcodes or argcodes.index('>') == len(argcodes) - 2 + if '>' in argcodes: + assert argcodes.index('>') == len(argcodes) - 2 + self.resulttypes[len(self.code)] = argcodes[-1] key = opname + '/' + ''.join(argcodes) num = self.insns.setdefault(key, len(self.insns)) self.code[startposition] = chr(num) @@ -212,7 +215,8 @@ self.count_regs['float'], liveness=self.liveness, startpoints=self.startpoints, - alllabels=self.alllabels) + alllabels=self.alllabels, + resulttypes=self.resulttypes) def see_raw_object(self, value): if value._obj not in self._seen_raw_objects: Modified: pypy/branch/jit-bounds/pypy/jit/codewriter/jitcode.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/codewriter/jitcode.py (original) +++ pypy/branch/jit-bounds/pypy/jit/codewriter/jitcode.py Tue Aug 24 18:54:23 2010 @@ -19,7 +19,8 @@ def setup(self, code='', constants_i=[], constants_r=[], constants_f=[], num_regs_i=255, num_regs_r=255, num_regs_f=255, - liveness=None, startpoints=None, alllabels=None): + liveness=None, startpoints=None, alllabels=None, + resulttypes=None): self.code = code # if the following lists are empty, use a single shared empty list self.constants_i = constants_i or self._empty_i @@ -33,6 +34,7 @@ self.liveness = make_liveness_cache(liveness) self._startpoints = startpoints # debugging self._alllabels = alllabels # debugging + self._resulttypes = resulttypes # debugging def get_fnaddr_as_int(self): return heaptracker.adr2int(self.fnaddr) Modified: pypy/branch/jit-bounds/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/jit-bounds/pypy/jit/codewriter/jtransform.py Tue Aug 24 18:54:23 2010 @@ -829,13 +829,20 @@ self.make_three_lists(op.args[2:2+num_green_args]) + self.make_three_lists(op.args[2+num_green_args:])) op1 = SpaceOperation('jit_merge_point', args, None) - return ops + [op1] + op2 = SpaceOperation('-live-', [], None) + # ^^^ we need a -live- for the case of do_recursive_call() + return ops + [op1, op2] - def handle_jit_marker__can_enter_jit(self, op, jitdriver): + def handle_jit_marker__loop_header(self, op, jitdriver): jd = self.callcontrol.jitdriver_sd_from_jitdriver(jitdriver) assert jd is not None c_index = Constant(jd.index, lltype.Signed) - return SpaceOperation('can_enter_jit', [c_index], None) + return SpaceOperation('loop_header', [c_index], None) + + # a 'can_enter_jit' in the source graph becomes a 'loop_header' + # operation in the transformed graph, as its only purpose in + # the transformed graph is to detect loops. + handle_jit_marker__can_enter_jit = handle_jit_marker__loop_header def rewrite_op_debug_assert(self, op): log.WARNING("found debug_assert in %r; should have be removed" % Modified: pypy/branch/jit-bounds/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/jit-bounds/pypy/jit/codewriter/test/test_flatten.py Tue Aug 24 18:54:23 2010 @@ -593,7 +593,8 @@ -live- %i0, %i1 int_guard_value %i0 jit_merge_point $27, I[%i0], R[], F[], I[%i1], R[], F[] - can_enter_jit $27 + -live- + loop_header $27 void_return """, transform=True, liveness=True, cc=MyFakeCallControl(), jd=jd) Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/blackhole.py Tue Aug 24 18:54:23 2010 @@ -2,7 +2,7 @@ from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.debug import debug_start, debug_stop -from pypy.rlib.debug import make_sure_not_resized +from pypy.rlib.debug import make_sure_not_resized, fatalerror from pypy.rpython.lltypesystem import lltype, llmemory, rclass from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.llinterp import LLException @@ -756,11 +756,15 @@ assert e reraise(e) + @arguments("r") + def bhimpl_debug_fatalerror(msg): + llop.debug_fatalerror(lltype.Void, msg) + # ---------- # the main hints and recursive calls @arguments("i") - def bhimpl_can_enter_jit(jdindex): + def bhimpl_loop_header(jdindex): pass @arguments("self", "i", "I", "R", "F", "I", "R", "F") @@ -1164,7 +1168,7 @@ # we now proceed to interpret the bytecode in this frame self.run() # - except JitException: + except JitException, e: raise # go through except Exception, e: # if we get an exception, return it to the caller frame @@ -1266,6 +1270,33 @@ e = lltype.cast_opaque_ptr(llmemory.GCREF, e) raise sd.ExitFrameWithExceptionRef(self.cpu, e) + def _handle_jitexception_in_portal(self, e): + # This case is really rare, but can occur if + # convert_and_run_from_pyjitpl() gets called in this situation: + # + # [function 1] <---- top BlackholeInterpreter() + # [recursive portal jit code] + # ... + # [bottom portal jit code] <---- bottom BlackholeInterpreter() + # + # and then "function 1" contains a call to "function 2", which + # calls "can_enter_jit". The latter can terminate by raising a + # JitException. In that case, the JitException is not supposed + # to fall through the whole chain of BlackholeInterpreters, but + # be caught and handled just below the level "recursive portal + # jit code". The present function is called to handle the case + # of recursive portal jit codes. + for jd in self.builder.metainterp_sd.jitdrivers_sd: + if jd.mainjitcode is self.jitcode: + break + else: + assert 0, "portal jitcode not found??" + # call the helper in warmspot.py. It might either raise a + # regular exception (which should then be propagated outside + # of 'self', not caught inside), or return (the return value + # gets stored in nextblackholeinterp). + jd.handle_jitexc_from_bh(self.nextblackholeinterp, e) + def _copy_data_from_miframe(self, miframe): self.setposition(miframe.jitcode, miframe.pc) for i in range(self.jitcode.num_regs_i()): @@ -1287,9 +1318,31 @@ while True: try: current_exc = blackholeinterp._resume_mainloop(current_exc) - finally: - blackholeinterp.builder.release_interp(blackholeinterp) + except JitException, e: + blackholeinterp, current_exc = _handle_jitexception( + blackholeinterp, e) + blackholeinterp.builder.release_interp(blackholeinterp) + blackholeinterp = blackholeinterp.nextblackholeinterp + +def _handle_jitexception(blackholeinterp, jitexc): + # See comments in _handle_jitexception_in_portal(). + while not blackholeinterp.jitcode.is_portal: + blackholeinterp.builder.release_interp(blackholeinterp) blackholeinterp = blackholeinterp.nextblackholeinterp + if blackholeinterp.nextblackholeinterp is None: + blackholeinterp.builder.release_interp(blackholeinterp) + raise jitexc # bottommost entry: go through + # We have reached a recursive portal level. + try: + blackholeinterp._handle_jitexception_in_portal(jitexc) + except Exception, e: + # It raised a general exception (it should not be a JitException here). + lle = get_llexception(blackholeinterp.cpu, e) + else: + # It set up the nextblackholeinterp to contain the return value. + lle = lltype.nullptr(rclass.OBJECTPTR.TO) + # We will continue to loop in _run_forever() from the parent level. + return blackholeinterp, lle def resume_in_blackhole(metainterp_sd, jitdriver_sd, resumedescr, all_virtuals=None): Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/executor.py Tue Aug 24 18:54:23 2010 @@ -172,34 +172,36 @@ [x1box.getref_base(), x2box.getref_base()], None) def do_int_add_ovf(cpu, metainterp, box1, box2): - assert metainterp is not None + # the overflow operations can be called without a metainterp, if an + # overflow cannot occur a = box1.getint() b = box2.getint() try: z = ovfcheck(a + b) except OverflowError: + assert metainterp is not None metainterp.execute_raised(OverflowError(), constant=True) z = 0 return BoxInt(z) def do_int_sub_ovf(cpu, metainterp, box1, box2): - assert metainterp is not None a = box1.getint() b = box2.getint() try: z = ovfcheck(a - b) except OverflowError: + assert metainterp is not None metainterp.execute_raised(OverflowError(), constant=True) z = 0 return BoxInt(z) def do_int_mul_ovf(cpu, metainterp, box1, box2): - assert metainterp is not None a = box1.getint() b = box2.getint() try: z = ovfcheck(a * b) except OverflowError: + assert metainterp is not None metainterp.execute_raised(OverflowError(), constant=True) z = 0 return BoxInt(z) Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/history.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/history.py Tue Aug 24 18:54:23 2010 @@ -919,11 +919,12 @@ "found %d %r, expected %d" % (found, insn, expected_count)) return insns - def check_loops(self, expected=None, **check): + def check_loops(self, expected=None, everywhere=False, **check): insns = {} for loop in self.loops: - if getattr(loop, '_ignore_during_counting', False): - continue + if not everywhere: + if getattr(loop, '_ignore_during_counting', False): + continue insns = loop.summary(adding_insns=insns) if expected is not None: insns.pop('debug_merge_point', None) Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/jitdriver.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/jitdriver.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/jitdriver.py Tue Aug 24 18:54:23 2010 @@ -12,6 +12,7 @@ # self.result_type ... pypy.jit.metainterp.warmspot # self.virtualizable_info... pypy.jit.metainterp.warmspot # self.warmstate ... pypy.jit.metainterp.warmspot + # self.handle_jitexc_from_bh pypy.jit.metainterp.warmspot # self.index ... pypy.jit.codewriter.call # self.mainjitcode ... pypy.jit.codewriter.call Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/optimizeopt.py Tue Aug 24 18:54:23 2010 @@ -820,7 +820,12 @@ return args def optimize_default(self, op): - if op.is_always_pure(): + canfold = op.is_always_pure() + is_ovf = op.is_ovf() + if is_ovf: + nextop = self.loop.operations[self.i + 1] + canfold = nextop.opnum == rop.GUARD_NO_OVERFLOW + if canfold: for arg in op.args: if self.get_constant_box(arg) is None: break @@ -830,6 +835,8 @@ resbox = execute_nonspec(self.cpu, None, op.opnum, argboxes, op.descr) self.make_constant(op.result, resbox.constbox()) + if is_ovf: + self.i += 1 # skip next operation, it is the unneeded guard return # did we do the exact same operation already? @@ -838,6 +845,8 @@ if oldop is not None and oldop.descr is op.descr: assert oldop.opnum == op.opnum self.make_equal_to(op.result, self.getvalue(oldop.result)) + if is_ovf: + self.i += 1 # skip next operation, it is the unneeded guard return elif self.find_rewritable_bool(op, args): return Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/pyjitpl.py Tue Aug 24 18:54:23 2010 @@ -149,16 +149,20 @@ assert oldbox not in registers[count:] def make_result_of_lastop(self, resultbox): - if resultbox is None: - return + got_type = resultbox.type + if not we_are_translated(): + typeof = {'i': history.INT, + 'r': history.REF, + 'f': history.FLOAT} + assert typeof[self.jitcode._resulttypes[self.pc]] == got_type target_index = ord(self.bytecode[self.pc-1]) - if resultbox.type == history.INT: + if got_type == history.INT: self.registers_i[target_index] = resultbox - elif resultbox.type == history.REF: + elif got_type == history.REF: #debug_print(' ->', # llmemory.cast_ptr_to_adr(resultbox.getref_base())) self.registers_r[target_index] = resultbox - elif resultbox.type == history.FLOAT: + elif got_type == history.FLOAT: self.registers_f[target_index] = resultbox else: raise AssertionError("bad result box type") @@ -685,11 +689,11 @@ def _opimpl_recursive_call(self, jdindex, greenboxes, redboxes): targetjitdriver_sd = self.metainterp.staticdata.jitdrivers_sd[jdindex] allboxes = greenboxes + redboxes - portal_code = targetjitdriver_sd.mainjitcode warmrunnerstate = targetjitdriver_sd.warmstate token = None if warmrunnerstate.inlining: if warmrunnerstate.can_inline_callable(greenboxes): + portal_code = targetjitdriver_sd.mainjitcode return self.metainterp.perform_call(portal_code, allboxes, greenkey=greenboxes) token = warmrunnerstate.get_assembler_token(greenboxes) @@ -697,6 +701,10 @@ # that assembler that we call is still correct self.verify_green_args(targetjitdriver_sd, greenboxes) # + return self.do_recursive_call(targetjitdriver_sd, allboxes, token) + + def do_recursive_call(self, targetjitdriver_sd, allboxes, token=None): + portal_code = targetjitdriver_sd.mainjitcode k = targetjitdriver_sd.portal_runner_adr funcbox = ConstInt(heaptracker.adr2int(k)) return self.do_residual_call(funcbox, portal_code.calldescr, @@ -786,13 +794,8 @@ return clsbox @arguments("int") - def opimpl_can_enter_jit(self, jdindex): - if self.metainterp.in_recursion: - from pypy.jit.metainterp.warmspot import CannotInlineCanEnterJit - raise CannotInlineCanEnterJit() - assert jdindex == self.metainterp.jitdriver_sd.index, ( - "found a can_enter_jit that does not match the current jitdriver") - self.metainterp.seen_can_enter_jit = True + def opimpl_loop_header(self, jdindex): + self.metainterp.seen_loop_header_for_jdindex = jdindex def verify_green_args(self, jitdriver_sd, varargs): num_green_args = jitdriver_sd.num_green_args @@ -806,22 +809,42 @@ self.verify_green_args(jitdriver_sd, greenboxes) # xxx we may disable the following line in some context later self.debug_merge_point(jitdriver_sd, greenboxes) - if self.metainterp.seen_can_enter_jit: - self.metainterp.seen_can_enter_jit = False - # Assert that it's impossible to arrive here with in_recursion - # set to a non-zero value: seen_can_enter_jit can only be set - # to True by opimpl_can_enter_jit, which should be executed - # just before opimpl_jit_merge_point (no recursion inbetween). - assert not self.metainterp.in_recursion + if self.metainterp.seen_loop_header_for_jdindex < 0: + return + # + assert self.metainterp.seen_loop_header_for_jdindex == jdindex, ( + "found a loop_header for a JitDriver that does not match " + "the following jit_merge_point's") + self.metainterp.seen_loop_header_for_jdindex = -1 + # + if not self.metainterp.in_recursion: assert jitdriver_sd is self.metainterp.jitdriver_sd # Set self.pc to point to jit_merge_point instead of just after: - # if reached_can_enter_jit() raises SwitchToBlackhole, then the + # if reached_loop_header() raises SwitchToBlackhole, then the # pc is still at the jit_merge_point, which is a point that is # much less expensive to blackhole out of. saved_pc = self.pc self.pc = orgpc - self.metainterp.reached_can_enter_jit(greenboxes, redboxes) + self.metainterp.reached_loop_header(greenboxes, redboxes) self.pc = saved_pc + else: + warmrunnerstate = jitdriver_sd.warmstate + token = warmrunnerstate.get_assembler_token(greenboxes) + # warning! careful here. We have to return from the current + # frame containing the jit_merge_point, and then use + # do_recursive_call() to follow the recursive call. This is + # needed because do_recursive_call() will write its result + # with make_result_of_lastop(), so the lastop must be right: + # it must be the call to 'self', and not the jit_merge_point + # itself, which has no result at all. + assert len(self.metainterp.framestack) >= 2 + try: + self.metainterp.finishframe(None) + except ChangeFrame: + pass + frame = self.metainterp.framestack[-1] + frame.do_recursive_call(jitdriver_sd, greenboxes + redboxes, token) + raise ChangeFrame def debug_merge_point(self, jitdriver_sd, greenkey): # debugging: produce a DEBUG_MERGE_POINT operation @@ -872,6 +895,12 @@ return exc_value_box @arguments("box") + def opimpl_debug_fatalerror(self, box): + from pypy.rpython.lltypesystem import rstr, lloperation + msg = box.getref(lltype.Ptr(rstr.STR)) + lloperation.llop.debug_fatalerror(msg) + + @arguments("box") def opimpl_virtual_ref(self, box): # Details on the content of metainterp.virtualref_boxes: # @@ -1018,9 +1047,10 @@ self.metainterp.clear_exception() resbox = self.metainterp.execute_and_record_varargs(opnum, argboxes, descr=descr) - self.make_result_of_lastop(resbox) - # ^^^ this is done before handle_possible_exception() because we need - # the box to show up in get_list_of_active_boxes() + if resbox is not None: + self.make_result_of_lastop(resbox) + # ^^^ this is done before handle_possible_exception() because we + # need the box to show up in get_list_of_active_boxes() if exc: self.metainterp.handle_possible_exception() else: @@ -1323,7 +1353,8 @@ self.last_exc_value_box = None self.popframe() if self.framestack: - self.framestack[-1].make_result_of_lastop(resultbox) + if resultbox is not None: + self.framestack[-1].make_result_of_lastop(resultbox) raise ChangeFrame else: try: @@ -1552,7 +1583,7 @@ redkey = original_boxes[num_green_args:] self.resumekey = compile.ResumeFromInterpDescr(original_greenkey, redkey) - self.seen_can_enter_jit = False + self.seen_loop_header_for_jdindex = -1 try: self.interpret() except GenerateMergePoint, gmp: @@ -1579,7 +1610,7 @@ # because we cannot reconstruct the beginning of the proper loop self.current_merge_points = [(original_greenkey, -1)] self.resumekey = key - self.seen_can_enter_jit = False + self.seen_loop_header_for_jdindex = -1 try: self.prepare_resume_from_failure(key.guard_opnum) self.interpret() @@ -1609,7 +1640,7 @@ else: duplicates[box] = None - def reached_can_enter_jit(self, greenboxes, redboxes): + def reached_loop_header(self, greenboxes, redboxes): duplicates = {} self.remove_consts_and_duplicates(redboxes, len(redboxes), duplicates) @@ -1623,7 +1654,7 @@ live_arg_boxes += self.virtualizable_boxes live_arg_boxes.pop() assert len(self.virtualref_boxes) == 0, "missing virtual_ref_finish()?" - # Called whenever we reach the 'can_enter_jit' hint. + # Called whenever we reach the 'loop_header' hint. # First, attempt to make a bridge: # - if self.resumekey is a ResumeGuardDescr, it starts from a guard # that failed; @@ -2232,7 +2263,10 @@ else: resultbox = unboundmethod(self, *args) # - self.make_result_of_lastop(resultbox) + if resultbox is not None: + self.make_result_of_lastop(resultbox) + elif not we_are_translated(): + assert self._result_argcode in 'v?' # unboundmethod = getattr(MIFrame, 'opimpl_' + name).im_func argtypes = unrolling_iterable(unboundmethod.argtypes) Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_basic.py Tue Aug 24 18:54:23 2010 @@ -116,8 +116,9 @@ class JitMixin: basic = True - def check_loops(self, expected=None, **check): - get_stats().check_loops(expected=expected, **check) + def check_loops(self, expected=None, everywhere=False, **check): + get_stats().check_loops(expected=expected, everywhere=everywhere, + **check) def check_loop_count(self, count): """NB. This is a hack; use check_tree_loop_count() or check_enter_count() for the real thing. Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_immutable.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_immutable.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_immutable.py Tue Aug 24 18:54:23 2010 @@ -17,6 +17,38 @@ assert res == 28 self.check_operations_history(getfield_gc=0, getfield_gc_pure=1, int_add=1) + def test_fields_subclass(self): + class X(object): + _immutable_fields_ = ["x"] + + def __init__(self, x): + self.x = x + + class Y(X): + _immutable_fields_ = ["y"] + + def __init__(self, x, y): + X.__init__(self, x) + self.y = y + + def f(x, y): + X(x) # force the field 'x' to be on class 'X' + z = Y(x, y) + return z.x + z.y + 5 + res = self.interp_operations(f, [23, 11]) + assert res == 39 + self.check_operations_history(getfield_gc=0, getfield_gc_pure=2, + int_add=2) + + def f(x, y): + # this time, the field 'x' only shows up on subclass 'Y' + z = Y(x, y) + return z.x + z.y + 5 + res = self.interp_operations(f, [23, 11]) + assert res == 39 + self.check_operations_history(getfield_gc=0, getfield_gc_pure=2, + int_add=2) + def test_array(self): class X(object): _immutable_fields_ = ["y[*]"] Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_jitdriver.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_jitdriver.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_jitdriver.py Tue Aug 24 18:54:23 2010 @@ -14,7 +14,6 @@ def test_simple(self): myjitdriver1 = JitDriver(greens=[], reds=['n', 'm'], - can_inline = lambda *args: False, get_printable_location = getloc1) myjitdriver2 = JitDriver(greens=['g'], reds=['r'], get_printable_location = getloc2) @@ -30,11 +29,14 @@ while r > 0: myjitdriver2.can_enter_jit(g=g, r=r) myjitdriver2.jit_merge_point(g=g, r=r) - r += loop1(r, g) - 1 + r += loop1(r, g) + (-1) return r # res = self.meta_interp(loop2, [4, 40], repeat=7, inline=True) assert res == loop2(4, 40) + # we expect only one int_sub, corresponding to the single + # compiled instance of loop1() + self.check_loops(int_sub=1) # the following numbers are not really expectations of the test # itself, but just the numbers that we got after looking carefully # at the generated machine code @@ -42,11 +44,10 @@ self.check_tree_loop_count(4) # 2 x loop, 2 x enter bridge self.check_enter_count(7) - def test_simple_inline(self): + def test_inline(self): # this is not an example of reasonable code: loop1() is unrolled # 'n/m' times, where n and m are given as red arguments. myjitdriver1 = JitDriver(greens=[], reds=['n', 'm'], - can_inline = lambda *args: True, get_printable_location = getloc1) myjitdriver2 = JitDriver(greens=['g'], reds=['r'], get_printable_location = getloc2) Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_optimizeopt.py Tue Aug 24 18:54:23 2010 @@ -291,6 +291,24 @@ """ self.optimize_loop(ops, '', expected) + def test_constant_propagate_ovf(self): + ops = """ + [] + i0 = int_add_ovf(2, 3) + guard_no_overflow() [] + i1 = int_is_true(i0) + guard_true(i1) [] + i2 = int_is_zero(i1) + guard_false(i2) [] + guard_value(i0, 5) [] + jump() + """ + expected = """ + [] + jump() + """ + self.optimize_loop(ops, '', expected) + def test_constfold_all(self): from pypy.jit.backend.llgraph.llimpl import TYPES # xxx fish from pypy.jit.metainterp.executor import execute_nonspec @@ -2110,6 +2128,33 @@ """ self.optimize_loop(ops, 'Not', expected) + def test_remove_duplicate_pure_op_ovf(self): + ops = """ + [i1] + i3 = int_add_ovf(i1, 1) + guard_no_overflow() [] + i3b = int_is_true(i3) + guard_true(i3b) [] + i4 = int_add_ovf(i1, 1) + guard_no_overflow() [] + i4b = int_is_true(i4) + guard_true(i4b) [] + escape(i3) + escape(i4) + jump(i1) + """ + expected = """ + [i1] + i3 = int_add_ovf(i1, 1) + guard_no_overflow() [] + i3b = int_is_true(i3) + guard_true(i3b) [] + escape(i3) + escape(i3) + jump(i1) + """ + self.optimize_loop(ops, "Not", expected) + def test_int_and_or_with_zero(self): ops = """ [i0, i1] Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_recursive.py Tue Aug 24 18:54:23 2010 @@ -2,10 +2,11 @@ from pypy.rlib.jit import JitDriver, we_are_jitted, OPTIMIZER_SIMPLE, hint from pypy.rlib.jit import unroll_safe, dont_look_inside from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.debug import fatalerror from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin from pypy.jit.codewriter.policy import StopAtXPolicy from pypy.rpython.annlowlevel import hlstr -from pypy.jit.metainterp.warmspot import CannotInlineCanEnterJit, get_stats +from pypy.jit.metainterp.warmspot import get_stats class RecursiveTests: @@ -98,23 +99,18 @@ policy=StopAtXPolicy(opaque)) assert res == 1 - def get_interpreter(self, codes, always_inline=False): + def get_interpreter(self, codes): ADD = "0" JUMP_BACK = "1" CALL = "2" EXIT = "3" - if always_inline: - def can_inline(*args): - return True - else: - def can_inline(i, code): - code = hlstr(code) - return not JUMP_BACK in code + def getloc(i, code): + return 'code="%s", i=%d' % (code, i) jitdriver = JitDriver(greens = ['i', 'code'], reds = ['n'], - can_inline = can_inline) - + get_printable_location = getloc) + def interpret(codenum, n, i): code = codes[codenum] while i < len(code): @@ -162,31 +158,16 @@ assert self.meta_interp(f, [0, 0, 0], optimizer=OPTIMIZER_SIMPLE, inline=True) == 42 - self.check_loops(call_may_force = 1, call = 0) - - def test_inline_faulty_can_inline(self): - code = "021" - subcode = "301" - codes = [code, subcode] - - f = self.get_interpreter(codes, always_inline=True) - - try: - self.meta_interp(f, [0, 0, 0], optimizer=OPTIMIZER_SIMPLE, - inline=True) - except CannotInlineCanEnterJit: - pass - else: - py.test.fail("DID NOT RAISE") + # the call is fully inlined, because we jump to subcode[1], thus + # skipping completely the JUMP_BACK in subcode[0] + self.check_loops(call_may_force = 0, call_assembler = 0, call = 0) def test_guard_failure_in_inlined_function(self): def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(pc, code): - return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], - get_printable_location=p, can_inline=c) + get_printable_location=p) def f(code, n): pc = 0 while pc < len(code): @@ -219,10 +200,8 @@ def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(pc, code): - return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n', 'flag'], - get_printable_location=p, can_inline=c) + get_printable_location=p) def f(code, n): pc = 0 flag = False @@ -262,10 +241,8 @@ def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(pc, code): - return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], - get_printable_location=p, can_inline=c) + get_printable_location=p) class Exc(Exception): pass @@ -307,10 +284,8 @@ def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(pc, code): - return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], - get_printable_location=p, can_inline=c) + get_printable_location=p) def f(code, n): pc = 0 @@ -524,10 +499,8 @@ def p(pc, code): code = hlstr(code) return "'%s' at %d: %s" % (code, pc, code[pc]) - def c(pc, code): - return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], - get_printable_location=p, can_inline=c) + get_printable_location=p) def f(code, n): pc = 0 @@ -558,6 +531,8 @@ result += f('+-cl--', i) g(50) self.meta_interp(g, [50], backendopt=True) + py.test.skip("tracing from start is by now only longer enabled " + "if a trace gets too big") self.check_tree_loop_count(3) self.check_history(int_add=1) @@ -565,10 +540,8 @@ def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(pc, code): - return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], - get_printable_location=p, can_inline=c) + get_printable_location=p) def f(code, n): pc = 0 @@ -607,8 +580,7 @@ def test_directly_call_assembler(self): driver = JitDriver(greens = ['codeno'], reds = ['i'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) def portal(codeno): i = 0 @@ -624,28 +596,29 @@ def test_recursion_cant_call_assembler_directly(self): driver = JitDriver(greens = ['codeno'], reds = ['i', 'j'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) def portal(codeno, j): i = 0 - while i < 1: - driver.can_enter_jit(codeno=codeno, i=i, j=j) + while 1: driver.jit_merge_point(codeno=codeno, i=i, j=j) - i += 1 - if j == 0: + if i == 1: + if j == 0: + return + portal(2, j - 1) + elif i == 3: return - portal(2, j - 1) + i += 1 + driver.can_enter_jit(codeno=codeno, i=i, j=j) portal(2, 50) self.meta_interp(portal, [2, 20], inline=True) - self.check_history(call_assembler=0, call_may_force=1) - self.check_enter_count_at_most(1) + self.check_loops(call_assembler=0, call_may_force=1, + everywhere=True) def test_directly_call_assembler_return(self): driver = JitDriver(greens = ['codeno'], reds = ['i', 'k'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) def portal(codeno): i = 0 @@ -668,8 +641,7 @@ self.x = x driver = JitDriver(greens = ['codeno'], reds = ['i'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) def portal(codeno): i = 0 @@ -690,8 +662,7 @@ def test_directly_call_assembler_fail_guard(self): driver = JitDriver(greens = ['codeno'], reds = ['i', 'k'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) def portal(codeno, k): i = 0 @@ -722,8 +693,7 @@ driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'], virtualizables = ['frame'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) def main(codeno): frame = Frame() @@ -761,8 +731,7 @@ driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'], virtualizables = ['frame'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) @dont_look_inside def check_frame(subframe): @@ -802,7 +771,7 @@ res = self.meta_interp(main, [0], inline=True) assert res == main(0) - def test_directly_call_assembler_virtualizable_force(self): + def test_directly_call_assembler_virtualizable_force1(self): class Thing(object): def __init__(self, val): self.val = val @@ -812,8 +781,7 @@ driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'], virtualizables = ['frame'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) class SomewhereElse(object): pass @@ -830,6 +798,7 @@ return frame.thing.val def portal(codeno, frame): + print 'ENTER:', codeno, frame.thing.val i = 0 while i < 10: driver.can_enter_jit(frame=frame, codeno=codeno, i=i) @@ -839,11 +808,15 @@ subframe = Frame() subframe.thing = Thing(nextval) nextval = portal(1, subframe) - elif frame.thing.val > 40: - change(Thing(13)) - nextval = 13 + elif codeno == 1: + if frame.thing.val > 40: + change(Thing(13)) + nextval = 13 + else: + fatalerror("bad codeno = " + str(codeno)) frame.thing = Thing(nextval + 1) i += 1 + print 'LEAVE:', codeno, frame.thing.val return frame.thing.val res = self.meta_interp(main, [0], inline=True, @@ -852,8 +825,7 @@ def test_directly_call_assembler_virtualizable_with_array(self): myjitdriver = JitDriver(greens = ['codeno'], reds = ['n', 'x', 'frame'], - virtualizables = ['frame'], - can_inline = lambda codeno : False) + virtualizables = ['frame']) class Frame(object): _virtualizable2_ = ['l[*]', 's'] @@ -899,8 +871,7 @@ driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'], virtualizables = ['frame'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) class SomewhereElse(object): pass @@ -942,17 +913,16 @@ def test_assembler_call_red_args(self): driver = JitDriver(greens = ['codeno'], reds = ['i', 'k'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) def residual(k): - if k > 40: + if k > 150: return 0 return 1 def portal(codeno, k): i = 0 - while i < 10: + while i < 15: driver.can_enter_jit(codeno=codeno, i=i, k=k) driver.jit_merge_point(codeno=codeno, i=i, k=k) if codeno == 2: @@ -969,10 +939,130 @@ assert res == portal(2, 0) self.check_loops(call_assembler=2) - # There is a test which I fail to write. - # * what happens if we call recursive_call while blackholing - # this seems to be completely corner case and not really happening - # in the wild + def test_inline_without_hitting_the_loop(self): + driver = JitDriver(greens = ['codeno'], reds = ['i'], + get_printable_location = lambda codeno : str(codeno)) + + def portal(codeno): + i = 0 + while True: + driver.jit_merge_point(codeno=codeno, i=i) + if codeno < 10: + i += portal(20) + codeno += 1 + elif codeno == 10: + if i > 63: + return i + codeno = 0 + driver.can_enter_jit(codeno=codeno, i=i) + else: + return 1 + + assert portal(0) == 70 + res = self.meta_interp(portal, [0], inline=True) + assert res == 70 + self.check_loops(call_assembler=0) + + def test_inline_with_hitting_the_loop_sometimes(self): + driver = JitDriver(greens = ['codeno'], reds = ['i', 'k'], + get_printable_location = lambda codeno : str(codeno)) + + def portal(codeno, k): + if k > 2: + return 1 + i = 0 + while True: + driver.jit_merge_point(codeno=codeno, i=i, k=k) + if codeno < 10: + i += portal(codeno + 5, k+1) + codeno += 1 + elif codeno == 10: + if i > [-1, 2000, 63][k]: + return i + codeno = 0 + driver.can_enter_jit(codeno=codeno, i=i, k=k) + else: + return 1 + + assert portal(0, 1) == 2095 + res = self.meta_interp(portal, [0, 1], inline=True) + assert res == 2095 + self.check_loops(call_assembler=6, everywhere=True) + + def test_inline_with_hitting_the_loop_sometimes_exc(self): + driver = JitDriver(greens = ['codeno'], reds = ['i', 'k'], + get_printable_location = lambda codeno : str(codeno)) + class GotValue(Exception): + def __init__(self, result): + self.result = result + + def portal(codeno, k): + if k > 2: + raise GotValue(1) + i = 0 + while True: + driver.jit_merge_point(codeno=codeno, i=i, k=k) + if codeno < 10: + try: + portal(codeno + 5, k+1) + except GotValue, e: + i += e.result + codeno += 1 + elif codeno == 10: + if i > [-1, 2000, 63][k]: + raise GotValue(i) + codeno = 0 + driver.can_enter_jit(codeno=codeno, i=i, k=k) + else: + raise GotValue(1) + + def main(codeno, k): + try: + portal(codeno, k) + except GotValue, e: + return e.result + + assert main(0, 1) == 2095 + res = self.meta_interp(main, [0, 1], inline=True) + assert res == 2095 + self.check_loops(call_assembler=6, everywhere=True) + + def test_handle_jitexception_in_portal(self): + # a test for _handle_jitexception_in_portal in blackhole.py + driver = JitDriver(greens = ['codeno'], reds = ['i', 'str'], + get_printable_location = lambda codeno: str(codeno)) + def do_can_enter_jit(codeno, i, str): + i = (i+1)-1 # some operations + driver.can_enter_jit(codeno=codeno, i=i, str=str) + def intermediate(codeno, i, str): + if i == 9: + do_can_enter_jit(codeno, i, str) + def portal(codeno, str): + i = value.initial + while i < 10: + intermediate(codeno, i, str) + driver.jit_merge_point(codeno=codeno, i=i, str=str) + i += 1 + if codeno == 64 and i == 10: + str = portal(96, str) + str += chr(codeno+i) + return str + class Value: + initial = -1 + value = Value() + def main(): + value.initial = 0 + return (portal(64, '') + + portal(64, '') + + portal(64, '') + + portal(64, '') + + portal(64, '')) + assert main() == 'ABCDEFGHIabcdefghijJ' * 5 + for tlimit in [95, 90, 102]: + print 'tlimit =', tlimit + res = self.meta_interp(main, [], inline=True, trace_limit=tlimit) + assert ''.join(res.chars) == 'ABCDEFGHIabcdefghijJ' * 5 + class TestLLtype(RecursiveTests, LLJitMixin): pass Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_virtualizable.py Tue Aug 24 18:54:23 2010 @@ -1330,11 +1330,9 @@ def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(pc, code): - return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['frame'], virtualizables=["frame"], - get_printable_location=p, can_inline=c) + get_printable_location=p) def f(code, frame): pc = 0 while pc < len(code): Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_warmspot.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_warmspot.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_warmspot.py Tue Aug 24 18:54:23 2010 @@ -272,6 +272,30 @@ self.check_enter_count_at_most(2) self.check_loops(call=0) + def test_loop_header(self): + # artificial test: we enter into the JIT only when can_enter_jit() + # is seen, but we close a loop in the JIT much more quickly + # because of loop_header(). + mydriver = JitDriver(reds = ['n', 'm'], greens = []) + + def f(m): + n = 0 + while True: + mydriver.jit_merge_point(n=n, m=m) + if n > m: + m -= 1 + if m < 0: + return n + n = 0 + mydriver.can_enter_jit(n=n, m=m) + else: + n += 1 + mydriver.loop_header() + assert f(15) == 1 + res = self.meta_interp(f, [15], backendopt=True) + assert res == 1 + self.check_loops(int_add=1) # I get 13 without the loop_header() + class TestLLWarmspot(WarmspotTests, LLJitMixin): CPUClass = runner.LLtypeCPU Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_warmstate.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_warmstate.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_warmstate.py Tue Aug 24 18:54:23 2010 @@ -61,12 +61,14 @@ _green_args_spec = [lltype.Signed, lltype.Float] state = WarmEnterState(None, FakeJitDriverSD()) get_jitcell = state._make_jitcell_getter_default() - cell1 = get_jitcell(42, 42.5) + cell1 = get_jitcell(True, 42, 42.5) assert isinstance(cell1, JitCell) - cell2 = get_jitcell(42, 42.5) + cell2 = get_jitcell(True, 42, 42.5) assert cell1 is cell2 - cell3 = get_jitcell(41, 42.5) - cell4 = get_jitcell(42, 0.25) + cell3 = get_jitcell(True, 41, 42.5) + assert get_jitcell(False, 42, 0.25) is None + cell4 = get_jitcell(True, 42, 0.25) + assert get_jitcell(False, 42, 0.25) is cell4 assert cell1 is not cell3 is not cell4 is not cell1 def test_make_jitcell_getter(): @@ -75,8 +77,8 @@ _get_jitcell_at_ptr = None state = WarmEnterState(None, FakeJitDriverSD()) get_jitcell = state.make_jitcell_getter() - cell1 = get_jitcell(1.75) - cell2 = get_jitcell(1.75) + cell1 = get_jitcell(True, 1.75) + cell2 = get_jitcell(True, 1.75) assert cell1 is cell2 assert get_jitcell is state.make_jitcell_getter() @@ -103,14 +105,16 @@ # state = WarmEnterState(FakeWarmRunnerDesc(), FakeJitDriverSD()) get_jitcell = state._make_jitcell_getter_custom() - cell1 = get_jitcell(5, 42.5) + cell1 = get_jitcell(True, 5, 42.5) assert isinstance(cell1, JitCell) assert cell1.x == 5 assert cell1.y == 42.5 - cell2 = get_jitcell(5, 42.5) + cell2 = get_jitcell(True, 5, 42.5) assert cell2 is cell1 - cell3 = get_jitcell(41, 42.5) - cell4 = get_jitcell(42, 0.25) + cell3 = get_jitcell(True, 41, 42.5) + assert get_jitcell(False, 42, 0.25) is None + cell4 = get_jitcell(True, 42, 0.25) + assert get_jitcell(False, 42, 0.25) is cell4 assert cell1 is not cell3 is not cell4 is not cell1 def test_make_set_future_values(): @@ -153,52 +157,25 @@ state.attach_unoptimized_bridge_from_interp([ConstInt(5), ConstFloat(2.25)], "entry loop token") - cell1 = get_jitcell(5, 2.25) + cell1 = get_jitcell(True, 5, 2.25) assert cell1.counter < 0 assert cell1.entry_loop_token == "entry loop token" def test_make_jitdriver_callbacks_1(): class FakeJitDriverSD: _green_args_spec = [lltype.Signed, lltype.Float] - _can_inline_ptr = None _get_printable_location_ptr = None _confirm_enter_jit_ptr = None class FakeCell: dont_trace_here = False state = WarmEnterState(None, FakeJitDriverSD()) - def jit_getter(*args): + def jit_getter(build, *args): return FakeCell() state.jit_getter = jit_getter state.make_jitdriver_callbacks() - res = state.can_inline_callable([ConstInt(5), ConstFloat(42.5)]) - assert res is True res = state.get_location_str([ConstInt(5), ConstFloat(42.5)]) assert res == '(no jitdriver.get_printable_location!)' -def test_make_jitdriver_callbacks_2(): - def can_inline(x, y): - assert x == 5 - assert y == 42.5 - return False - CAN_INLINE = lltype.Ptr(lltype.FuncType([lltype.Signed, lltype.Float], - lltype.Bool)) - class FakeCell: - dont_trace_here = False - class FakeWarmRunnerDesc: - rtyper = None - class FakeJitDriverSD: - _green_args_spec = [lltype.Signed, lltype.Float] - _can_inline_ptr = llhelper(CAN_INLINE, can_inline) - _get_printable_location_ptr = None - _confirm_enter_jit_ptr = None - state = WarmEnterState(FakeWarmRunnerDesc(), FakeJitDriverSD()) - def jit_getter(*args): - return FakeCell() - state.jit_getter = jit_getter - state.make_jitdriver_callbacks() - res = state.can_inline_callable([ConstInt(5), ConstFloat(42.5)]) - assert res is False - def test_make_jitdriver_callbacks_3(): def get_location(x, y): assert x == 5 @@ -210,7 +187,6 @@ rtyper = None class FakeJitDriverSD: _green_args_spec = [lltype.Signed, lltype.Float] - _can_inline_ptr = None _get_printable_location_ptr = llhelper(GET_LOCATION, get_location) _confirm_enter_jit_ptr = None _get_jitcell_at_ptr = None @@ -231,7 +207,6 @@ rtyper = None class FakeJitDriverSD: _green_args_spec = [lltype.Signed, lltype.Float] - _can_inline_ptr = None _get_printable_location_ptr = None _confirm_enter_jit_ptr = llhelper(ENTER_JIT, confirm_enter_jit) _get_jitcell_at_ptr = None Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_ztranslation.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_ztranslation.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/test/test_ztranslation.py Tue Aug 24 18:54:23 2010 @@ -37,15 +37,12 @@ return jitcellcache.entry def get_printable_location(): return '(hello world)' - def can_inline(): - return False jitdriver = JitDriver(greens = [], reds = ['total', 'frame'], virtualizables = ['frame'], get_jitcell_at=get_jitcell_at, set_jitcell_at=set_jitcell_at, - get_printable_location=get_printable_location, - can_inline=can_inline) + get_printable_location=get_printable_location) def f(i): for param in unroll_parameters: defl = PARAMETERS[param] @@ -63,8 +60,7 @@ frame.i -= 1 return total * 10 # - myjitdriver2 = JitDriver(greens = ['g'], reds = ['m', 'x'], - can_inline = lambda *args: False) + myjitdriver2 = JitDriver(greens = ['g'], reds = ['m', 'x']) def f2(g, m, x): while m > 0: myjitdriver2.can_enter_jit(g=g, m=m, x=x) Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/warmspot.py Tue Aug 24 18:54:23 2010 @@ -136,9 +136,6 @@ class ContinueRunningNormallyBase(JitException): pass -class CannotInlineCanEnterJit(JitException): - pass - # ____________________________________________________________ class WarmRunnerDesc(object): @@ -402,7 +399,7 @@ can_inline = state.can_inline_greenargs num_green_args = jd.num_green_args def maybe_enter_from_start(*args): - if can_inline is not None and not can_inline(*args[:num_green_args]): + if not can_inline(*args[:num_green_args]): maybe_compile_and_run(*args) maybe_enter_from_start._always_inline_ = True jd._maybe_enter_from_start_fn = maybe_enter_from_start @@ -423,8 +420,6 @@ s_BaseJitCell_not_None) jd._get_jitcell_at_ptr = self._make_hook_graph(jd, annhelper, jd.jitdriver.get_jitcell_at, s_BaseJitCell_or_None) - jd._can_inline_ptr = self._make_hook_graph(jd, - annhelper, jd.jitdriver.can_inline, annmodel.s_Bool) jd._get_printable_location_ptr = self._make_hook_graph(jd, annhelper, jd.jitdriver.get_printable_location, s_Str) jd._confirm_enter_jit_ptr = self._make_hook_graph(jd, @@ -478,7 +473,13 @@ jitdriver = op.args[1].value assert jitdriver in sublists, \ "can_enter_jit with no matching jit_merge_point" - sublists[jitdriver].append((graph, block, index)) + origportalgraph = jd._jit_merge_point_pos[0] + if graph is not origportalgraph: + sublists[jitdriver].append((graph, block, index)) + else: + pass # a 'can_enter_jit' before the 'jit-merge_point', but + # originally in the same function: we ignore it here + # see e.g. test_jitdriver.test_simple for jd in self.jitdrivers_sd: sublist = sublists[jd.jitdriver] assert len(sublist) > 0, \ @@ -557,6 +558,7 @@ # Prepare the portal_runner() helper # from pypy.jit.metainterp.warmstate import specialize_value + from pypy.jit.metainterp.warmstate import unspecialize_value portal_ptr = self.cpu.ts.functionptr(PORTALFUNC, 'portal', graph = portalgraph) jd._portal_ptr = portal_ptr @@ -611,6 +613,37 @@ value = cast_base_ptr_to_instance(Exception, value) raise Exception, value + def handle_jitexception(e): + # XXX the bulk of this function is a copy-paste from above :-( + try: + raise e + except self.ContinueRunningNormally, e: + args = () + for ARGTYPE, attrname, count in portalfunc_ARGS: + x = getattr(e, attrname)[count] + x = specialize_value(ARGTYPE, x) + args = args + (x,) + return ll_portal_runner(*args) + except self.DoneWithThisFrameVoid: + assert result_kind == 'void' + return + except self.DoneWithThisFrameInt, e: + assert result_kind == 'int' + return specialize_value(RESULT, e.result) + except self.DoneWithThisFrameRef, e: + assert result_kind == 'ref' + return specialize_value(RESULT, e.result) + except self.DoneWithThisFrameFloat, e: + assert result_kind == 'float' + return specialize_value(RESULT, e.result) + except self.ExitFrameWithExceptionRef, e: + value = ts.cast_to_baseclass(e.value) + if not we_are_translated(): + raise LLException(ts.get_typeptr(value), value) + else: + value = cast_base_ptr_to_instance(Exception, value) + raise Exception, value + jd._ll_portal_runner = ll_portal_runner # for debugging jd.portal_runner_ptr = self.helper_func(jd._PTR_PORTAL_FUNCTYPE, ll_portal_runner) @@ -631,32 +664,8 @@ vinfo.reset_vable_token(virtualizable) try: loop_token = fail_descr.handle_fail(self.metainterp_sd, jd) - except self.ContinueRunningNormally, e: - args = () - for ARGTYPE, attrname, count in portalfunc_ARGS: - x = getattr(e, attrname)[count] - x = specialize_value(ARGTYPE, x) - args = args + (x,) - return ll_portal_runner(*args) - except self.DoneWithThisFrameVoid: - assert result_kind == 'void' - return - except self.DoneWithThisFrameInt, e: - assert result_kind == 'int' - return specialize_value(RESULT, e.result) - except self.DoneWithThisFrameRef, e: - assert result_kind == 'ref' - return specialize_value(RESULT, e.result) - except self.DoneWithThisFrameFloat, e: - assert result_kind == 'float' - return specialize_value(RESULT, e.result) - except self.ExitFrameWithExceptionRef, e: - value = ts.cast_to_baseclass(e.value) - if not we_are_translated(): - raise LLException(ts.get_typeptr(value), value) - else: - value = cast_base_ptr_to_instance(Exception, value) - raise Exception, value + except JitException, e: + return handle_jitexception(e) fail_descr = self.cpu.execute_token(loop_token) jd._assembler_call_helper = assembler_call_helper # for debugging @@ -668,6 +677,21 @@ if vinfo is not None: jd.vable_token_descr = vinfo.vable_token_descr + def handle_jitexception_from_blackhole(bhcaller, e): + result = handle_jitexception(e) + # + if result_kind != 'void': + result = unspecialize_value(result) + if result_kind == 'int': + bhcaller._setup_return_value_i(result) + elif result_kind == 'ref': + bhcaller._setup_return_value_r(result) + elif result_kind == 'float': + bhcaller._setup_return_value_f(result) + else: + assert False + jd.handle_jitexc_from_bh = handle_jitexception_from_blackhole + # ____________________________________________________________ # Now mutate origportalgraph to end with a call to portal_runner_ptr # @@ -687,17 +711,6 @@ origblock.exitswitch = None origblock.recloseblock(Link([v_result], origportalgraph.returnblock)) # - # Also kill any can_enter_jit left behind (example: see - # test_jitdriver.test_simple, which has a can_enter_jit in - # loop1's origportalgraph) - can_enter_jits = _find_jit_marker([origportalgraph], 'can_enter_jit') - for _, block, i in can_enter_jits: - op = block.operations[i] - assert op.opname == 'jit_marker' - block.operations[i] = SpaceOperation('same_as', - [Constant(None, lltype.Void)], - op.result) - # checkgraph(origportalgraph) def add_finish(self): Modified: pypy/branch/jit-bounds/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/jit-bounds/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/jit-bounds/pypy/jit/metainterp/warmstate.py Tue Aug 24 18:54:23 2010 @@ -30,6 +30,22 @@ else: return lltype.cast_opaque_ptr(TYPE, x) + at specialize.ll() +def unspecialize_value(value): + """Casts 'value' to a Signed, a GCREF or a Float.""" + if isinstance(lltype.typeOf(value), lltype.Ptr): + if lltype.typeOf(value).TO._gckind == 'gc': + return lltype.cast_opaque_ptr(llmemory.GCREF, value) + else: + adr = llmemory.cast_ptr_to_adr(value) + return heaptracker.adr2int(adr) + elif isinstance(lltype.typeOf(value), ootype.OOType): + return ootype.cast_to_object(value) + elif isinstance(value, float): + return value + else: + return intmask(value) + @specialize.arg(0) def unwrap(TYPE, box): if TYPE is lltype.Void: @@ -232,7 +248,7 @@ # look for the cell corresponding to the current greenargs greenargs = args[:num_green_args] - cell = get_jitcell(*greenargs) + cell = get_jitcell(True, *greenargs) if cell.counter >= 0: # update the profiling counter @@ -324,7 +340,7 @@ # def jit_cell_at_key(greenkey): greenargs = unwrap_greenkey(greenkey) - return jit_getter(*greenargs) + return jit_getter(True, *greenargs) self.jit_cell_at_key = jit_cell_at_key self.jit_getter = jit_getter # @@ -355,10 +371,12 @@ # jitcell_dict = r_dict(comparekey, hashkey) # - def get_jitcell(*greenargs): + def get_jitcell(build, *greenargs): try: cell = jitcell_dict[greenargs] except KeyError: + if not build: + return None cell = JitCell() jitcell_dict[greenargs] = cell return cell @@ -371,38 +389,41 @@ set_jitcell_at_ptr = self.jitdriver_sd._set_jitcell_at_ptr lltohlhack = {} # - def get_jitcell(*greenargs): + def get_jitcell(build, *greenargs): fn = support.maybe_on_top_of_llinterp(rtyper, get_jitcell_at_ptr) cellref = fn(*greenargs) # if we_are_translated(): BASEJITCELL = lltype.typeOf(cellref) cell = cast_base_ptr_to_instance(JitCell, cellref) - elif isinstance(cellref, (BaseJitCell, type(None))): - BASEJITCELL = None - cell = cellref else: - BASEJITCELL = lltype.typeOf(cellref) - if cellref: - cell = lltohlhack[rtyper.type_system.deref(cellref)] + if isinstance(cellref, (BaseJitCell, type(None))): + BASEJITCELL = None + cell = cellref else: - cell = None - # + BASEJITCELL = lltype.typeOf(cellref) + if cellref: + cell = lltohlhack[rtyper.type_system.deref(cellref)] + else: + cell = None + if not build: + return cell if cell is None: cell = JitCell() # if we_are_translated(): cellref = cast_object_to_ptr(BASEJITCELL, cell) - elif BASEJITCELL is None: - cellref = cell else: - if isinstance(BASEJITCELL, lltype.Ptr): - cellref = lltype.malloc(BASEJITCELL.TO) - elif isinstance(BASEJITCELL, ootype.Instance): - cellref = ootype.new(BASEJITCELL) + if BASEJITCELL is None: + cellref = cell else: - assert False, "no clue" - lltohlhack[rtyper.type_system.deref(cellref)] = cell + if isinstance(BASEJITCELL, lltype.Ptr): + cellref = lltype.malloc(BASEJITCELL.TO) + elif isinstance(BASEJITCELL, ootype.Instance): + cellref = ootype.new(BASEJITCELL) + else: + assert False, "no clue" + lltohlhack[rtyper.type_system.deref(cellref)] = cell # fn = support.maybe_on_top_of_llinterp(rtyper, set_jitcell_at_ptr) @@ -468,34 +489,24 @@ if hasattr(self, 'get_location_str'): return # - can_inline_ptr = self.jitdriver_sd._can_inline_ptr unwrap_greenkey = self.make_unwrap_greenkey() jit_getter = self.make_jitcell_getter() - if can_inline_ptr is None: - def can_inline_callable(*greenargs): - # XXX shouldn't it be False by default? - return True - else: - rtyper = self.warmrunnerdesc.rtyper - # - def can_inline_callable(*greenargs): - fn = support.maybe_on_top_of_llinterp(rtyper, can_inline_ptr) - return fn(*greenargs) - def can_inline(*greenargs): - cell = jit_getter(*greenargs) - if cell.dont_trace_here: + + def can_inline_greenargs(*greenargs): + cell = jit_getter(False, *greenargs) + if cell is not None and cell.dont_trace_here: return False - return can_inline_callable(*greenargs) - self.can_inline_greenargs = can_inline - def can_inline_greenkey(greenkey): + return True + def can_inline_callable(greenkey): greenargs = unwrap_greenkey(greenkey) - return can_inline(*greenargs) - self.can_inline_callable = can_inline_greenkey + return can_inline_greenargs(*greenargs) + self.can_inline_greenargs = can_inline_greenargs + self.can_inline_callable = can_inline_callable def get_assembler_token(greenkey): greenargs = unwrap_greenkey(greenkey) - cell = jit_getter(*greenargs) - if cell.counter >= 0: + cell = jit_getter(False, *greenargs) + if cell is None or cell.counter >= 0: return None return cell.entry_loop_token self.get_assembler_token = get_assembler_token Modified: pypy/branch/jit-bounds/pypy/module/__builtin__/functional.py ============================================================================== --- pypy/branch/jit-bounds/pypy/module/__builtin__/functional.py (original) +++ pypy/branch/jit-bounds/pypy/module/__builtin__/functional.py Tue Aug 24 18:54:23 2010 @@ -221,8 +221,7 @@ from pypy.rlib.jit import JitDriver mapjitdriver = JitDriver(greens = ['code'], - reds = ['w_func', 'w_iter', 'result_w'], - can_inline = lambda *args: False) + reds = ['w_func', 'w_iter', 'result_w']) def map_single_user_function(code, w_func, w_iter): result_w = [] while True: Modified: pypy/branch/jit-bounds/pypy/module/_locale/__init__.py ============================================================================== --- pypy/branch/jit-bounds/pypy/module/_locale/__init__.py (original) +++ pypy/branch/jit-bounds/pypy/module/_locale/__init__.py Tue Aug 24 18:54:23 2010 @@ -1,5 +1,4 @@ from pypy.interpreter.mixedmodule import MixedModule -from pypy.module._locale import interp_locale from pypy.rlib import rlocale import sys Modified: pypy/branch/jit-bounds/pypy/module/_sre/__init__.py ============================================================================== --- pypy/branch/jit-bounds/pypy/module/_sre/__init__.py (original) +++ pypy/branch/jit-bounds/pypy/module/_sre/__init__.py Tue Aug 24 18:54:23 2010 @@ -1,24 +1,14 @@ from pypy.interpreter.mixedmodule import MixedModule class Module(MixedModule): - """A pure Python reimplementation of the _sre module from CPython 2.4 -Copyright 2005 Nik Haldimann, licensed under the MIT license -This code is based on material licensed under CNRI's Python 1.6 license and -copyrighted by: Copyright (c) 1997-2001 by Secret Labs AB -""" - appleveldefs = { - 'compile': 'app_sre.compile', } interpleveldefs = { 'CODESIZE': 'space.wrap(interp_sre.CODESIZE)', 'MAGIC': 'space.wrap(interp_sre.MAGIC)', - 'copyright': 'space.wrap(interp_sre.copyright)', + 'compile': 'interp_sre.W_SRE_Pattern', 'getlower': 'interp_sre.w_getlower', 'getcodesize': 'interp_sre.w_getcodesize', - '_State': 'interp_sre.make_state', - '_match': 'interp_sre.w_match', - '_search': 'interp_sre.w_search', } Modified: pypy/branch/jit-bounds/pypy/module/_sre/interp_sre.py ============================================================================== --- pypy/branch/jit-bounds/pypy/module/_sre/interp_sre.py (original) +++ pypy/branch/jit-bounds/pypy/module/_sre/interp_sre.py Tue Aug 24 18:54:23 2010 @@ -1,27 +1,20 @@ +import sys from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.typedef import GetSetProperty, TypeDef from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w +from pypy.interpreter.typedef import make_weakref_descr from pypy.interpreter.gateway import interp2app, ObjSpace, W_Root from pypy.interpreter.error import OperationError from pypy.rlib.rarithmetic import intmask +from pypy.tool.pairtype import extendabletype -# This can be compiled in two ways: -# -# * THREE_VERSIONS_OF_CORE=True: you get three copies of the whole -# regexp searching and matching code: for strings, for unicode strings, -# and for generic buffer objects (like mmap.mmap or array.array). -# -# * THREE_VERSIONS_OF_CORE=False: there is only one copy of the code, -# at the cost of an indirect method call to fetch each character. - -THREE_VERSIONS_OF_CORE = True +# ____________________________________________________________ +# +# Constants and exposed functions -#### Constants and exposed functions - -from pypy.rlib.rsre import rsre -from pypy.rlib.rsre.rsre_char import MAGIC, CODESIZE, getlower -copyright = "_sre.py 2.4 Copyright 2005 by Nik Haldimann" +from pypy.rlib.rsre import rsre_core +from pypy.rlib.rsre.rsre_char import MAGIC, CODESIZE, getlower, set_unicode_db def w_getlower(space, char_ord, flags): return space.wrap(getlower(char_ord, flags)) @@ -31,166 +24,529 @@ return space.wrap(CODESIZE) # use the same version of unicodedb as the standard objspace -from pypy.objspace.std.unicodeobject import unicodedb -rsre.set_unicode_db(unicodedb) +import pypy.objspace.std.unicodeobject +set_unicode_db(pypy.objspace.std.unicodeobject.unicodedb) -#### State classes +# ____________________________________________________________ +# +# Additional methods on the classes XxxMatchContext + +class __extend__(rsre_core.AbstractMatchContext): + __metaclass__ = extendabletype + def _w_slice(self, space, start, end): + raise NotImplementedError + def _w_string(self, space): + raise NotImplementedError -def make_state(space, w_string, start, end, flags): - # XXX maybe turn this into a __new__ method of W_State - if space.is_true(space.isinstance(w_string, space.w_str)): - cls = W_StringState - elif space.is_true(space.isinstance(w_string, space.w_unicode)): - cls = W_UnicodeState - else: - cls = W_GenericState - return space.wrap(cls(space, w_string, start, end, flags)) -make_state.unwrap_spec = [ObjSpace, W_Root, int, int, int] - - -class W_State(Wrappable): - if not THREE_VERSIONS_OF_CORE: - rsre.insert_sre_methods(locals(), 'all') - - def __init__(self, space, w_string, start, end, flags): - self.space = space - self.w_string = w_string - length = self.unwrap_object() - if start < 0: - start = 0 - if end > length: - end = length - self.start = start - self.pos = start # records the original start position - self.end = end - self.flags = flags - self.reset() +class __extend__(rsre_core.StrMatchContext): + __metaclass__ = extendabletype + def _w_slice(self, space, start, end): + return space.wrap(self._string[start:end]) + def _w_string(self, space): + return space.wrap(self._string) + +class __extend__(rsre_core.UnicodeMatchContext): + __metaclass__ = extendabletype + def _w_slice(self, space, start, end): + return space.wrap(self._unicodestr[start:end]) + def _w_string(self, space): + return space.wrap(self._unicodestr) + +def slice_w(space, ctx, start, end, w_default): + if 0 <= start <= end: + return ctx._w_slice(space, start, end) + return w_default + +def do_flatten_marks(ctx, num_groups): + # Returns a list of RPython-level integers. + # Unlike the app-level groups() method, groups are numbered from 0 + # and the returned list does not start with the whole match range. + if num_groups == 0: + return None + result = [-1] * (2*num_groups) + mark = ctx.match_marks + while mark is not None: + index = mark.gid + if result[index] == -1: + result[index] = mark.position + mark = mark.prev + return result + +def allgroups_w(space, ctx, fmarks, num_groups, w_default): + grps = [slice_w(space, ctx, fmarks[i*2], fmarks[i*2+1], w_default) + for i in range(num_groups)] + return space.newtuple(grps) + +def import_re(space): + w_builtin = space.getbuiltinmodule('__builtin__') + w_import = space.getattr(w_builtin, space.wrap("__import__")) + return space.call_function(w_import, space.wrap("re")) - def lower(self, char_ord): - return getlower(char_ord, self.flags) +def matchcontext(space, ctx): + try: + return rsre_core.match_context(ctx) + except rsre_core.Error, e: + raise OperationError(space.w_RuntimeError, space.wrap(e.msg)) - # methods overridden by subclasses +def searchcontext(space, ctx): + try: + return rsre_core.search_context(ctx) + except rsre_core.Error, e: + raise OperationError(space.w_RuntimeError, space.wrap(e.msg)) - def unwrap_object(self): - raise NotImplementedError +# ____________________________________________________________ +# +# SRE_Pattern class - if 'reset' not in locals(): - def reset(self): - raise NotImplementedError - - if 'search' not in locals(): - def search(self, pattern_codes): - raise NotImplementedError - - if 'match' not in locals(): - def match(self, pattern_codes): - raise NotImplementedError - - # Accessors for the typedef - - def w_reset(self): - self.reset() - - def create_regs(self, group_count): - """ Purely abstract method - """ - raise NotImplementedError +class W_SRE_Pattern(Wrappable): - def w_create_regs(self, group_count): - """Creates a tuple of index pairs representing matched groups, a format - that's convenient for SRE_Match.""" + def cannot_copy_w(self): space = self.space - return space.newtuple([ - space.newtuple([space.wrap(value1), - space.wrap(value2)]) - for value1, value2 in self.create_regs(group_count)]) - w_create_regs.unwrap_spec = ['self', int] + raise OperationError(space.w_TypeError, + space.wrap("cannot copy this pattern object")) - def fget_start(space, self): - return space.wrap(self.start) - - def fset_start(space, self, w_value): - self.start = space.int_w(w_value) + def make_ctx(self, w_string, pos=0, endpos=sys.maxint): + """Make a StrMatchContext or a UnicodeMatchContext for searching + in the given w_string object.""" + space = self.space + if pos < 0: pos = 0 + if endpos < pos: endpos = pos + if space.is_true(space.isinstance(w_string, space.w_unicode)): + unicodestr = space.unicode_w(w_string) + if pos > len(unicodestr): pos = len(unicodestr) + if endpos > len(unicodestr): endpos = len(unicodestr) + return rsre_core.UnicodeMatchContext(self.code, unicodestr, + pos, endpos, self.flags) + else: + str = space.bufferstr_w(w_string) + if pos > len(str): pos = len(str) + if endpos > len(str): endpos = len(str) + return rsre_core.StrMatchContext(self.code, str, + pos, endpos, self.flags) + + def getmatch(self, ctx, found): + if found: + return W_SRE_Match(self, ctx) + else: + return self.space.w_None + + def match_w(self, w_string, pos=0, endpos=sys.maxint): + ctx = self.make_ctx(w_string, pos, endpos) + return self.getmatch(ctx, matchcontext(self.space, ctx)) + match_w.unwrap_spec = ['self', W_Root, int, int] + + def search_w(self, w_string, pos=0, endpos=sys.maxint): + ctx = self.make_ctx(w_string, pos, endpos) + return self.getmatch(ctx, searchcontext(self.space, ctx)) + search_w.unwrap_spec = ['self', W_Root, int, int] - def fget_string_position(space, self): - return space.wrap(self.string_position) + def findall_w(self, w_string, pos=0, endpos=sys.maxint): + space = self.space + matchlist_w = [] + ctx = self.make_ctx(w_string, pos, endpos) + while ctx.match_start <= ctx.end: + if not searchcontext(space, ctx): + break + num_groups = self.num_groups + w_emptystr = space.wrap("") + if num_groups == 0: + w_item = slice_w(space, ctx, ctx.match_start, ctx.match_end, + w_emptystr) + else: + fmarks = do_flatten_marks(ctx, num_groups) + if num_groups == 1: + w_item = slice_w(space, ctx, fmarks[0], fmarks[1], + w_emptystr) + else: + w_item = allgroups_w(space, ctx, fmarks, num_groups, + w_emptystr) + matchlist_w.append(w_item) + no_progress = (ctx.match_start == ctx.match_end) + ctx.reset(ctx.match_end + no_progress) + return space.newlist(matchlist_w) + findall_w.unwrap_spec = ['self', W_Root, int, int] + + def finditer_w(self, w_string, pos=0, endpos=sys.maxint): + # this also works as the implementation of the undocumented + # scanner() method. + ctx = self.make_ctx(w_string, pos, endpos) + scanner = W_SRE_Scanner(self, ctx) + return self.space.wrap(scanner) + finditer_w.unwrap_spec = ['self', W_Root, int, int] - def fset_string_position(space, self, w_value): - self.start = space.int_w(w_value) + def split_w(self, w_string, maxsplit=0): + space = self.space + splitlist = [] + n = 0 + last = 0 + ctx = self.make_ctx(w_string) + while not maxsplit or n < maxsplit: + if not searchcontext(space, ctx): + break + if ctx.match_start == ctx.match_end: # zero-width match + if ctx.match_start == ctx.end: # or end of string + break + ctx.reset(ctx.match_end + 1) + continue + splitlist.append(slice_w(space, ctx, last, ctx.match_start, + space.w_None)) + # add groups (if any) + fmarks = do_flatten_marks(ctx, self.num_groups) + for groupnum in range(self.num_groups): + groupstart, groupend = fmarks[groupnum*2], fmarks[groupnum*2+1] + splitlist.append(slice_w(space, ctx, groupstart, groupend, + space.w_None)) + n += 1 + last = ctx.match_end + ctx.reset(last) + splitlist.append(slice_w(space, ctx, last, ctx.end, space.w_None)) + return space.newlist(splitlist) + split_w.unwrap_spec = ['self', W_Root, int] + + def sub_w(self, w_repl, w_string, count=0): + w_item, n = self.subx(w_repl, w_string, count) + return w_item + sub_w.unwrap_spec = ['self', W_Root, W_Root, int] - def get_char_ord(self, p): - raise NotImplementedError + def subn_w(self, w_repl, w_string, count=0): + w_item, n = self.subx(w_repl, w_string, count) + space = self.space + return space.newtuple([w_item, space.wrap(n)]) + subn_w.unwrap_spec = ['self', W_Root, W_Root, int] -getset_start = GetSetProperty(W_State.fget_start, W_State.fset_start, cls=W_State) -getset_string_position = GetSetProperty(W_State.fget_string_position, - W_State.fset_string_position, cls=W_State) - -W_State.typedef = TypeDef("W_State", - string = interp_attrproperty_w("w_string", W_State), - start = getset_start, - end = interp_attrproperty("end", W_State), - string_position = getset_string_position, - pos = interp_attrproperty("pos", W_State), - lastindex = interp_attrproperty("lastindex", W_State), - reset = interp2app(W_State.w_reset), - create_regs = interp2app(W_State.w_create_regs), + def subx(self, w_ptemplate, w_string, count): + space = self.space + if space.is_true(space.callable(w_ptemplate)): + w_filter = w_ptemplate + filter_is_callable = True + else: + if space.is_true(space.isinstance(w_ptemplate, space.w_unicode)): + filter_as_unicode = space.unicode_w(w_ptemplate) + literal = u'\\' not in filter_as_unicode + else: + try: + filter_as_string = space.str_w(w_ptemplate) + except OperationError, e: + if e.async(space): + raise + literal = False + else: + literal = '\\' not in filter_as_string + if literal: + w_filter = w_ptemplate + filter_is_callable = False + else: + # not a literal; hand it over to the template compiler + w_re = import_re(space) + w_filter = space.call_method(w_re, '_subx', + space.wrap(self), w_ptemplate) + filter_is_callable = space.is_true(space.callable(w_filter)) + # + ctx = self.make_ctx(w_string) + sublist_w = [] + n = last_pos = 0 + while not count or n < count: + if not searchcontext(space, ctx): + break + if last_pos < ctx.match_start: + sublist_w.append(slice_w(space, ctx, last_pos, + ctx.match_start, space.w_None)) + start = ctx.match_end + if start == ctx.match_start: + start += 1 + nextctx = ctx.fresh_copy(start) + if not (last_pos == ctx.match_start + == ctx.match_end and n > 0): + # the above ignores empty matches on latest position + if filter_is_callable: + w_match = self.getmatch(ctx, True) + w_piece = space.call_function(w_filter, w_match) + if not space.is_w(w_piece, space.w_None): + sublist_w.append(w_piece) + else: + sublist_w.append(w_filter) + last_pos = ctx.match_end + n += 1 + elif last_pos >= ctx.end: + break # empty match at the end: finished + ctx = nextctx + + if last_pos < ctx.end: + sublist_w.append(slice_w(space, ctx, last_pos, ctx.end, + space.w_None)) + if n == 0: + # not just an optimization -- see test_sub_unicode + return w_string, n + + if space.is_true(space.isinstance(w_string, space.w_unicode)): + w_emptystr = space.wrap(u'') + else: + w_emptystr = space.wrap('') + w_item = space.call_method(w_emptystr, 'join', + space.newlist(sublist_w)) + return w_item, n + + +def SRE_Pattern__new__(space, w_subtype, w_pattern, flags, w_code, + groups=0, w_groupindex=None, w_indexgroup=None): + n = space.int_w(space.len(w_code)) + code = [0] * n + for i in range(n): + x = space.uint_w(space.getitem(w_code, space.wrap(i))) + code[i] = intmask(x) + # + w_srepat = space.allocate_instance(W_SRE_Pattern, w_subtype) + srepat = space.interp_w(W_SRE_Pattern, w_srepat) + srepat.space = space + srepat.w_pattern = w_pattern # the original uncompiled pattern + srepat.flags = flags + srepat.code = code + srepat.num_groups = groups + srepat.w_groupindex = w_groupindex + srepat.w_indexgroup = w_indexgroup + return w_srepat +SRE_Pattern__new__.unwrap_spec = [ObjSpace, W_Root, W_Root, int, W_Root, + int, W_Root, W_Root] + + +W_SRE_Pattern.typedef = TypeDef( + 'SRE_Pattern', + __new__ = interp2app(SRE_Pattern__new__), + __copy__ = interp2app(W_SRE_Pattern.cannot_copy_w), + __deepcopy__ = interp2app(W_SRE_Pattern.cannot_copy_w), + __weakref__ = make_weakref_descr(W_SRE_Pattern), + findall = interp2app(W_SRE_Pattern.findall_w), + finditer = interp2app(W_SRE_Pattern.finditer_w), + match = interp2app(W_SRE_Pattern.match_w), + scanner = interp2app(W_SRE_Pattern.finditer_w), # reuse finditer() + search = interp2app(W_SRE_Pattern.search_w), + split = interp2app(W_SRE_Pattern.split_w), + sub = interp2app(W_SRE_Pattern.sub_w), + subn = interp2app(W_SRE_Pattern.subn_w), + flags = interp_attrproperty('flags', W_SRE_Pattern), + groupindex = interp_attrproperty_w('w_groupindex', W_SRE_Pattern), + groups = interp_attrproperty('num_groups', W_SRE_Pattern), + pattern = interp_attrproperty_w('w_pattern', W_SRE_Pattern), ) +# ____________________________________________________________ +# +# SRE_Match class -class W_StringState(W_State): - if THREE_VERSIONS_OF_CORE: - rsre.insert_sre_methods(locals(), 'str') - - def unwrap_object(self): - self.string = self.space.str_w(self.w_string) - return len(self.string) - - def get_char_ord(self, p): - return ord(self.string[p]) +class W_SRE_Match(Wrappable): + flatten_cache = None + def __init__(self, srepat, ctx): + self.space = srepat.space + self.srepat = srepat + self.ctx = ctx -class W_UnicodeState(W_State): - if THREE_VERSIONS_OF_CORE: - rsre.insert_sre_methods(locals(), 'unicode') + def cannot_copy_w(self): + space = self.space + raise OperationError(space.w_TypeError, + space.wrap("cannot copy this match object")) - def unwrap_object(self): - self.unicode = self.space.unicode_w(self.w_string) - return len(self.unicode) + def group_w(self, args_w): + space = self.space + ctx = self.ctx + if len(args_w) <= 1: + if len(args_w) == 0: + start, end = ctx.match_start, ctx.match_end + else: + start, end = self.do_span(args_w[0]) + return slice_w(space, ctx, start, end, space.w_None) + else: + results = [None] * len(args_w) + for i in range(len(args_w)): + start, end = self.do_span(args_w[i]) + results[i] = slice_w(space, ctx, start, end, space.w_None) + return space.newtuple(results) + group_w.unwrap_spec = ['self', 'args_w'] + + def groups_w(self, w_default=None): + fmarks = self.flatten_marks() + num_groups = self.srepat.num_groups + return allgroups_w(self.space, self.ctx, fmarks, num_groups, w_default) - def get_char_ord(self, p): - return ord(self.unicode[p]) + def groupdict_w(self, w_default=None): + space = self.space + w_dict = space.newdict() + w_groupindex = self.srepat.w_groupindex + w_iterator = space.iter(w_groupindex) + while True: + try: + w_key = space.next(w_iterator) + except OperationError, e: + if not e.match(space, space.w_StopIteration): + raise + break # done + w_value = space.getitem(w_groupindex, w_key) + start, end = self.do_span(w_value) + w_grp = slice_w(space, self.ctx, start, end, w_default) + space.setitem(w_dict, w_key, w_grp) + return w_dict + def expand_w(self, w_template): + space = self.space + w_re = import_re(space) + return space.call_method(w_re, '_expand', space.wrap(self.srepat), + space.wrap(self), w_template) + + def start_w(self, w_groupnum=0): + return self.space.wrap(self.do_span(w_groupnum)[0]) + + def end_w(self, w_groupnum=0): + return self.space.wrap(self.do_span(w_groupnum)[1]) + + def span_w(self, w_groupnum=0): + start, end = self.do_span(w_groupnum) + return self.space.newtuple([self.space.wrap(start), + self.space.wrap(end)]) + + def flatten_marks(self): + if self.flatten_cache is None: + num_groups = self.srepat.num_groups + self.flatten_cache = do_flatten_marks(self.ctx, num_groups) + return self.flatten_cache -class W_GenericState(W_State): - if THREE_VERSIONS_OF_CORE: - rsre.insert_sre_methods(locals(), 'generic') + def do_span(self, w_arg): + space = self.space + try: + groupnum = space.int_w(w_arg) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + w_groupnum = space.getitem(self.srepat.w_groupindex, w_arg) + groupnum = space.int_w(w_groupnum) + if groupnum == 0: + return self.ctx.match_start, self.ctx.match_end + elif 1 <= groupnum <= self.srepat.num_groups: + fmarks = self.flatten_marks() + idx = 2*(groupnum-1) + assert idx >= 0 + return fmarks[idx], fmarks[idx+1] + else: + raise OperationError(space.w_IndexError, + space.wrap("group index out of range")) + + def _last_index(self): + mark = self.ctx.match_marks + if mark is not None: + return mark.gid // 2 + 1 + return -1 + + def fget_lastgroup(space, self): + lastindex = self._last_index() + if lastindex < 0: + return space.w_None + w_result = space.finditem(self.srepat.w_indexgroup, + space.wrap(lastindex)) + if w_result is None: + return space.w_None + return w_result + + def fget_lastindex(space, self): + lastindex = self._last_index() + if lastindex >= 0: + return space.wrap(lastindex) + return space.w_None - def unwrap_object(self): - self.buffer = self.space.buffer_w(self.w_string) - return self.buffer.getlength() + def fget_pos(space, self): + return space.wrap(self.ctx.original_pos) - def get_char_ord(self, p): - return ord(self.buffer.getitem(p)) + def fget_endpos(space, self): + return space.wrap(self.ctx.end) + def fget_regs(space, self): + space = self.space + fmarks = self.flatten_marks() + num_groups = self.srepat.num_groups + result_w = [None] * (num_groups + 1) + ctx = self.ctx + result_w[0] = space.newtuple([space.wrap(ctx.match_start), + space.wrap(ctx.match_end)]) + for i in range(num_groups): + result_w[i + 1] = space.newtuple([space.wrap(fmarks[i*2]), + space.wrap(fmarks[i*2+1])]) + return space.newtuple(result_w) + + def fget_string(space, self): + return self.ctx._w_string(space) + + +W_SRE_Match.typedef = TypeDef( + 'SRE_Match', + __copy__ = interp2app(W_SRE_Match.cannot_copy_w), + __deepcopy__ = interp2app(W_SRE_Match.cannot_copy_w), + group = interp2app(W_SRE_Match.group_w), + groups = interp2app(W_SRE_Match.groups_w), + groupdict = interp2app(W_SRE_Match.groupdict_w), + start = interp2app(W_SRE_Match.start_w), + end = interp2app(W_SRE_Match.end_w), + span = interp2app(W_SRE_Match.span_w), + expand = interp2app(W_SRE_Match.expand_w), + # + re = interp_attrproperty('srepat', W_SRE_Match), + string = GetSetProperty(W_SRE_Match.fget_string), + pos = GetSetProperty(W_SRE_Match.fget_pos), + endpos = GetSetProperty(W_SRE_Match.fget_endpos), + lastgroup = GetSetProperty(W_SRE_Match.fget_lastgroup), + lastindex = GetSetProperty(W_SRE_Match.fget_lastindex), + regs = GetSetProperty(W_SRE_Match.fget_regs), +) -def w_search(space, w_state, w_pattern_codes): - state = space.interp_w(W_State, w_state) - pattern_codes = [intmask(space.uint_w(code)) for code - in space.unpackiterable(w_pattern_codes)] - try: - res = state.search(pattern_codes) - except RuntimeError: - raise OperationError(space.w_RuntimeError, - space.wrap("Internal re error")) - return space.newbool(res) - -def w_match(space, w_state, w_pattern_codes): - state = space.interp_w(W_State, w_state) - pattern_codes = [intmask(space.uint_w(code)) for code - in space.unpackiterable(w_pattern_codes)] - try: - res = state.match(pattern_codes) - except RuntimeError: - raise OperationError(space.w_RuntimeError, - space.wrap("Internal re error")) - return space.newbool(res) +# ____________________________________________________________ +# +# SRE_Scanner class +# This is mostly an internal class in CPython. +# Our version is also directly iterable, to make finditer() easier. + +class W_SRE_Scanner(Wrappable): + + def __init__(self, pattern, ctx): + self.space = pattern.space + self.srepat = pattern + self.ctx = ctx + # 'self.ctx' is always a fresh context in which no searching + # or matching succeeded so far. + + def iter_w(self): + return self.space.wrap(self) + + def next_w(self): + if self.ctx.match_start > self.ctx.end: + raise OperationError(self.space.w_StopIteration, self.space.w_None) + if not searchcontext(self.space, self.ctx): + raise OperationError(self.space.w_StopIteration, self.space.w_None) + return self.getmatch(True) + + def match_w(self): + if self.ctx.match_start > self.ctx.end: + return self.space.w_None + return self.getmatch(matchcontext(self.space, self.ctx)) + + def search_w(self): + if self.ctx.match_start > self.ctx.end: + return self.space.w_None + return self.getmatch(searchcontext(self.space, self.ctx)) + + def getmatch(self, found): + if found: + ctx = self.ctx + nextstart = ctx.match_end + nextstart += (ctx.match_start == nextstart) + self.ctx = ctx.fresh_copy(nextstart) + match = W_SRE_Match(self.srepat, ctx) + return self.space.wrap(match) + else: + self.ctx.match_start += 1 # obscure corner case + return None + +W_SRE_Scanner.typedef = TypeDef( + 'SRE_Scanner', + __iter__ = interp2app(W_SRE_Scanner.iter_w, unwrap_spec=['self']), + next = interp2app(W_SRE_Scanner.next_w, unwrap_spec=['self']), + match = interp2app(W_SRE_Scanner.match_w, unwrap_spec=['self']), + search = interp2app(W_SRE_Scanner.search_w, unwrap_spec=['self']), + pattern = interp_attrproperty('srepat', W_SRE_Scanner), +) Modified: pypy/branch/jit-bounds/pypy/module/_sre/test/test_app_sre.py ============================================================================== --- pypy/branch/jit-bounds/pypy/module/_sre/test/test_app_sre.py (original) +++ pypy/branch/jit-bounds/pypy/module/_sre/test/test_app_sre.py Tue Aug 24 18:54:23 2010 @@ -33,8 +33,8 @@ # copy support is disabled by default in _sre.c import re p = re.compile("b") - raises(TypeError, p.__copy__) - raises(TypeError, p.__deepcopy__) + raises(TypeError, p.__copy__) # p.__copy__() should raise + raises(TypeError, p.__deepcopy__) # p.__deepcopy__() should raise def test_creation_attributes(self): import re @@ -85,6 +85,10 @@ assert ['', 'a', None, 'l', 'u', None, 'lla'] == ( re.split("b([ua]|(s))", "balbulla")) + def test_weakref(self): + import re, _weakref + _weakref.ref(re.compile(r"")) + class AppTestSreMatch: def setup_class(cls): @@ -202,6 +206,20 @@ return ret assert ("bbbbb", 3) == re.subn("a", call_me, "ababa") + def test_sub_callable_returns_none(self): + import re + def call_me(match): + return None + assert "acd" == re.sub("b", call_me, "abcd") + + def test_sub_callable_suddenly_unicode(self): + import re + def call_me(match): + if match.group() == 'A': + return unichr(0x3039) + return '' + assert (u"bb\u3039b", 2) == re.subn("[aA]", call_me, "babAb") + def test_match_array(self): import re, array a = array.array('c', 'hello') @@ -268,6 +286,18 @@ p.match().group(0), p.match().group(0)) assert None == p.match() + def test_scanner_match_detail(self): + import re + p = re.compile("a").scanner("aaXaa") + assert "a" == p.match().group(0) + assert "a" == p.match().group(0) + assert None == p.match() + assert "a" == p.match().group(0) + assert "a" == p.match().group(0) + assert None == p.match() + assert None == p.match() + assert None == p.match() + def test_scanner_search(self): import re p = re.compile("\d").scanner("bla23c5a") @@ -653,69 +683,6 @@ s.ATCODES["at_uni_non_boundary"], s.OPCODES["success"]] s.assert_match(opcodes, ["blaha", u"bl%sja" % UPPER_PI]) - def test_category_digit(self): - INDIAN_DIGIT = u"\u0966" - opcodes = [s.OPCODES["category"], s.CHCODES["category_digit"]] \ - + s.encode_literal("b") + [s.OPCODES["success"]] - s.assert_match(opcodes, ["1b", "a1b"]) - s.assert_no_match(opcodes, ["bb", "b1", u"%sb" % INDIAN_DIGIT]) - - def test_category_not_digit(self): - INDIAN_DIGIT = u"\u0966" - opcodes = [s.OPCODES["category"], s.CHCODES["category_not_digit"]] \ - + s.encode_literal("b") + [s.OPCODES["success"]] - s.assert_match(opcodes, ["bb", "1ab", u"%sb" % INDIAN_DIGIT]) - s.assert_no_match(opcodes, ["1b", "a1b"]) - - def test_category_space(self): - EM_SPACE = u"\u2001" - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_space"], s.OPCODES["success"]] - s.assert_match(opcodes, ["b ", "b\n", "b\t", "b\r", "b\v", "b\f"]) - s.assert_no_match(opcodes, ["bb", "b1", u"b%s" % EM_SPACE]) - - def test_category_not_space(self): - EM_SPACE = u"\u2001" - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_not_space"], s.OPCODES["success"]] - s.assert_match(opcodes, ["bb", "b1", u"b%s" % EM_SPACE]) - s.assert_no_match(opcodes, ["b ", "b\n", "b\t", "b\r", "b\v", "b\f"]) - - def test_category_word(self): - LOWER_PI = u"\u03c0" - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_word"], s.OPCODES["success"]] - s.assert_match(opcodes, ["bl", "b4", "b_"]) - s.assert_no_match(opcodes, ["b ", "b\n", u"b%s" % LOWER_PI]) - - def test_category_not_word(self): - LOWER_PI = u"\u03c0" - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_not_word"], s.OPCODES["success"]] - s.assert_match(opcodes, ["b ", "b\n", u"b%s" % LOWER_PI]) - s.assert_no_match(opcodes, ["bl", "b4", "b_"]) - - def test_category_linebreak(self): - LINE_SEP = u"\u2028" - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_linebreak"], s.OPCODES["success"]] - s.assert_match(opcodes, ["b\n"]) - s.assert_no_match(opcodes, ["b ", "bs", "b\r", u"b%s" % LINE_SEP]) - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_uni_linebreak"], s.OPCODES["success"]] - s.assert_match(opcodes, ["b\n", u"b%s" % LINE_SEP]) - - def test_category_not_linebreak(self): - LINE_SEP = u"\u2028" - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_not_linebreak"], s.OPCODES["success"]] - s.assert_match(opcodes, ["b ", "bs", u"b%s" % LINE_SEP]) - s.assert_no_match(opcodes, ["b\n"]) - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_uni_not_linebreak"], s.OPCODES["success"]] - s.assert_match(opcodes, ["b ", "bs"]) - s.assert_no_match(opcodes, ["b\n", u"b%s" % LINE_SEP, "b\r"]) - def test_category_loc_word(self): import locale try: @@ -873,10 +840,6 @@ s.assert_match(opcodes, ["ab", "aaaab", "baabb"]) s.assert_no_match(opcodes, ["aaa", "", "ac"]) - def test_max_until_error(self): - opcodes = [s.OPCODES["max_until"], s.OPCODES["success"]] - raises(RuntimeError, s.search, opcodes, "a") - def test_max_until_zero_width_match(self): # re.compile won't compile prospective zero-with matches (all of them?), # so we can only produce an example by directly constructing bytecodes. @@ -896,10 +859,6 @@ s.assert_no_match(opcodes, ["b"]) assert "aab" == s.search(opcodes, "aabb").group(0) - def test_min_until_error(self): - opcodes = [s.OPCODES["min_until"], s.OPCODES["success"]] - raises(RuntimeError, s.search, opcodes, "a") - def test_groupref(self): opcodes = [s.OPCODES["mark"], 0, s.OPCODES["any"], s.OPCODES["mark"], 1] \ + s.encode_literal("a") + [s.OPCODES["groupref"], 0, s.OPCODES["success"]] Modified: pypy/branch/jit-bounds/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/jit-bounds/pypy/module/array/interp_array.py (original) +++ pypy/branch/jit-bounds/pypy/module/array/interp_array.py Tue Aug 24 18:54:23 2010 @@ -15,6 +15,7 @@ from pypy.interpreter.argument import Arguments, Signature from pypy.module._file.interp_file import W_File from pypy.interpreter.buffer import RWBuffer +from pypy.objspace.std.multimethod import FailedToImplement def w_array(space, w_cls, typecode, w_args=None): if len(w_args.arguments_w) > 1: @@ -476,9 +477,11 @@ def mul__Array_ANY(space, self, w_repeat): try: - repeat = space.int_w(w_repeat) - except OperationError: - return space.w_NotImplemented + repeat = space.getindex_w(w_repeat, space.w_OverflowError) + except OperationError, e: + if e.match(space, space.w_TypeError): + raise FailedToImplement + raise a = mytype.w_class(space) repeat = max(repeat, 0) a.setlen(self.len * repeat) @@ -492,9 +495,11 @@ def inplace_mul__Array_ANY(space, self, w_repeat): try: - repeat = space.int_w(w_repeat) - except OperationError: - return space.w_NotImplemented + repeat = space.getindex_w(w_repeat, space.w_OverflowError) + except OperationError, e: + if e.match(space, space.w_TypeError): + raise FailedToImplement + raise oldlen = self.len repeat = max(repeat, 0) self.setlen(self.len * repeat) Modified: pypy/branch/jit-bounds/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/jit-bounds/pypy/module/array/test/test_array.py (original) +++ pypy/branch/jit-bounds/pypy/module/array/test/test_array.py Tue Aug 24 18:54:23 2010 @@ -628,7 +628,8 @@ a = self.array('i') raises(TypeError, "a * 'hi'") raises(TypeError, "'hi' * a") - + raises(TypeError, "a *= 'hi'") + class mulable(object): def __mul__(self, other): return "mul" Modified: pypy/branch/jit-bounds/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/jit-bounds/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/jit-bounds/pypy/module/cpyext/stubs.py Tue Aug 24 18:54:23 2010 @@ -2874,36 +2874,6 @@ """ raise NotImplementedError - at cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.INTP], PyObject) -def PyUnicode_DecodeUTF16(space, s, size, errors, byteorder): - """Decode length bytes from a UTF-16 encoded buffer string and return the - corresponding Unicode object. errors (if non-NULL) defines the error - handling. It defaults to "strict". - - If byteorder is non-NULL, the decoder starts decoding using the given byte - order: - - *byteorder == -1: little endian - *byteorder == 0: native order - *byteorder == 1: big endian - - If *byteorder is zero, and the first two bytes of the input data are a - byte order mark (BOM), the decoder switches to this byte order and the BOM is - not copied into the resulting Unicode string. If *byteorder is -1 or - 1, any byte order mark is copied to the output (where it will result in - either a \ufeff or a \ufffe character). - - After completion, *byteorder is set to the current byte order at the end - of input data. - - If byteorder is NULL, the codec starts in native order mode. - - Return NULL if an exception was raised by the codec. - - This function used an int type for size. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.INTP, Py_ssize_t], PyObject) def PyUnicode_DecodeUTF16Stateful(space, s, size, errors, byteorder, consumed): """If consumed is NULL, behave like PyUnicode_DecodeUTF16(). If Modified: pypy/branch/jit-bounds/pypy/module/cpyext/test/test_unicodeobject.py ============================================================================== --- pypy/branch/jit-bounds/pypy/module/cpyext/test/test_unicodeobject.py (original) +++ pypy/branch/jit-bounds/pypy/module/cpyext/test/test_unicodeobject.py Tue Aug 24 18:54:23 2010 @@ -172,4 +172,37 @@ result = api.PyUnicode_AsASCIIString(w_ustr) assert result is None + def test_decode_utf16(self, space, api): + def test(encoded, endian, realendian=None): + encoded_charp = rffi.str2charp(encoded) + strict_charp = rffi.str2charp("strict") + if endian is not None: + pendian = lltype.malloc(rffi.INTP.TO, 1, flavor='raw') + if endian < 0: + pendian[0] = -1 + elif endian > 0: + pendian[0] = 1 + else: + pendian[0] = 0 + else: + pendian = None + w_ustr = api.PyUnicode_DecodeUTF16(encoded_charp, len(encoded), strict_charp, pendian) + assert space.eq_w(space.call_method(w_ustr, 'encode', space.wrap('ascii')), + space.wrap("abcd")) + + rffi.free_charp(encoded_charp) + rffi.free_charp(strict_charp) + if pendian: + if realendian is not None: + assert rffi.cast(rffi.INT, realendian) == pendian[0] + lltype.free(pendian, flavor='raw') + + test("\x61\x00\x62\x00\x63\x00\x64\x00", -1) + + test("\x61\x00\x62\x00\x63\x00\x64\x00", None) + + test("\x00\x61\x00\x62\x00\x63\x00\x64", 1) + + test("\xFE\xFF\x00\x61\x00\x62\x00\x63\x00\x64", 0, 1) + test("\xFF\xFE\x61\x00\x62\x00\x63\x00\x64\x00", 0, -1) Modified: pypy/branch/jit-bounds/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/jit-bounds/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/jit-bounds/pypy/module/cpyext/unicodeobject.py Tue Aug 24 18:54:23 2010 @@ -9,6 +9,7 @@ from pypy.module.cpyext.pyobject import PyObject, from_ref, make_typedescr from pypy.module.sys.interp_encoding import setdefaultencoding from pypy.objspace.std import unicodeobject, unicodetype +from pypy.rlib import runicode import sys ## See comment in stringobject.py. PyUnicode_FromUnicode(NULL, size) is not @@ -307,6 +308,64 @@ w_errors = space.w_None return space.call_method(w_str, 'decode', space.wrap("utf-8"), w_errors) + at cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.INTP], PyObject) +def PyUnicode_DecodeUTF16(space, s, size, llerrors, pbyteorder): + """Decode length bytes from a UTF-16 encoded buffer string and return the + corresponding Unicode object. errors (if non-NULL) defines the error + handling. It defaults to "strict". + + If byteorder is non-NULL, the decoder starts decoding using the given byte + order: + + *byteorder == -1: little endian + *byteorder == 0: native order + *byteorder == 1: big endian + + If *byteorder is zero, and the first two bytes of the input data are a + byte order mark (BOM), the decoder switches to this byte order and the BOM is + not copied into the resulting Unicode string. If *byteorder is -1 or + 1, any byte order mark is copied to the output (where it will result in + either a \ufeff or a \ufffe character). + + After completion, *byteorder is set to the current byte order at the end + of input data. + + If byteorder is NULL, the codec starts in native order mode. + + Return NULL if an exception was raised by the codec. + + This function used an int type for size. This might require + changes in your code for properly supporting 64-bit systems.""" + + string = rffi.charpsize2str(s, size) + + #FIXME: I don't like these prefixes + if pbyteorder is not None: # correct NULL check? + llbyteorder = rffi.cast(lltype.Signed, pbyteorder[0]) # compatible with int? + if llbyteorder < 0: + byteorder = "little" + elif llbyteorder > 0: + byteorder = "big" + else: + byteorder = "native" + else: + byteorder = "native" + + if llerrors: + errors = rffi.charp2str(llerrors) + else: + errors = None + + result, length, byteorder = runicode.str_decode_utf_16_helper(string, size, + errors, + True, # final ? false for multiple passes? + None, # errorhandler + byteorder) + if pbyteorder is not None: + pbyteorder[0] = rffi.cast(rffi.INT, byteorder) + + return space.wrap(result) + @cpython_api([PyObject], PyObject) def PyUnicode_AsASCIIString(space, w_unicode): """Encode a Unicode object using ASCII and return the result as Python string Modified: pypy/branch/jit-bounds/pypy/module/pypyjit/interp_jit.py ============================================================================== --- pypy/branch/jit-bounds/pypy/module/pypyjit/interp_jit.py (original) +++ pypy/branch/jit-bounds/pypy/module/pypyjit/interp_jit.py Tue Aug 24 18:54:23 2010 @@ -9,7 +9,7 @@ import pypy.interpreter.pyopcode # for side-effects from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import ObjSpace, Arguments -from pypy.interpreter.pycode import PyCode, CO_CONTAINSLOOP +from pypy.interpreter.pycode import PyCode, CO_GENERATOR from pypy.interpreter.pyframe import PyFrame from pypy.interpreter.pyopcode import ExitFrame from opcode import opmap @@ -24,9 +24,6 @@ JUMP_ABSOLUTE = opmap['JUMP_ABSOLUTE'] -def can_inline(next_instr, bytecode): - return not bool(bytecode.co_flags & CO_CONTAINSLOOP) - def get_printable_location(next_instr, bytecode): from pypy.tool.stdlib_opcode import opcode_method_names name = opcode_method_names[ord(bytecode.co_code[next_instr])] @@ -39,7 +36,8 @@ bytecode.jit_cells[next_instr] = newcell def confirm_enter_jit(next_instr, bytecode, frame, ec): - return (frame.w_f_trace is None and + return (not (bytecode.co_flags & CO_GENERATOR) and + frame.w_f_trace is None and ec.profilefunc is None and ec.w_tracefunc is None) @@ -57,8 +55,7 @@ ## blockstack = frame.blockstack ## return (valuestackdepth, blockstack) -pypyjitdriver = PyPyJitDriver(can_inline = can_inline, - get_printable_location = get_printable_location, +pypyjitdriver = PyPyJitDriver(get_printable_location = get_printable_location, get_jitcell_at = get_jitcell_at, set_jitcell_at = set_jitcell_at, confirm_enter_jit = confirm_enter_jit) Modified: pypy/branch/jit-bounds/pypy/module/pypyjit/policy.py ============================================================================== --- pypy/branch/jit-bounds/pypy/module/pypyjit/policy.py (original) +++ pypy/branch/jit-bounds/pypy/module/pypyjit/policy.py Tue Aug 24 18:54:23 2010 @@ -32,6 +32,8 @@ return False if mod.startswith('pypy.interpreter.pyparser.'): return False + if mod == 'pypy.interpreter.generator': + return False if mod.startswith('pypy.module.'): modname = mod[len('pypy.module.'):] if not self.look_inside_pypy_module(modname): Modified: pypy/branch/jit-bounds/pypy/objspace/std/intobject.py ============================================================================== --- pypy/branch/jit-bounds/pypy/objspace/std/intobject.py (original) +++ pypy/branch/jit-bounds/pypy/objspace/std/intobject.py Tue Aug 24 18:54:23 2010 @@ -240,34 +240,36 @@ def lshift__Int_Int(space, w_int1, w_int2): a = w_int1.intval b = w_int2.intval + if r_uint(b) < LONG_BIT: # 0 <= b < LONG_BIT + try: + c = ovfcheck_lshift(a, b) + except OverflowError: + raise FailedToImplementArgs(space.w_OverflowError, + space.wrap("integer left shift")) + return wrapint(space, c) if b < 0: raise OperationError(space.w_ValueError, space.wrap("negative shift count")) - if a == 0 or b == 0: - return get_integer(space, w_int1) - if b >= LONG_BIT: + else: #b >= LONG_BIT + if a == 0: + return get_integer(space, w_int1) raise FailedToImplementArgs(space.w_OverflowError, space.wrap("integer left shift")) - try: - c = ovfcheck_lshift(a, b) - except OverflowError: - raise FailedToImplementArgs(space.w_OverflowError, - space.wrap("integer left shift")) - return wrapint(space, c) def rshift__Int_Int(space, w_int1, w_int2): a = w_int1.intval b = w_int2.intval - if b < 0: - raise OperationError(space.w_ValueError, - space.wrap("negative shift count")) - if a == 0 or b == 0: - return get_integer(space, w_int1) - if b >= LONG_BIT: - if a < 0: - a = -1 - else: - a = 0 + if r_uint(b) >= LONG_BIT: # not (0 <= b < LONG_BIT) + if b < 0: + raise OperationError(space.w_ValueError, + space.wrap("negative shift count")) + else: # b >= LONG_BIT + if a == 0: + return get_integer(space, w_int1) + if a < 0: + a = -1 + else: + a = 0 else: a = a >> b return wrapint(space, a) Modified: pypy/branch/jit-bounds/pypy/objspace/std/model.py ============================================================================== --- pypy/branch/jit-bounds/pypy/objspace/std/model.py (original) +++ pypy/branch/jit-bounds/pypy/objspace/std/model.py Tue Aug 24 18:54:23 2010 @@ -88,7 +88,8 @@ import pypy.objspace.std.default # register a few catch-all multimethods import pypy.objspace.std.marshal_impl # install marshal multimethods - import pypy.module.array + if config.objspace.usemodules.array: + import pypy.module.array # the set of implementation types self.typeorder = { Modified: pypy/branch/jit-bounds/pypy/rlib/debug.py ============================================================================== --- pypy/branch/jit-bounds/pypy/rlib/debug.py (original) +++ pypy/branch/jit-bounds/pypy/rlib/debug.py Tue Aug 24 18:54:23 2010 @@ -255,11 +255,32 @@ class IntegerCanBeNegative(Exception): pass -def _check_nonneg(ann, bk): - from pypy.annotation.model import SomeInteger - s_nonneg = SomeInteger(nonneg=True) - if not s_nonneg.contains(ann): - raise IntegerCanBeNegative +class UnexpectedRUInt(Exception): + pass def check_nonneg(x): - check_annotation(x, _check_nonneg) + """Give a translation-time error if 'x' is not known to be non-negative. + To help debugging, this also gives a translation-time error if 'x' is + actually typed as an r_uint (in which case the call to check_nonneg() + is a bit strange and probably unexpected). + """ + assert type(x)(-1) < 0 # otherwise, 'x' is a r_uint or similar + assert x >= 0 + return x + +class Entry(ExtRegistryEntry): + _about_ = check_nonneg + + def compute_result_annotation(self, s_arg): + from pypy.annotation.model import SomeInteger + if isinstance(s_arg, SomeInteger) and s_arg.unsigned: + raise UnexpectedRUInt("check_nonneg() arg is a %s" % ( + s_arg.knowntype,)) + s_nonneg = SomeInteger(nonneg=True) + if not s_nonneg.contains(s_arg): + raise IntegerCanBeNegative + return s_arg + + def specialize_call(self, hop): + hop.exception_cannot_occur() + return hop.inputarg(hop.args_r[0], arg=0) Modified: pypy/branch/jit-bounds/pypy/rlib/jit.py ============================================================================== --- pypy/branch/jit-bounds/pypy/rlib/jit.py (original) +++ pypy/branch/jit-bounds/pypy/rlib/jit.py Tue Aug 24 18:54:23 2010 @@ -253,8 +253,7 @@ def __init__(self, greens=None, reds=None, virtualizables=None, get_jitcell_at=None, set_jitcell_at=None, - can_inline=None, get_printable_location=None, - confirm_enter_jit=None): + get_printable_location=None, confirm_enter_jit=None): if greens is not None: self.greens = greens if reds is not None: @@ -270,7 +269,6 @@ self.get_jitcell_at = get_jitcell_at self.set_jitcell_at = set_jitcell_at self.get_printable_location = get_printable_location - self.can_inline = can_inline self.confirm_enter_jit = confirm_enter_jit def _freeze_(self): @@ -284,6 +282,10 @@ # special-cased by ExtRegistryEntry assert dict.fromkeys(livevars) == _self._alllivevars + def loop_header(self): + # special-cased by ExtRegistryEntry + pass + def _set_param(self, name, value): # special-cased by ExtRegistryEntry # (internal, must receive a constant 'name') @@ -323,11 +325,15 @@ # specifically for them. self.jit_merge_point = self.jit_merge_point self.can_enter_jit = self.can_enter_jit + self.loop_header = self.loop_header self._set_param = self._set_param class Entry(ExtEnterLeaveMarker): _about_ = (self.jit_merge_point, self.can_enter_jit) + class Entry(ExtLoopHeader): + _about_ = self.loop_header + class Entry(ExtSetParam): _about_ = self._set_param @@ -384,7 +390,6 @@ self.annotate_hook(driver.get_jitcell_at, driver.greens, **kwds_s) self.annotate_hook(driver.set_jitcell_at, driver.greens, [s_jitcell], **kwds_s) - self.annotate_hook(driver.can_inline, driver.greens, **kwds_s) self.annotate_hook(driver.get_printable_location, driver.greens, **kwds_s) def annotate_hook(self, func, variables, args_s=[], **kwds_s): @@ -425,6 +430,23 @@ return hop.genop('jit_marker', vlist, resulttype=lltype.Void) +class ExtLoopHeader(ExtRegistryEntry): + # Replace a call to myjitdriver.loop_header() + # with an operation jit_marker('loop_header', myjitdriver). + + def compute_result_annotation(self, **kwds_s): + from pypy.annotation import model as annmodel + return annmodel.s_None + + def specialize_call(self, hop): + from pypy.rpython.lltypesystem import lltype + driver = self.instance.im_self + hop.exception_cannot_occur() + vlist = [hop.inputconst(lltype.Void, 'loop_header'), + hop.inputconst(lltype.Void, driver)] + return hop.genop('jit_marker', vlist, + resulttype=lltype.Void) + class ExtSetParam(ExtRegistryEntry): def compute_result_annotation(self, s_name, s_value): Modified: pypy/branch/jit-bounds/pypy/rlib/test/test_jit.py ============================================================================== --- pypy/branch/jit-bounds/pypy/rlib/test/test_jit.py (original) +++ pypy/branch/jit-bounds/pypy/rlib/test/test_jit.py Tue Aug 24 18:54:23 2010 @@ -59,11 +59,9 @@ def test_annotate_hooks(self): - def can_inline(m): pass def get_printable_location(m): pass myjitdriver = JitDriver(greens=['m'], reds=['n'], - can_inline=can_inline, get_printable_location=get_printable_location) def fn(n): m = 42.5 @@ -81,9 +79,8 @@ return [v.concretetype for v in graph.getargs()] raise Exception, 'function %r has not been annotated' % func - can_inline_args = getargs(can_inline) get_printable_location_args = getargs(get_printable_location) - assert can_inline_args == get_printable_location_args == [lltype.Float] + assert get_printable_location_args == [lltype.Float] def test_annotate_argumenterror(self): myjitdriver = JitDriver(greens=['m'], reds=['n']) Modified: pypy/branch/jit-bounds/pypy/rpython/rclass.py ============================================================================== --- pypy/branch/jit-bounds/pypy/rpython/rclass.py (original) +++ pypy/branch/jit-bounds/pypy/rpython/rclass.py Tue Aug 24 18:54:23 2010 @@ -156,9 +156,13 @@ if '_immutable_' in self.classdef.classdesc.classdict: hints = hints.copy() hints['immutable'] = True - if '_immutable_fields_' in self.classdef.classdesc.classdict: + self.immutable_field_list = [] # unless overwritten below + if self.classdef.classdesc.lookup('_immutable_fields_') is not None: hints = hints.copy() - self.immutable_field_list = self.classdef.classdesc.classdict['_immutable_fields_'].value + immutable_fields = self.classdef.classdesc.classdict.get( + '_immutable_fields_') + if immutable_fields is not None: + self.immutable_field_list = immutable_fields.value accessor = FieldListAccessor() hints['immutable_fields'] = accessor return hints @@ -181,7 +185,13 @@ hints = self.object_type._hints if "immutable_fields" in hints: accessor = hints["immutable_fields"] - self._parse_field_list(self.immutable_field_list, accessor) + immutable_fields = {} + rbase = self + while rbase.classdef is not None: + immutable_fields.update( + dict.fromkeys(rbase.immutable_field_list)) + rbase = rbase.rbase + self._parse_field_list(immutable_fields, accessor) def _parse_field_list(self, fields, accessor): with_suffix = {} @@ -191,7 +201,10 @@ suffix = '[*]' else: suffix = '' - mangled_name, r = self._get_field(name) + try: + mangled_name, r = self._get_field(name) + except KeyError: + continue with_suffix[mangled_name] = suffix accessor.initialize(self.object_type, with_suffix) return with_suffix Modified: pypy/branch/jit-bounds/pypy/rpython/test/test_rclass.py ============================================================================== --- pypy/branch/jit-bounds/pypy/rpython/test/test_rclass.py (original) +++ pypy/branch/jit-bounds/pypy/rpython/test/test_rclass.py Tue Aug 24 18:54:23 2010 @@ -738,6 +738,64 @@ assert accessor.fields == {"inst_x" : "", "inst_y" : "[*]"} or \ accessor.fields == {"ox" : "", "oy" : "[*]"} # for ootype + def test_immutable_fields_subclass_1(self): + from pypy.jit.metainterp.typesystem import deref + class A(object): + _immutable_fields_ = ["x"] + def __init__(self, x): + self.x = x + class B(A): + def __init__(self, x, y): + A.__init__(self, x) + self.y = y + + def f(): + return B(3, 5) + t, typer, graph = self.gengraph(f, []) + B_TYPE = deref(graph.getreturnvar().concretetype) + accessor = B_TYPE._hints["immutable_fields"] + assert accessor.fields == {"inst_x" : ""} or \ + accessor.fields == {"ox" : ""} # for ootype + + def test_immutable_fields_subclass_2(self): + from pypy.jit.metainterp.typesystem import deref + class A(object): + _immutable_fields_ = ["x"] + def __init__(self, x): + self.x = x + class B(A): + _immutable_fields_ = ["y"] + def __init__(self, x, y): + A.__init__(self, x) + self.y = y + + def f(): + return B(3, 5) + t, typer, graph = self.gengraph(f, []) + B_TYPE = deref(graph.getreturnvar().concretetype) + accessor = B_TYPE._hints["immutable_fields"] + assert accessor.fields == {"inst_x" : "", "inst_y" : ""} or \ + accessor.fields == {"ox" : "", "oy" : ""} # for ootype + + def test_immutable_fields_only_in_subclass(self): + from pypy.jit.metainterp.typesystem import deref + class A(object): + def __init__(self, x): + self.x = x + class B(A): + _immutable_fields_ = ["y"] + def __init__(self, x, y): + A.__init__(self, x) + self.y = y + + def f(): + return B(3, 5) + t, typer, graph = self.gengraph(f, []) + B_TYPE = deref(graph.getreturnvar().concretetype) + accessor = B_TYPE._hints["immutable_fields"] + assert accessor.fields == {"inst_y" : ""} or \ + accessor.fields == {"oy" : ""} # for ootype + def test_immutable_inheritance(self): class I(object): def __init__(self, v): Modified: pypy/branch/jit-bounds/pypy/tool/release/package.py ============================================================================== --- pypy/branch/jit-bounds/pypy/tool/release/package.py (original) +++ pypy/branch/jit-bounds/pypy/tool/release/package.py Tue Aug 24 18:54:23 2010 @@ -11,11 +11,12 @@ import py import os import fnmatch -import tarfile from pypy.tool.udir import udir if sys.version_info < (2,6): py.test.skip("requires 2.6 so far") +USE_TARFILE_MODULE = sys.platform == 'win32' + def ignore_patterns(*patterns): """Function that can be used as copytree() ignore parameter. @@ -69,9 +70,17 @@ old_dir = os.getcwd() try: os.chdir(str(builddir)) - os.system("strip " + str(archive_pypy_c)) - os.system('tar cvjf ' + str(builddir.join(name + '.tar.bz2')) + - " " + name) + os.system("strip " + str(archive_pypy_c)) # ignore errors + if USE_TARFILE_MODULE: + import tarfile + tf = tarfile.open(str(builddir.join(name + '.tar.bz2')), 'w:bz2') + tf.add(name) + tf.close() + else: + e = os.system('tar cvjf ' + str(builddir.join(name + '.tar.bz2')) + + " " + name) + if e: + raise OSError('"tar" returned exit status %r' % e) finally: os.chdir(old_dir) if copy_to_dir is not None: Modified: pypy/branch/jit-bounds/pypy/tool/release/test/test_package.py ============================================================================== --- pypy/branch/jit-bounds/pypy/tool/release/test/test_package.py (original) +++ pypy/branch/jit-bounds/pypy/tool/release/test/test_package.py Tue Aug 24 18:54:23 2010 @@ -5,7 +5,7 @@ from pypy.module.sys.version import CPYTHON_VERSION import tarfile, os -def test_dir_structure(): +def test_dir_structure(test='test'): # make sure we have sort of pypy-c pypy_c = py.path.local(pypydir).join('translator', 'goal', 'pypy-c') if not pypy_c.check(): @@ -14,8 +14,8 @@ else: fake_pypy_c = False try: - builddir = package(py.path.local(pypydir).dirpath(), 'test') - prefix = builddir.join('test') + builddir = package(py.path.local(pypydir).dirpath(), test) + prefix = builddir.join(test) cpyver = '%d.%d.%d' % CPYTHON_VERSION[:3] assert prefix.join('lib-python', cpyver, 'test').check() assert prefix.join('bin', 'pypy-c').check() @@ -24,18 +24,27 @@ assert not prefix.join('lib_pypy', 'ctypes_configure').check() assert prefix.join('LICENSE').check() assert prefix.join('README').check() - th = tarfile.open(str(builddir.join('test.tar.bz2'))) - assert th.getmember('test/lib_pypy/syslog.py') + th = tarfile.open(str(builddir.join('%s.tar.bz2' % test))) + assert th.getmember('%s/lib_pypy/syslog.py' % test) # the headers file could be not there, because they are copied into # trunk/include only during translation includedir = py.path.local(pypydir).dirpath().join('include') def check_include(name): if includedir.join(name).check(file=True): - assert th.getmember('test/include/%s' % name) + assert th.getmember('%s/include/%s' % (test, name)) check_include('Python.h') check_include('modsupport.inl') check_include('pypy_decl.h') finally: if fake_pypy_c: pypy_c.remove() + +def test_with_tarfile_module(): + from pypy.tool.release import package + prev = package.USE_TARFILE_MODULE + try: + package.USE_TARFILE_MODULE = True + test_dir_structure(test='testtarfile') + finally: + package.USE_TARFILE_MODULE = prev Modified: pypy/branch/jit-bounds/pypy/translator/platform/__init__.py ============================================================================== --- pypy/branch/jit-bounds/pypy/translator/platform/__init__.py (original) +++ pypy/branch/jit-bounds/pypy/translator/platform/__init__.py Tue Aug 24 18:54:23 2010 @@ -38,9 +38,9 @@ name = "abstract platform" c_environ = None - relevant_environ = [] + relevant_environ = () - so_prefixes = [''] + so_prefixes = ('',) def __init__(self, cc): if self.__class__ is Platform: @@ -146,7 +146,7 @@ extra = self.standalone_only else: extra = self.shared_only - cflags = self.cflags + extra + cflags = list(self.cflags) + list(extra) return (cflags + list(eci.compile_extra) + args) def _preprocess_library_dirs(self, library_dirs): @@ -158,7 +158,7 @@ libraries = self._libs(eci.libraries) link_files = self._linkfiles(eci.link_files) export_flags = self._exportsymbols_link_flags(eci) - return (library_dirs + self.link_flags + export_flags + + return (library_dirs + list(self.link_flags) + export_flags + link_files + list(eci.link_extra) + libraries) def _exportsymbols_link_flags(self, eci, relto=None): Modified: pypy/branch/jit-bounds/pypy/translator/platform/darwin.py ============================================================================== --- pypy/branch/jit-bounds/pypy/translator/platform/darwin.py (original) +++ pypy/branch/jit-bounds/pypy/translator/platform/darwin.py Tue Aug 24 18:54:23 2010 @@ -5,10 +5,10 @@ class Darwin(posix.BasePosix): name = "darwin" - link_flags = ['-mmacosx-version-min=10.4'] - cflags = ['-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4'] - standalone_only = ['-mdynamic-no-pic'] - shared_only = [] + link_flags = ('-mmacosx-version-min=10.4',) + cflags = ('-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4') + standalone_only = ('-mdynamic-no-pic',) + shared_only = () so_ext = 'so' @@ -18,8 +18,9 @@ self.cc = cc def _args_for_shared(self, args): - return (self.shared_only + ['-dynamiclib', '-undefined', 'dynamic_lookup'] - + args) + return (list(self.shared_only) + + ['-dynamiclib', '-undefined', 'dynamic_lookup'] + + args) def _preprocess_include_dirs(self, include_dirs): res_incl_dirs = list(include_dirs) @@ -72,11 +73,12 @@ class Darwin_i386(Darwin): name = "darwin_i386" - link_flags = ['-arch', 'i386', '-mmacosx-version-min=10.4'] - cflags = ['-arch', 'i386', '-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4'] + link_flags = ('-arch', 'i386', '-mmacosx-version-min=10.4') + cflags = ('-arch', 'i386', '-O3', '-fomit-frame-pointer', + '-mmacosx-version-min=10.4') class Darwin_x86_64(Darwin): name = "darwin_x86_64" - link_flags = ['-arch', 'x86_64', '-mmacosx-version-min=10.4'] - cflags = ['-arch', 'x86_64', '-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4'] - + link_flags = ('-arch', 'x86_64', '-mmacosx-version-min=10.4') + cflags = ('-arch', 'x86_64', '-O3', '-fomit-frame-pointer', + '-mmacosx-version-min=10.4') Modified: pypy/branch/jit-bounds/pypy/translator/platform/freebsd7.py ============================================================================== --- pypy/branch/jit-bounds/pypy/translator/platform/freebsd7.py (original) +++ pypy/branch/jit-bounds/pypy/translator/platform/freebsd7.py Tue Aug 24 18:54:23 2010 @@ -5,10 +5,10 @@ class Freebsd7(posix.BasePosix): name = "freebsd7" - link_flags = ['-pthread'] - cflags = ['-O3', '-pthread', '-fomit-frame-pointer'] - standalone_only = [] - shared_only = [] + link_flags = ('-pthread',) + cflags = ('-O3', '-pthread', '-fomit-frame-pointer') + standalone_only = () + shared_only = () so_ext = 'so' make_cmd = 'gmake' @@ -32,4 +32,4 @@ return ['/usr/local/lib'] class Freebsd7_64(Freebsd7): - shared_only = ['-fPIC'] + shared_only = ('-fPIC',) Modified: pypy/branch/jit-bounds/pypy/translator/platform/linux.py ============================================================================== --- pypy/branch/jit-bounds/pypy/translator/platform/linux.py (original) +++ pypy/branch/jit-bounds/pypy/translator/platform/linux.py Tue Aug 24 18:54:23 2010 @@ -6,12 +6,13 @@ class Linux(BasePosix): name = "linux" - link_flags = ['-pthread', '-lrt'] - cflags = ['-O3', '-pthread', '-fomit-frame-pointer', '-Wall', '-Wno-unused'] - standalone_only = [] - shared_only = ['-fPIC'] + link_flags = ('-pthread', '-lrt') + cflags = ('-O3', '-pthread', '-fomit-frame-pointer', + '-Wall', '-Wno-unused') + standalone_only = () + shared_only = ('-fPIC',) so_ext = 'so' - so_prefixes = ['lib', ''] + so_prefixes = ('lib', '') def _args_for_shared(self, args): return ['-shared'] + args @@ -30,4 +31,4 @@ class Linux64(Linux): - shared_only = ['-fPIC'] + shared_only = ('-fPIC',) Modified: pypy/branch/jit-bounds/pypy/translator/platform/maemo.py ============================================================================== --- pypy/branch/jit-bounds/pypy/translator/platform/maemo.py (original) +++ pypy/branch/jit-bounds/pypy/translator/platform/maemo.py Tue Aug 24 18:54:23 2010 @@ -13,7 +13,7 @@ class Maemo(Linux): name = "maemo" - available_includedirs = ['/usr/include', '/tmp'] + available_includedirs = ('/usr/include', '/tmp') copied_cache = {} def _invent_new_name(self, basepath, base): Modified: pypy/branch/jit-bounds/pypy/translator/platform/posix.py ============================================================================== --- pypy/branch/jit-bounds/pypy/translator/platform/posix.py (original) +++ pypy/branch/jit-bounds/pypy/translator/platform/posix.py Tue Aug 24 18:54:23 2010 @@ -10,7 +10,7 @@ exe_ext = '' make_cmd = 'make' - relevant_environ=['CPATH', 'LIBRARY_PATH', 'C_INCLUDE_PATH'] + relevant_environ = ('CPATH', 'LIBRARY_PATH', 'C_INCLUDE_PATH') def __init__(self, cc=None): if cc is None: @@ -92,7 +92,7 @@ else: exe_name = exe_name.new(ext=self.exe_ext) - linkflags = self.link_flags[:] + linkflags = list(self.link_flags) if shared: linkflags = self._args_for_shared(linkflags) Modified: pypy/branch/jit-bounds/pypy/translator/platform/windows.py ============================================================================== --- pypy/branch/jit-bounds/pypy/translator/platform/windows.py (original) +++ pypy/branch/jit-bounds/pypy/translator/platform/windows.py Tue Aug 24 18:54:23 2010 @@ -72,10 +72,10 @@ cc = 'cl.exe' link = 'link.exe' - cflags = ['/MD', '/O2'] - link_flags = [] - standalone_only = [] - shared_only = [] + cflags = ('/MD', '/O2') + link_flags = () + standalone_only = () + shared_only = () environ = None def __init__(self, cc=None): @@ -218,7 +218,7 @@ m.exe_name = exe_name m.eci = eci - linkflags = self.link_flags[:] + linkflags = list(self.link_flags) if shared: linkflags = self._args_for_shared(linkflags) + [ '/EXPORT:$(PYPY_MAIN_FUNCTION)'] @@ -336,10 +336,10 @@ class MingwPlatform(posix.BasePosix): name = 'mingw32' - standalone_only = [] - shared_only = [] - cflags = ['-O3'] - link_flags = [] + standalone_only = () + shared_only = () + cflags = ('-O3',) + link_flags = () exe_ext = 'exe' so_ext = 'dll' From dan at codespeak.net Wed Aug 25 04:02:02 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Wed, 25 Aug 2010 04:02:02 +0200 (CEST) Subject: [pypy-svn] r76719 - pypy/branch/micronumpy/pypy/module/micronumpy Message-ID: <20100825020202.13864282BAD@codespeak.net> Author: dan Date: Wed Aug 25 04:01:59 2010 New Revision: 76719 Modified: pypy/branch/micronumpy/pypy/module/micronumpy/array.py pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py Log: Cleaning up. Killed ndarray.py Modified: pypy/branch/micronumpy/pypy/module/micronumpy/array.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/array.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/array.py Wed Aug 25 04:01:59 2010 @@ -76,6 +76,12 @@ suffix = 0 return suffix +def unpack_shape(space, w_shape): + if space.is_true(space.isinstance(w_shape, space.w_int)): + return [space.int_w(w_shape)] + shape_w = space.fixedview(w_shape) + return [space.int_w(w_i) for w_i in shape_w] + def validate_index(array, space, w_i): index_dimensionality = space.int_w(space.len(w_i)) array_dimensionality = len(array.shape) Modified: pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/dtype.py Wed Aug 25 04:01:59 2010 @@ -93,13 +93,13 @@ def dump(self, data): return ', '.join([str(x) for x in self.cast(data)]) - def str(self): + def typestr(self): if self is float_descr: code = 'f' else: code = self.typecode - return ''.join([byteorder, code, self.itemsize()]) + return ''.join([byteorder, code, str(self.itemsize())]) for type in [lltype.Signed, lltype.Float]: def get_type(self, data, index): Modified: pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py ============================================================================== --- pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py (original) +++ pypy/branch/micronumpy/pypy/module/micronumpy/microarray.py Wed Aug 25 04:01:59 2010 @@ -18,6 +18,8 @@ from pypy.module.micronumpy.array import squeeze_slice, squeeze_shape from pypy.module.micronumpy.array import shape_prefix +from pypy.rpython.lltypesystem.lltype import cast_ptr_to_int + class MicroIter(Wrappable): _immutable_fields_ = ['step', 'stop', 'ndim'] # XXX: removed array def __init__(self, array): @@ -372,13 +374,13 @@ def descr_get_shape(space, self): return space.newtuple([space.wrap(x) for x in self.shape[self.offset:]]) -def descr_array_interface(space, self): +def descr_get_array_interface(space, self): w_dict = space.newdict() - data_ptr = space.wrap(lltype.cast_ptr_to_int(self.data)) + data_ptr = space.wrap(cast_ptr_to_int(self.data)) data = [data_ptr, space.w_False] content = [(space.wrap('shape'), descr_get_shape(space, self)), (space.wrap('data'), space.newtuple(data)), - (space.wrap('typestr'), space.wrap(self.dtype.dtype.str())), + (space.wrap('typestr'), space.wrap(self.dtype.dtype.typestr())), (space.wrap('version'), space.wrap(3))] w_dict.initialize_content(content) return w_dict From arigo at codespeak.net Wed Aug 25 11:33:05 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 25 Aug 2010 11:33:05 +0200 (CEST) Subject: [pypy-svn] r76720 - pypy/branch/better-map-instances/pypy/objspace/std Message-ID: <20100825093305.1F92E282BAD@codespeak.net> Author: arigo Date: Wed Aug 25 11:33:02 2010 New Revision: 76720 Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py Log: Mostly add XXXes. Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py Wed Aug 25 11:33:02 2010 @@ -45,6 +45,7 @@ oldattr = obj._get_mapdict_map() oldattr._size_estimate += attr.size_estimate() - oldattr.size_estimate() if attr.length() > len(obj._get_mapdict_storage()): + # note that attr.size_estimate() is always at least attr.length() new_storage = [None] * attr.size_estimate() for i in range(len(obj._get_mapdict_storage())): new_storage[i] = obj._get_mapdict_storage()[i] @@ -200,7 +201,7 @@ # ____________________________________________________________ # object implementation -DICT = 6 +DICT = 6 # XXX meant '0'? SLOT = 1 SPECIAL = 2 @@ -354,13 +355,13 @@ return MapDictIteratorImplementation(self.space, self) def impl_clear(self): - raise NotImplementedError("abstract base class") + XXX # implement me! def _clear_fields(self): self.w_obj = None def _as_rdict(self): - r_dict_content = self.initialize_as_rdict() + self.initialize_as_rdict() space = self.space w_obj = self.w_obj materialize_r_dict(space, w_obj, self) From arigo at codespeak.net Wed Aug 25 13:30:34 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 25 Aug 2010 13:30:34 +0200 (CEST) Subject: [pypy-svn] r76721 - in pypy/trunk/pypy: config translator/goal Message-ID: <20100825113034.A8405282BAD@codespeak.net> Author: arigo Date: Wed Aug 25 13:30:31 2010 New Revision: 76721 Modified: pypy/trunk/pypy/config/translationoption.py pypy/trunk/pypy/translator/goal/translate.py Log: Make the command "translate.py -Ojit" crash on 64-bit machines for now, instead of automatically switching to Boehm. You have to say explicitly "translate.py -Ojit --gc=boehm". Modified: pypy/trunk/pypy/config/translationoption.py ============================================================================== --- pypy/trunk/pypy/config/translationoption.py (original) +++ pypy/trunk/pypy/config/translationoption.py Wed Aug 25 13:30:31 2010 @@ -342,9 +342,12 @@ 'jit': 'hybrid extraopts jit', } -# For now, 64-bit JIT requires boehm -if IS_64_BITS: - OPT_TABLE['jit'] = OPT_TABLE['jit'].replace('hybrid', 'boehm') +def final_check_config(config): + # For now, 64-bit JIT requires boehm. You have to say it explicitly + # with --gc=boehm, so that you don't get boehm by mistake. + if IS_64_BITS: + if config.translation.jit and config.translation.gc != 'boehm': + raise ConfigError("for now, 64-bit JIT requires --gc=boehm") def set_opt_level(config, level): """Apply optimization suggestions on the 'config'. Modified: pypy/trunk/pypy/translator/goal/translate.py ============================================================================== --- pypy/trunk/pypy/translator/goal/translate.py (original) +++ pypy/trunk/pypy/translator/goal/translate.py Wed Aug 25 13:30:31 2010 @@ -18,7 +18,7 @@ ArbitraryOption, StrOption, IntOption, Config, \ ChoiceOption, OptHelpFormatter from pypy.config.translationoption import get_combined_translation_config -from pypy.config.translationoption import set_opt_level +from pypy.config.translationoption import set_opt_level, final_check_config from pypy.config.translationoption import OPT_LEVELS, DEFAULT_OPT_LEVEL from pypy.config.translationoption import PLATFORMS, set_platform @@ -175,6 +175,9 @@ if 'handle_config' in targetspec_dic: targetspec_dic['handle_config'](config, translateconfig) + # perform checks (if any) on the final config + final_check_config(config) + if translateconfig.help: opt_parser.print_help() if 'print_help' in targetspec_dic: From arigo at codespeak.net Wed Aug 25 14:06:08 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 25 Aug 2010 14:06:08 +0200 (CEST) Subject: [pypy-svn] r76722 - pypy/build/bot2/pypybuildbot Message-ID: <20100825120608.EE852282BAD@codespeak.net> Author: arigo Date: Wed Aug 25 14:06:07 2010 New Revision: 76722 Modified: pypy/build/bot2/pypybuildbot/master.py Log: Reorder the nightly runs. Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Wed Aug 25 14:06:07 2010 @@ -133,15 +133,27 @@ 'change_source': [], 'schedulers': [ - Nightly("nightly-first", [LINUX32, LINUX64], - hour=4, minute=44), - Nightly("nightly", [APPLVLLINUX32, APPLVLLINUX64, APPLVLWIN32, - STACKLESSAPPLVLLINUX32, STACKLESSAPPLVLFREEBSD64, - JITLINUX32, OJITLINUX32, - MACOSX32], - hour=4, minute=45), - Nightly("nightly-benchmark", [JITBENCH], - hour=6, minute=15), + Nightly("nightly-1-00", [ + JITBENCH, # on tannit -- nothing else there during first round! + ], hour=1, minute=0), + Nightly("nightly-4-00", [ + # rule: what we pick here on tannit should take at most 8 cores + # and be hopefully finished after 2 hours 50 minutes + LINUX32, # on tannit32, uses 4 cores + JITLINUX32, # on tannit32, uses 1 core + OJITLINUX32, # on tannit32, uses 1 core + MACOSX32, # on minime + APPLVLWIN32, # on bigboard + STACKLESSAPPLVLFREEBSD64, # on headless + ], hour=4, minute=0), + Nightly("nightly-6-50", [ + # the remaining stuff on tannit, which should also take at most + # 8 cores + LINUX64, # on tannit64, uses 4 cores + APPLVLLINUX32, # on tannit32, uses 1 core + APPLVLLINUX64, # on tannit64, uses 1 core + STACKLESSAPPLVLLINUX32, # on tannit32, uses 1 core + ], hour=6, minute=50), ], 'status': [status], From arigo at codespeak.net Wed Aug 25 14:36:05 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 25 Aug 2010 14:36:05 +0200 (CEST) Subject: [pypy-svn] r76723 - pypy/trunk/pypy/tool Message-ID: <20100825123605.958F5282BAD@codespeak.net> Author: arigo Date: Wed Aug 25 14:36:03 2010 New Revision: 76723 Modified: pypy/trunk/pypy/tool/runsubprocess.py Log: Raise an OSError that contains the name of the command that we tried to run, at least. Modified: pypy/trunk/pypy/tool/runsubprocess.py ============================================================================== --- pypy/trunk/pypy/tool/runsubprocess.py (original) +++ pypy/trunk/pypy/tool/runsubprocess.py Wed Aug 25 14:36:03 2010 @@ -70,5 +70,5 @@ assert results.startswith('(') results = eval(results) if results[0] is None: - raise OSError(results[1]) + raise OSError('%s: %s' % (args[0], results[1])) return results From arigo at codespeak.net Wed Aug 25 14:39:34 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 25 Aug 2010 14:39:34 +0200 (CEST) Subject: [pypy-svn] r76724 - pypy/build/bot2/pypybuildbot Message-ID: <20100825123934.B94B3282BAD@codespeak.net> Author: arigo Date: Wed Aug 25 14:39:33 2010 New Revision: 76724 Modified: pypy/build/bot2/pypybuildbot/master.py Log: Give it 15 more minutes. Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Wed Aug 25 14:39:33 2010 @@ -133,9 +133,9 @@ 'change_source': [], 'schedulers': [ - Nightly("nightly-1-00", [ + Nightly("nightly-0-45", [ JITBENCH, # on tannit -- nothing else there during first round! - ], hour=1, minute=0), + ], hour=0, minute=45), Nightly("nightly-4-00", [ # rule: what we pick here on tannit should take at most 8 cores # and be hopefully finished after 2 hours 50 minutes From arigo at codespeak.net Wed Aug 25 15:27:33 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 25 Aug 2010 15:27:33 +0200 (CEST) Subject: [pypy-svn] r76725 - pypy/branch/better-map-instances/pypy/objspace/std/test Message-ID: <20100825132733.857FB282BAD@codespeak.net> Author: arigo Date: Wed Aug 25 15:27:31 2010 New Revision: 76725 Modified: pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py Log: Start integration tests. XXX write more. One is obscure but failing. Modified: pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py Wed Aug 25 15:27:31 2010 @@ -1,3 +1,4 @@ +from pypy.conftest import gettestobjspace from pypy.objspace.std.test.test_dictmultiobject import FakeSpace, W_DictMultiObject from pypy.objspace.std.mapdict import * @@ -344,3 +345,34 @@ assert w_d.getitem_str("a") == 5 assert w_d.getitem_str("b") == 6 assert w_d.getitem_str("c") == 7 + +# ___________________________________________________________ +# integration tests + +# XXX write more + +class AppTestWithMapDict(object): + def setup_class(cls): + cls.space = gettestobjspace(**{"objspace.std.withmapdict": True}) + + def test_simple(self): + class A(object): + pass + a = A() + a.x = 5 + a.y = 6 + a.zz = 7 + assert a.x == 5 + assert a.y == 6 + assert a.zz == 7 + + def test_slot_name_conflict(self): + class A(object): + __slots__ = 'slot1' + class B(A): + __slots__ = 'slot1' + x = B() + x.slot1 = 'child' # using B.slot1 + A.slot1.__set__(x, 'parent') # using A.slot1 + assert x.slot1 == 'child' # B.slot1 should still have its old value + assert A.slot1.__get__(x) == 'parent' From arigo at codespeak.net Wed Aug 25 15:43:30 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 25 Aug 2010 15:43:30 +0200 (CEST) Subject: [pypy-svn] r76726 - pypy/trunk/pypy/rlib Message-ID: <20100825134330.7F3BE282BAD@codespeak.net> Author: arigo Date: Wed Aug 25 15:43:28 2010 New Revision: 76726 Modified: pypy/trunk/pypy/rlib/rlocale.py Log: Disable #include on platforms that don't have it, notably some Mac OS/X. Modified: pypy/trunk/pypy/rlib/rlocale.py ============================================================================== --- pypy/trunk/pypy/rlib/rlocale.py (original) +++ pypy/trunk/pypy/rlib/rlocale.py Wed Aug 25 15:43:28 2010 @@ -15,6 +15,12 @@ HAVE_LANGINFO = sys.platform != 'win32' HAVE_LIBINTL = sys.platform != 'win32' +if HAVE_LIBINTL: + try: + platform.verify_eci(ExternalCompilationInfo(includes=['libintl.h'])) + except platform.CompilationError: + HAVE_LIBINTL = False + class CConfig: includes = ['locale.h', 'limits.h'] From arigo at codespeak.net Wed Aug 25 17:20:00 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 25 Aug 2010 17:20:00 +0200 (CEST) Subject: [pypy-svn] r76729 - pypy/branch/better-map-instances/pypy/objspace/std Message-ID: <20100825152000.8638F282BAD@codespeak.net> Author: arigo Date: Wed Aug 25 17:19:58 2010 New Revision: 76729 Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py Log: Comments, (bad) implementation of clear(). Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py Wed Aug 25 17:19:58 2010 @@ -355,7 +355,9 @@ return MapDictIteratorImplementation(self.space, self) def impl_clear(self): - XXX # implement me! + # XXX implement me better, or provide a reasonable default + # XXX implementation in W_DictMultiObject + self._as_rdict().clear() def _clear_fields(self): self.w_obj = None @@ -367,6 +369,9 @@ materialize_r_dict(space, w_obj, self) self._clear_fields() return self + # XXX then the calls self._as_rdict().method() from above look like + # recursive calls, and a stack check is inserted, which is pointless. + # It would be better to return self.r_dict_content, I think def materialize_r_dict(space, obj, w_d): From arigo at codespeak.net Wed Aug 25 17:24:07 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 25 Aug 2010 17:24:07 +0200 (CEST) Subject: [pypy-svn] r76730 - pypy/branch/better-map-instances/pypy/objspace/std Message-ID: <20100825152407.79401282BAD@codespeak.net> Author: arigo Date: Wed Aug 25 17:24:06 2010 New Revision: 76730 Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py Log: RPython normalization fix. A bit obscure. Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py Wed Aug 25 17:24:06 2010 @@ -320,7 +320,7 @@ return self.w_obj.getdictvalue(self.space, key) def impl_setitem_str(self, key, w_value, shadows_type=True): - flag = self.w_obj.setdictvalue(self.space, key, w_value) + flag = self.w_obj.setdictvalue(self.space, key, w_value, shadows_type) assert flag def impl_setitem(self, w_key, w_value): From jcreigh at codespeak.net Wed Aug 25 18:44:01 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Wed, 25 Aug 2010 18:44:01 +0200 (CEST) Subject: [pypy-svn] r76731 - pypy/branch/asmgcc-64/pypy/jit/backend/x86 Message-ID: <20100825164401.7752D282BAD@codespeak.net> Author: jcreigh Date: Wed Aug 25 18:43:59 2010 New Revision: 76731 Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/assembler.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/rx86.py Log: fix bug that was causing guard_class to always fail on 64-bit Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/assembler.py Wed Aug 25 18:43:59 2010 @@ -1251,8 +1251,12 @@ sizeof_ti = rffi.sizeof(GCData.TYPE_INFO) type_info_group = llop.gc_get_type_info_group(llmemory.Address) type_info_group = rffi.cast(lltype.Signed, type_info_group) - expected_typeid = (classptr - sizeof_ti - type_info_group) >> 2 - self.mc.CMP16(mem(locs[0], 0), ImmedLoc(expected_typeid)) + expected_typeid = classptr - sizeof_ti - type_info_group + if IS_X86_32: + expected_typeid >>= 2 + self.mc.CMP16(mem(locs[0], 0), ImmedLoc(expected_typeid)) + elif IS_X86_64: + self.mc.CMP32_mi((locs[0].value, 0), expected_typeid) def genop_guard_guard_class(self, ign_1, guard_op, guard_token, locs, ign_2): self.mc.ensure_bytes_available(256) Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/rx86.py Wed Aug 25 18:43:59 2010 @@ -462,6 +462,8 @@ CMP_ji = select_8_or_32_bit_immed(CMP_ji8, CMP_ji32) CMP_rj = insn(rex_w, '\x3B', register(1, 8), '\x05', immediate(2)) + CMP32_mi = insn(rex_nw, '\x81', orbyte(7<<3), mem_reg_plus_const(1), immediate(2)) + AND8_rr = insn(rex_w, '\x20', byte_register(1), byte_register(2,8), '\xC0') OR8_rr = insn(rex_w, '\x08', byte_register(1), byte_register(2,8), '\xC0') From jcreigh at codespeak.net Wed Aug 25 19:06:51 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Wed, 25 Aug 2010 19:06:51 +0200 (CEST) Subject: [pypy-svn] r76732 - in pypy/branch/asmgcc-64: . lib_pypy pypy/annotation pypy/interpreter pypy/interpreter/astcompiler pypy/interpreter/test pypy/jit/backend/llgraph pypy/jit/backend/llsupport pypy/jit/backend/test pypy/jit/backend/x86 pypy/jit/backend/x86/test pypy/jit/codewriter pypy/jit/codewriter/test pypy/jit/metainterp pypy/jit/metainterp/test pypy/module/__builtin__ pypy/module/_locale pypy/module/_sre pypy/module/_sre/test pypy/module/array pypy/module/array/benchmark pypy/module/array/test pypy/module/cpyext pypy/module/cpyext/test pypy/module/pypyjit pypy/module/pypyjit/test pypy/module/test_lib_pypy pypy/objspace/std pypy/rlib pypy/rlib/rsre pypy/rlib/rsre/test pypy/rlib/test pypy/rpython pypy/rpython/test pypy/tool pypy/tool/release pypy/tool/release/test pypy/translator/goal pypy/translator/platform Message-ID: <20100825170651.0A62A282BAD@codespeak.net> Author: jcreigh Date: Wed Aug 25 19:06:43 2010 New Revision: 76732 Added: pypy/branch/asmgcc-64/pypy/module/test_lib_pypy/test_msvcrt.py - copied unchanged from r76731, pypy/trunk/pypy/module/test_lib_pypy/test_msvcrt.py pypy/branch/asmgcc-64/pypy/rlib/rsre/ (props changed) - copied from r76731, pypy/trunk/pypy/rlib/rsre/ pypy/branch/asmgcc-64/pypy/rlib/rsre/__init__.py - copied unchanged from r76731, pypy/trunk/pypy/rlib/rsre/__init__.py pypy/branch/asmgcc-64/pypy/rlib/rsre/rsre_char.py - copied unchanged from r76731, pypy/trunk/pypy/rlib/rsre/rsre_char.py pypy/branch/asmgcc-64/pypy/rlib/rsre/rsre_core.py - copied unchanged from r76731, pypy/trunk/pypy/rlib/rsre/rsre_core.py pypy/branch/asmgcc-64/pypy/rlib/rsre/test/ (props changed) - copied from r76731, pypy/trunk/pypy/rlib/rsre/test/ pypy/branch/asmgcc-64/pypy/rlib/rsre/test/__init__.py - copied unchanged from r76731, pypy/trunk/pypy/rlib/rsre/test/__init__.py pypy/branch/asmgcc-64/pypy/rlib/rsre/test/targetrsre.py - copied unchanged from r76731, pypy/trunk/pypy/rlib/rsre/test/targetrsre.py pypy/branch/asmgcc-64/pypy/rlib/rsre/test/test_search.py - copied unchanged from r76731, pypy/trunk/pypy/rlib/rsre/test/test_search.py pypy/branch/asmgcc-64/pypy/rlib/rsre/test/test_zinterp.py - copied unchanged from r76731, pypy/trunk/pypy/rlib/rsre/test/test_zinterp.py Removed: pypy/branch/asmgcc-64/lib_pypy/_sre.py pypy/branch/asmgcc-64/pypy/module/_sre/app_sre.py pypy/branch/asmgcc-64/pypy/module/pypyjit/test/test_can_inline.py Modified: pypy/branch/asmgcc-64/ (props changed) pypy/branch/asmgcc-64/lib_pypy/msvcrt.py pypy/branch/asmgcc-64/pypy/annotation/specialize.py pypy/branch/asmgcc-64/pypy/interpreter/astcompiler/consts.py pypy/branch/asmgcc-64/pypy/interpreter/pycode.py pypy/branch/asmgcc-64/pypy/interpreter/test/test_code.py pypy/branch/asmgcc-64/pypy/interpreter/test/test_zpy.py pypy/branch/asmgcc-64/pypy/jit/backend/llgraph/llimpl.py pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/descr.py pypy/branch/asmgcc-64/pypy/jit/backend/test/runner_test.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/assembler.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_runner.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_ztranslation.py pypy/branch/asmgcc-64/pypy/jit/codewriter/assembler.py pypy/branch/asmgcc-64/pypy/jit/codewriter/jitcode.py pypy/branch/asmgcc-64/pypy/jit/codewriter/jtransform.py pypy/branch/asmgcc-64/pypy/jit/codewriter/test/test_flatten.py pypy/branch/asmgcc-64/pypy/jit/metainterp/blackhole.py pypy/branch/asmgcc-64/pypy/jit/metainterp/executor.py pypy/branch/asmgcc-64/pypy/jit/metainterp/history.py pypy/branch/asmgcc-64/pypy/jit/metainterp/jitdriver.py pypy/branch/asmgcc-64/pypy/jit/metainterp/optimizeopt.py pypy/branch/asmgcc-64/pypy/jit/metainterp/pyjitpl.py pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_basic.py pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_immutable.py pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_jitdriver.py pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_recursive.py pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_virtualizable.py pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_warmspot.py pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_warmstate.py pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_ztranslation.py pypy/branch/asmgcc-64/pypy/jit/metainterp/warmspot.py pypy/branch/asmgcc-64/pypy/jit/metainterp/warmstate.py pypy/branch/asmgcc-64/pypy/module/__builtin__/functional.py pypy/branch/asmgcc-64/pypy/module/_locale/__init__.py pypy/branch/asmgcc-64/pypy/module/_sre/__init__.py pypy/branch/asmgcc-64/pypy/module/_sre/interp_sre.py pypy/branch/asmgcc-64/pypy/module/_sre/test/test_app_sre.py pypy/branch/asmgcc-64/pypy/module/array/benchmark/Makefile (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/intimg.c (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/intimgtst.c (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/intimgtst.py (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/loop.c (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/sum.c (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/sumtst.c (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/sumtst.py (props changed) pypy/branch/asmgcc-64/pypy/module/array/interp_array.py pypy/branch/asmgcc-64/pypy/module/array/test/test_array.py pypy/branch/asmgcc-64/pypy/module/array/test/test_array_old.py (props changed) pypy/branch/asmgcc-64/pypy/module/cpyext/stubs.py pypy/branch/asmgcc-64/pypy/module/cpyext/test/test_unicodeobject.py pypy/branch/asmgcc-64/pypy/module/cpyext/unicodeobject.py pypy/branch/asmgcc-64/pypy/module/pypyjit/interp_jit.py pypy/branch/asmgcc-64/pypy/module/pypyjit/policy.py pypy/branch/asmgcc-64/pypy/objspace/std/intobject.py pypy/branch/asmgcc-64/pypy/objspace/std/model.py pypy/branch/asmgcc-64/pypy/rlib/debug.py pypy/branch/asmgcc-64/pypy/rlib/jit.py pypy/branch/asmgcc-64/pypy/rlib/rlocale.py pypy/branch/asmgcc-64/pypy/rlib/test/test_jit.py pypy/branch/asmgcc-64/pypy/rpython/rclass.py pypy/branch/asmgcc-64/pypy/rpython/test/test_rclass.py pypy/branch/asmgcc-64/pypy/tool/release/package.py pypy/branch/asmgcc-64/pypy/tool/release/test/test_package.py pypy/branch/asmgcc-64/pypy/tool/runsubprocess.py pypy/branch/asmgcc-64/pypy/translator/goal/translate.py pypy/branch/asmgcc-64/pypy/translator/platform/__init__.py pypy/branch/asmgcc-64/pypy/translator/platform/darwin.py pypy/branch/asmgcc-64/pypy/translator/platform/freebsd7.py pypy/branch/asmgcc-64/pypy/translator/platform/linux.py pypy/branch/asmgcc-64/pypy/translator/platform/maemo.py pypy/branch/asmgcc-64/pypy/translator/platform/posix.py pypy/branch/asmgcc-64/pypy/translator/platform/windows.py Log: merge changes from trunk through r76731 Modified: pypy/branch/asmgcc-64/lib_pypy/msvcrt.py ============================================================================== --- pypy/branch/asmgcc-64/lib_pypy/msvcrt.py (original) +++ pypy/branch/asmgcc-64/lib_pypy/msvcrt.py Wed Aug 25 19:06:43 2010 @@ -5,9 +5,12 @@ """ # XXX incomplete: implemented only functions needed by subprocess.py +# PAC: 2010/08 added MS locking for Whoosh import ctypes from ctypes_support import standard_c_lib as _c +from ctypes_support import get_errno +import errno try: open_osfhandle = _c._open_osfhandle @@ -25,4 +28,17 @@ setmode.argtypes = [ctypes.c_int, ctypes.c_int] setmode.restype = ctypes.c_int +LK_UNLCK, LK_LOCK, LK_NBLCK, LK_RLCK, LK_NBRLCK = range(5) + +_locking = _c._locking +_locking.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int] +_locking.restype = ctypes.c_int + +def locking(fd, mode, nbytes): + '''lock or unlock a number of bytes in a file.''' + rv = _locking(fd, mode, nbytes) + if rv != 0: + e = get_errno() + raise IOError(e, errno.errorcode[e]) + del ctypes Modified: pypy/branch/asmgcc-64/pypy/annotation/specialize.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/annotation/specialize.py (original) +++ pypy/branch/asmgcc-64/pypy/annotation/specialize.py Wed Aug 25 19:06:43 2010 @@ -354,6 +354,12 @@ def specialize_argtype(funcdesc, args_s, *argindices): key = tuple([args_s[i].knowntype for i in argindices]) + for cls in key: + try: + assert '_must_specialize_' not in cls.classdesc.pyobj.__dict__, ( + "%s has the tag _must_specialize_" % (cls,)) + except AttributeError: + pass return maybe_star_args(funcdesc, key, args_s) def specialize_arglistitemtype(funcdesc, args_s, i): Modified: pypy/branch/asmgcc-64/pypy/interpreter/astcompiler/consts.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/interpreter/astcompiler/consts.py (original) +++ pypy/branch/asmgcc-64/pypy/interpreter/astcompiler/consts.py Wed Aug 25 19:06:43 2010 @@ -9,7 +9,6 @@ CO_NESTED = 0x0010 CO_GENERATOR = 0x0020 CO_NOFREE = 0x0040 -CO_CONTAINSLOOP = 0x0080 CO_CONTAINSGLOBALS = 0x0800 CO_GENERATOR_ALLOWED = 0x1000 CO_FUTURE_DIVISION = 0x2000 Modified: pypy/branch/asmgcc-64/pypy/interpreter/pycode.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/interpreter/pycode.py (original) +++ pypy/branch/asmgcc-64/pypy/interpreter/pycode.py Wed Aug 25 19:06:43 2010 @@ -13,7 +13,7 @@ from pypy.interpreter.baseobjspace import ObjSpace, W_Root from pypy.interpreter.astcompiler.consts import (CO_OPTIMIZED, CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED, - CO_GENERATOR, CO_CONTAINSLOOP, CO_CONTAINSGLOBALS) + CO_GENERATOR, CO_CONTAINSGLOBALS) from pypy.rlib.rarithmetic import intmask from pypy.rlib.debug import make_sure_not_resized, make_sure_not_modified from pypy.rlib import jit @@ -133,9 +133,7 @@ while opcode == opcodedesc.EXTENDED_ARG.index: opcode = ord(co_code[next_instr]) next_instr += 3 - if opcode == opcodedesc.JUMP_ABSOLUTE.index: - self.co_flags |= CO_CONTAINSLOOP - elif opcode == opcodedesc.LOAD_GLOBAL.index: + if opcode == opcodedesc.LOAD_GLOBAL.index: self.co_flags |= CO_CONTAINSGLOBALS elif opcode == opcodedesc.LOAD_NAME.index: self.co_flags |= CO_CONTAINSGLOBALS Modified: pypy/branch/asmgcc-64/pypy/interpreter/test/test_code.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/interpreter/test/test_code.py (original) +++ pypy/branch/asmgcc-64/pypy/interpreter/test/test_code.py Wed Aug 25 19:06:43 2010 @@ -184,8 +184,6 @@ # CO_NESTED assert f(4).func_code.co_flags & 0x10 assert f.func_code.co_flags & 0x10 == 0 - # check for CO_CONTAINSLOOP - assert not f.func_code.co_flags & 0x0080 # check for CO_CONTAINSGLOBALS assert not f.func_code.co_flags & 0x0800 @@ -198,9 +196,6 @@ return [l for l in [1, 2, 3, 4]] """ - # check for CO_CONTAINSLOOP - assert f.func_code.co_flags & 0x0080 - assert g.func_code.co_flags & 0x0080 # check for CO_CONTAINSGLOBALS assert f.func_code.co_flags & 0x0800 assert not g.func_code.co_flags & 0x0800 Modified: pypy/branch/asmgcc-64/pypy/interpreter/test/test_zpy.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/interpreter/test/test_zpy.py (original) +++ pypy/branch/asmgcc-64/pypy/interpreter/test/test_zpy.py Wed Aug 25 19:06:43 2010 @@ -3,27 +3,28 @@ import py import sys import pypy +import subprocess pypypath = py.path.local(pypy.__file__).dirpath("bin", "py.py") -def cmdexec(s): - if sys.platform == 'win32': - s = '"%s"' % s # double double quotes - return py.process.cmdexec(s) +def run(*args): + argslist = map(str, args) + popen = subprocess.Popen(argslist, stdout=subprocess.PIPE) + stdout, stderr = popen.communicate() + return stdout def test_executable(): """Ensures sys.executable points to the py.py script""" # TODO : watch out for spaces/special chars in pypypath - output = cmdexec( '''"%s" "%s" -c "import sys;print sys.executable" ''' % - (sys.executable, pypypath) ) + output = run(sys.executable, pypypath, + "-c", "import sys;print sys.executable") assert output.splitlines()[-1] == pypypath def test_special_names(): """Test the __name__ and __file__ special global names""" cmd = "print __name__; print '__file__' in globals()" - output = cmdexec( '''"%s" "%s" -c "%s" ''' % - (sys.executable, pypypath, cmd) ) + output = run(sys.executable, pypypath, '-c', cmd) assert output.splitlines()[-2] == '__main__' assert output.splitlines()[-1] == 'False' @@ -32,26 +33,25 @@ tmpfile.write("print __name__; print __file__\n") tmpfile.close() - output = cmdexec( '''"%s" "%s" "%s" ''' % - (sys.executable, pypypath, tmpfilepath) ) + output = run(sys.executable, pypypath, tmpfilepath) assert output.splitlines()[-2] == '__main__' assert output.splitlines()[-1] == str(tmpfilepath) def test_argv_command(): """Some tests on argv""" # test 1 : no arguments - output = cmdexec( '''"%s" "%s" -c "import sys;print sys.argv" ''' % - (sys.executable, pypypath) ) + output = run(sys.executable, pypypath, + "-c", "import sys;print sys.argv") assert output.splitlines()[-1] == str(['-c']) # test 2 : some arguments after - output = cmdexec( '''"%s" "%s" -c "import sys;print sys.argv" hello''' % - (sys.executable, pypypath) ) + output = run(sys.executable, pypypath, + "-c", "import sys;print sys.argv", "hello") assert output.splitlines()[-1] == str(['-c','hello']) # test 3 : additionnal pypy parameters - output = cmdexec( '''"%s" "%s" -O -c "import sys;print sys.argv" hello''' % - (sys.executable, pypypath) ) + output = run(sys.executable, pypypath, + "-O", "-c", "import sys;print sys.argv", "hello") assert output.splitlines()[-1] == str(['-c','hello']) SCRIPT_1 = """ @@ -65,18 +65,15 @@ tmpfile.close() # test 1 : no arguments - output = cmdexec( '''"%s" "%s" "%s" ''' % - (sys.executable, pypypath, tmpfilepath) ) + output = run(sys.executable, pypypath, tmpfilepath) assert output.splitlines()[-1] == str([tmpfilepath]) # test 2 : some arguments after - output = cmdexec( '''"%s" "%s" "%s" hello''' % - (sys.executable, pypypath, tmpfilepath) ) + output = run(sys.executable, pypypath, tmpfilepath, "hello") assert output.splitlines()[-1] == str([tmpfilepath,'hello']) # test 3 : additionnal pypy parameters - output = cmdexec( '''"%s" "%s" -O "%s" hello''' % - (sys.executable, pypypath, tmpfilepath) ) + output = run(sys.executable, pypypath, "-O", tmpfilepath, "hello") assert output.splitlines()[-1] == str([tmpfilepath,'hello']) @@ -98,11 +95,7 @@ tmpfile.write(TB_NORMALIZATION_CHK) tmpfile.close() - e = None - try: - output = cmdexec( '''"%s" "%s" "%s" ''' % - (sys.executable, pypypath, tmpfilepath) ) - except py.process.cmdexec.Error, e: - pass - assert e," expected failure" - assert e.err.splitlines()[-1] == 'KeyError: ' + popen = subprocess.Popen([sys.executable, str(pypypath), tmpfilepath], + stderr=subprocess.PIPE) + _, stderr = popen.communicate() + assert stderr.endswith('KeyError: \n') Modified: pypy/branch/asmgcc-64/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/llgraph/llimpl.py Wed Aug 25 19:06:43 2010 @@ -123,6 +123,9 @@ 'setarrayitem_gc' : (('ref', 'int', 'intorptr'), None), 'getarrayitem_gc' : (('ref', 'int'), 'intorptr'), 'getarrayitem_gc_pure' : (('ref', 'int'), 'intorptr'), + 'setarrayitem_raw' : (('ref', 'int', 'intorptr'), None), + 'getarrayitem_raw' : (('ref', 'int'), 'intorptr'), + 'getarrayitem_raw_pure' : (('ref', 'int'), 'intorptr'), 'arraylen_gc' : (('ref',), 'int'), 'call' : (('ref', 'varargs'), 'intorptr'), 'call_assembler' : (('ref', 'varargs'), 'intorptr'), @@ -689,6 +692,18 @@ op_getarrayitem_gc_pure = op_getarrayitem_gc + def op_getarrayitem_raw(self, arraydescr, array, index): + if arraydescr.typeinfo == REF: + raise NotImplementedError("getarrayitem_raw -> gcref") + elif arraydescr.typeinfo == INT: + return do_getarrayitem_raw_int(array, index) + elif arraydescr.typeinfo == FLOAT: + return do_getarrayitem_raw_float(array, index) + else: + raise NotImplementedError + + op_getarrayitem_raw_pure = op_getarrayitem_raw + def op_getfield_gc(self, fielddescr, struct): if fielddescr.typeinfo == REF: return do_getfield_gc_ptr(struct, fielddescr.ofs) @@ -734,6 +749,16 @@ else: raise NotImplementedError + def op_setarrayitem_raw(self, arraydescr, array, index, newvalue): + if arraydescr.typeinfo == REF: + raise NotImplementedError("setarrayitem_raw <- gcref") + elif arraydescr.typeinfo == INT: + do_setarrayitem_raw_int(array, index, newvalue) + elif arraydescr.typeinfo == FLOAT: + do_setarrayitem_raw_float(array, index, newvalue) + else: + raise NotImplementedError + def op_setfield_gc(self, fielddescr, struct, newvalue): if fielddescr.typeinfo == REF: do_setfield_gc_ptr(struct, fielddescr.ofs, newvalue) Modified: pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/descr.py Wed Aug 25 19:06:43 2010 @@ -198,9 +198,13 @@ try: return cache[ARRAY] except KeyError: + # we only support Arrays that are either GcArrays, or raw no-length + # non-gc Arrays. if ARRAY._hints.get('nolength', False): + assert not isinstance(ARRAY, lltype.GcArray) arraydescr = getArrayNoLengthDescrClass(ARRAY)() else: + assert isinstance(ARRAY, lltype.GcArray) arraydescr = getArrayDescrClass(ARRAY)() # verify basic assumption that all arrays' basesize and ofslength # are equal Modified: pypy/branch/asmgcc-64/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/test/runner_test.py Wed Aug 25 19:06:43 2010 @@ -1832,6 +1832,31 @@ assert self.cpu.get_latest_value_float(0) == 13.5 assert called + def test_raw_malloced_getarrayitem(self): + ARRAY = rffi.CArray(lltype.Signed) + descr = self.cpu.arraydescrof(ARRAY) + a = lltype.malloc(ARRAY, 10, flavor='raw') + a[7] = -4242 + addr = llmemory.cast_ptr_to_adr(a) + abox = BoxInt(heaptracker.adr2int(addr)) + r1 = self.execute_operation(rop.GETARRAYITEM_RAW, [abox, BoxInt(7)], + 'int', descr=descr) + assert r1.getint() == -4242 + lltype.free(a, flavor='raw') + + def test_raw_malloced_setarrayitem(self): + ARRAY = rffi.CArray(lltype.Signed) + descr = self.cpu.arraydescrof(ARRAY) + a = lltype.malloc(ARRAY, 10, flavor='raw') + addr = llmemory.cast_ptr_to_adr(a) + abox = BoxInt(heaptracker.adr2int(addr)) + self.execute_operation(rop.SETARRAYITEM_RAW, [abox, BoxInt(5), + BoxInt(12345)], + 'void', descr=descr) + assert a[5] == 12345 + lltype.free(a, flavor='raw') + + class OOtypeBackendTest(BaseBackendTest): type_system = 'ootype' Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/assembler.py Wed Aug 25 19:06:43 2010 @@ -241,7 +241,7 @@ f = open_file_as_stream(output_log, "w") for i in range(len(self.loop_run_counters)): name, struct = self.loop_run_counters[i] - f.write(name + ":" + str(struct.i) + "\n") + f.write(str(struct.i) + " " * (8 - len(str(struct.i))) + name + "\n") f.close() def _build_float_constants(self): Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_runner.py Wed Aug 25 19:06:43 2010 @@ -503,4 +503,4 @@ assert struct.i == 10 self.cpu.finish_once() lines = py.path.local(self.logfile + ".count").readlines() - assert lines[0] == 'xyz:10\n' + assert lines[0] == '10 xyz\n' Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_ztranslation.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_ztranslation.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_ztranslation.py Wed Aug 25 19:06:43 2010 @@ -75,8 +75,7 @@ driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'], virtualizables = ['frame'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno: str(codeno)) class SomewhereElse(object): pass Modified: pypy/branch/asmgcc-64/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/codewriter/assembler.py Wed Aug 25 19:06:43 2010 @@ -53,6 +53,7 @@ self.liveness = {} self.startpoints = set() self.alllabels = set() + self.resulttypes = {} def emit_reg(self, reg): if reg.index >= self.count_regs[reg.kind]: @@ -165,7 +166,9 @@ raise NotImplementedError(x) # opname = insn[0] - assert '>' not in argcodes or argcodes.index('>') == len(argcodes) - 2 + if '>' in argcodes: + assert argcodes.index('>') == len(argcodes) - 2 + self.resulttypes[len(self.code)] = argcodes[-1] key = opname + '/' + ''.join(argcodes) num = self.insns.setdefault(key, len(self.insns)) self.code[startposition] = chr(num) @@ -212,7 +215,8 @@ self.count_regs['float'], liveness=self.liveness, startpoints=self.startpoints, - alllabels=self.alllabels) + alllabels=self.alllabels, + resulttypes=self.resulttypes) def see_raw_object(self, value): if value._obj not in self._seen_raw_objects: Modified: pypy/branch/asmgcc-64/pypy/jit/codewriter/jitcode.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/codewriter/jitcode.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/codewriter/jitcode.py Wed Aug 25 19:06:43 2010 @@ -19,7 +19,8 @@ def setup(self, code='', constants_i=[], constants_r=[], constants_f=[], num_regs_i=255, num_regs_r=255, num_regs_f=255, - liveness=None, startpoints=None, alllabels=None): + liveness=None, startpoints=None, alllabels=None, + resulttypes=None): self.code = code # if the following lists are empty, use a single shared empty list self.constants_i = constants_i or self._empty_i @@ -33,6 +34,7 @@ self.liveness = make_liveness_cache(liveness) self._startpoints = startpoints # debugging self._alllabels = alllabels # debugging + self._resulttypes = resulttypes # debugging def get_fnaddr_as_int(self): return heaptracker.adr2int(self.fnaddr) Modified: pypy/branch/asmgcc-64/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/codewriter/jtransform.py Wed Aug 25 19:06:43 2010 @@ -829,13 +829,20 @@ self.make_three_lists(op.args[2:2+num_green_args]) + self.make_three_lists(op.args[2+num_green_args:])) op1 = SpaceOperation('jit_merge_point', args, None) - return ops + [op1] + op2 = SpaceOperation('-live-', [], None) + # ^^^ we need a -live- for the case of do_recursive_call() + return ops + [op1, op2] - def handle_jit_marker__can_enter_jit(self, op, jitdriver): + def handle_jit_marker__loop_header(self, op, jitdriver): jd = self.callcontrol.jitdriver_sd_from_jitdriver(jitdriver) assert jd is not None c_index = Constant(jd.index, lltype.Signed) - return SpaceOperation('can_enter_jit', [c_index], None) + return SpaceOperation('loop_header', [c_index], None) + + # a 'can_enter_jit' in the source graph becomes a 'loop_header' + # operation in the transformed graph, as its only purpose in + # the transformed graph is to detect loops. + handle_jit_marker__can_enter_jit = handle_jit_marker__loop_header def rewrite_op_debug_assert(self, op): log.WARNING("found debug_assert in %r; should have be removed" % Modified: pypy/branch/asmgcc-64/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/codewriter/test/test_flatten.py Wed Aug 25 19:06:43 2010 @@ -593,7 +593,8 @@ -live- %i0, %i1 int_guard_value %i0 jit_merge_point $27, I[%i0], R[], F[], I[%i1], R[], F[] - can_enter_jit $27 + -live- + loop_header $27 void_return """, transform=True, liveness=True, cc=MyFakeCallControl(), jd=jd) Modified: pypy/branch/asmgcc-64/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/metainterp/blackhole.py Wed Aug 25 19:06:43 2010 @@ -2,7 +2,7 @@ from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.debug import debug_start, debug_stop -from pypy.rlib.debug import make_sure_not_resized +from pypy.rlib.debug import make_sure_not_resized, fatalerror from pypy.rpython.lltypesystem import lltype, llmemory, rclass from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.llinterp import LLException @@ -756,11 +756,15 @@ assert e reraise(e) + @arguments("r") + def bhimpl_debug_fatalerror(msg): + llop.debug_fatalerror(lltype.Void, msg) + # ---------- # the main hints and recursive calls @arguments("i") - def bhimpl_can_enter_jit(jdindex): + def bhimpl_loop_header(jdindex): pass @arguments("self", "i", "I", "R", "F", "I", "R", "F") @@ -1164,7 +1168,7 @@ # we now proceed to interpret the bytecode in this frame self.run() # - except JitException: + except JitException, e: raise # go through except Exception, e: # if we get an exception, return it to the caller frame @@ -1266,6 +1270,33 @@ e = lltype.cast_opaque_ptr(llmemory.GCREF, e) raise sd.ExitFrameWithExceptionRef(self.cpu, e) + def _handle_jitexception_in_portal(self, e): + # This case is really rare, but can occur if + # convert_and_run_from_pyjitpl() gets called in this situation: + # + # [function 1] <---- top BlackholeInterpreter() + # [recursive portal jit code] + # ... + # [bottom portal jit code] <---- bottom BlackholeInterpreter() + # + # and then "function 1" contains a call to "function 2", which + # calls "can_enter_jit". The latter can terminate by raising a + # JitException. In that case, the JitException is not supposed + # to fall through the whole chain of BlackholeInterpreters, but + # be caught and handled just below the level "recursive portal + # jit code". The present function is called to handle the case + # of recursive portal jit codes. + for jd in self.builder.metainterp_sd.jitdrivers_sd: + if jd.mainjitcode is self.jitcode: + break + else: + assert 0, "portal jitcode not found??" + # call the helper in warmspot.py. It might either raise a + # regular exception (which should then be propagated outside + # of 'self', not caught inside), or return (the return value + # gets stored in nextblackholeinterp). + jd.handle_jitexc_from_bh(self.nextblackholeinterp, e) + def _copy_data_from_miframe(self, miframe): self.setposition(miframe.jitcode, miframe.pc) for i in range(self.jitcode.num_regs_i()): @@ -1287,9 +1318,31 @@ while True: try: current_exc = blackholeinterp._resume_mainloop(current_exc) - finally: - blackholeinterp.builder.release_interp(blackholeinterp) + except JitException, e: + blackholeinterp, current_exc = _handle_jitexception( + blackholeinterp, e) + blackholeinterp.builder.release_interp(blackholeinterp) + blackholeinterp = blackholeinterp.nextblackholeinterp + +def _handle_jitexception(blackholeinterp, jitexc): + # See comments in _handle_jitexception_in_portal(). + while not blackholeinterp.jitcode.is_portal: + blackholeinterp.builder.release_interp(blackholeinterp) blackholeinterp = blackholeinterp.nextblackholeinterp + if blackholeinterp.nextblackholeinterp is None: + blackholeinterp.builder.release_interp(blackholeinterp) + raise jitexc # bottommost entry: go through + # We have reached a recursive portal level. + try: + blackholeinterp._handle_jitexception_in_portal(jitexc) + except Exception, e: + # It raised a general exception (it should not be a JitException here). + lle = get_llexception(blackholeinterp.cpu, e) + else: + # It set up the nextblackholeinterp to contain the return value. + lle = lltype.nullptr(rclass.OBJECTPTR.TO) + # We will continue to loop in _run_forever() from the parent level. + return blackholeinterp, lle def resume_in_blackhole(metainterp_sd, jitdriver_sd, resumedescr, all_virtuals=None): Modified: pypy/branch/asmgcc-64/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/metainterp/executor.py Wed Aug 25 19:06:43 2010 @@ -172,34 +172,36 @@ [x1box.getref_base(), x2box.getref_base()], None) def do_int_add_ovf(cpu, metainterp, box1, box2): - assert metainterp is not None + # the overflow operations can be called without a metainterp, if an + # overflow cannot occur a = box1.getint() b = box2.getint() try: z = ovfcheck(a + b) except OverflowError: + assert metainterp is not None metainterp.execute_raised(OverflowError(), constant=True) z = 0 return BoxInt(z) def do_int_sub_ovf(cpu, metainterp, box1, box2): - assert metainterp is not None a = box1.getint() b = box2.getint() try: z = ovfcheck(a - b) except OverflowError: + assert metainterp is not None metainterp.execute_raised(OverflowError(), constant=True) z = 0 return BoxInt(z) def do_int_mul_ovf(cpu, metainterp, box1, box2): - assert metainterp is not None a = box1.getint() b = box2.getint() try: z = ovfcheck(a * b) except OverflowError: + assert metainterp is not None metainterp.execute_raised(OverflowError(), constant=True) z = 0 return BoxInt(z) Modified: pypy/branch/asmgcc-64/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/metainterp/history.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/metainterp/history.py Wed Aug 25 19:06:43 2010 @@ -919,11 +919,12 @@ "found %d %r, expected %d" % (found, insn, expected_count)) return insns - def check_loops(self, expected=None, **check): + def check_loops(self, expected=None, everywhere=False, **check): insns = {} for loop in self.loops: - if getattr(loop, '_ignore_during_counting', False): - continue + if not everywhere: + if getattr(loop, '_ignore_during_counting', False): + continue insns = loop.summary(adding_insns=insns) if expected is not None: insns.pop('debug_merge_point', None) Modified: pypy/branch/asmgcc-64/pypy/jit/metainterp/jitdriver.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/metainterp/jitdriver.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/metainterp/jitdriver.py Wed Aug 25 19:06:43 2010 @@ -12,6 +12,7 @@ # self.result_type ... pypy.jit.metainterp.warmspot # self.virtualizable_info... pypy.jit.metainterp.warmspot # self.warmstate ... pypy.jit.metainterp.warmspot + # self.handle_jitexc_from_bh pypy.jit.metainterp.warmspot # self.index ... pypy.jit.codewriter.call # self.mainjitcode ... pypy.jit.codewriter.call Modified: pypy/branch/asmgcc-64/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/metainterp/optimizeopt.py Wed Aug 25 19:06:43 2010 @@ -526,7 +526,9 @@ def propagate_forward(self): self.exception_might_have_happened = False self.newoperations = [] - for op in self.loop.operations: + self.i = 0 + while self.i < len(self.loop.operations): + op = self.loop.operations[self.i] opnum = op.opnum for value, func in optimize_ops: if opnum == value: @@ -534,6 +536,7 @@ break else: self.optimize_default(op) + self.i += 1 self.loop.operations = self.newoperations # accumulate counters self.resumedata_memo.update_counters(self.metainterp_sd.profiler) @@ -588,7 +591,12 @@ descr.make_a_counter_per_value(op) def optimize_default(self, op): - if op.is_always_pure(): + canfold = op.is_always_pure() + is_ovf = op.is_ovf() + if is_ovf: + nextop = self.loop.operations[self.i + 1] + canfold = nextop.opnum == rop.GUARD_NO_OVERFLOW + if canfold: for arg in op.args: if self.get_constant_box(arg) is None: break @@ -598,6 +606,8 @@ resbox = execute_nonspec(self.cpu, None, op.opnum, argboxes, op.descr) self.make_constant(op.result, resbox.constbox()) + if is_ovf: + self.i += 1 # skip next operation, it is the unneeded guard return # did we do the exact same operation already? @@ -611,6 +621,8 @@ if oldop is not None and oldop.descr is op.descr: assert oldop.opnum == op.opnum self.make_equal_to(op.result, self.getvalue(oldop.result)) + if is_ovf: + self.i += 1 # skip next operation, it is the unneeded guard return elif self.find_rewritable_bool(op, args): return Modified: pypy/branch/asmgcc-64/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/metainterp/pyjitpl.py Wed Aug 25 19:06:43 2010 @@ -149,16 +149,20 @@ assert oldbox not in registers[count:] def make_result_of_lastop(self, resultbox): - if resultbox is None: - return + got_type = resultbox.type + if not we_are_translated(): + typeof = {'i': history.INT, + 'r': history.REF, + 'f': history.FLOAT} + assert typeof[self.jitcode._resulttypes[self.pc]] == got_type target_index = ord(self.bytecode[self.pc-1]) - if resultbox.type == history.INT: + if got_type == history.INT: self.registers_i[target_index] = resultbox - elif resultbox.type == history.REF: + elif got_type == history.REF: #debug_print(' ->', # llmemory.cast_ptr_to_adr(resultbox.getref_base())) self.registers_r[target_index] = resultbox - elif resultbox.type == history.FLOAT: + elif got_type == history.FLOAT: self.registers_f[target_index] = resultbox else: raise AssertionError("bad result box type") @@ -685,11 +689,11 @@ def _opimpl_recursive_call(self, jdindex, greenboxes, redboxes): targetjitdriver_sd = self.metainterp.staticdata.jitdrivers_sd[jdindex] allboxes = greenboxes + redboxes - portal_code = targetjitdriver_sd.mainjitcode warmrunnerstate = targetjitdriver_sd.warmstate token = None if warmrunnerstate.inlining: if warmrunnerstate.can_inline_callable(greenboxes): + portal_code = targetjitdriver_sd.mainjitcode return self.metainterp.perform_call(portal_code, allboxes, greenkey=greenboxes) token = warmrunnerstate.get_assembler_token(greenboxes) @@ -697,6 +701,10 @@ # that assembler that we call is still correct self.verify_green_args(targetjitdriver_sd, greenboxes) # + return self.do_recursive_call(targetjitdriver_sd, allboxes, token) + + def do_recursive_call(self, targetjitdriver_sd, allboxes, token=None): + portal_code = targetjitdriver_sd.mainjitcode k = targetjitdriver_sd.portal_runner_adr funcbox = ConstInt(heaptracker.adr2int(k)) return self.do_residual_call(funcbox, portal_code.calldescr, @@ -786,13 +794,8 @@ return clsbox @arguments("int") - def opimpl_can_enter_jit(self, jdindex): - if self.metainterp.in_recursion: - from pypy.jit.metainterp.warmspot import CannotInlineCanEnterJit - raise CannotInlineCanEnterJit() - assert jdindex == self.metainterp.jitdriver_sd.index, ( - "found a can_enter_jit that does not match the current jitdriver") - self.metainterp.seen_can_enter_jit = True + def opimpl_loop_header(self, jdindex): + self.metainterp.seen_loop_header_for_jdindex = jdindex def verify_green_args(self, jitdriver_sd, varargs): num_green_args = jitdriver_sd.num_green_args @@ -806,22 +809,42 @@ self.verify_green_args(jitdriver_sd, greenboxes) # xxx we may disable the following line in some context later self.debug_merge_point(jitdriver_sd, greenboxes) - if self.metainterp.seen_can_enter_jit: - self.metainterp.seen_can_enter_jit = False - # Assert that it's impossible to arrive here with in_recursion - # set to a non-zero value: seen_can_enter_jit can only be set - # to True by opimpl_can_enter_jit, which should be executed - # just before opimpl_jit_merge_point (no recursion inbetween). - assert not self.metainterp.in_recursion + if self.metainterp.seen_loop_header_for_jdindex < 0: + return + # + assert self.metainterp.seen_loop_header_for_jdindex == jdindex, ( + "found a loop_header for a JitDriver that does not match " + "the following jit_merge_point's") + self.metainterp.seen_loop_header_for_jdindex = -1 + # + if not self.metainterp.in_recursion: assert jitdriver_sd is self.metainterp.jitdriver_sd # Set self.pc to point to jit_merge_point instead of just after: - # if reached_can_enter_jit() raises SwitchToBlackhole, then the + # if reached_loop_header() raises SwitchToBlackhole, then the # pc is still at the jit_merge_point, which is a point that is # much less expensive to blackhole out of. saved_pc = self.pc self.pc = orgpc - self.metainterp.reached_can_enter_jit(greenboxes, redboxes) + self.metainterp.reached_loop_header(greenboxes, redboxes) self.pc = saved_pc + else: + warmrunnerstate = jitdriver_sd.warmstate + token = warmrunnerstate.get_assembler_token(greenboxes) + # warning! careful here. We have to return from the current + # frame containing the jit_merge_point, and then use + # do_recursive_call() to follow the recursive call. This is + # needed because do_recursive_call() will write its result + # with make_result_of_lastop(), so the lastop must be right: + # it must be the call to 'self', and not the jit_merge_point + # itself, which has no result at all. + assert len(self.metainterp.framestack) >= 2 + try: + self.metainterp.finishframe(None) + except ChangeFrame: + pass + frame = self.metainterp.framestack[-1] + frame.do_recursive_call(jitdriver_sd, greenboxes + redboxes, token) + raise ChangeFrame def debug_merge_point(self, jitdriver_sd, greenkey): # debugging: produce a DEBUG_MERGE_POINT operation @@ -872,6 +895,12 @@ return exc_value_box @arguments("box") + def opimpl_debug_fatalerror(self, box): + from pypy.rpython.lltypesystem import rstr, lloperation + msg = box.getref(lltype.Ptr(rstr.STR)) + lloperation.llop.debug_fatalerror(msg) + + @arguments("box") def opimpl_virtual_ref(self, box): # Details on the content of metainterp.virtualref_boxes: # @@ -1018,9 +1047,10 @@ self.metainterp.clear_exception() resbox = self.metainterp.execute_and_record_varargs(opnum, argboxes, descr=descr) - self.make_result_of_lastop(resbox) - # ^^^ this is done before handle_possible_exception() because we need - # the box to show up in get_list_of_active_boxes() + if resbox is not None: + self.make_result_of_lastop(resbox) + # ^^^ this is done before handle_possible_exception() because we + # need the box to show up in get_list_of_active_boxes() if exc: self.metainterp.handle_possible_exception() else: @@ -1323,7 +1353,8 @@ self.last_exc_value_box = None self.popframe() if self.framestack: - self.framestack[-1].make_result_of_lastop(resultbox) + if resultbox is not None: + self.framestack[-1].make_result_of_lastop(resultbox) raise ChangeFrame else: try: @@ -1552,7 +1583,7 @@ redkey = original_boxes[num_green_args:] self.resumekey = compile.ResumeFromInterpDescr(original_greenkey, redkey) - self.seen_can_enter_jit = False + self.seen_loop_header_for_jdindex = -1 try: self.interpret() except GenerateMergePoint, gmp: @@ -1579,7 +1610,7 @@ # because we cannot reconstruct the beginning of the proper loop self.current_merge_points = [(original_greenkey, -1)] self.resumekey = key - self.seen_can_enter_jit = False + self.seen_loop_header_for_jdindex = -1 try: self.prepare_resume_from_failure(key.guard_opnum) self.interpret() @@ -1609,7 +1640,7 @@ else: duplicates[box] = None - def reached_can_enter_jit(self, greenboxes, redboxes): + def reached_loop_header(self, greenboxes, redboxes): duplicates = {} self.remove_consts_and_duplicates(redboxes, len(redboxes), duplicates) @@ -1623,7 +1654,7 @@ live_arg_boxes += self.virtualizable_boxes live_arg_boxes.pop() assert len(self.virtualref_boxes) == 0, "missing virtual_ref_finish()?" - # Called whenever we reach the 'can_enter_jit' hint. + # Called whenever we reach the 'loop_header' hint. # First, attempt to make a bridge: # - if self.resumekey is a ResumeGuardDescr, it starts from a guard # that failed; @@ -2232,7 +2263,10 @@ else: resultbox = unboundmethod(self, *args) # - self.make_result_of_lastop(resultbox) + if resultbox is not None: + self.make_result_of_lastop(resultbox) + elif not we_are_translated(): + assert self._result_argcode in 'v?' # unboundmethod = getattr(MIFrame, 'opimpl_' + name).im_func argtypes = unrolling_iterable(unboundmethod.argtypes) Modified: pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_basic.py Wed Aug 25 19:06:43 2010 @@ -116,8 +116,9 @@ class JitMixin: basic = True - def check_loops(self, expected=None, **check): - get_stats().check_loops(expected=expected, **check) + def check_loops(self, expected=None, everywhere=False, **check): + get_stats().check_loops(expected=expected, everywhere=everywhere, + **check) def check_loop_count(self, count): """NB. This is a hack; use check_tree_loop_count() or check_enter_count() for the real thing. Modified: pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_immutable.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_immutable.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_immutable.py Wed Aug 25 19:06:43 2010 @@ -17,6 +17,38 @@ assert res == 28 self.check_operations_history(getfield_gc=0, getfield_gc_pure=1, int_add=1) + def test_fields_subclass(self): + class X(object): + _immutable_fields_ = ["x"] + + def __init__(self, x): + self.x = x + + class Y(X): + _immutable_fields_ = ["y"] + + def __init__(self, x, y): + X.__init__(self, x) + self.y = y + + def f(x, y): + X(x) # force the field 'x' to be on class 'X' + z = Y(x, y) + return z.x + z.y + 5 + res = self.interp_operations(f, [23, 11]) + assert res == 39 + self.check_operations_history(getfield_gc=0, getfield_gc_pure=2, + int_add=2) + + def f(x, y): + # this time, the field 'x' only shows up on subclass 'Y' + z = Y(x, y) + return z.x + z.y + 5 + res = self.interp_operations(f, [23, 11]) + assert res == 39 + self.check_operations_history(getfield_gc=0, getfield_gc_pure=2, + int_add=2) + def test_array(self): class X(object): _immutable_fields_ = ["y[*]"] Modified: pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_jitdriver.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_jitdriver.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_jitdriver.py Wed Aug 25 19:06:43 2010 @@ -14,7 +14,6 @@ def test_simple(self): myjitdriver1 = JitDriver(greens=[], reds=['n', 'm'], - can_inline = lambda *args: False, get_printable_location = getloc1) myjitdriver2 = JitDriver(greens=['g'], reds=['r'], get_printable_location = getloc2) @@ -30,11 +29,14 @@ while r > 0: myjitdriver2.can_enter_jit(g=g, r=r) myjitdriver2.jit_merge_point(g=g, r=r) - r += loop1(r, g) - 1 + r += loop1(r, g) + (-1) return r # res = self.meta_interp(loop2, [4, 40], repeat=7, inline=True) assert res == loop2(4, 40) + # we expect only one int_sub, corresponding to the single + # compiled instance of loop1() + self.check_loops(int_sub=1) # the following numbers are not really expectations of the test # itself, but just the numbers that we got after looking carefully # at the generated machine code @@ -42,11 +44,10 @@ self.check_tree_loop_count(4) # 2 x loop, 2 x enter bridge self.check_enter_count(7) - def test_simple_inline(self): + def test_inline(self): # this is not an example of reasonable code: loop1() is unrolled # 'n/m' times, where n and m are given as red arguments. myjitdriver1 = JitDriver(greens=[], reds=['n', 'm'], - can_inline = lambda *args: True, get_printable_location = getloc1) myjitdriver2 = JitDriver(greens=['g'], reds=['r'], get_printable_location = getloc2) Modified: pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_optimizeopt.py Wed Aug 25 19:06:43 2010 @@ -285,6 +285,24 @@ """ self.optimize_loop(ops, '', expected) + def test_constant_propagate_ovf(self): + ops = """ + [] + i0 = int_add_ovf(2, 3) + guard_no_overflow() [] + i1 = int_is_true(i0) + guard_true(i1) [] + i2 = int_is_zero(i1) + guard_false(i2) [] + guard_value(i0, 5) [] + jump() + """ + expected = """ + [] + jump() + """ + self.optimize_loop(ops, '', expected) + def test_constfold_all(self): from pypy.jit.backend.llgraph.llimpl import TYPES # xxx fish from pypy.jit.metainterp.executor import execute_nonspec @@ -2098,6 +2116,33 @@ """ self.optimize_loop(ops, 'Not', expected) + def test_remove_duplicate_pure_op_ovf(self): + ops = """ + [i1] + i3 = int_add_ovf(i1, 1) + guard_no_overflow() [] + i3b = int_is_true(i3) + guard_true(i3b) [] + i4 = int_add_ovf(i1, 1) + guard_no_overflow() [] + i4b = int_is_true(i4) + guard_true(i4b) [] + escape(i3) + escape(i4) + jump(i1) + """ + expected = """ + [i1] + i3 = int_add_ovf(i1, 1) + guard_no_overflow() [] + i3b = int_is_true(i3) + guard_true(i3b) [] + escape(i3) + escape(i3) + jump(i1) + """ + self.optimize_loop(ops, "Not", expected) + def test_int_and_or_with_zero(self): ops = """ [i0, i1] Modified: pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_recursive.py Wed Aug 25 19:06:43 2010 @@ -2,10 +2,11 @@ from pypy.rlib.jit import JitDriver, we_are_jitted, OPTIMIZER_SIMPLE, hint from pypy.rlib.jit import unroll_safe, dont_look_inside from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.debug import fatalerror from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin from pypy.jit.codewriter.policy import StopAtXPolicy from pypy.rpython.annlowlevel import hlstr -from pypy.jit.metainterp.warmspot import CannotInlineCanEnterJit, get_stats +from pypy.jit.metainterp.warmspot import get_stats class RecursiveTests: @@ -98,23 +99,18 @@ policy=StopAtXPolicy(opaque)) assert res == 1 - def get_interpreter(self, codes, always_inline=False): + def get_interpreter(self, codes): ADD = "0" JUMP_BACK = "1" CALL = "2" EXIT = "3" - if always_inline: - def can_inline(*args): - return True - else: - def can_inline(i, code): - code = hlstr(code) - return not JUMP_BACK in code + def getloc(i, code): + return 'code="%s", i=%d' % (code, i) jitdriver = JitDriver(greens = ['i', 'code'], reds = ['n'], - can_inline = can_inline) - + get_printable_location = getloc) + def interpret(codenum, n, i): code = codes[codenum] while i < len(code): @@ -162,31 +158,16 @@ assert self.meta_interp(f, [0, 0, 0], optimizer=OPTIMIZER_SIMPLE, inline=True) == 42 - self.check_loops(call_may_force = 1, call = 0) - - def test_inline_faulty_can_inline(self): - code = "021" - subcode = "301" - codes = [code, subcode] - - f = self.get_interpreter(codes, always_inline=True) - - try: - self.meta_interp(f, [0, 0, 0], optimizer=OPTIMIZER_SIMPLE, - inline=True) - except CannotInlineCanEnterJit: - pass - else: - py.test.fail("DID NOT RAISE") + # the call is fully inlined, because we jump to subcode[1], thus + # skipping completely the JUMP_BACK in subcode[0] + self.check_loops(call_may_force = 0, call_assembler = 0, call = 0) def test_guard_failure_in_inlined_function(self): def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(pc, code): - return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], - get_printable_location=p, can_inline=c) + get_printable_location=p) def f(code, n): pc = 0 while pc < len(code): @@ -219,10 +200,8 @@ def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(pc, code): - return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n', 'flag'], - get_printable_location=p, can_inline=c) + get_printable_location=p) def f(code, n): pc = 0 flag = False @@ -262,10 +241,8 @@ def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(pc, code): - return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], - get_printable_location=p, can_inline=c) + get_printable_location=p) class Exc(Exception): pass @@ -307,10 +284,8 @@ def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(pc, code): - return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], - get_printable_location=p, can_inline=c) + get_printable_location=p) def f(code, n): pc = 0 @@ -524,10 +499,8 @@ def p(pc, code): code = hlstr(code) return "'%s' at %d: %s" % (code, pc, code[pc]) - def c(pc, code): - return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], - get_printable_location=p, can_inline=c) + get_printable_location=p) def f(code, n): pc = 0 @@ -558,6 +531,8 @@ result += f('+-cl--', i) g(50) self.meta_interp(g, [50], backendopt=True) + py.test.skip("tracing from start is by now only longer enabled " + "if a trace gets too big") self.check_tree_loop_count(3) self.check_history(int_add=1) @@ -565,10 +540,8 @@ def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(pc, code): - return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], - get_printable_location=p, can_inline=c) + get_printable_location=p) def f(code, n): pc = 0 @@ -607,8 +580,7 @@ def test_directly_call_assembler(self): driver = JitDriver(greens = ['codeno'], reds = ['i'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) def portal(codeno): i = 0 @@ -624,28 +596,29 @@ def test_recursion_cant_call_assembler_directly(self): driver = JitDriver(greens = ['codeno'], reds = ['i', 'j'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) def portal(codeno, j): i = 0 - while i < 1: - driver.can_enter_jit(codeno=codeno, i=i, j=j) + while 1: driver.jit_merge_point(codeno=codeno, i=i, j=j) - i += 1 - if j == 0: + if i == 1: + if j == 0: + return + portal(2, j - 1) + elif i == 3: return - portal(2, j - 1) + i += 1 + driver.can_enter_jit(codeno=codeno, i=i, j=j) portal(2, 50) self.meta_interp(portal, [2, 20], inline=True) - self.check_history(call_assembler=0, call_may_force=1) - self.check_enter_count_at_most(1) + self.check_loops(call_assembler=0, call_may_force=1, + everywhere=True) def test_directly_call_assembler_return(self): driver = JitDriver(greens = ['codeno'], reds = ['i', 'k'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) def portal(codeno): i = 0 @@ -668,8 +641,7 @@ self.x = x driver = JitDriver(greens = ['codeno'], reds = ['i'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) def portal(codeno): i = 0 @@ -690,8 +662,7 @@ def test_directly_call_assembler_fail_guard(self): driver = JitDriver(greens = ['codeno'], reds = ['i', 'k'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) def portal(codeno, k): i = 0 @@ -722,8 +693,7 @@ driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'], virtualizables = ['frame'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) def main(codeno): frame = Frame() @@ -761,8 +731,7 @@ driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'], virtualizables = ['frame'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) @dont_look_inside def check_frame(subframe): @@ -802,7 +771,7 @@ res = self.meta_interp(main, [0], inline=True) assert res == main(0) - def test_directly_call_assembler_virtualizable_force(self): + def test_directly_call_assembler_virtualizable_force1(self): class Thing(object): def __init__(self, val): self.val = val @@ -812,8 +781,7 @@ driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'], virtualizables = ['frame'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) class SomewhereElse(object): pass @@ -830,6 +798,7 @@ return frame.thing.val def portal(codeno, frame): + print 'ENTER:', codeno, frame.thing.val i = 0 while i < 10: driver.can_enter_jit(frame=frame, codeno=codeno, i=i) @@ -839,11 +808,15 @@ subframe = Frame() subframe.thing = Thing(nextval) nextval = portal(1, subframe) - elif frame.thing.val > 40: - change(Thing(13)) - nextval = 13 + elif codeno == 1: + if frame.thing.val > 40: + change(Thing(13)) + nextval = 13 + else: + fatalerror("bad codeno = " + str(codeno)) frame.thing = Thing(nextval + 1) i += 1 + print 'LEAVE:', codeno, frame.thing.val return frame.thing.val res = self.meta_interp(main, [0], inline=True, @@ -852,8 +825,7 @@ def test_directly_call_assembler_virtualizable_with_array(self): myjitdriver = JitDriver(greens = ['codeno'], reds = ['n', 'x', 'frame'], - virtualizables = ['frame'], - can_inline = lambda codeno : False) + virtualizables = ['frame']) class Frame(object): _virtualizable2_ = ['l[*]', 's'] @@ -899,8 +871,7 @@ driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'], virtualizables = ['frame'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) class SomewhereElse(object): pass @@ -942,17 +913,16 @@ def test_assembler_call_red_args(self): driver = JitDriver(greens = ['codeno'], reds = ['i', 'k'], - get_printable_location = lambda codeno : str(codeno), - can_inline = lambda codeno : False) + get_printable_location = lambda codeno : str(codeno)) def residual(k): - if k > 40: + if k > 150: return 0 return 1 def portal(codeno, k): i = 0 - while i < 10: + while i < 15: driver.can_enter_jit(codeno=codeno, i=i, k=k) driver.jit_merge_point(codeno=codeno, i=i, k=k) if codeno == 2: @@ -969,10 +939,130 @@ assert res == portal(2, 0) self.check_loops(call_assembler=2) - # There is a test which I fail to write. - # * what happens if we call recursive_call while blackholing - # this seems to be completely corner case and not really happening - # in the wild + def test_inline_without_hitting_the_loop(self): + driver = JitDriver(greens = ['codeno'], reds = ['i'], + get_printable_location = lambda codeno : str(codeno)) + + def portal(codeno): + i = 0 + while True: + driver.jit_merge_point(codeno=codeno, i=i) + if codeno < 10: + i += portal(20) + codeno += 1 + elif codeno == 10: + if i > 63: + return i + codeno = 0 + driver.can_enter_jit(codeno=codeno, i=i) + else: + return 1 + + assert portal(0) == 70 + res = self.meta_interp(portal, [0], inline=True) + assert res == 70 + self.check_loops(call_assembler=0) + + def test_inline_with_hitting_the_loop_sometimes(self): + driver = JitDriver(greens = ['codeno'], reds = ['i', 'k'], + get_printable_location = lambda codeno : str(codeno)) + + def portal(codeno, k): + if k > 2: + return 1 + i = 0 + while True: + driver.jit_merge_point(codeno=codeno, i=i, k=k) + if codeno < 10: + i += portal(codeno + 5, k+1) + codeno += 1 + elif codeno == 10: + if i > [-1, 2000, 63][k]: + return i + codeno = 0 + driver.can_enter_jit(codeno=codeno, i=i, k=k) + else: + return 1 + + assert portal(0, 1) == 2095 + res = self.meta_interp(portal, [0, 1], inline=True) + assert res == 2095 + self.check_loops(call_assembler=6, everywhere=True) + + def test_inline_with_hitting_the_loop_sometimes_exc(self): + driver = JitDriver(greens = ['codeno'], reds = ['i', 'k'], + get_printable_location = lambda codeno : str(codeno)) + class GotValue(Exception): + def __init__(self, result): + self.result = result + + def portal(codeno, k): + if k > 2: + raise GotValue(1) + i = 0 + while True: + driver.jit_merge_point(codeno=codeno, i=i, k=k) + if codeno < 10: + try: + portal(codeno + 5, k+1) + except GotValue, e: + i += e.result + codeno += 1 + elif codeno == 10: + if i > [-1, 2000, 63][k]: + raise GotValue(i) + codeno = 0 + driver.can_enter_jit(codeno=codeno, i=i, k=k) + else: + raise GotValue(1) + + def main(codeno, k): + try: + portal(codeno, k) + except GotValue, e: + return e.result + + assert main(0, 1) == 2095 + res = self.meta_interp(main, [0, 1], inline=True) + assert res == 2095 + self.check_loops(call_assembler=6, everywhere=True) + + def test_handle_jitexception_in_portal(self): + # a test for _handle_jitexception_in_portal in blackhole.py + driver = JitDriver(greens = ['codeno'], reds = ['i', 'str'], + get_printable_location = lambda codeno: str(codeno)) + def do_can_enter_jit(codeno, i, str): + i = (i+1)-1 # some operations + driver.can_enter_jit(codeno=codeno, i=i, str=str) + def intermediate(codeno, i, str): + if i == 9: + do_can_enter_jit(codeno, i, str) + def portal(codeno, str): + i = value.initial + while i < 10: + intermediate(codeno, i, str) + driver.jit_merge_point(codeno=codeno, i=i, str=str) + i += 1 + if codeno == 64 and i == 10: + str = portal(96, str) + str += chr(codeno+i) + return str + class Value: + initial = -1 + value = Value() + def main(): + value.initial = 0 + return (portal(64, '') + + portal(64, '') + + portal(64, '') + + portal(64, '') + + portal(64, '')) + assert main() == 'ABCDEFGHIabcdefghijJ' * 5 + for tlimit in [95, 90, 102]: + print 'tlimit =', tlimit + res = self.meta_interp(main, [], inline=True, trace_limit=tlimit) + assert ''.join(res.chars) == 'ABCDEFGHIabcdefghijJ' * 5 + class TestLLtype(RecursiveTests, LLJitMixin): pass Modified: pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_virtualizable.py Wed Aug 25 19:06:43 2010 @@ -1330,11 +1330,9 @@ def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(pc, code): - return "l" not in hlstr(code) myjitdriver = JitDriver(greens=['pc', 'code'], reds=['frame'], virtualizables=["frame"], - get_printable_location=p, can_inline=c) + get_printable_location=p) def f(code, frame): pc = 0 while pc < len(code): Modified: pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_warmspot.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_warmspot.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_warmspot.py Wed Aug 25 19:06:43 2010 @@ -272,6 +272,30 @@ self.check_enter_count_at_most(2) self.check_loops(call=0) + def test_loop_header(self): + # artificial test: we enter into the JIT only when can_enter_jit() + # is seen, but we close a loop in the JIT much more quickly + # because of loop_header(). + mydriver = JitDriver(reds = ['n', 'm'], greens = []) + + def f(m): + n = 0 + while True: + mydriver.jit_merge_point(n=n, m=m) + if n > m: + m -= 1 + if m < 0: + return n + n = 0 + mydriver.can_enter_jit(n=n, m=m) + else: + n += 1 + mydriver.loop_header() + assert f(15) == 1 + res = self.meta_interp(f, [15], backendopt=True) + assert res == 1 + self.check_loops(int_add=1) # I get 13 without the loop_header() + class TestLLWarmspot(WarmspotTests, LLJitMixin): CPUClass = runner.LLtypeCPU Modified: pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_warmstate.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_warmstate.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_warmstate.py Wed Aug 25 19:06:43 2010 @@ -61,12 +61,14 @@ _green_args_spec = [lltype.Signed, lltype.Float] state = WarmEnterState(None, FakeJitDriverSD()) get_jitcell = state._make_jitcell_getter_default() - cell1 = get_jitcell(42, 42.5) + cell1 = get_jitcell(True, 42, 42.5) assert isinstance(cell1, JitCell) - cell2 = get_jitcell(42, 42.5) + cell2 = get_jitcell(True, 42, 42.5) assert cell1 is cell2 - cell3 = get_jitcell(41, 42.5) - cell4 = get_jitcell(42, 0.25) + cell3 = get_jitcell(True, 41, 42.5) + assert get_jitcell(False, 42, 0.25) is None + cell4 = get_jitcell(True, 42, 0.25) + assert get_jitcell(False, 42, 0.25) is cell4 assert cell1 is not cell3 is not cell4 is not cell1 def test_make_jitcell_getter(): @@ -75,8 +77,8 @@ _get_jitcell_at_ptr = None state = WarmEnterState(None, FakeJitDriverSD()) get_jitcell = state.make_jitcell_getter() - cell1 = get_jitcell(1.75) - cell2 = get_jitcell(1.75) + cell1 = get_jitcell(True, 1.75) + cell2 = get_jitcell(True, 1.75) assert cell1 is cell2 assert get_jitcell is state.make_jitcell_getter() @@ -103,14 +105,16 @@ # state = WarmEnterState(FakeWarmRunnerDesc(), FakeJitDriverSD()) get_jitcell = state._make_jitcell_getter_custom() - cell1 = get_jitcell(5, 42.5) + cell1 = get_jitcell(True, 5, 42.5) assert isinstance(cell1, JitCell) assert cell1.x == 5 assert cell1.y == 42.5 - cell2 = get_jitcell(5, 42.5) + cell2 = get_jitcell(True, 5, 42.5) assert cell2 is cell1 - cell3 = get_jitcell(41, 42.5) - cell4 = get_jitcell(42, 0.25) + cell3 = get_jitcell(True, 41, 42.5) + assert get_jitcell(False, 42, 0.25) is None + cell4 = get_jitcell(True, 42, 0.25) + assert get_jitcell(False, 42, 0.25) is cell4 assert cell1 is not cell3 is not cell4 is not cell1 def test_make_set_future_values(): @@ -153,52 +157,25 @@ state.attach_unoptimized_bridge_from_interp([ConstInt(5), ConstFloat(2.25)], "entry loop token") - cell1 = get_jitcell(5, 2.25) + cell1 = get_jitcell(True, 5, 2.25) assert cell1.counter < 0 assert cell1.entry_loop_token == "entry loop token" def test_make_jitdriver_callbacks_1(): class FakeJitDriverSD: _green_args_spec = [lltype.Signed, lltype.Float] - _can_inline_ptr = None _get_printable_location_ptr = None _confirm_enter_jit_ptr = None class FakeCell: dont_trace_here = False state = WarmEnterState(None, FakeJitDriverSD()) - def jit_getter(*args): + def jit_getter(build, *args): return FakeCell() state.jit_getter = jit_getter state.make_jitdriver_callbacks() - res = state.can_inline_callable([ConstInt(5), ConstFloat(42.5)]) - assert res is True res = state.get_location_str([ConstInt(5), ConstFloat(42.5)]) assert res == '(no jitdriver.get_printable_location!)' -def test_make_jitdriver_callbacks_2(): - def can_inline(x, y): - assert x == 5 - assert y == 42.5 - return False - CAN_INLINE = lltype.Ptr(lltype.FuncType([lltype.Signed, lltype.Float], - lltype.Bool)) - class FakeCell: - dont_trace_here = False - class FakeWarmRunnerDesc: - rtyper = None - class FakeJitDriverSD: - _green_args_spec = [lltype.Signed, lltype.Float] - _can_inline_ptr = llhelper(CAN_INLINE, can_inline) - _get_printable_location_ptr = None - _confirm_enter_jit_ptr = None - state = WarmEnterState(FakeWarmRunnerDesc(), FakeJitDriverSD()) - def jit_getter(*args): - return FakeCell() - state.jit_getter = jit_getter - state.make_jitdriver_callbacks() - res = state.can_inline_callable([ConstInt(5), ConstFloat(42.5)]) - assert res is False - def test_make_jitdriver_callbacks_3(): def get_location(x, y): assert x == 5 @@ -210,7 +187,6 @@ rtyper = None class FakeJitDriverSD: _green_args_spec = [lltype.Signed, lltype.Float] - _can_inline_ptr = None _get_printable_location_ptr = llhelper(GET_LOCATION, get_location) _confirm_enter_jit_ptr = None _get_jitcell_at_ptr = None @@ -231,7 +207,6 @@ rtyper = None class FakeJitDriverSD: _green_args_spec = [lltype.Signed, lltype.Float] - _can_inline_ptr = None _get_printable_location_ptr = None _confirm_enter_jit_ptr = llhelper(ENTER_JIT, confirm_enter_jit) _get_jitcell_at_ptr = None Modified: pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_ztranslation.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_ztranslation.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/metainterp/test/test_ztranslation.py Wed Aug 25 19:06:43 2010 @@ -37,15 +37,12 @@ return jitcellcache.entry def get_printable_location(): return '(hello world)' - def can_inline(): - return False jitdriver = JitDriver(greens = [], reds = ['total', 'frame'], virtualizables = ['frame'], get_jitcell_at=get_jitcell_at, set_jitcell_at=set_jitcell_at, - get_printable_location=get_printable_location, - can_inline=can_inline) + get_printable_location=get_printable_location) def f(i): for param in unroll_parameters: defl = PARAMETERS[param] @@ -63,8 +60,7 @@ frame.i -= 1 return total * 10 # - myjitdriver2 = JitDriver(greens = ['g'], reds = ['m', 'x'], - can_inline = lambda *args: False) + myjitdriver2 = JitDriver(greens = ['g'], reds = ['m', 'x']) def f2(g, m, x): while m > 0: myjitdriver2.can_enter_jit(g=g, m=m, x=x) Modified: pypy/branch/asmgcc-64/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/metainterp/warmspot.py Wed Aug 25 19:06:43 2010 @@ -136,9 +136,6 @@ class ContinueRunningNormallyBase(JitException): pass -class CannotInlineCanEnterJit(JitException): - pass - # ____________________________________________________________ class WarmRunnerDesc(object): @@ -402,7 +399,7 @@ can_inline = state.can_inline_greenargs num_green_args = jd.num_green_args def maybe_enter_from_start(*args): - if can_inline is not None and not can_inline(*args[:num_green_args]): + if not can_inline(*args[:num_green_args]): maybe_compile_and_run(*args) maybe_enter_from_start._always_inline_ = True jd._maybe_enter_from_start_fn = maybe_enter_from_start @@ -423,8 +420,6 @@ s_BaseJitCell_not_None) jd._get_jitcell_at_ptr = self._make_hook_graph(jd, annhelper, jd.jitdriver.get_jitcell_at, s_BaseJitCell_or_None) - jd._can_inline_ptr = self._make_hook_graph(jd, - annhelper, jd.jitdriver.can_inline, annmodel.s_Bool) jd._get_printable_location_ptr = self._make_hook_graph(jd, annhelper, jd.jitdriver.get_printable_location, s_Str) jd._confirm_enter_jit_ptr = self._make_hook_graph(jd, @@ -478,7 +473,13 @@ jitdriver = op.args[1].value assert jitdriver in sublists, \ "can_enter_jit with no matching jit_merge_point" - sublists[jitdriver].append((graph, block, index)) + origportalgraph = jd._jit_merge_point_pos[0] + if graph is not origportalgraph: + sublists[jitdriver].append((graph, block, index)) + else: + pass # a 'can_enter_jit' before the 'jit-merge_point', but + # originally in the same function: we ignore it here + # see e.g. test_jitdriver.test_simple for jd in self.jitdrivers_sd: sublist = sublists[jd.jitdriver] assert len(sublist) > 0, \ @@ -557,6 +558,7 @@ # Prepare the portal_runner() helper # from pypy.jit.metainterp.warmstate import specialize_value + from pypy.jit.metainterp.warmstate import unspecialize_value portal_ptr = self.cpu.ts.functionptr(PORTALFUNC, 'portal', graph = portalgraph) jd._portal_ptr = portal_ptr @@ -611,6 +613,37 @@ value = cast_base_ptr_to_instance(Exception, value) raise Exception, value + def handle_jitexception(e): + # XXX the bulk of this function is a copy-paste from above :-( + try: + raise e + except self.ContinueRunningNormally, e: + args = () + for ARGTYPE, attrname, count in portalfunc_ARGS: + x = getattr(e, attrname)[count] + x = specialize_value(ARGTYPE, x) + args = args + (x,) + return ll_portal_runner(*args) + except self.DoneWithThisFrameVoid: + assert result_kind == 'void' + return + except self.DoneWithThisFrameInt, e: + assert result_kind == 'int' + return specialize_value(RESULT, e.result) + except self.DoneWithThisFrameRef, e: + assert result_kind == 'ref' + return specialize_value(RESULT, e.result) + except self.DoneWithThisFrameFloat, e: + assert result_kind == 'float' + return specialize_value(RESULT, e.result) + except self.ExitFrameWithExceptionRef, e: + value = ts.cast_to_baseclass(e.value) + if not we_are_translated(): + raise LLException(ts.get_typeptr(value), value) + else: + value = cast_base_ptr_to_instance(Exception, value) + raise Exception, value + jd._ll_portal_runner = ll_portal_runner # for debugging jd.portal_runner_ptr = self.helper_func(jd._PTR_PORTAL_FUNCTYPE, ll_portal_runner) @@ -631,32 +664,8 @@ vinfo.reset_vable_token(virtualizable) try: loop_token = fail_descr.handle_fail(self.metainterp_sd, jd) - except self.ContinueRunningNormally, e: - args = () - for ARGTYPE, attrname, count in portalfunc_ARGS: - x = getattr(e, attrname)[count] - x = specialize_value(ARGTYPE, x) - args = args + (x,) - return ll_portal_runner(*args) - except self.DoneWithThisFrameVoid: - assert result_kind == 'void' - return - except self.DoneWithThisFrameInt, e: - assert result_kind == 'int' - return specialize_value(RESULT, e.result) - except self.DoneWithThisFrameRef, e: - assert result_kind == 'ref' - return specialize_value(RESULT, e.result) - except self.DoneWithThisFrameFloat, e: - assert result_kind == 'float' - return specialize_value(RESULT, e.result) - except self.ExitFrameWithExceptionRef, e: - value = ts.cast_to_baseclass(e.value) - if not we_are_translated(): - raise LLException(ts.get_typeptr(value), value) - else: - value = cast_base_ptr_to_instance(Exception, value) - raise Exception, value + except JitException, e: + return handle_jitexception(e) fail_descr = self.cpu.execute_token(loop_token) jd._assembler_call_helper = assembler_call_helper # for debugging @@ -668,6 +677,21 @@ if vinfo is not None: jd.vable_token_descr = vinfo.vable_token_descr + def handle_jitexception_from_blackhole(bhcaller, e): + result = handle_jitexception(e) + # + if result_kind != 'void': + result = unspecialize_value(result) + if result_kind == 'int': + bhcaller._setup_return_value_i(result) + elif result_kind == 'ref': + bhcaller._setup_return_value_r(result) + elif result_kind == 'float': + bhcaller._setup_return_value_f(result) + else: + assert False + jd.handle_jitexc_from_bh = handle_jitexception_from_blackhole + # ____________________________________________________________ # Now mutate origportalgraph to end with a call to portal_runner_ptr # @@ -687,17 +711,6 @@ origblock.exitswitch = None origblock.recloseblock(Link([v_result], origportalgraph.returnblock)) # - # Also kill any can_enter_jit left behind (example: see - # test_jitdriver.test_simple, which has a can_enter_jit in - # loop1's origportalgraph) - can_enter_jits = _find_jit_marker([origportalgraph], 'can_enter_jit') - for _, block, i in can_enter_jits: - op = block.operations[i] - assert op.opname == 'jit_marker' - block.operations[i] = SpaceOperation('same_as', - [Constant(None, lltype.Void)], - op.result) - # checkgraph(origportalgraph) def add_finish(self): Modified: pypy/branch/asmgcc-64/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/metainterp/warmstate.py Wed Aug 25 19:06:43 2010 @@ -30,6 +30,22 @@ else: return lltype.cast_opaque_ptr(TYPE, x) + at specialize.ll() +def unspecialize_value(value): + """Casts 'value' to a Signed, a GCREF or a Float.""" + if isinstance(lltype.typeOf(value), lltype.Ptr): + if lltype.typeOf(value).TO._gckind == 'gc': + return lltype.cast_opaque_ptr(llmemory.GCREF, value) + else: + adr = llmemory.cast_ptr_to_adr(value) + return heaptracker.adr2int(adr) + elif isinstance(lltype.typeOf(value), ootype.OOType): + return ootype.cast_to_object(value) + elif isinstance(value, float): + return value + else: + return intmask(value) + @specialize.arg(0) def unwrap(TYPE, box): if TYPE is lltype.Void: @@ -232,7 +248,7 @@ # look for the cell corresponding to the current greenargs greenargs = args[:num_green_args] - cell = get_jitcell(*greenargs) + cell = get_jitcell(True, *greenargs) if cell.counter >= 0: # update the profiling counter @@ -324,7 +340,7 @@ # def jit_cell_at_key(greenkey): greenargs = unwrap_greenkey(greenkey) - return jit_getter(*greenargs) + return jit_getter(True, *greenargs) self.jit_cell_at_key = jit_cell_at_key self.jit_getter = jit_getter # @@ -355,10 +371,12 @@ # jitcell_dict = r_dict(comparekey, hashkey) # - def get_jitcell(*greenargs): + def get_jitcell(build, *greenargs): try: cell = jitcell_dict[greenargs] except KeyError: + if not build: + return None cell = JitCell() jitcell_dict[greenargs] = cell return cell @@ -371,38 +389,41 @@ set_jitcell_at_ptr = self.jitdriver_sd._set_jitcell_at_ptr lltohlhack = {} # - def get_jitcell(*greenargs): + def get_jitcell(build, *greenargs): fn = support.maybe_on_top_of_llinterp(rtyper, get_jitcell_at_ptr) cellref = fn(*greenargs) # if we_are_translated(): BASEJITCELL = lltype.typeOf(cellref) cell = cast_base_ptr_to_instance(JitCell, cellref) - elif isinstance(cellref, (BaseJitCell, type(None))): - BASEJITCELL = None - cell = cellref else: - BASEJITCELL = lltype.typeOf(cellref) - if cellref: - cell = lltohlhack[rtyper.type_system.deref(cellref)] + if isinstance(cellref, (BaseJitCell, type(None))): + BASEJITCELL = None + cell = cellref else: - cell = None - # + BASEJITCELL = lltype.typeOf(cellref) + if cellref: + cell = lltohlhack[rtyper.type_system.deref(cellref)] + else: + cell = None + if not build: + return cell if cell is None: cell = JitCell() # if we_are_translated(): cellref = cast_object_to_ptr(BASEJITCELL, cell) - elif BASEJITCELL is None: - cellref = cell else: - if isinstance(BASEJITCELL, lltype.Ptr): - cellref = lltype.malloc(BASEJITCELL.TO) - elif isinstance(BASEJITCELL, ootype.Instance): - cellref = ootype.new(BASEJITCELL) + if BASEJITCELL is None: + cellref = cell else: - assert False, "no clue" - lltohlhack[rtyper.type_system.deref(cellref)] = cell + if isinstance(BASEJITCELL, lltype.Ptr): + cellref = lltype.malloc(BASEJITCELL.TO) + elif isinstance(BASEJITCELL, ootype.Instance): + cellref = ootype.new(BASEJITCELL) + else: + assert False, "no clue" + lltohlhack[rtyper.type_system.deref(cellref)] = cell # fn = support.maybe_on_top_of_llinterp(rtyper, set_jitcell_at_ptr) @@ -468,34 +489,24 @@ if hasattr(self, 'get_location_str'): return # - can_inline_ptr = self.jitdriver_sd._can_inline_ptr unwrap_greenkey = self.make_unwrap_greenkey() jit_getter = self.make_jitcell_getter() - if can_inline_ptr is None: - def can_inline_callable(*greenargs): - # XXX shouldn't it be False by default? - return True - else: - rtyper = self.warmrunnerdesc.rtyper - # - def can_inline_callable(*greenargs): - fn = support.maybe_on_top_of_llinterp(rtyper, can_inline_ptr) - return fn(*greenargs) - def can_inline(*greenargs): - cell = jit_getter(*greenargs) - if cell.dont_trace_here: + + def can_inline_greenargs(*greenargs): + cell = jit_getter(False, *greenargs) + if cell is not None and cell.dont_trace_here: return False - return can_inline_callable(*greenargs) - self.can_inline_greenargs = can_inline - def can_inline_greenkey(greenkey): + return True + def can_inline_callable(greenkey): greenargs = unwrap_greenkey(greenkey) - return can_inline(*greenargs) - self.can_inline_callable = can_inline_greenkey + return can_inline_greenargs(*greenargs) + self.can_inline_greenargs = can_inline_greenargs + self.can_inline_callable = can_inline_callable def get_assembler_token(greenkey): greenargs = unwrap_greenkey(greenkey) - cell = jit_getter(*greenargs) - if cell.counter >= 0: + cell = jit_getter(False, *greenargs) + if cell is None or cell.counter >= 0: return None return cell.entry_loop_token self.get_assembler_token = get_assembler_token Modified: pypy/branch/asmgcc-64/pypy/module/__builtin__/functional.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/__builtin__/functional.py (original) +++ pypy/branch/asmgcc-64/pypy/module/__builtin__/functional.py Wed Aug 25 19:06:43 2010 @@ -221,8 +221,7 @@ from pypy.rlib.jit import JitDriver mapjitdriver = JitDriver(greens = ['code'], - reds = ['w_func', 'w_iter', 'result_w'], - can_inline = lambda *args: False) + reds = ['w_func', 'w_iter', 'result_w']) def map_single_user_function(code, w_func, w_iter): result_w = [] while True: Modified: pypy/branch/asmgcc-64/pypy/module/_locale/__init__.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/_locale/__init__.py (original) +++ pypy/branch/asmgcc-64/pypy/module/_locale/__init__.py Wed Aug 25 19:06:43 2010 @@ -1,5 +1,4 @@ from pypy.interpreter.mixedmodule import MixedModule -from pypy.module._locale import interp_locale from pypy.rlib import rlocale import sys Modified: pypy/branch/asmgcc-64/pypy/module/_sre/__init__.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/_sre/__init__.py (original) +++ pypy/branch/asmgcc-64/pypy/module/_sre/__init__.py Wed Aug 25 19:06:43 2010 @@ -1,24 +1,14 @@ from pypy.interpreter.mixedmodule import MixedModule class Module(MixedModule): - """A pure Python reimplementation of the _sre module from CPython 2.4 -Copyright 2005 Nik Haldimann, licensed under the MIT license -This code is based on material licensed under CNRI's Python 1.6 license and -copyrighted by: Copyright (c) 1997-2001 by Secret Labs AB -""" - appleveldefs = { - 'compile': 'app_sre.compile', } interpleveldefs = { 'CODESIZE': 'space.wrap(interp_sre.CODESIZE)', 'MAGIC': 'space.wrap(interp_sre.MAGIC)', - 'copyright': 'space.wrap(interp_sre.copyright)', + 'compile': 'interp_sre.W_SRE_Pattern', 'getlower': 'interp_sre.w_getlower', 'getcodesize': 'interp_sre.w_getcodesize', - '_State': 'interp_sre.make_state', - '_match': 'interp_sre.w_match', - '_search': 'interp_sre.w_search', } Modified: pypy/branch/asmgcc-64/pypy/module/_sre/interp_sre.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/_sre/interp_sre.py (original) +++ pypy/branch/asmgcc-64/pypy/module/_sre/interp_sre.py Wed Aug 25 19:06:43 2010 @@ -1,27 +1,20 @@ +import sys from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.typedef import GetSetProperty, TypeDef from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w +from pypy.interpreter.typedef import make_weakref_descr from pypy.interpreter.gateway import interp2app, ObjSpace, W_Root from pypy.interpreter.error import OperationError from pypy.rlib.rarithmetic import intmask +from pypy.tool.pairtype import extendabletype -# This can be compiled in two ways: -# -# * THREE_VERSIONS_OF_CORE=True: you get three copies of the whole -# regexp searching and matching code: for strings, for unicode strings, -# and for generic buffer objects (like mmap.mmap or array.array). -# -# * THREE_VERSIONS_OF_CORE=False: there is only one copy of the code, -# at the cost of an indirect method call to fetch each character. - -THREE_VERSIONS_OF_CORE = True +# ____________________________________________________________ +# +# Constants and exposed functions -#### Constants and exposed functions - -from pypy.rlib.rsre import rsre -from pypy.rlib.rsre.rsre_char import MAGIC, CODESIZE, getlower -copyright = "_sre.py 2.4 Copyright 2005 by Nik Haldimann" +from pypy.rlib.rsre import rsre_core +from pypy.rlib.rsre.rsre_char import MAGIC, CODESIZE, getlower, set_unicode_db def w_getlower(space, char_ord, flags): return space.wrap(getlower(char_ord, flags)) @@ -31,166 +24,529 @@ return space.wrap(CODESIZE) # use the same version of unicodedb as the standard objspace -from pypy.objspace.std.unicodeobject import unicodedb -rsre.set_unicode_db(unicodedb) +import pypy.objspace.std.unicodeobject +set_unicode_db(pypy.objspace.std.unicodeobject.unicodedb) -#### State classes +# ____________________________________________________________ +# +# Additional methods on the classes XxxMatchContext + +class __extend__(rsre_core.AbstractMatchContext): + __metaclass__ = extendabletype + def _w_slice(self, space, start, end): + raise NotImplementedError + def _w_string(self, space): + raise NotImplementedError -def make_state(space, w_string, start, end, flags): - # XXX maybe turn this into a __new__ method of W_State - if space.is_true(space.isinstance(w_string, space.w_str)): - cls = W_StringState - elif space.is_true(space.isinstance(w_string, space.w_unicode)): - cls = W_UnicodeState - else: - cls = W_GenericState - return space.wrap(cls(space, w_string, start, end, flags)) -make_state.unwrap_spec = [ObjSpace, W_Root, int, int, int] - - -class W_State(Wrappable): - if not THREE_VERSIONS_OF_CORE: - rsre.insert_sre_methods(locals(), 'all') - - def __init__(self, space, w_string, start, end, flags): - self.space = space - self.w_string = w_string - length = self.unwrap_object() - if start < 0: - start = 0 - if end > length: - end = length - self.start = start - self.pos = start # records the original start position - self.end = end - self.flags = flags - self.reset() +class __extend__(rsre_core.StrMatchContext): + __metaclass__ = extendabletype + def _w_slice(self, space, start, end): + return space.wrap(self._string[start:end]) + def _w_string(self, space): + return space.wrap(self._string) + +class __extend__(rsre_core.UnicodeMatchContext): + __metaclass__ = extendabletype + def _w_slice(self, space, start, end): + return space.wrap(self._unicodestr[start:end]) + def _w_string(self, space): + return space.wrap(self._unicodestr) + +def slice_w(space, ctx, start, end, w_default): + if 0 <= start <= end: + return ctx._w_slice(space, start, end) + return w_default + +def do_flatten_marks(ctx, num_groups): + # Returns a list of RPython-level integers. + # Unlike the app-level groups() method, groups are numbered from 0 + # and the returned list does not start with the whole match range. + if num_groups == 0: + return None + result = [-1] * (2*num_groups) + mark = ctx.match_marks + while mark is not None: + index = mark.gid + if result[index] == -1: + result[index] = mark.position + mark = mark.prev + return result + +def allgroups_w(space, ctx, fmarks, num_groups, w_default): + grps = [slice_w(space, ctx, fmarks[i*2], fmarks[i*2+1], w_default) + for i in range(num_groups)] + return space.newtuple(grps) + +def import_re(space): + w_builtin = space.getbuiltinmodule('__builtin__') + w_import = space.getattr(w_builtin, space.wrap("__import__")) + return space.call_function(w_import, space.wrap("re")) - def lower(self, char_ord): - return getlower(char_ord, self.flags) +def matchcontext(space, ctx): + try: + return rsre_core.match_context(ctx) + except rsre_core.Error, e: + raise OperationError(space.w_RuntimeError, space.wrap(e.msg)) - # methods overridden by subclasses +def searchcontext(space, ctx): + try: + return rsre_core.search_context(ctx) + except rsre_core.Error, e: + raise OperationError(space.w_RuntimeError, space.wrap(e.msg)) - def unwrap_object(self): - raise NotImplementedError +# ____________________________________________________________ +# +# SRE_Pattern class - if 'reset' not in locals(): - def reset(self): - raise NotImplementedError - - if 'search' not in locals(): - def search(self, pattern_codes): - raise NotImplementedError - - if 'match' not in locals(): - def match(self, pattern_codes): - raise NotImplementedError - - # Accessors for the typedef - - def w_reset(self): - self.reset() - - def create_regs(self, group_count): - """ Purely abstract method - """ - raise NotImplementedError +class W_SRE_Pattern(Wrappable): - def w_create_regs(self, group_count): - """Creates a tuple of index pairs representing matched groups, a format - that's convenient for SRE_Match.""" + def cannot_copy_w(self): space = self.space - return space.newtuple([ - space.newtuple([space.wrap(value1), - space.wrap(value2)]) - for value1, value2 in self.create_regs(group_count)]) - w_create_regs.unwrap_spec = ['self', int] + raise OperationError(space.w_TypeError, + space.wrap("cannot copy this pattern object")) - def fget_start(space, self): - return space.wrap(self.start) - - def fset_start(space, self, w_value): - self.start = space.int_w(w_value) + def make_ctx(self, w_string, pos=0, endpos=sys.maxint): + """Make a StrMatchContext or a UnicodeMatchContext for searching + in the given w_string object.""" + space = self.space + if pos < 0: pos = 0 + if endpos < pos: endpos = pos + if space.is_true(space.isinstance(w_string, space.w_unicode)): + unicodestr = space.unicode_w(w_string) + if pos > len(unicodestr): pos = len(unicodestr) + if endpos > len(unicodestr): endpos = len(unicodestr) + return rsre_core.UnicodeMatchContext(self.code, unicodestr, + pos, endpos, self.flags) + else: + str = space.bufferstr_w(w_string) + if pos > len(str): pos = len(str) + if endpos > len(str): endpos = len(str) + return rsre_core.StrMatchContext(self.code, str, + pos, endpos, self.flags) + + def getmatch(self, ctx, found): + if found: + return W_SRE_Match(self, ctx) + else: + return self.space.w_None + + def match_w(self, w_string, pos=0, endpos=sys.maxint): + ctx = self.make_ctx(w_string, pos, endpos) + return self.getmatch(ctx, matchcontext(self.space, ctx)) + match_w.unwrap_spec = ['self', W_Root, int, int] + + def search_w(self, w_string, pos=0, endpos=sys.maxint): + ctx = self.make_ctx(w_string, pos, endpos) + return self.getmatch(ctx, searchcontext(self.space, ctx)) + search_w.unwrap_spec = ['self', W_Root, int, int] - def fget_string_position(space, self): - return space.wrap(self.string_position) + def findall_w(self, w_string, pos=0, endpos=sys.maxint): + space = self.space + matchlist_w = [] + ctx = self.make_ctx(w_string, pos, endpos) + while ctx.match_start <= ctx.end: + if not searchcontext(space, ctx): + break + num_groups = self.num_groups + w_emptystr = space.wrap("") + if num_groups == 0: + w_item = slice_w(space, ctx, ctx.match_start, ctx.match_end, + w_emptystr) + else: + fmarks = do_flatten_marks(ctx, num_groups) + if num_groups == 1: + w_item = slice_w(space, ctx, fmarks[0], fmarks[1], + w_emptystr) + else: + w_item = allgroups_w(space, ctx, fmarks, num_groups, + w_emptystr) + matchlist_w.append(w_item) + no_progress = (ctx.match_start == ctx.match_end) + ctx.reset(ctx.match_end + no_progress) + return space.newlist(matchlist_w) + findall_w.unwrap_spec = ['self', W_Root, int, int] + + def finditer_w(self, w_string, pos=0, endpos=sys.maxint): + # this also works as the implementation of the undocumented + # scanner() method. + ctx = self.make_ctx(w_string, pos, endpos) + scanner = W_SRE_Scanner(self, ctx) + return self.space.wrap(scanner) + finditer_w.unwrap_spec = ['self', W_Root, int, int] - def fset_string_position(space, self, w_value): - self.start = space.int_w(w_value) + def split_w(self, w_string, maxsplit=0): + space = self.space + splitlist = [] + n = 0 + last = 0 + ctx = self.make_ctx(w_string) + while not maxsplit or n < maxsplit: + if not searchcontext(space, ctx): + break + if ctx.match_start == ctx.match_end: # zero-width match + if ctx.match_start == ctx.end: # or end of string + break + ctx.reset(ctx.match_end + 1) + continue + splitlist.append(slice_w(space, ctx, last, ctx.match_start, + space.w_None)) + # add groups (if any) + fmarks = do_flatten_marks(ctx, self.num_groups) + for groupnum in range(self.num_groups): + groupstart, groupend = fmarks[groupnum*2], fmarks[groupnum*2+1] + splitlist.append(slice_w(space, ctx, groupstart, groupend, + space.w_None)) + n += 1 + last = ctx.match_end + ctx.reset(last) + splitlist.append(slice_w(space, ctx, last, ctx.end, space.w_None)) + return space.newlist(splitlist) + split_w.unwrap_spec = ['self', W_Root, int] + + def sub_w(self, w_repl, w_string, count=0): + w_item, n = self.subx(w_repl, w_string, count) + return w_item + sub_w.unwrap_spec = ['self', W_Root, W_Root, int] - def get_char_ord(self, p): - raise NotImplementedError + def subn_w(self, w_repl, w_string, count=0): + w_item, n = self.subx(w_repl, w_string, count) + space = self.space + return space.newtuple([w_item, space.wrap(n)]) + subn_w.unwrap_spec = ['self', W_Root, W_Root, int] -getset_start = GetSetProperty(W_State.fget_start, W_State.fset_start, cls=W_State) -getset_string_position = GetSetProperty(W_State.fget_string_position, - W_State.fset_string_position, cls=W_State) - -W_State.typedef = TypeDef("W_State", - string = interp_attrproperty_w("w_string", W_State), - start = getset_start, - end = interp_attrproperty("end", W_State), - string_position = getset_string_position, - pos = interp_attrproperty("pos", W_State), - lastindex = interp_attrproperty("lastindex", W_State), - reset = interp2app(W_State.w_reset), - create_regs = interp2app(W_State.w_create_regs), + def subx(self, w_ptemplate, w_string, count): + space = self.space + if space.is_true(space.callable(w_ptemplate)): + w_filter = w_ptemplate + filter_is_callable = True + else: + if space.is_true(space.isinstance(w_ptemplate, space.w_unicode)): + filter_as_unicode = space.unicode_w(w_ptemplate) + literal = u'\\' not in filter_as_unicode + else: + try: + filter_as_string = space.str_w(w_ptemplate) + except OperationError, e: + if e.async(space): + raise + literal = False + else: + literal = '\\' not in filter_as_string + if literal: + w_filter = w_ptemplate + filter_is_callable = False + else: + # not a literal; hand it over to the template compiler + w_re = import_re(space) + w_filter = space.call_method(w_re, '_subx', + space.wrap(self), w_ptemplate) + filter_is_callable = space.is_true(space.callable(w_filter)) + # + ctx = self.make_ctx(w_string) + sublist_w = [] + n = last_pos = 0 + while not count or n < count: + if not searchcontext(space, ctx): + break + if last_pos < ctx.match_start: + sublist_w.append(slice_w(space, ctx, last_pos, + ctx.match_start, space.w_None)) + start = ctx.match_end + if start == ctx.match_start: + start += 1 + nextctx = ctx.fresh_copy(start) + if not (last_pos == ctx.match_start + == ctx.match_end and n > 0): + # the above ignores empty matches on latest position + if filter_is_callable: + w_match = self.getmatch(ctx, True) + w_piece = space.call_function(w_filter, w_match) + if not space.is_w(w_piece, space.w_None): + sublist_w.append(w_piece) + else: + sublist_w.append(w_filter) + last_pos = ctx.match_end + n += 1 + elif last_pos >= ctx.end: + break # empty match at the end: finished + ctx = nextctx + + if last_pos < ctx.end: + sublist_w.append(slice_w(space, ctx, last_pos, ctx.end, + space.w_None)) + if n == 0: + # not just an optimization -- see test_sub_unicode + return w_string, n + + if space.is_true(space.isinstance(w_string, space.w_unicode)): + w_emptystr = space.wrap(u'') + else: + w_emptystr = space.wrap('') + w_item = space.call_method(w_emptystr, 'join', + space.newlist(sublist_w)) + return w_item, n + + +def SRE_Pattern__new__(space, w_subtype, w_pattern, flags, w_code, + groups=0, w_groupindex=None, w_indexgroup=None): + n = space.int_w(space.len(w_code)) + code = [0] * n + for i in range(n): + x = space.uint_w(space.getitem(w_code, space.wrap(i))) + code[i] = intmask(x) + # + w_srepat = space.allocate_instance(W_SRE_Pattern, w_subtype) + srepat = space.interp_w(W_SRE_Pattern, w_srepat) + srepat.space = space + srepat.w_pattern = w_pattern # the original uncompiled pattern + srepat.flags = flags + srepat.code = code + srepat.num_groups = groups + srepat.w_groupindex = w_groupindex + srepat.w_indexgroup = w_indexgroup + return w_srepat +SRE_Pattern__new__.unwrap_spec = [ObjSpace, W_Root, W_Root, int, W_Root, + int, W_Root, W_Root] + + +W_SRE_Pattern.typedef = TypeDef( + 'SRE_Pattern', + __new__ = interp2app(SRE_Pattern__new__), + __copy__ = interp2app(W_SRE_Pattern.cannot_copy_w), + __deepcopy__ = interp2app(W_SRE_Pattern.cannot_copy_w), + __weakref__ = make_weakref_descr(W_SRE_Pattern), + findall = interp2app(W_SRE_Pattern.findall_w), + finditer = interp2app(W_SRE_Pattern.finditer_w), + match = interp2app(W_SRE_Pattern.match_w), + scanner = interp2app(W_SRE_Pattern.finditer_w), # reuse finditer() + search = interp2app(W_SRE_Pattern.search_w), + split = interp2app(W_SRE_Pattern.split_w), + sub = interp2app(W_SRE_Pattern.sub_w), + subn = interp2app(W_SRE_Pattern.subn_w), + flags = interp_attrproperty('flags', W_SRE_Pattern), + groupindex = interp_attrproperty_w('w_groupindex', W_SRE_Pattern), + groups = interp_attrproperty('num_groups', W_SRE_Pattern), + pattern = interp_attrproperty_w('w_pattern', W_SRE_Pattern), ) +# ____________________________________________________________ +# +# SRE_Match class -class W_StringState(W_State): - if THREE_VERSIONS_OF_CORE: - rsre.insert_sre_methods(locals(), 'str') - - def unwrap_object(self): - self.string = self.space.str_w(self.w_string) - return len(self.string) - - def get_char_ord(self, p): - return ord(self.string[p]) +class W_SRE_Match(Wrappable): + flatten_cache = None + def __init__(self, srepat, ctx): + self.space = srepat.space + self.srepat = srepat + self.ctx = ctx -class W_UnicodeState(W_State): - if THREE_VERSIONS_OF_CORE: - rsre.insert_sre_methods(locals(), 'unicode') + def cannot_copy_w(self): + space = self.space + raise OperationError(space.w_TypeError, + space.wrap("cannot copy this match object")) - def unwrap_object(self): - self.unicode = self.space.unicode_w(self.w_string) - return len(self.unicode) + def group_w(self, args_w): + space = self.space + ctx = self.ctx + if len(args_w) <= 1: + if len(args_w) == 0: + start, end = ctx.match_start, ctx.match_end + else: + start, end = self.do_span(args_w[0]) + return slice_w(space, ctx, start, end, space.w_None) + else: + results = [None] * len(args_w) + for i in range(len(args_w)): + start, end = self.do_span(args_w[i]) + results[i] = slice_w(space, ctx, start, end, space.w_None) + return space.newtuple(results) + group_w.unwrap_spec = ['self', 'args_w'] + + def groups_w(self, w_default=None): + fmarks = self.flatten_marks() + num_groups = self.srepat.num_groups + return allgroups_w(self.space, self.ctx, fmarks, num_groups, w_default) - def get_char_ord(self, p): - return ord(self.unicode[p]) + def groupdict_w(self, w_default=None): + space = self.space + w_dict = space.newdict() + w_groupindex = self.srepat.w_groupindex + w_iterator = space.iter(w_groupindex) + while True: + try: + w_key = space.next(w_iterator) + except OperationError, e: + if not e.match(space, space.w_StopIteration): + raise + break # done + w_value = space.getitem(w_groupindex, w_key) + start, end = self.do_span(w_value) + w_grp = slice_w(space, self.ctx, start, end, w_default) + space.setitem(w_dict, w_key, w_grp) + return w_dict + def expand_w(self, w_template): + space = self.space + w_re = import_re(space) + return space.call_method(w_re, '_expand', space.wrap(self.srepat), + space.wrap(self), w_template) + + def start_w(self, w_groupnum=0): + return self.space.wrap(self.do_span(w_groupnum)[0]) + + def end_w(self, w_groupnum=0): + return self.space.wrap(self.do_span(w_groupnum)[1]) + + def span_w(self, w_groupnum=0): + start, end = self.do_span(w_groupnum) + return self.space.newtuple([self.space.wrap(start), + self.space.wrap(end)]) + + def flatten_marks(self): + if self.flatten_cache is None: + num_groups = self.srepat.num_groups + self.flatten_cache = do_flatten_marks(self.ctx, num_groups) + return self.flatten_cache -class W_GenericState(W_State): - if THREE_VERSIONS_OF_CORE: - rsre.insert_sre_methods(locals(), 'generic') + def do_span(self, w_arg): + space = self.space + try: + groupnum = space.int_w(w_arg) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + w_groupnum = space.getitem(self.srepat.w_groupindex, w_arg) + groupnum = space.int_w(w_groupnum) + if groupnum == 0: + return self.ctx.match_start, self.ctx.match_end + elif 1 <= groupnum <= self.srepat.num_groups: + fmarks = self.flatten_marks() + idx = 2*(groupnum-1) + assert idx >= 0 + return fmarks[idx], fmarks[idx+1] + else: + raise OperationError(space.w_IndexError, + space.wrap("group index out of range")) + + def _last_index(self): + mark = self.ctx.match_marks + if mark is not None: + return mark.gid // 2 + 1 + return -1 + + def fget_lastgroup(space, self): + lastindex = self._last_index() + if lastindex < 0: + return space.w_None + w_result = space.finditem(self.srepat.w_indexgroup, + space.wrap(lastindex)) + if w_result is None: + return space.w_None + return w_result + + def fget_lastindex(space, self): + lastindex = self._last_index() + if lastindex >= 0: + return space.wrap(lastindex) + return space.w_None - def unwrap_object(self): - self.buffer = self.space.buffer_w(self.w_string) - return self.buffer.getlength() + def fget_pos(space, self): + return space.wrap(self.ctx.original_pos) - def get_char_ord(self, p): - return ord(self.buffer.getitem(p)) + def fget_endpos(space, self): + return space.wrap(self.ctx.end) + def fget_regs(space, self): + space = self.space + fmarks = self.flatten_marks() + num_groups = self.srepat.num_groups + result_w = [None] * (num_groups + 1) + ctx = self.ctx + result_w[0] = space.newtuple([space.wrap(ctx.match_start), + space.wrap(ctx.match_end)]) + for i in range(num_groups): + result_w[i + 1] = space.newtuple([space.wrap(fmarks[i*2]), + space.wrap(fmarks[i*2+1])]) + return space.newtuple(result_w) + + def fget_string(space, self): + return self.ctx._w_string(space) + + +W_SRE_Match.typedef = TypeDef( + 'SRE_Match', + __copy__ = interp2app(W_SRE_Match.cannot_copy_w), + __deepcopy__ = interp2app(W_SRE_Match.cannot_copy_w), + group = interp2app(W_SRE_Match.group_w), + groups = interp2app(W_SRE_Match.groups_w), + groupdict = interp2app(W_SRE_Match.groupdict_w), + start = interp2app(W_SRE_Match.start_w), + end = interp2app(W_SRE_Match.end_w), + span = interp2app(W_SRE_Match.span_w), + expand = interp2app(W_SRE_Match.expand_w), + # + re = interp_attrproperty('srepat', W_SRE_Match), + string = GetSetProperty(W_SRE_Match.fget_string), + pos = GetSetProperty(W_SRE_Match.fget_pos), + endpos = GetSetProperty(W_SRE_Match.fget_endpos), + lastgroup = GetSetProperty(W_SRE_Match.fget_lastgroup), + lastindex = GetSetProperty(W_SRE_Match.fget_lastindex), + regs = GetSetProperty(W_SRE_Match.fget_regs), +) -def w_search(space, w_state, w_pattern_codes): - state = space.interp_w(W_State, w_state) - pattern_codes = [intmask(space.uint_w(code)) for code - in space.unpackiterable(w_pattern_codes)] - try: - res = state.search(pattern_codes) - except RuntimeError: - raise OperationError(space.w_RuntimeError, - space.wrap("Internal re error")) - return space.newbool(res) - -def w_match(space, w_state, w_pattern_codes): - state = space.interp_w(W_State, w_state) - pattern_codes = [intmask(space.uint_w(code)) for code - in space.unpackiterable(w_pattern_codes)] - try: - res = state.match(pattern_codes) - except RuntimeError: - raise OperationError(space.w_RuntimeError, - space.wrap("Internal re error")) - return space.newbool(res) +# ____________________________________________________________ +# +# SRE_Scanner class +# This is mostly an internal class in CPython. +# Our version is also directly iterable, to make finditer() easier. + +class W_SRE_Scanner(Wrappable): + + def __init__(self, pattern, ctx): + self.space = pattern.space + self.srepat = pattern + self.ctx = ctx + # 'self.ctx' is always a fresh context in which no searching + # or matching succeeded so far. + + def iter_w(self): + return self.space.wrap(self) + + def next_w(self): + if self.ctx.match_start > self.ctx.end: + raise OperationError(self.space.w_StopIteration, self.space.w_None) + if not searchcontext(self.space, self.ctx): + raise OperationError(self.space.w_StopIteration, self.space.w_None) + return self.getmatch(True) + + def match_w(self): + if self.ctx.match_start > self.ctx.end: + return self.space.w_None + return self.getmatch(matchcontext(self.space, self.ctx)) + + def search_w(self): + if self.ctx.match_start > self.ctx.end: + return self.space.w_None + return self.getmatch(searchcontext(self.space, self.ctx)) + + def getmatch(self, found): + if found: + ctx = self.ctx + nextstart = ctx.match_end + nextstart += (ctx.match_start == nextstart) + self.ctx = ctx.fresh_copy(nextstart) + match = W_SRE_Match(self.srepat, ctx) + return self.space.wrap(match) + else: + self.ctx.match_start += 1 # obscure corner case + return None + +W_SRE_Scanner.typedef = TypeDef( + 'SRE_Scanner', + __iter__ = interp2app(W_SRE_Scanner.iter_w, unwrap_spec=['self']), + next = interp2app(W_SRE_Scanner.next_w, unwrap_spec=['self']), + match = interp2app(W_SRE_Scanner.match_w, unwrap_spec=['self']), + search = interp2app(W_SRE_Scanner.search_w, unwrap_spec=['self']), + pattern = interp_attrproperty('srepat', W_SRE_Scanner), +) Modified: pypy/branch/asmgcc-64/pypy/module/_sre/test/test_app_sre.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/_sre/test/test_app_sre.py (original) +++ pypy/branch/asmgcc-64/pypy/module/_sre/test/test_app_sre.py Wed Aug 25 19:06:43 2010 @@ -33,8 +33,8 @@ # copy support is disabled by default in _sre.c import re p = re.compile("b") - raises(TypeError, p.__copy__) - raises(TypeError, p.__deepcopy__) + raises(TypeError, p.__copy__) # p.__copy__() should raise + raises(TypeError, p.__deepcopy__) # p.__deepcopy__() should raise def test_creation_attributes(self): import re @@ -85,6 +85,10 @@ assert ['', 'a', None, 'l', 'u', None, 'lla'] == ( re.split("b([ua]|(s))", "balbulla")) + def test_weakref(self): + import re, _weakref + _weakref.ref(re.compile(r"")) + class AppTestSreMatch: def setup_class(cls): @@ -202,6 +206,20 @@ return ret assert ("bbbbb", 3) == re.subn("a", call_me, "ababa") + def test_sub_callable_returns_none(self): + import re + def call_me(match): + return None + assert "acd" == re.sub("b", call_me, "abcd") + + def test_sub_callable_suddenly_unicode(self): + import re + def call_me(match): + if match.group() == 'A': + return unichr(0x3039) + return '' + assert (u"bb\u3039b", 2) == re.subn("[aA]", call_me, "babAb") + def test_match_array(self): import re, array a = array.array('c', 'hello') @@ -268,6 +286,18 @@ p.match().group(0), p.match().group(0)) assert None == p.match() + def test_scanner_match_detail(self): + import re + p = re.compile("a").scanner("aaXaa") + assert "a" == p.match().group(0) + assert "a" == p.match().group(0) + assert None == p.match() + assert "a" == p.match().group(0) + assert "a" == p.match().group(0) + assert None == p.match() + assert None == p.match() + assert None == p.match() + def test_scanner_search(self): import re p = re.compile("\d").scanner("bla23c5a") @@ -653,69 +683,6 @@ s.ATCODES["at_uni_non_boundary"], s.OPCODES["success"]] s.assert_match(opcodes, ["blaha", u"bl%sja" % UPPER_PI]) - def test_category_digit(self): - INDIAN_DIGIT = u"\u0966" - opcodes = [s.OPCODES["category"], s.CHCODES["category_digit"]] \ - + s.encode_literal("b") + [s.OPCODES["success"]] - s.assert_match(opcodes, ["1b", "a1b"]) - s.assert_no_match(opcodes, ["bb", "b1", u"%sb" % INDIAN_DIGIT]) - - def test_category_not_digit(self): - INDIAN_DIGIT = u"\u0966" - opcodes = [s.OPCODES["category"], s.CHCODES["category_not_digit"]] \ - + s.encode_literal("b") + [s.OPCODES["success"]] - s.assert_match(opcodes, ["bb", "1ab", u"%sb" % INDIAN_DIGIT]) - s.assert_no_match(opcodes, ["1b", "a1b"]) - - def test_category_space(self): - EM_SPACE = u"\u2001" - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_space"], s.OPCODES["success"]] - s.assert_match(opcodes, ["b ", "b\n", "b\t", "b\r", "b\v", "b\f"]) - s.assert_no_match(opcodes, ["bb", "b1", u"b%s" % EM_SPACE]) - - def test_category_not_space(self): - EM_SPACE = u"\u2001" - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_not_space"], s.OPCODES["success"]] - s.assert_match(opcodes, ["bb", "b1", u"b%s" % EM_SPACE]) - s.assert_no_match(opcodes, ["b ", "b\n", "b\t", "b\r", "b\v", "b\f"]) - - def test_category_word(self): - LOWER_PI = u"\u03c0" - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_word"], s.OPCODES["success"]] - s.assert_match(opcodes, ["bl", "b4", "b_"]) - s.assert_no_match(opcodes, ["b ", "b\n", u"b%s" % LOWER_PI]) - - def test_category_not_word(self): - LOWER_PI = u"\u03c0" - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_not_word"], s.OPCODES["success"]] - s.assert_match(opcodes, ["b ", "b\n", u"b%s" % LOWER_PI]) - s.assert_no_match(opcodes, ["bl", "b4", "b_"]) - - def test_category_linebreak(self): - LINE_SEP = u"\u2028" - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_linebreak"], s.OPCODES["success"]] - s.assert_match(opcodes, ["b\n"]) - s.assert_no_match(opcodes, ["b ", "bs", "b\r", u"b%s" % LINE_SEP]) - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_uni_linebreak"], s.OPCODES["success"]] - s.assert_match(opcodes, ["b\n", u"b%s" % LINE_SEP]) - - def test_category_not_linebreak(self): - LINE_SEP = u"\u2028" - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_not_linebreak"], s.OPCODES["success"]] - s.assert_match(opcodes, ["b ", "bs", u"b%s" % LINE_SEP]) - s.assert_no_match(opcodes, ["b\n"]) - opcodes = s.encode_literal("b") \ - + [s.OPCODES["category"], s.CHCODES["category_uni_not_linebreak"], s.OPCODES["success"]] - s.assert_match(opcodes, ["b ", "bs"]) - s.assert_no_match(opcodes, ["b\n", u"b%s" % LINE_SEP, "b\r"]) - def test_category_loc_word(self): import locale try: @@ -873,10 +840,6 @@ s.assert_match(opcodes, ["ab", "aaaab", "baabb"]) s.assert_no_match(opcodes, ["aaa", "", "ac"]) - def test_max_until_error(self): - opcodes = [s.OPCODES["max_until"], s.OPCODES["success"]] - raises(RuntimeError, s.search, opcodes, "a") - def test_max_until_zero_width_match(self): # re.compile won't compile prospective zero-with matches (all of them?), # so we can only produce an example by directly constructing bytecodes. @@ -896,10 +859,6 @@ s.assert_no_match(opcodes, ["b"]) assert "aab" == s.search(opcodes, "aabb").group(0) - def test_min_until_error(self): - opcodes = [s.OPCODES["min_until"], s.OPCODES["success"]] - raises(RuntimeError, s.search, opcodes, "a") - def test_groupref(self): opcodes = [s.OPCODES["mark"], 0, s.OPCODES["any"], s.OPCODES["mark"], 1] \ + s.encode_literal("a") + [s.OPCODES["groupref"], 0, s.OPCODES["success"]] Modified: pypy/branch/asmgcc-64/pypy/module/array/interp_array.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/array/interp_array.py (original) +++ pypy/branch/asmgcc-64/pypy/module/array/interp_array.py Wed Aug 25 19:06:43 2010 @@ -15,6 +15,7 @@ from pypy.interpreter.argument import Arguments, Signature from pypy.module._file.interp_file import W_File from pypy.interpreter.buffer import RWBuffer +from pypy.objspace.std.multimethod import FailedToImplement def w_array(space, w_cls, typecode, w_args=None): if len(w_args.arguments_w) > 1: @@ -476,9 +477,11 @@ def mul__Array_ANY(space, self, w_repeat): try: - repeat = space.int_w(w_repeat) - except OperationError: - return space.w_NotImplemented + repeat = space.getindex_w(w_repeat, space.w_OverflowError) + except OperationError, e: + if e.match(space, space.w_TypeError): + raise FailedToImplement + raise a = mytype.w_class(space) repeat = max(repeat, 0) a.setlen(self.len * repeat) @@ -492,9 +495,11 @@ def inplace_mul__Array_ANY(space, self, w_repeat): try: - repeat = space.int_w(w_repeat) - except OperationError: - return space.w_NotImplemented + repeat = space.getindex_w(w_repeat, space.w_OverflowError) + except OperationError, e: + if e.match(space, space.w_TypeError): + raise FailedToImplement + raise oldlen = self.len repeat = max(repeat, 0) self.setlen(self.len * repeat) Modified: pypy/branch/asmgcc-64/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/array/test/test_array.py (original) +++ pypy/branch/asmgcc-64/pypy/module/array/test/test_array.py Wed Aug 25 19:06:43 2010 @@ -628,7 +628,8 @@ a = self.array('i') raises(TypeError, "a * 'hi'") raises(TypeError, "'hi' * a") - + raises(TypeError, "a *= 'hi'") + class mulable(object): def __mul__(self, other): return "mul" Modified: pypy/branch/asmgcc-64/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/asmgcc-64/pypy/module/cpyext/stubs.py Wed Aug 25 19:06:43 2010 @@ -2874,36 +2874,6 @@ """ raise NotImplementedError - at cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.INTP], PyObject) -def PyUnicode_DecodeUTF16(space, s, size, errors, byteorder): - """Decode length bytes from a UTF-16 encoded buffer string and return the - corresponding Unicode object. errors (if non-NULL) defines the error - handling. It defaults to "strict". - - If byteorder is non-NULL, the decoder starts decoding using the given byte - order: - - *byteorder == -1: little endian - *byteorder == 0: native order - *byteorder == 1: big endian - - If *byteorder is zero, and the first two bytes of the input data are a - byte order mark (BOM), the decoder switches to this byte order and the BOM is - not copied into the resulting Unicode string. If *byteorder is -1 or - 1, any byte order mark is copied to the output (where it will result in - either a \ufeff or a \ufffe character). - - After completion, *byteorder is set to the current byte order at the end - of input data. - - If byteorder is NULL, the codec starts in native order mode. - - Return NULL if an exception was raised by the codec. - - This function used an int type for size. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.INTP, Py_ssize_t], PyObject) def PyUnicode_DecodeUTF16Stateful(space, s, size, errors, byteorder, consumed): """If consumed is NULL, behave like PyUnicode_DecodeUTF16(). If Modified: pypy/branch/asmgcc-64/pypy/module/cpyext/test/test_unicodeobject.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/cpyext/test/test_unicodeobject.py (original) +++ pypy/branch/asmgcc-64/pypy/module/cpyext/test/test_unicodeobject.py Wed Aug 25 19:06:43 2010 @@ -172,4 +172,37 @@ result = api.PyUnicode_AsASCIIString(w_ustr) assert result is None + def test_decode_utf16(self, space, api): + def test(encoded, endian, realendian=None): + encoded_charp = rffi.str2charp(encoded) + strict_charp = rffi.str2charp("strict") + if endian is not None: + pendian = lltype.malloc(rffi.INTP.TO, 1, flavor='raw') + if endian < 0: + pendian[0] = -1 + elif endian > 0: + pendian[0] = 1 + else: + pendian[0] = 0 + else: + pendian = None + w_ustr = api.PyUnicode_DecodeUTF16(encoded_charp, len(encoded), strict_charp, pendian) + assert space.eq_w(space.call_method(w_ustr, 'encode', space.wrap('ascii')), + space.wrap("abcd")) + + rffi.free_charp(encoded_charp) + rffi.free_charp(strict_charp) + if pendian: + if realendian is not None: + assert rffi.cast(rffi.INT, realendian) == pendian[0] + lltype.free(pendian, flavor='raw') + + test("\x61\x00\x62\x00\x63\x00\x64\x00", -1) + + test("\x61\x00\x62\x00\x63\x00\x64\x00", None) + + test("\x00\x61\x00\x62\x00\x63\x00\x64", 1) + + test("\xFE\xFF\x00\x61\x00\x62\x00\x63\x00\x64", 0, 1) + test("\xFF\xFE\x61\x00\x62\x00\x63\x00\x64\x00", 0, -1) Modified: pypy/branch/asmgcc-64/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/asmgcc-64/pypy/module/cpyext/unicodeobject.py Wed Aug 25 19:06:43 2010 @@ -9,6 +9,7 @@ from pypy.module.cpyext.pyobject import PyObject, from_ref, make_typedescr from pypy.module.sys.interp_encoding import setdefaultencoding from pypy.objspace.std import unicodeobject, unicodetype +from pypy.rlib import runicode import sys ## See comment in stringobject.py. PyUnicode_FromUnicode(NULL, size) is not @@ -307,6 +308,64 @@ w_errors = space.w_None return space.call_method(w_str, 'decode', space.wrap("utf-8"), w_errors) + at cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.INTP], PyObject) +def PyUnicode_DecodeUTF16(space, s, size, llerrors, pbyteorder): + """Decode length bytes from a UTF-16 encoded buffer string and return the + corresponding Unicode object. errors (if non-NULL) defines the error + handling. It defaults to "strict". + + If byteorder is non-NULL, the decoder starts decoding using the given byte + order: + + *byteorder == -1: little endian + *byteorder == 0: native order + *byteorder == 1: big endian + + If *byteorder is zero, and the first two bytes of the input data are a + byte order mark (BOM), the decoder switches to this byte order and the BOM is + not copied into the resulting Unicode string. If *byteorder is -1 or + 1, any byte order mark is copied to the output (where it will result in + either a \ufeff or a \ufffe character). + + After completion, *byteorder is set to the current byte order at the end + of input data. + + If byteorder is NULL, the codec starts in native order mode. + + Return NULL if an exception was raised by the codec. + + This function used an int type for size. This might require + changes in your code for properly supporting 64-bit systems.""" + + string = rffi.charpsize2str(s, size) + + #FIXME: I don't like these prefixes + if pbyteorder is not None: # correct NULL check? + llbyteorder = rffi.cast(lltype.Signed, pbyteorder[0]) # compatible with int? + if llbyteorder < 0: + byteorder = "little" + elif llbyteorder > 0: + byteorder = "big" + else: + byteorder = "native" + else: + byteorder = "native" + + if llerrors: + errors = rffi.charp2str(llerrors) + else: + errors = None + + result, length, byteorder = runicode.str_decode_utf_16_helper(string, size, + errors, + True, # final ? false for multiple passes? + None, # errorhandler + byteorder) + if pbyteorder is not None: + pbyteorder[0] = rffi.cast(rffi.INT, byteorder) + + return space.wrap(result) + @cpython_api([PyObject], PyObject) def PyUnicode_AsASCIIString(space, w_unicode): """Encode a Unicode object using ASCII and return the result as Python string Modified: pypy/branch/asmgcc-64/pypy/module/pypyjit/interp_jit.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/pypyjit/interp_jit.py (original) +++ pypy/branch/asmgcc-64/pypy/module/pypyjit/interp_jit.py Wed Aug 25 19:06:43 2010 @@ -9,7 +9,7 @@ import pypy.interpreter.pyopcode # for side-effects from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import ObjSpace, Arguments -from pypy.interpreter.pycode import PyCode, CO_CONTAINSLOOP +from pypy.interpreter.pycode import PyCode, CO_GENERATOR from pypy.interpreter.pyframe import PyFrame from pypy.interpreter.pyopcode import ExitFrame from opcode import opmap @@ -24,9 +24,6 @@ JUMP_ABSOLUTE = opmap['JUMP_ABSOLUTE'] -def can_inline(next_instr, bytecode): - return not bool(bytecode.co_flags & CO_CONTAINSLOOP) - def get_printable_location(next_instr, bytecode): from pypy.tool.stdlib_opcode import opcode_method_names name = opcode_method_names[ord(bytecode.co_code[next_instr])] @@ -39,7 +36,8 @@ bytecode.jit_cells[next_instr] = newcell def confirm_enter_jit(next_instr, bytecode, frame, ec): - return (frame.w_f_trace is None and + return (not (bytecode.co_flags & CO_GENERATOR) and + frame.w_f_trace is None and ec.profilefunc is None and ec.w_tracefunc is None) @@ -57,8 +55,7 @@ ## blockstack = frame.blockstack ## return (valuestackdepth, blockstack) -pypyjitdriver = PyPyJitDriver(can_inline = can_inline, - get_printable_location = get_printable_location, +pypyjitdriver = PyPyJitDriver(get_printable_location = get_printable_location, get_jitcell_at = get_jitcell_at, set_jitcell_at = set_jitcell_at, confirm_enter_jit = confirm_enter_jit) Modified: pypy/branch/asmgcc-64/pypy/module/pypyjit/policy.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/pypyjit/policy.py (original) +++ pypy/branch/asmgcc-64/pypy/module/pypyjit/policy.py Wed Aug 25 19:06:43 2010 @@ -32,6 +32,8 @@ return False if mod.startswith('pypy.interpreter.pyparser.'): return False + if mod == 'pypy.interpreter.generator': + return False if mod.startswith('pypy.module.'): modname = mod[len('pypy.module.'):] if not self.look_inside_pypy_module(modname): Modified: pypy/branch/asmgcc-64/pypy/objspace/std/intobject.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/objspace/std/intobject.py (original) +++ pypy/branch/asmgcc-64/pypy/objspace/std/intobject.py Wed Aug 25 19:06:43 2010 @@ -240,34 +240,36 @@ def lshift__Int_Int(space, w_int1, w_int2): a = w_int1.intval b = w_int2.intval + if r_uint(b) < LONG_BIT: # 0 <= b < LONG_BIT + try: + c = ovfcheck_lshift(a, b) + except OverflowError: + raise FailedToImplementArgs(space.w_OverflowError, + space.wrap("integer left shift")) + return wrapint(space, c) if b < 0: raise OperationError(space.w_ValueError, space.wrap("negative shift count")) - if a == 0 or b == 0: - return get_integer(space, w_int1) - if b >= LONG_BIT: + else: #b >= LONG_BIT + if a == 0: + return get_integer(space, w_int1) raise FailedToImplementArgs(space.w_OverflowError, space.wrap("integer left shift")) - try: - c = ovfcheck_lshift(a, b) - except OverflowError: - raise FailedToImplementArgs(space.w_OverflowError, - space.wrap("integer left shift")) - return wrapint(space, c) def rshift__Int_Int(space, w_int1, w_int2): a = w_int1.intval b = w_int2.intval - if b < 0: - raise OperationError(space.w_ValueError, - space.wrap("negative shift count")) - if a == 0 or b == 0: - return get_integer(space, w_int1) - if b >= LONG_BIT: - if a < 0: - a = -1 - else: - a = 0 + if r_uint(b) >= LONG_BIT: # not (0 <= b < LONG_BIT) + if b < 0: + raise OperationError(space.w_ValueError, + space.wrap("negative shift count")) + else: # b >= LONG_BIT + if a == 0: + return get_integer(space, w_int1) + if a < 0: + a = -1 + else: + a = 0 else: a = a >> b return wrapint(space, a) Modified: pypy/branch/asmgcc-64/pypy/objspace/std/model.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/objspace/std/model.py (original) +++ pypy/branch/asmgcc-64/pypy/objspace/std/model.py Wed Aug 25 19:06:43 2010 @@ -88,7 +88,8 @@ import pypy.objspace.std.default # register a few catch-all multimethods import pypy.objspace.std.marshal_impl # install marshal multimethods - import pypy.module.array + if config.objspace.usemodules.array: + import pypy.module.array # the set of implementation types self.typeorder = { Modified: pypy/branch/asmgcc-64/pypy/rlib/debug.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rlib/debug.py (original) +++ pypy/branch/asmgcc-64/pypy/rlib/debug.py Wed Aug 25 19:06:43 2010 @@ -255,11 +255,32 @@ class IntegerCanBeNegative(Exception): pass -def _check_nonneg(ann, bk): - from pypy.annotation.model import SomeInteger - s_nonneg = SomeInteger(nonneg=True) - if not s_nonneg.contains(ann): - raise IntegerCanBeNegative +class UnexpectedRUInt(Exception): + pass def check_nonneg(x): - check_annotation(x, _check_nonneg) + """Give a translation-time error if 'x' is not known to be non-negative. + To help debugging, this also gives a translation-time error if 'x' is + actually typed as an r_uint (in which case the call to check_nonneg() + is a bit strange and probably unexpected). + """ + assert type(x)(-1) < 0 # otherwise, 'x' is a r_uint or similar + assert x >= 0 + return x + +class Entry(ExtRegistryEntry): + _about_ = check_nonneg + + def compute_result_annotation(self, s_arg): + from pypy.annotation.model import SomeInteger + if isinstance(s_arg, SomeInteger) and s_arg.unsigned: + raise UnexpectedRUInt("check_nonneg() arg is a %s" % ( + s_arg.knowntype,)) + s_nonneg = SomeInteger(nonneg=True) + if not s_nonneg.contains(s_arg): + raise IntegerCanBeNegative + return s_arg + + def specialize_call(self, hop): + hop.exception_cannot_occur() + return hop.inputarg(hop.args_r[0], arg=0) Modified: pypy/branch/asmgcc-64/pypy/rlib/jit.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rlib/jit.py (original) +++ pypy/branch/asmgcc-64/pypy/rlib/jit.py Wed Aug 25 19:06:43 2010 @@ -253,8 +253,7 @@ def __init__(self, greens=None, reds=None, virtualizables=None, get_jitcell_at=None, set_jitcell_at=None, - can_inline=None, get_printable_location=None, - confirm_enter_jit=None): + get_printable_location=None, confirm_enter_jit=None): if greens is not None: self.greens = greens if reds is not None: @@ -270,7 +269,6 @@ self.get_jitcell_at = get_jitcell_at self.set_jitcell_at = set_jitcell_at self.get_printable_location = get_printable_location - self.can_inline = can_inline self.confirm_enter_jit = confirm_enter_jit def _freeze_(self): @@ -284,6 +282,10 @@ # special-cased by ExtRegistryEntry assert dict.fromkeys(livevars) == _self._alllivevars + def loop_header(self): + # special-cased by ExtRegistryEntry + pass + def _set_param(self, name, value): # special-cased by ExtRegistryEntry # (internal, must receive a constant 'name') @@ -323,11 +325,15 @@ # specifically for them. self.jit_merge_point = self.jit_merge_point self.can_enter_jit = self.can_enter_jit + self.loop_header = self.loop_header self._set_param = self._set_param class Entry(ExtEnterLeaveMarker): _about_ = (self.jit_merge_point, self.can_enter_jit) + class Entry(ExtLoopHeader): + _about_ = self.loop_header + class Entry(ExtSetParam): _about_ = self._set_param @@ -384,7 +390,6 @@ self.annotate_hook(driver.get_jitcell_at, driver.greens, **kwds_s) self.annotate_hook(driver.set_jitcell_at, driver.greens, [s_jitcell], **kwds_s) - self.annotate_hook(driver.can_inline, driver.greens, **kwds_s) self.annotate_hook(driver.get_printable_location, driver.greens, **kwds_s) def annotate_hook(self, func, variables, args_s=[], **kwds_s): @@ -425,6 +430,23 @@ return hop.genop('jit_marker', vlist, resulttype=lltype.Void) +class ExtLoopHeader(ExtRegistryEntry): + # Replace a call to myjitdriver.loop_header() + # with an operation jit_marker('loop_header', myjitdriver). + + def compute_result_annotation(self, **kwds_s): + from pypy.annotation import model as annmodel + return annmodel.s_None + + def specialize_call(self, hop): + from pypy.rpython.lltypesystem import lltype + driver = self.instance.im_self + hop.exception_cannot_occur() + vlist = [hop.inputconst(lltype.Void, 'loop_header'), + hop.inputconst(lltype.Void, driver)] + return hop.genop('jit_marker', vlist, + resulttype=lltype.Void) + class ExtSetParam(ExtRegistryEntry): def compute_result_annotation(self, s_name, s_value): Modified: pypy/branch/asmgcc-64/pypy/rlib/rlocale.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rlib/rlocale.py (original) +++ pypy/branch/asmgcc-64/pypy/rlib/rlocale.py Wed Aug 25 19:06:43 2010 @@ -15,6 +15,12 @@ HAVE_LANGINFO = sys.platform != 'win32' HAVE_LIBINTL = sys.platform != 'win32' +if HAVE_LIBINTL: + try: + platform.verify_eci(ExternalCompilationInfo(includes=['libintl.h'])) + except platform.CompilationError: + HAVE_LIBINTL = False + class CConfig: includes = ['locale.h', 'limits.h'] Modified: pypy/branch/asmgcc-64/pypy/rlib/test/test_jit.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rlib/test/test_jit.py (original) +++ pypy/branch/asmgcc-64/pypy/rlib/test/test_jit.py Wed Aug 25 19:06:43 2010 @@ -59,11 +59,9 @@ def test_annotate_hooks(self): - def can_inline(m): pass def get_printable_location(m): pass myjitdriver = JitDriver(greens=['m'], reds=['n'], - can_inline=can_inline, get_printable_location=get_printable_location) def fn(n): m = 42.5 @@ -81,9 +79,8 @@ return [v.concretetype for v in graph.getargs()] raise Exception, 'function %r has not been annotated' % func - can_inline_args = getargs(can_inline) get_printable_location_args = getargs(get_printable_location) - assert can_inline_args == get_printable_location_args == [lltype.Float] + assert get_printable_location_args == [lltype.Float] def test_annotate_argumenterror(self): myjitdriver = JitDriver(greens=['m'], reds=['n']) Modified: pypy/branch/asmgcc-64/pypy/rpython/rclass.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rpython/rclass.py (original) +++ pypy/branch/asmgcc-64/pypy/rpython/rclass.py Wed Aug 25 19:06:43 2010 @@ -156,9 +156,13 @@ if '_immutable_' in self.classdef.classdesc.classdict: hints = hints.copy() hints['immutable'] = True - if '_immutable_fields_' in self.classdef.classdesc.classdict: + self.immutable_field_list = [] # unless overwritten below + if self.classdef.classdesc.lookup('_immutable_fields_') is not None: hints = hints.copy() - self.immutable_field_list = self.classdef.classdesc.classdict['_immutable_fields_'].value + immutable_fields = self.classdef.classdesc.classdict.get( + '_immutable_fields_') + if immutable_fields is not None: + self.immutable_field_list = immutable_fields.value accessor = FieldListAccessor() hints['immutable_fields'] = accessor return hints @@ -181,7 +185,13 @@ hints = self.object_type._hints if "immutable_fields" in hints: accessor = hints["immutable_fields"] - self._parse_field_list(self.immutable_field_list, accessor) + immutable_fields = {} + rbase = self + while rbase.classdef is not None: + immutable_fields.update( + dict.fromkeys(rbase.immutable_field_list)) + rbase = rbase.rbase + self._parse_field_list(immutable_fields, accessor) def _parse_field_list(self, fields, accessor): with_suffix = {} @@ -191,7 +201,10 @@ suffix = '[*]' else: suffix = '' - mangled_name, r = self._get_field(name) + try: + mangled_name, r = self._get_field(name) + except KeyError: + continue with_suffix[mangled_name] = suffix accessor.initialize(self.object_type, with_suffix) return with_suffix Modified: pypy/branch/asmgcc-64/pypy/rpython/test/test_rclass.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rpython/test/test_rclass.py (original) +++ pypy/branch/asmgcc-64/pypy/rpython/test/test_rclass.py Wed Aug 25 19:06:43 2010 @@ -738,6 +738,64 @@ assert accessor.fields == {"inst_x" : "", "inst_y" : "[*]"} or \ accessor.fields == {"ox" : "", "oy" : "[*]"} # for ootype + def test_immutable_fields_subclass_1(self): + from pypy.jit.metainterp.typesystem import deref + class A(object): + _immutable_fields_ = ["x"] + def __init__(self, x): + self.x = x + class B(A): + def __init__(self, x, y): + A.__init__(self, x) + self.y = y + + def f(): + return B(3, 5) + t, typer, graph = self.gengraph(f, []) + B_TYPE = deref(graph.getreturnvar().concretetype) + accessor = B_TYPE._hints["immutable_fields"] + assert accessor.fields == {"inst_x" : ""} or \ + accessor.fields == {"ox" : ""} # for ootype + + def test_immutable_fields_subclass_2(self): + from pypy.jit.metainterp.typesystem import deref + class A(object): + _immutable_fields_ = ["x"] + def __init__(self, x): + self.x = x + class B(A): + _immutable_fields_ = ["y"] + def __init__(self, x, y): + A.__init__(self, x) + self.y = y + + def f(): + return B(3, 5) + t, typer, graph = self.gengraph(f, []) + B_TYPE = deref(graph.getreturnvar().concretetype) + accessor = B_TYPE._hints["immutable_fields"] + assert accessor.fields == {"inst_x" : "", "inst_y" : ""} or \ + accessor.fields == {"ox" : "", "oy" : ""} # for ootype + + def test_immutable_fields_only_in_subclass(self): + from pypy.jit.metainterp.typesystem import deref + class A(object): + def __init__(self, x): + self.x = x + class B(A): + _immutable_fields_ = ["y"] + def __init__(self, x, y): + A.__init__(self, x) + self.y = y + + def f(): + return B(3, 5) + t, typer, graph = self.gengraph(f, []) + B_TYPE = deref(graph.getreturnvar().concretetype) + accessor = B_TYPE._hints["immutable_fields"] + assert accessor.fields == {"inst_y" : ""} or \ + accessor.fields == {"oy" : ""} # for ootype + def test_immutable_inheritance(self): class I(object): def __init__(self, v): Modified: pypy/branch/asmgcc-64/pypy/tool/release/package.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/tool/release/package.py (original) +++ pypy/branch/asmgcc-64/pypy/tool/release/package.py Wed Aug 25 19:06:43 2010 @@ -11,11 +11,12 @@ import py import os import fnmatch -import tarfile from pypy.tool.udir import udir if sys.version_info < (2,6): py.test.skip("requires 2.6 so far") +USE_TARFILE_MODULE = sys.platform == 'win32' + def ignore_patterns(*patterns): """Function that can be used as copytree() ignore parameter. @@ -69,9 +70,17 @@ old_dir = os.getcwd() try: os.chdir(str(builddir)) - os.system("strip " + str(archive_pypy_c)) - os.system('tar cvjf ' + str(builddir.join(name + '.tar.bz2')) + - " " + name) + os.system("strip " + str(archive_pypy_c)) # ignore errors + if USE_TARFILE_MODULE: + import tarfile + tf = tarfile.open(str(builddir.join(name + '.tar.bz2')), 'w:bz2') + tf.add(name) + tf.close() + else: + e = os.system('tar cvjf ' + str(builddir.join(name + '.tar.bz2')) + + " " + name) + if e: + raise OSError('"tar" returned exit status %r' % e) finally: os.chdir(old_dir) if copy_to_dir is not None: Modified: pypy/branch/asmgcc-64/pypy/tool/release/test/test_package.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/tool/release/test/test_package.py (original) +++ pypy/branch/asmgcc-64/pypy/tool/release/test/test_package.py Wed Aug 25 19:06:43 2010 @@ -5,7 +5,7 @@ from pypy.module.sys.version import CPYTHON_VERSION import tarfile, os -def test_dir_structure(): +def test_dir_structure(test='test'): # make sure we have sort of pypy-c pypy_c = py.path.local(pypydir).join('translator', 'goal', 'pypy-c') if not pypy_c.check(): @@ -14,8 +14,8 @@ else: fake_pypy_c = False try: - builddir = package(py.path.local(pypydir).dirpath(), 'test') - prefix = builddir.join('test') + builddir = package(py.path.local(pypydir).dirpath(), test) + prefix = builddir.join(test) cpyver = '%d.%d.%d' % CPYTHON_VERSION[:3] assert prefix.join('lib-python', cpyver, 'test').check() assert prefix.join('bin', 'pypy-c').check() @@ -24,18 +24,27 @@ assert not prefix.join('lib_pypy', 'ctypes_configure').check() assert prefix.join('LICENSE').check() assert prefix.join('README').check() - th = tarfile.open(str(builddir.join('test.tar.bz2'))) - assert th.getmember('test/lib_pypy/syslog.py') + th = tarfile.open(str(builddir.join('%s.tar.bz2' % test))) + assert th.getmember('%s/lib_pypy/syslog.py' % test) # the headers file could be not there, because they are copied into # trunk/include only during translation includedir = py.path.local(pypydir).dirpath().join('include') def check_include(name): if includedir.join(name).check(file=True): - assert th.getmember('test/include/%s' % name) + assert th.getmember('%s/include/%s' % (test, name)) check_include('Python.h') check_include('modsupport.inl') check_include('pypy_decl.h') finally: if fake_pypy_c: pypy_c.remove() + +def test_with_tarfile_module(): + from pypy.tool.release import package + prev = package.USE_TARFILE_MODULE + try: + package.USE_TARFILE_MODULE = True + test_dir_structure(test='testtarfile') + finally: + package.USE_TARFILE_MODULE = prev Modified: pypy/branch/asmgcc-64/pypy/tool/runsubprocess.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/tool/runsubprocess.py (original) +++ pypy/branch/asmgcc-64/pypy/tool/runsubprocess.py Wed Aug 25 19:06:43 2010 @@ -70,5 +70,5 @@ assert results.startswith('(') results = eval(results) if results[0] is None: - raise OSError(results[1]) + raise OSError('%s: %s' % (args[0], results[1])) return results Modified: pypy/branch/asmgcc-64/pypy/translator/goal/translate.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/goal/translate.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/goal/translate.py Wed Aug 25 19:06:43 2010 @@ -18,7 +18,7 @@ ArbitraryOption, StrOption, IntOption, Config, \ ChoiceOption, OptHelpFormatter from pypy.config.translationoption import get_combined_translation_config -from pypy.config.translationoption import set_opt_level +from pypy.config.translationoption import set_opt_level, final_check_config from pypy.config.translationoption import OPT_LEVELS, DEFAULT_OPT_LEVEL from pypy.config.translationoption import PLATFORMS, set_platform @@ -175,6 +175,9 @@ if 'handle_config' in targetspec_dic: targetspec_dic['handle_config'](config, translateconfig) + # perform checks (if any) on the final config + final_check_config(config) + if translateconfig.help: opt_parser.print_help() if 'print_help' in targetspec_dic: Modified: pypy/branch/asmgcc-64/pypy/translator/platform/__init__.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/platform/__init__.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/platform/__init__.py Wed Aug 25 19:06:43 2010 @@ -38,9 +38,9 @@ name = "abstract platform" c_environ = None - relevant_environ = [] + relevant_environ = () - so_prefixes = [''] + so_prefixes = ('',) def __init__(self, cc): if self.__class__ is Platform: @@ -146,7 +146,7 @@ extra = self.standalone_only else: extra = self.shared_only - cflags = self.cflags + extra + cflags = list(self.cflags) + list(extra) return (cflags + list(eci.compile_extra) + args) def _preprocess_library_dirs(self, library_dirs): @@ -158,7 +158,7 @@ libraries = self._libs(eci.libraries) link_files = self._linkfiles(eci.link_files) export_flags = self._exportsymbols_link_flags(eci) - return (library_dirs + self.link_flags + export_flags + + return (library_dirs + list(self.link_flags) + export_flags + link_files + list(eci.link_extra) + libraries) def _exportsymbols_link_flags(self, eci, relto=None): Modified: pypy/branch/asmgcc-64/pypy/translator/platform/darwin.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/platform/darwin.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/platform/darwin.py Wed Aug 25 19:06:43 2010 @@ -5,10 +5,10 @@ class Darwin(posix.BasePosix): name = "darwin" - link_flags = ['-mmacosx-version-min=10.4'] - cflags = ['-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4'] - standalone_only = ['-mdynamic-no-pic'] - shared_only = [] + link_flags = ('-mmacosx-version-min=10.4',) + cflags = ('-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4') + standalone_only = ('-mdynamic-no-pic',) + shared_only = () so_ext = 'so' @@ -18,8 +18,9 @@ self.cc = cc def _args_for_shared(self, args): - return (self.shared_only + ['-dynamiclib', '-undefined', 'dynamic_lookup'] - + args) + return (list(self.shared_only) + + ['-dynamiclib', '-undefined', 'dynamic_lookup'] + + args) def _preprocess_include_dirs(self, include_dirs): res_incl_dirs = list(include_dirs) @@ -72,11 +73,12 @@ class Darwin_i386(Darwin): name = "darwin_i386" - link_flags = ['-arch', 'i386', '-mmacosx-version-min=10.4'] - cflags = ['-arch', 'i386', '-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4'] + link_flags = ('-arch', 'i386', '-mmacosx-version-min=10.4') + cflags = ('-arch', 'i386', '-O3', '-fomit-frame-pointer', + '-mmacosx-version-min=10.4') class Darwin_x86_64(Darwin): name = "darwin_x86_64" - link_flags = ['-arch', 'x86_64', '-mmacosx-version-min=10.4'] - cflags = ['-arch', 'x86_64', '-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4'] - + link_flags = ('-arch', 'x86_64', '-mmacosx-version-min=10.4') + cflags = ('-arch', 'x86_64', '-O3', '-fomit-frame-pointer', + '-mmacosx-version-min=10.4') Modified: pypy/branch/asmgcc-64/pypy/translator/platform/freebsd7.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/platform/freebsd7.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/platform/freebsd7.py Wed Aug 25 19:06:43 2010 @@ -5,10 +5,10 @@ class Freebsd7(posix.BasePosix): name = "freebsd7" - link_flags = ['-pthread'] - cflags = ['-O3', '-pthread', '-fomit-frame-pointer'] - standalone_only = [] - shared_only = [] + link_flags = ('-pthread',) + cflags = ('-O3', '-pthread', '-fomit-frame-pointer') + standalone_only = () + shared_only = () so_ext = 'so' make_cmd = 'gmake' @@ -32,4 +32,4 @@ return ['/usr/local/lib'] class Freebsd7_64(Freebsd7): - shared_only = ['-fPIC'] + shared_only = ('-fPIC',) Modified: pypy/branch/asmgcc-64/pypy/translator/platform/linux.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/platform/linux.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/platform/linux.py Wed Aug 25 19:06:43 2010 @@ -6,12 +6,13 @@ class BaseLinux(BasePosix): name = "linux" - link_flags = ['-pthread', '-lrt'] - cflags = ['-O3', '-pthread', '-fomit-frame-pointer', '-Wall', '-Wno-unused'] - standalone_only = [] - shared_only = ['-fPIC'] + link_flags = ('-pthread', '-lrt') + cflags = ('-O3', '-pthread', '-fomit-frame-pointer', + '-Wall', '-Wno-unused') + standalone_only = () + shared_only = ('-fPIC',) so_ext = 'so' - so_prefixes = ['lib', ''] + so_prefixes = ('lib', '') def _args_for_shared(self, args): return ['-shared'] + args Modified: pypy/branch/asmgcc-64/pypy/translator/platform/maemo.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/platform/maemo.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/platform/maemo.py Wed Aug 25 19:06:43 2010 @@ -13,7 +13,7 @@ class Maemo(Linux): name = "maemo" - available_includedirs = ['/usr/include', '/tmp'] + available_includedirs = ('/usr/include', '/tmp') copied_cache = {} def _invent_new_name(self, basepath, base): Modified: pypy/branch/asmgcc-64/pypy/translator/platform/posix.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/platform/posix.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/platform/posix.py Wed Aug 25 19:06:43 2010 @@ -10,7 +10,7 @@ exe_ext = '' make_cmd = 'make' - relevant_environ=['CPATH', 'LIBRARY_PATH', 'C_INCLUDE_PATH'] + relevant_environ = ('CPATH', 'LIBRARY_PATH', 'C_INCLUDE_PATH') def __init__(self, cc=None): if cc is None: @@ -92,7 +92,7 @@ else: exe_name = exe_name.new(ext=self.exe_ext) - linkflags = self.link_flags[:] + linkflags = list(self.link_flags) if shared: linkflags = self._args_for_shared(linkflags) Modified: pypy/branch/asmgcc-64/pypy/translator/platform/windows.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/platform/windows.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/platform/windows.py Wed Aug 25 19:06:43 2010 @@ -72,10 +72,10 @@ cc = 'cl.exe' link = 'link.exe' - cflags = ['/MD', '/O2'] - link_flags = [] - standalone_only = [] - shared_only = [] + cflags = ('/MD', '/O2') + link_flags = () + standalone_only = () + shared_only = () environ = None def __init__(self, cc=None): @@ -218,7 +218,7 @@ m.exe_name = exe_name m.eci = eci - linkflags = self.link_flags[:] + linkflags = list(self.link_flags) if shared: linkflags = self._args_for_shared(linkflags) + [ '/EXPORT:$(PYPY_MAIN_FUNCTION)'] @@ -336,10 +336,10 @@ class MingwPlatform(posix.BasePosix): name = 'mingw32' - standalone_only = [] - shared_only = [] - cflags = ['-O3'] - link_flags = [] + standalone_only = () + shared_only = () + cflags = ('-O3',) + link_flags = () exe_ext = 'exe' so_ext = 'dll' From jcreigh at codespeak.net Wed Aug 25 19:13:00 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Wed, 25 Aug 2010 19:13:00 +0200 (CEST) Subject: [pypy-svn] r76733 - pypy/branch/asmgcc-64/pypy/config Message-ID: <20100825171300.36076282BAD@codespeak.net> Author: jcreigh Date: Wed Aug 25 19:12:58 2010 New Revision: 76733 Modified: pypy/branch/asmgcc-64/pypy/config/translationoption.py Log: add stub final_check_config that was accidentally removed in merge Modified: pypy/branch/asmgcc-64/pypy/config/translationoption.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/config/translationoption.py (original) +++ pypy/branch/asmgcc-64/pypy/config/translationoption.py Wed Aug 25 19:12:58 2010 @@ -342,6 +342,9 @@ 'jit': 'hybrid extraopts jit', } +def final_check_config(config): + pass + def set_opt_level(config, level): """Apply optimization suggestions on the 'config'. The optimizations depend on the selected level and possibly on the backend. From jcreigh at codespeak.net Wed Aug 25 21:09:13 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Wed, 25 Aug 2010 21:09:13 +0200 (CEST) Subject: [pypy-svn] r76734 - pypy/branch/asmgcc-64/pypy/jit/backend/x86/test Message-ID: <20100825190913.2D6F3282BAD@codespeak.net> Author: jcreigh Date: Wed Aug 25 21:08:51 2010 New Revision: 76734 Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_ztranslation.py Log: enable removetypeptr test Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_ztranslation.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_ztranslation.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_ztranslation.py Wed Aug 25 21:08:51 2010 @@ -125,10 +125,6 @@ return t def test_external_exception_handling_translates(self): - # FIXME - if IS_X86_64: - import py.test; py.test.skip() - jitdriver = JitDriver(greens = [], reds = ['n', 'total']) class ImDone(Exception): From cfbolz at codespeak.net Thu Aug 26 00:03:03 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 26 Aug 2010 00:03:03 +0200 (CEST) Subject: [pypy-svn] r76735 - in pypy/branch/better-map-instances/pypy/objspace/std: . test Message-ID: <20100825220303.8DAAD282BAD@codespeak.net> Author: cfbolz Date: Thu Aug 26 00:02:59 2010 New Revision: 76735 Modified: pypy/branch/better-map-instances/pypy/objspace/std/celldict.py pypy/branch/better-map-instances/pypy/objspace/std/dictmultiobject.py pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py pypy/branch/better-map-instances/pypy/objspace/std/sharingdict.py pypy/branch/better-map-instances/pypy/objspace/std/test/test_dictmultiobject.py pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py Log: fix some XXXs: - a better clear - fix test about slots with the same names - when calling _as_rdict, always call the correct impl_fallback_* method Modified: pypy/branch/better-map-instances/pypy/objspace/std/celldict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/celldict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/celldict.py Thu Aug 26 00:02:59 2010 @@ -45,7 +45,7 @@ if space.is_w(space.type(w_key), space.w_str): self.impl_setitem_str(self.space.str_w(w_key), w_value) else: - self._as_rdict().setitem(w_key, w_value) + self._as_rdict().impl_fallback_setitem(w_key, w_value) def impl_setitem_str(self, name, w_value, shadows_type=True): self.getcell(name, True).w_value = w_value @@ -66,7 +66,7 @@ elif _is_sane_hash(space, w_key_type): raise KeyError else: - self._as_rdict().delitem(w_key) + self._as_rdict().impl_fallback_delitem(w_key) def impl_length(self): # inefficient, but do we care? @@ -85,7 +85,7 @@ elif _is_sane_hash(space, w_lookup_type): return None else: - return self._as_rdict().getitem(w_lookup) + return self._as_rdict().impl_fallback_getitem(w_lookup) def impl_getitem_str(self, lookup): res = self.getcell(lookup, False) Modified: pypy/branch/better-map-instances/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/dictmultiobject.py Thu Aug 26 00:02:59 2010 @@ -310,7 +310,7 @@ if space.is_w(space.type(w_key), space.w_str): self.impl_setitem_str(self.space.str_w(w_key), w_value) else: - self._as_rdict().setitem(w_key, w_value) + self._as_rdict().impl_fallback_setitem(w_key, w_value) def impl_setitem_str(self, key, w_value, shadows_type=True): self.content[key] = w_value @@ -324,7 +324,7 @@ elif _is_sane_hash(space, w_key_type): raise KeyError else: - self._as_rdict().delitem(w_key) + self._as_rdict().impl_fallback_delitem(w_key) def impl_length(self): return len(self.content) @@ -344,7 +344,7 @@ elif _is_sane_hash(space, w_lookup_type): return None else: - return self._as_rdict().getitem(w_key) + return self._as_rdict().impl_fallback_getitem(w_key) def impl_iter(self): return StrIteratorImplementation(self.space, self) @@ -414,7 +414,7 @@ StrDictImplementation.impl_setitem_str( self, self.space.str_w(w_key), w_value, False) else: - self._as_rdict().setitem(w_key, w_value) + self._as_rdict().impl_fallback_setitem(w_key, w_value) def impl_shadows_anything(self): return (self._shadows_anything or @@ -446,7 +446,7 @@ elif _is_sane_hash(space, w_key_type): raise KeyError else: - self._as_rdict().delitem(w_key) + self._as_rdict().impl_fallback_delitem(w_key) def impl_get_builtin_indexed(self, i): return self.shadowed[i] Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py Thu Aug 26 00:02:59 2010 @@ -57,6 +57,9 @@ def materialize_r_dict(self, space, obj, w_d): raise NotImplementedError("abstract base class") + def remove_dict_entries(self, obj): + raise NotImplementedError("abstract base class") + class Terminator(AbstractAttribute): def __init__(self, w_cls, space): @@ -88,6 +91,8 @@ result._init_empty(terminator) return result + def remove_dict_entries(self, obj): + return self.copy(obj) class DictTerminator(Terminator): def __init__(self, w_cls, space): @@ -135,6 +140,8 @@ return Terminator.copy(self, obj) return Terminator.delete(self, obj, selector) + def remove_dict_entries(self, obj): + assert 0, "should be unreachable" class PlainAttribute(AbstractAttribute): def __init__(self, selector, back): @@ -197,13 +204,23 @@ self._copy_attr(obj, new_obj) return new_obj + def remove_dict_entries(self, obj): + new_obj = self.back.remove_dict_entries(obj) + if self.selector[1] != DICT: + self._copy_attr(obj, new_obj) + return new_obj +def _become(w_obj, new_obj): + # this is like the _become method, really, but we cannot use that due to + # RPython reasons + w_obj._set_mapdict_map(new_obj.map) + w_obj._set_mapdict_storage(new_obj.storage) # ____________________________________________________________ # object implementation -DICT = 6 # XXX meant '0'? -SLOT = 1 -SPECIAL = 2 +DICT = 0 +SPECIAL = 1 +SLOTS_STARTING_FROM = 2 from pypy.interpreter.baseobjspace import W_Root @@ -273,10 +290,12 @@ self._init_empty(w_subtype.terminator) def getslotvalue(self, member): - return self.map.read(self, (member.name, SLOT)) + key = (member.name, SLOTS_STARTING_FROM + member.index) + return self.map.read(self, key) def setslotvalue(self, member, w_value): - self.map.write(self, (member.name, SLOT), w_value) + key = (member.name, SLOTS_STARTING_FROM + member.index) + self.map.write(self, key, w_value) # used by _weakref implemenation @@ -314,7 +333,7 @@ elif _is_sane_hash(space, w_lookup_type): return None else: - return self._as_rdict().getitem(w_lookup) + return self._as_rdict().impl_fallback_getitem(w_lookup) def impl_getitem_str(self, key): return self.w_obj.getdictvalue(self.space, key) @@ -328,7 +347,7 @@ if space.is_w(space.type(w_key), space.w_str): self.impl_setitem_str(self.space.str_w(w_key), w_value) else: - self._as_rdict().setitem(w_key, w_value) + self._as_rdict().impl_fallback_setitem(w_key, w_value) def impl_delitem(self, w_key): space = self.space @@ -340,7 +359,7 @@ elif _is_sane_hash(space, w_key_type): raise KeyError else: - self._as_rdict().delitem(w_key) + self._as_rdict().impl_fallback_delitem(w_key) def impl_length(self): res = 0 @@ -355,9 +374,9 @@ return MapDictIteratorImplementation(self.space, self) def impl_clear(self): - # XXX implement me better, or provide a reasonable default - # XXX implementation in W_DictMultiObject - self._as_rdict().clear() + w_obj = self.w_obj + new_obj = w_obj._get_mapdict_map().remove_dict_entries(w_obj) + _become(w_obj, new_obj) def _clear_fields(self): self.w_obj = None @@ -369,19 +388,13 @@ materialize_r_dict(space, w_obj, self) self._clear_fields() return self - # XXX then the calls self._as_rdict().method() from above look like - # recursive calls, and a stack check is inserted, which is pointless. - # It would be better to return self.r_dict_content, I think def materialize_r_dict(space, obj, w_d): map = obj._get_mapdict_map() assert obj.getdict() is w_d new_obj = map.materialize_r_dict(space, obj, w_d) - # XXX this is like _become, really, but we cannot use that due to RPython - # reasons - obj._set_mapdict_map(new_obj.map) - obj._set_mapdict_storage(new_obj.storage) + _become(obj, new_obj) class MapDictIteratorImplementation(IteratorImplementation): def __init__(self, space, dictimplementation): Modified: pypy/branch/better-map-instances/pypy/objspace/std/sharingdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/sharingdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/sharingdict.py Thu Aug 26 00:02:59 2010 @@ -71,7 +71,7 @@ elif _is_sane_hash(space, w_lookup_type): return None else: - return self._as_rdict().getitem(w_lookup) + return self._as_rdict().impl_fallback_getitem(w_lookup) def impl_getitem_str(self, lookup): i = self.structure.lookup_position(lookup) @@ -84,7 +84,7 @@ if space.is_w(space.type(w_key), space.w_str): self.impl_setitem_str(self.space.str_w(w_key), w_value) else: - self._as_rdict().setitem(w_key, w_value) + self._as_rdict().impl_fallback_setitem(w_key, w_value) @unroll_safe def impl_setitem_str(self, key, w_value, shadows_type=True): @@ -132,7 +132,7 @@ elif _is_sane_hash(space, w_key_type): raise KeyError else: - self._as_rdict().delitem(w_key) + self._as_rdict().impl_fallback_delitem(w_key) def impl_length(self): return self.structure.length Modified: pypy/branch/better-map-instances/pypy/objspace/std/test/test_dictmultiobject.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/test/test_dictmultiobject.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/test/test_dictmultiobject.py Thu Aug 26 00:02:59 2010 @@ -696,6 +696,14 @@ assert self.impl.length() == 0 self.check_not_devolved() + def test_clear(self): + self.fill_impl() + assert self.impl.length() == 2 + self.impl.clear() + assert self.impl.length() == 0 + self.check_not_devolved() + + def test_keys(self): self.fill_impl() keys = self.impl.keys() Modified: pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py Thu Aug 26 00:02:59 2010 @@ -5,8 +5,9 @@ space = FakeSpace() class FakeMember(object): - def __init__(self, name): + def __init__(self, name, index=0): self.name = name + self.index = index class Class(object): def __init__(self, hasdict=True): @@ -51,11 +52,11 @@ def test_search(): aa = PlainAttribute(("b", DICT), PlainAttribute(("a", DICT), Terminator(None, None))) assert aa.search(DICT) is aa - assert aa.search(SLOT) is None + assert aa.search(SLOTS_STARTING_FROM) is None assert aa.search(SPECIAL) is None - bb = PlainAttribute(("C", SPECIAL), PlainAttribute(("A", SLOT), aa)) + bb = PlainAttribute(("C", SPECIAL), PlainAttribute(("A", SLOTS_STARTING_FROM), aa)) assert bb.search(DICT) is aa - assert bb.search(SLOT) is bb.back + assert bb.search(SLOTS_STARTING_FROM) is bb.back assert bb.search(SPECIAL) is bb def test_add_attribute(): @@ -193,6 +194,21 @@ assert obj2.storage == [501, 601, 701, 51, 61, 71] assert obj.map is obj2.map +def test_slots_same_name(): + cls = Class() + obj = cls.instantiate() + a = FakeMember("a") + b = FakeMember("a", 1) + c = FakeMember("a", 2) + obj.setslotvalue(a, 50) + obj.setslotvalue(b, 60) + obj.setslotvalue(c, 70) + assert obj.getslotvalue(a) == 50 + assert obj.getslotvalue(b) == 60 + assert obj.getslotvalue(c) == 70 + assert obj.storage == [50, 60, 70] + + def test_slots_no_dict(): cls = Class(hasdict=False) obj = cls.instantiate() @@ -366,6 +382,29 @@ assert a.y == 6 assert a.zz == 7 + def test_read_write_dict(self): + class A(object): + pass + a = A() + a.x = 5 + a.y = 6 + a.zz = 7 + d = a.__dict__ + assert d == {"x": 5, "y": 6, "zz": 7} + d['dd'] = 41 + assert a.dd == 41 + del a.x + assert d == {"y": 6, "zz": 7, 'dd': 41} + d2 = d.copy() + d2[1] = 2 + a.__dict__ = d2 + assert a.y == 6 + assert a.zz == 7 + assert a.dd == 41 + d['dd'] = 43 + assert a.dd == 41 + + def test_slot_name_conflict(self): class A(object): __slots__ = 'slot1' From arigo at codespeak.net Thu Aug 26 09:36:51 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 26 Aug 2010 09:36:51 +0200 (CEST) Subject: [pypy-svn] r76736 - in pypy/branch/better-map-instances/pypy/objspace/std: . test Message-ID: <20100826073651.7868E282BAD@codespeak.net> Author: arigo Date: Thu Aug 26 09:36:49 2010 New Revision: 76736 Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py Log: Test and fix. Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py Thu Aug 26 09:36:49 2010 @@ -115,9 +115,11 @@ class DevolvedDictTerminator(Terminator): def read(self, obj, selector): - w_dict = obj.getdict() - space = self.space - return space.finditem_str(w_dict, selector[0]) + if selector[1] == DICT: + w_dict = obj.getdict() + space = self.space + return space.finditem_str(w_dict, selector[0]) + return Terminator.read(self, obj, selector) def write(self, obj, selector, w_value): if selector[1] == DICT: Modified: pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py Thu Aug 26 09:36:49 2010 @@ -314,6 +314,7 @@ obj.setdictvalue(space, "a", 5) obj.setdictvalue(space, "b", 6) obj.setdictvalue(space, "c", 7) + obj.setdictvalue(space, "weakref", 42) devolve_dict(obj) assert obj.getdictvalue(space, "a") == 5 assert obj.getdictvalue(space, "b") == 6 @@ -321,6 +322,8 @@ assert obj.getslotvalue(a) == 50 assert obj.getslotvalue(b) == 60 assert obj.getslotvalue(c) == 70 + assert obj.getdictvalue(space, "weakref") == 42 + assert obj.getweakref() is None obj.setslotvalue(a, 501) obj.setslotvalue(b, 601) From arigo at codespeak.net Thu Aug 26 10:12:07 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 26 Aug 2010 10:12:07 +0200 (CEST) Subject: [pypy-svn] r76737 - in pypy/trunk/pypy/translator/goal: . test2 Message-ID: <20100826081207.76D5E282BAD@codespeak.net> Author: arigo Date: Thu Aug 26 10:12:05 2010 New Revision: 76737 Modified: pypy/trunk/pypy/translator/goal/app_main.py pypy/trunk/pypy/translator/goal/test2/test_app_main.py Log: Don't change to which lists sys.argv and sys.path are bound. Add tests for that case. Modified: pypy/trunk/pypy/translator/goal/app_main.py ============================================================================== --- pypy/trunk/pypy/translator/goal/app_main.py (original) +++ pypy/trunk/pypy/translator/goal/app_main.py Thu Aug 26 10:12:05 2010 @@ -292,8 +292,8 @@ else: raise CommandLineError('unrecognized option %r' % (arg,)) i += 1 - sys.argv = argv[i:] - if not sys.argv: + sys.argv[:] = argv[i:] # don't change the list that sys.argv is bound to + if not sys.argv: # (relevant in case of "reload(sys)") sys.argv.append('') run_stdin = True return locals() @@ -478,6 +478,10 @@ reset.append(('PYTHONINSPECT', os.environ.get('PYTHONINSPECT', ''))) os.environ['PYTHONINSPECT'] = os.environ['PYTHONINSPECT_'] + # no one should change to which lists sys.argv and sys.path are bound + old_argv = sys.argv + old_path = sys.path + from pypy.module.sys.version import PYPY_VERSION sys.pypy_version_info = PYPY_VERSION sys.pypy_initial_path = pypy_initial_path @@ -490,3 +494,5 @@ sys.ps1 = '>>> ' # restore the normal ones, in case sys.ps2 = '... ' # we are dropping to CPython's prompt import os; os.environ.update(reset) + assert old_argv is sys.argv + assert old_path is sys.path Modified: pypy/trunk/pypy/translator/goal/test2/test_app_main.py ============================================================================== --- pypy/trunk/pypy/translator/goal/test2/test_app_main.py (original) +++ pypy/trunk/pypy/translator/goal/test2/test_app_main.py Thu Aug 26 10:12:05 2010 @@ -1,6 +1,7 @@ """ Tests for the entry point of pypy-c, app_main.py. """ +from __future__ import with_statement import py import sys, os, re import autopath From arigo at codespeak.net Thu Aug 26 10:33:55 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 26 Aug 2010 10:33:55 +0200 (CEST) Subject: [pypy-svn] r76738 - in pypy/trunk/pypy/translator: c/test platform platform/test Message-ID: <20100826083355.6E98B282BAD@codespeak.net> Author: arigo Date: Thu Aug 26 10:33:52 2010 New Revision: 76738 Modified: pypy/trunk/pypy/translator/c/test/test_standalone.py pypy/trunk/pypy/translator/platform/__init__.py pypy/trunk/pypy/translator/platform/test/test_platform.py Log: Support for the env var CC containing the name of the compiler followed by some compiler options. Modified: pypy/trunk/pypy/translator/c/test/test_standalone.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_standalone.py (original) +++ pypy/trunk/pypy/translator/c/test/test_standalone.py Thu Aug 26 10:33:52 2010 @@ -604,6 +604,33 @@ out, err = cbuilder.cmdexec("a b") assert out == "3" + def test_gcc_options(self): + # check that the env var CC is correctly interpreted, even if + # it contains the compiler name followed by some options. + if sys.platform == 'win32': + py.test.skip("only for gcc") + + from pypy.rpython.lltypesystem import lltype, rffi + dir = udir.ensure('test_gcc_options', dir=1) + dir.join('someextraheader.h').write('#define someextrafunc() 42\n') + eci = ExternalCompilationInfo(includes=['someextraheader.h']) + someextrafunc = rffi.llexternal('someextrafunc', [], lltype.Signed, + compilation_info=eci) + + def entry_point(argv): + return someextrafunc() + + old_cc = os.environ.get('CC') + try: + os.environ['CC'] = 'gcc -I%s' % dir + t, cbuilder = self.compile(entry_point) + finally: + if old_cc is None: + del os.environ['CC'] + else: + os.environ['CC'] = old_cc + + class TestMaemo(TestStandalone): def setup_class(cls): py.test.skip("TestMaemo: tests skipped for now") Modified: pypy/trunk/pypy/translator/platform/__init__.py ============================================================================== --- pypy/trunk/pypy/translator/platform/__init__.py (original) +++ pypy/trunk/pypy/translator/platform/__init__.py Thu Aug 26 10:33:52 2010 @@ -99,15 +99,20 @@ self.__dict__ == other.__dict__) def key(self): - bits = [self.__class__.__name__, 'cc=%s' % self.cc] + bits = [self.__class__.__name__, 'cc=%r' % self.cc] for varname in self.relevant_environ: - bits.append('%s=%s' % (varname, os.environ.get(varname))) + bits.append('%s=%r' % (varname, os.environ.get(varname))) return ' '.join(bits) # some helpers which seem to be cross-platform enough def _execute_c_compiler(self, cc, args, outname, cwd=None): log.execute(cc + ' ' + ' '.join(args)) + # 'cc' can also contain some options for the C compiler; + # e.g. it can be "gcc -m32". We handle it by splitting on ' '. + cclist = cc.split() + cc = cclist[0] + args = cclist[1:] + args returncode, stdout, stderr = _run_subprocess(cc, args, self.c_environ, cwd) self._handle_error(returncode, stderr, stdout, outname) Modified: pypy/trunk/pypy/translator/platform/test/test_platform.py ============================================================================== --- pypy/trunk/pypy/translator/platform/test/test_platform.py (original) +++ pypy/trunk/pypy/translator/platform/test/test_platform.py Thu Aug 26 10:33:52 2010 @@ -131,7 +131,7 @@ self.cc = 'xcc' x = XPlatform() res = x.key() - assert res.startswith('XPlatform cc=xcc CPATH=') + assert res.startswith("XPlatform cc='xcc' CPATH=") def test_equality(): class X(Platform): From arigo at codespeak.net Thu Aug 26 12:08:20 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 26 Aug 2010 12:08:20 +0200 (CEST) Subject: [pypy-svn] r76739 - pypy/build/bot2/pypybuildbot Message-ID: <20100826100820.18EAF282B9D@codespeak.net> Author: arigo Date: Thu Aug 26 12:08:19 2010 New Revision: 76739 Modified: pypy/build/bot2/pypybuildbot/master.py Log: Tweak the times. Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Thu Aug 26 12:08:19 2010 @@ -138,7 +138,7 @@ ], hour=0, minute=45), Nightly("nightly-4-00", [ # rule: what we pick here on tannit should take at most 8 cores - # and be hopefully finished after 2 hours 50 minutes + # and be hopefully finished after 2 hours LINUX32, # on tannit32, uses 4 cores JITLINUX32, # on tannit32, uses 1 core OJITLINUX32, # on tannit32, uses 1 core @@ -146,14 +146,17 @@ APPLVLWIN32, # on bigboard STACKLESSAPPLVLFREEBSD64, # on headless ], hour=4, minute=0), - Nightly("nightly-6-50", [ - # the remaining stuff on tannit, which should also take at most - # 8 cores - LINUX64, # on tannit64, uses 4 cores + Nightly("nightly-6-00", [ + # there should be only JITLINUX32 that takes a bit longer than + # that. We can use a few more cores now. APPLVLLINUX32, # on tannit32, uses 1 core APPLVLLINUX64, # on tannit64, uses 1 core STACKLESSAPPLVLLINUX32, # on tannit32, uses 1 core - ], hour=6, minute=50), + ], hour=6, minute=0), + Nightly("nightly-7-00", [ + # the remaining quickly-run stuff on tannit + LINUX64, # on tannit64, uses 4 cores + ], hour=7, minute=0), ], 'status': [status], From arigo at codespeak.net Thu Aug 26 12:09:30 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 26 Aug 2010 12:09:30 +0200 (CEST) Subject: [pypy-svn] r76740 - pypy/build/bot2/pypybuildbot Message-ID: <20100826100930.259A0282B9D@codespeak.net> Author: arigo Date: Thu Aug 26 12:09:26 2010 New Revision: 76740 Modified: pypy/build/bot2/pypybuildbot/builds.py Log: It does not make sense to run Psyco nightly. Modified: pypy/build/bot2/pypybuildbot/builds.py ============================================================================== --- pypy/build/bot2/pypybuildbot/builds.py (original) +++ pypy/build/bot2/pypybuildbot/builds.py Thu Aug 26 12:09:26 2010 @@ -199,13 +199,13 @@ masterdest=WithProperties(resfile), workdir=".")) - self.addStep(ShellCmd( - description="run on top of python with psyco", - command=["python", "runner.py", '--output-filename', 'result.json', - '--pypy-c', 'psyco/python_with_psyco.sh', - '--revision', WithProperties('%(got_revision)s'), - '--upload', #'--force-host', 'bigdog', - '--branch', WithProperties('%(branch)s'), - ], - workdir='./benchmarks', - haltOnFailure=True)) +## self.addStep(ShellCmd( +## description="run on top of python with psyco", +## command=["python", "runner.py", '--output-filename', 'result.json', +## '--pypy-c', 'psyco/python_with_psyco.sh', +## '--revision', WithProperties('%(got_revision)s'), +## '--upload', #'--force-host', 'bigdog', +## '--branch', WithProperties('%(branch)s'), +## ], +## workdir='./benchmarks', +## haltOnFailure=True)) From arigo at codespeak.net Thu Aug 26 12:13:23 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 26 Aug 2010 12:13:23 +0200 (CEST) Subject: [pypy-svn] r76741 - pypy/extradoc/pypy.org/source Message-ID: <20100826101323.AB884282B9D@codespeak.net> Author: arigo Date: Thu Aug 26 12:13:21 2010 New Revision: 76741 Modified: pypy/extradoc/pypy.org/source/download.txt Log: Updates. Modified: pypy/extradoc/pypy.org/source/download.txt ============================================================================== --- pypy/extradoc/pypy.org/source/download.txt (original) +++ pypy/extradoc/pypy.org/source/download.txt Thu Aug 26 12:13:21 2010 @@ -198,7 +198,8 @@ +++++++++++++++++++++++++++++++++++++++++++++++++++ To build 32-bit versions of ``pypy-c`` you need to run ``translate.py`` -in a 32-bit version of Python. You can check with:: +in a 32-bit version of Python, and to make sure that the C compiler it +uses is also 32-bit. You can check the first condition with:: $ python Python 2.6.2 (...) @@ -206,22 +207,36 @@ >>> sys.maxint This prints 9223372036854775807 in 64-bit versions and 2147483647 in -32-bit versions. +32-bit versions. Moreover, if your Python is 32-bit but your C compiler +is 64-bit, you will get ``AssertionError: Mixed configuration of the +word size of the machine`` when running ``translate.py.`` + +**On Linux,** the recommended way is to install and run in a fully +32-bit chroot (e.g. with the ``schroot`` Ubuntu package). The +alternative has issues about building some extension modules, but if you +want to try it, first compile yourself a 32-bit Python, e.g.:: -On Linux, you may have to compile yourself a 32-bit Python, e.g.:: + # on Ubuntu Linux, you need at least: + apt-get install libc6-dev-i386 ia32-libs lib32z1-dev lib32ncurses5-dev cd Python-2.6.4 CC="gcc -m32" LDFLAGS="-L/lib32 -L/usr/lib32 \ - -L`pwd`/lib32 -Wl,-rpath,/lib32 -Wl,-rpath,/usr/lib32" \ + -Wl,-rpath,/lib32 -Wl,-rpath,/usr/lib32" \ ./configure make - # and then use this ./python to run translate.py -On Mac OS/X: the easiest is to systematically use Python 2.5 when +And then run ``translate.py`` as follows:: + + cd pypy/translator/goal + CC="gcc -m32" /path/to/32bit/python translate.py ... + + +**On Mac OS/X,** the easiest is to systematically use Python 2.5 when working with PyPy. Indeed, the standard Python 2.5 runs in 32-bit mode. +(XXX how to run the compiler in 32-bit mode??) -On Windows, the only known way is to install a 32-bit Python -manually. +**On Windows,** the only known way is to install a 32-bit Python +manually. (XXX how to run the compiler in 32-bit mode??) .. _`x86 (IA-32)`: http://en.wikipedia.org/wiki/IA-32 From arigo at codespeak.net Thu Aug 26 13:30:21 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 26 Aug 2010 13:30:21 +0200 (CEST) Subject: [pypy-svn] r76742 - pypy/extradoc/pypy.org/source Message-ID: <20100826113021.98513282B9D@codespeak.net> Author: arigo Date: Thu Aug 26 13:30:20 2010 New Revision: 76742 Modified: pypy/extradoc/pypy.org/source/download.txt Log: Fix py.test. Modified: pypy/extradoc/pypy.org/source/download.txt ============================================================================== --- pypy/extradoc/pypy.org/source/download.txt (original) +++ pypy/extradoc/pypy.org/source/download.txt Thu Aug 26 13:30:20 2010 @@ -247,6 +247,4 @@ .. _`sandboxing`: features.html#sandboxing .. _`stackless`: http://www.stackless.com/ .. _`greenlets`: http://codespeak.net/svn/greenlet/trunk/doc/greenlet.txt -.. _`pypy-1.3-src.tar.bz2`: release.html -.. _`pypy-1.3-src.zip`: release.html .. _Subversion: http://subversion.tigris.org/ From arigo at codespeak.net Thu Aug 26 13:42:51 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 26 Aug 2010 13:42:51 +0200 (CEST) Subject: [pypy-svn] r76743 - pypy/extradoc/pypy.org Message-ID: <20100826114251.C54BC282BAD@codespeak.net> Author: arigo Date: Thu Aug 26 13:42:49 2010 New Revision: 76743 Modified: pypy/extradoc/pypy.org/download.html Log: Re-run yatiblog. Modified: pypy/extradoc/pypy.org/download.html ============================================================================== --- pypy/extradoc/pypy.org/download.html (original) +++ pypy/extradoc/pypy.org/download.html Thu Aug 26 13:42:49 2010 @@ -187,7 +187,8 @@

Note on building a 32-bit version on 64-bit systems

To build 32-bit versions of pypy-c you need to run translate.py -in a 32-bit version of Python. You can check with:

+in a 32-bit version of Python, and to make sure that the C compiler it +uses is also 32-bit. You can check the first condition with:

 $ python
 Python 2.6.2 (...)
@@ -195,20 +196,32 @@
 >>> sys.maxint

This prints 9223372036854775807 in 64-bit versions and 2147483647 in -32-bit versions.

-

On Linux, you may have to compile yourself a 32-bit Python, e.g.:

+32-bit versions. Moreover, if your Python is 32-bit but your C compiler +is 64-bit, you will get AssertionError: Mixed configuration of the +word size of the machine when running translate.py.

+

On Linux, the recommended way is to install and run in a fully +32-bit chroot (e.g. with the schroot Ubuntu package). The +alternative has issues about building some extension modules, but if you +want to try it, first compile yourself a 32-bit Python, e.g.:

+# on Ubuntu Linux, you need at least:
+apt-get install libc6-dev-i386 ia32-libs lib32z1-dev lib32ncurses5-dev
 cd Python-2.6.4
 CC="gcc -m32" LDFLAGS="-L/lib32 -L/usr/lib32 \
-  -L`pwd`/lib32 -Wl,-rpath,/lib32 -Wl,-rpath,/usr/lib32" \
+  -Wl,-rpath,/lib32 -Wl,-rpath,/usr/lib32" \
   ./configure
 make
-# and then use this ./python to run translate.py
-

On Mac OS/X: the easiest is to systematically use Python 2.5 when -working with PyPy. Indeed, the standard Python 2.5 runs in 32-bit mode.

-

On Windows, the only known way is to install a 32-bit Python -manually.

+

And then run translate.py as follows:

+
+cd pypy/translator/goal
+CC="gcc -m32" /path/to/32bit/python translate.py ...
+
+

On Mac OS/X, the easiest is to systematically use Python 2.5 when +working with PyPy. Indeed, the standard Python 2.5 runs in 32-bit mode. +(XXX how to run the compiler in 32-bit mode??)

+

On Windows, the only known way is to install a 32-bit Python +manually. (XXX how to run the compiler in 32-bit mode??)

From arigo at codespeak.net Thu Aug 26 13:55:43 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 26 Aug 2010 13:55:43 +0200 (CEST) Subject: [pypy-svn] r76744 - pypy/trunk/pypy/rpython/memory/gc Message-ID: <20100826115543.8A5A7282BDC@codespeak.net> Author: arigo Date: Thu Aug 26 13:55:41 2010 New Revision: 76744 Modified: pypy/trunk/pypy/rpython/memory/gc/markcompact.py Log: Update the debug_prints in markcompact.py too. Modified: pypy/trunk/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/markcompact.py Thu Aug 26 13:55:41 2010 @@ -3,7 +3,8 @@ from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup from pypy.rpython.memory.gc.base import MovingGCBase -from pypy.rlib.debug import ll_assert +from pypy.rlib.debug import ll_assert, have_debug_prints +from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE from pypy.rpython.memory.support import get_address_stack, get_address_deque from pypy.rpython.memory.support import AddressDict @@ -90,7 +91,7 @@ total_collection_count = 0 def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE, space_size=4096): - import py; py.test.skip("Disabled for now, sorry") + #import py; py.test.skip("Disabled for now, sorry") self.param_space_size = space_size MovingGCBase.__init__(self, config, chunk_size) @@ -98,8 +99,7 @@ self.space_size = self.param_space_size self.next_collect_after = self.param_space_size/2 # whatever... - if self.config.gcconfig.debugprint: - self.program_start_time = time.time() + self.program_start_time = time.time() self.space = llarena.arena_malloc(self.space_size, True) ll_assert(bool(self.space), "couldn't allocate arena") self.free = self.space @@ -289,15 +289,16 @@ return weakref_offsets def debug_collect_start(self): - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void) - llop.debug_print(lltype.Void, - ".----------- Full collection ------------------") + if have_debug_prints(): + debug_start("gc-collect") + debug_print() + debug_print(".----------- Full collection ------------------") start_time = time.time() - return start_time + return start_time + return -1 def debug_collect_finish(self, start_time): - if self.config.gcconfig.debugprint: + if start_time != -1: end_time = time.time() elapsed_time = end_time - start_time self.total_collection_time += elapsed_time @@ -305,20 +306,16 @@ total_program_time = end_time - self.program_start_time ct = self.total_collection_time cc = self.total_collection_count - llop.debug_print(lltype.Void, - "| number of collections so far ", - cc) - llop.debug_print(lltype.Void, - "| total collections per second: ", - cc / total_program_time) - llop.debug_print(lltype.Void, - "| total time in markcompact-collect: ", - ct, "seconds") - llop.debug_print(lltype.Void, - "| percentage collection<->total time:", - ct * 100.0 / total_program_time, "%") - llop.debug_print(lltype.Void, - "`----------------------------------------------") + debug_print("| number of collections so far ", + cc) + debug_print("| total collections per second: ", + cc / total_program_time) + debug_print("| total time in markcompact-collect: ", + ct, "seconds") + debug_print("| percentage collection<->total time:", + ct * 100.0 / total_program_time, "%") + debug_print("`----------------------------------------------") + debug_stop("gc-collect") def update_run_finalizers(self): From arigo at codespeak.net Thu Aug 26 14:39:05 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 26 Aug 2010 14:39:05 +0200 (CEST) Subject: [pypy-svn] r76745 - pypy/trunk/pypy/rpython/memory/gc Message-ID: <20100826123905.6615E282B9D@codespeak.net> Author: arigo Date: Thu Aug 26 14:39:03 2010 New Revision: 76745 Modified: pypy/trunk/pypy/rpython/memory/gc/markcompact.py Log: Re-disable. Modified: pypy/trunk/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/markcompact.py Thu Aug 26 14:39:03 2010 @@ -91,7 +91,7 @@ total_collection_count = 0 def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE, space_size=4096): - #import py; py.test.skip("Disabled for now, sorry") + import py; py.test.skip("Disabled for now, sorry") self.param_space_size = space_size MovingGCBase.__init__(self, config, chunk_size) From arigo at codespeak.net Thu Aug 26 14:40:07 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 26 Aug 2010 14:40:07 +0200 (CEST) Subject: [pypy-svn] r76746 - pypy/branch/markcompact Message-ID: <20100826124007.D83FE282B9D@codespeak.net> Author: arigo Date: Thu Aug 26 14:40:05 2010 New Revision: 76746 Added: pypy/branch/markcompact/ - copied from r76745, pypy/trunk/ Log: A branch in which to try to re-enable at least some parts of the mark-and-compact GC. From arigo at codespeak.net Thu Aug 26 17:27:35 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 26 Aug 2010 17:27:35 +0200 (CEST) Subject: [pypy-svn] r76747 - pypy/branch/better-map-instances/pypy/objspace/std Message-ID: <20100826152735.4531C282B9C@codespeak.net> Author: arigo Date: Thu Aug 26 17:27:32 2010 New Revision: 76747 Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py Log: A note. Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py Thu Aug 26 17:27:32 2010 @@ -292,6 +292,8 @@ self._init_empty(w_subtype.terminator) def getslotvalue(self, member): + # XXX we don't need member here, it's enough to have member.index + # XXX and revert the change to the signature of getslotvalue(), maybe. key = (member.name, SLOTS_STARTING_FROM + member.index) return self.map.read(self, key) From arigo at codespeak.net Thu Aug 26 17:28:12 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 26 Aug 2010 17:28:12 +0200 (CEST) Subject: [pypy-svn] r76748 - in pypy/branch/markcompact/pypy/rpython/memory/gc: . test Message-ID: <20100826152812.79049282B9D@codespeak.net> Author: arigo Date: Thu Aug 26 17:28:09 2010 New Revision: 76748 Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/base.py pypy/branch/markcompact/pypy/rpython/memory/gc/generation.py pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py pypy/branch/markcompact/pypy/rpython/memory/gc/test/test_direct.py Log: Random changes, mostly killing support for resizing the spaces in markcompact. Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/base.py Thu Aug 26 17:28:09 2010 @@ -348,3 +348,23 @@ globals(), locals(), [classname]) GCClass = getattr(module, classname) return GCClass, GCClass.TRANSLATION_PARAMS + +def read_from_env(varname): + import os + value = os.environ.get(varname) + if value: + realvalue = value[:-1] + if value[-1] in 'kK': + factor = 1024 + elif value[-1] in 'mM': + factor = 1024*1024 + elif value[-1] in 'gG': + factor = 1024*1024*1024 + else: + factor = 1 + realvalue = value + try: + return int(float(realvalue) * factor) + except ValueError: + pass + return -1 Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/generation.py Thu Aug 26 17:28:09 2010 @@ -2,6 +2,7 @@ from pypy.rpython.memory.gc.semispace import SemiSpaceGC from pypy.rpython.memory.gc.semispace import GCFLAG_EXTERNAL, GCFLAG_FORWARDED from pypy.rpython.memory.gc.semispace import GC_HASH_TAKEN_ADDR +from pypy.rpython.memory.gc.base import read_from_env from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE @@ -625,18 +626,7 @@ import os def nursery_size_from_env(): - value = os.environ.get('PYPY_GENERATIONGC_NURSERY') - if value: - if value[-1] in 'kK': - factor = 1024 - value = value[:-1] - else: - factor = 1 - try: - return int(value) * factor - except ValueError: - pass - return -1 + return read_from_env('PYPY_GENERATIONGC_NURSERY') def best_nursery_size_for_L2cache(L2cache): # Heuristically, the best nursery size to choose is about half Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Thu Aug 26 17:28:09 2010 @@ -2,26 +2,19 @@ import time from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup -from pypy.rpython.memory.gc.base import MovingGCBase +from pypy.rpython.memory.gc.base import MovingGCBase, read_from_env from pypy.rlib.debug import ll_assert, have_debug_prints from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE from pypy.rpython.memory.support import get_address_stack, get_address_deque from pypy.rpython.memory.support import AddressDict from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage -from pypy.rlib.rarithmetic import ovfcheck +from pypy.rlib.rarithmetic import ovfcheck, LONG_BIT from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.objectmodel import we_are_translated from pypy.rpython.lltypesystem import rffi from pypy.rpython.memory.gcheader import GCHeaderBuilder -first_gcflag = 1 << 16 -GCFLAG_MARKBIT = first_gcflag << 0 -GCFLAG_HASHTAKEN = first_gcflag << 1 # someone already asked for the hash -GCFLAG_HASHFIELD = first_gcflag << 2 # we have an extra hash field - -memoryError = MemoryError() - # Mark'n'compact garbage collector # # main point of this GC is to save as much memory as possible @@ -34,41 +27,37 @@ # this gc works more or less like semispace, but has some essential # differencies. The main difference is that we have separate phases of # marking and assigning pointers, hence order of objects is preserved. -# This means we can reuse the same space if it did not grow enough. -# More importantly, in case we need to resize space we can copy it bit by -# bit, hence avoiding double memory consumption at peak times +# This means we can reuse the same space, overwriting it as we collect. -# so the algorithm itself is performed in 3 stages (module weakrefs and -# finalizers) +# so the algorithm itself is performed in 3 stages (modulo weakrefs and +# finalizers): # 1. We mark alive objects # 2. We walk all objects and assign forward pointers in the same order, # also updating all references -# 3. We compact the space by moving. In case we move to the same space, -# we use arena_new_view trick, which looks like new space to tests, -# but compiles to the same pointer. Also we use raw_memmove in case -# objects overlap. - -# Exact algorithm for space resizing: we keep allocated more space than needed -# (2x, can be even more), but it's full of zeroes. After each collection, -# we bump next_collect_after which is a marker where to start each collection. -# It should be exponential (but less than 2) from the size occupied by objects +# 3. We compact the space by moving. We use 'arena_new_view' trick, which +# looks like new space to tests, but compiles to the same pointer. +# Also we use raw_memmove in case the object overlaps with its destination. + +# After each collection, we bump 'next_collect_after' which is a marker +# where to start each collection. It should be exponential (but less +# than 2) from the size occupied by objects so far. # field optimization - we don't need forward pointer and flags at the same -# time. Instead we copy list of tids when we know how many objects are alive -# and store forward pointer there. +# time. Instead we copy the TIDs in a list when we know how many objects are +# alive, and store the forward pointer in the old object header. +first_gcflag_bit = LONG_BIT//2 +first_gcflag = 1 << first_gcflag_bit +GCFLAG_HASHTAKEN = first_gcflag << 0 # someone already asked for the hash +GCFLAG_HASHFIELD = first_gcflag << 1 # we have an extra hash field +GCFLAG_MARKBIT = first_gcflag << 2 +# note that only the first 2 bits are preserved during a collection! -# in case we need to grow space, we use -# current_space_size * FREE_SPACE_MULTIPLIER / FREE_SPACE_DIVIDER + needed -FREE_SPACE_MULTIPLIER = 3 -FREE_SPACE_DIVIDER = 2 -FREE_SPACE_ADD = 256 -# XXX adjust -GC_CLEARANCE = 32*1024 TID_TYPE = llgroup.HALFWORD BYTES_PER_TID = rffi.sizeof(TID_TYPE) +TID_BACKUP = lltype.FixedSizeArray(TID_TYPE, 1) class MarkCompactGC(MovingGCBase): @@ -77,37 +66,40 @@ withhash_flag_is_in_field = 'tid', GCFLAG_HASHFIELD # ^^^ all prebuilt objects have GCFLAG_HASHTAKEN, but only some have # GCFLAG_HASHFIELD (and then they are one word longer). - TID_BACKUP = lltype.Array(TID_TYPE, hints={'nolength':True}) - WEAKREF_OFFSETS = lltype.Array(lltype.Signed) + WEAKREF_OFFSETS = rffi.CArray(lltype.Signed) + # The default space size is 1.9375 GB, i.e. almost 2 GB, allocated as + # a big mmap. The process does not actually consume that space until + # needed, of course. + TRANSLATION_PARAMS = {'space_size': int((1 + 15.0/16)*1024*1024*1024)} - TRANSLATION_PARAMS = {'space_size': 8*1024*1024} # XXX adjust - - malloc_zero_filled = True + malloc_zero_filled = False inline_simple_malloc = True inline_simple_malloc_varsize = True - first_unused_gcflag = first_gcflag << 3 total_collection_time = 0.0 total_collection_count = 0 def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE, space_size=4096): - import py; py.test.skip("Disabled for now, sorry") - self.param_space_size = space_size MovingGCBase.__init__(self, config, chunk_size) + self.space_size = space_size def setup(self): - self.space_size = self.param_space_size - self.next_collect_after = self.param_space_size/2 # whatever... + envsize = max_size_from_env() + if envsize >= 4096: + self.space_size = envsize & ~4095 + + self.next_collect_after = min(self.space_size, + 4*1024*1024) # 4MB initially self.program_start_time = time.time() - self.space = llarena.arena_malloc(self.space_size, True) - ll_assert(bool(self.space), "couldn't allocate arena") + self.space = llarena.arena_malloc(self.space_size, False) + if not self.space: + raise CannotAllocateGCArena self.free = self.space - self.top_of_space = self.space + self.next_collect_after MovingGCBase.setup(self) self.objects_with_finalizers = self.AddressDeque() self.objects_with_weakrefs = self.AddressStack() - self.tid_backup = lltype.nullptr(self.TID_BACKUP) + self.tid_backup = lltype.nullptr(TID_BACKUP) def init_gc_object(self, addr, typeid16, flags=0): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) @@ -117,135 +109,106 @@ hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) flags |= GCFLAG_HASHTAKEN hdr.tid = self.combine(typeid16, flags) - # XXX we can store forward_ptr to itself, if we fix C backend - # so that get_forwarding_address(obj) returns - # obj itself if obj is a prebuilt object - def malloc_fixedsize_clear(self, typeid16, size, can_collect, - has_finalizer=False, contains_weakptr=False): - size_gc_header = self.gcheaderbuilder.size_gc_header - totalsize = size_gc_header + size + def _get_memory(self, totalsize): + # also counts the space that will be needed during the following + # collection to store the TID + requested_size = raw_malloc_usage(totalsize) + BYTES_PER_TID + self.next_collect_after -= requested_size + if self.next_collect_after < 0: + self.obtain_free_space(requested_size) result = self.free - if raw_malloc_usage(totalsize) > self.top_of_space - result: - result = self.obtain_free_space(totalsize) + self.free += totalsize llarena.arena_reserve(result, totalsize) + return result + _get_memory._always_inline_ = True + + def _get_totalsize_var(self, nonvarsize, itemsize, length): + try: + varsize = ovfcheck(itemsize * length) + except OverflowError: + raise MemoryError + totalsize = llarena.round_up_for_allocation(nonvarsize + varsize) + if totalsize < 0: # if wrapped around + raise MemoryError + return totalsize + _get_totalsize_var._always_inline_ = True + + def _setup_object(self, result, typeid16, has_finalizer, contains_weakptr): + size_gc_header = self.gcheaderbuilder.size_gc_header self.init_gc_object(result, typeid16) - self.free += totalsize if has_finalizer: self.objects_with_finalizers.append(result + size_gc_header) if contains_weakptr: self.objects_with_weakrefs.append(result + size_gc_header) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) - + _setup_object._always_inline_ = True + + def malloc_fixedsize(self, typeid16, size, can_collect, + has_finalizer=False, contains_weakptr=False): + size_gc_header = self.gcheaderbuilder.size_gc_header + totalsize = size_gc_header + size + result = self._get_memory(totalsize) + return self._setup_object(result, typeid16, has_finalizer, + contains_weakptr) + + def malloc_fixedsize_clear(self, typeid16, size, can_collect, + has_finalizer=False, contains_weakptr=False): + size_gc_header = self.gcheaderbuilder.size_gc_header + totalsize = size_gc_header + size + result = self._get_memory(totalsize) + llmemory.raw_memclear(result, totalsize) + return self._setup_object(result, typeid16, has_finalizer, + contains_weakptr) + def malloc_varsize_clear(self, typeid16, length, size, itemsize, offset_to_length, can_collect): size_gc_header = self.gcheaderbuilder.size_gc_header nonvarsize = size_gc_header + size - try: - varsize = ovfcheck(itemsize * length) - totalsize = ovfcheck(nonvarsize + varsize) - except OverflowError: - raise memoryError - result = self.free - if raw_malloc_usage(totalsize) > self.top_of_space - result: - result = self.obtain_free_space(totalsize) - llarena.arena_reserve(result, totalsize) - self.init_gc_object(result, typeid16) + totalsize = self._get_totalsize_var(nonvarsize, itemize, length) + result = self._get_memory(totalsize) + llmemory.raw_memclear(result, totalsize) (result + size_gc_header + offset_to_length).signed[0] = length - self.free = result + llarena.round_up_for_allocation(totalsize) - return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) + return self._setup_object(result, typeid16, False, False) - def obtain_free_space(self, totalsize): - # a bit of tweaking to maximize the performance and minimize the - # amount of code in an inlined version of malloc_fixedsize_clear() - if not self.try_obtain_free_space(totalsize): - raise memoryError - return self.free + def obtain_free_space(self, requested_size): + self.markcompactcollect() + self.next_collect_after -= requested_size + if self.next_collect_after < 0: + raise MemoryError obtain_free_space._dont_inline_ = True - def try_obtain_free_space(self, needed): - needed = raw_malloc_usage(needed) - while 1: - self.markcompactcollect(needed) - missing = needed - (self.top_of_space - self.free) - if missing < 0: - return True - - def new_space_size(self, occupied, needed): - res = (occupied * FREE_SPACE_MULTIPLIER / - FREE_SPACE_DIVIDER + FREE_SPACE_ADD + needed) - # align it to 4096, which is somewhat around page size - return ((res/4096) + 1) * 4096 - - def double_space_size(self, minimal_size): - while self.space_size <= minimal_size: - self.space_size *= 2 - toaddr = llarena.arena_malloc(self.space_size, True) - return toaddr - - def compute_alive_objects(self): - fromaddr = self.space - addraftercollect = self.space - num = 1 - while fromaddr < self.free: - size_gc_header = self.gcheaderbuilder.size_gc_header - tid = llmemory.cast_adr_to_ptr(fromaddr, lltype.Ptr(self.HDR)).tid - obj = fromaddr + size_gc_header - objsize = self.get_size(obj) - objtotalsize = size_gc_header + objsize - if self.marked(obj): - copy_has_hash_field = ((tid & GCFLAG_HASHFIELD) != 0 or - ((tid & GCFLAG_HASHTAKEN) != 0 and - addraftercollect < fromaddr)) - addraftercollect += raw_malloc_usage(objtotalsize) - if copy_has_hash_field: - addraftercollect += llmemory.sizeof(lltype.Signed) - num += 1 - fromaddr += objtotalsize - if tid & GCFLAG_HASHFIELD: - fromaddr += llmemory.sizeof(lltype.Signed) - ll_assert(addraftercollect <= fromaddr, - "markcompactcollect() is trying to increase memory usage") - self.totalsize_of_objs = addraftercollect - self.space - return num - def collect(self, gen=0): self.markcompactcollect() - - def markcompactcollect(self, needed=0): + + def markcompactcollect(self): start_time = self.debug_collect_start() self.debug_check_consistency() - self.to_see = self.AddressStack() - self.mark_roots_recursively() - if (self.objects_with_finalizers.non_empty() or - self.run_finalizers.non_empty()): - self.mark_objects_with_finalizers() - self._trace_and_mark() + # + # Mark alive objects + # + self.to_see = self.AddressDeque() + self.trace_from_roots() + #if (self.objects_with_finalizers.non_empty() or + # self.run_finalizers.non_empty()): + # self.mark_objects_with_finalizers() + # self._trace_and_mark() self.to_see.delete() - num_of_alive_objs = self.compute_alive_objects() - size_of_alive_objs = self.totalsize_of_objs - totalsize = self.new_space_size(size_of_alive_objs, needed + - num_of_alive_objs * BYTES_PER_TID) - tid_backup_size = (llmemory.sizeof(self.TID_BACKUP, 0) + - llmemory.sizeof(TID_TYPE) * num_of_alive_objs) - used_space_now = self.next_collect_after + raw_malloc_usage(tid_backup_size) - if totalsize >= self.space_size or used_space_now >= self.space_size: - toaddr = self.double_space_size(totalsize) - llarena.arena_reserve(toaddr + size_of_alive_objs, tid_backup_size) - self.tid_backup = llmemory.cast_adr_to_ptr( - toaddr + size_of_alive_objs, - lltype.Ptr(self.TID_BACKUP)) - resizing = True - else: - toaddr = llarena.arena_new_view(self.space) - llarena.arena_reserve(self.top_of_space, tid_backup_size) - self.tid_backup = llmemory.cast_adr_to_ptr( - self.top_of_space, - lltype.Ptr(self.TID_BACKUP)) - resizing = False - self.next_collect_after = totalsize - weakref_offsets = self.collect_weakref_offsets() - finaladdr = self.update_forward_pointers(toaddr, num_of_alive_objs) + # + # Prepare new views on the same memory + # + toaddr = llarena.arena_new_view(self.space) + llarena.arena_reserve(self.free, self.space_size - self.free) + self.tid_backup = llmemory.cast_adr_to_ptr( + self.free, + lltype.Ptr(self.TID_BACKUP)) + # + # Walk all objects and assign forward pointers in the same order, + # also updating all references + # + self.next_collect_after = self.space_size + #weakref_offsets = self.collect_weakref_offsets() + finaladdr = self.update_forward_pointers(toaddr) if (self.run_finalizers.non_empty() or self.objects_with_finalizers.non_empty()): self.update_run_finalizers() @@ -257,7 +220,7 @@ size = toaddr + self.space_size - finaladdr llarena.arena_reset(finaladdr, size, True) else: - if we_are_translated(): + if 1: # XXX? we_are_translated(): # because we free stuff already in raw_memmove, we # would get double free here. Let's free it anyway llarena.arena_free(self.space) @@ -353,18 +316,16 @@ tid = self.header(addr).tid return llop.extract_ushort(llgroup.HALFWORD, tid) - def mark_roots_recursively(self): + def trace_from_roots(self): self.root_walker.walk_roots( - MarkCompactGC._mark_root_recursively, # stack roots - MarkCompactGC._mark_root_recursively, # static in prebuilt non-gc structures - MarkCompactGC._mark_root_recursively) # static in prebuilt gc objects + MarkCompactGC._mark_root, # stack roots + MarkCompactGC._mark_root, # static in prebuilt non-gc structures + MarkCompactGC._mark_root) # static in prebuilt gc objects self._trace_and_mark() def _trace_and_mark(self): - # XXX depth-first tracing... it can consume a lot of rawmalloced - # memory for very long stacks in some cases while self.to_see.non_empty(): - obj = self.to_see.pop() + obj = self.to_see.popleft() self.trace(obj, self._mark_obj, None) def _mark_obj(self, pointer, ignored): @@ -374,7 +335,7 @@ self.mark(obj) self.to_see.append(obj) - def _mark_root_recursively(self, root): + def _mark_root(self, root): self.mark(root.address[0]) self.to_see.append(root.address[0]) @@ -384,24 +345,46 @@ def marked(self, obj): return self.header(obj).tid & GCFLAG_MARKBIT - def update_forward_pointers(self, toaddr, num_of_alive_objs): - self.base_forwarding_addr = toaddr + def update_forward_pointers(self, toaddr): + base_forwarding_addr = toaddr fromaddr = self.space size_gc_header = self.gcheaderbuilder.size_gc_header - i = 0 + p_tid_backup = self.tid_backup while fromaddr < self.free: hdr = llmemory.cast_adr_to_ptr(fromaddr, lltype.Ptr(self.HDR)) obj = fromaddr + size_gc_header - objsize = self.get_size(obj) - totalsize = size_gc_header + objsize + # compute the original and the new object size, including the + # optional hash field + baseobjsize = self.get_size(obj) + totalsrcsize = size_gc_header + baseobjsize + if hdr.tid & GCFLAG_HASHFIELD: # already a hash field, copy it too + totalsrcsize += llmemory.sizeof(lltype.Signed) + totaldstsize = totalsrcsize + elif hdr.tid & GCFLAG_HASHTAKEN: + # grow a new hash field -- with the exception: if the object + # actually doesn't move, don't (otherwise, toaddr > fromaddr) + if toaddr < fromaddr: + totaldstsize += llmemory.sizeof(lltype.Signed) + # if not self.marked(obj): - self.set_null_forwarding_address(obj, i) + hdr.tid = -1 # mark the object as dying else: llarena.arena_reserve(toaddr, totalsize) - self.set_forwarding_address(obj, toaddr, i) - toaddr += totalsize - i += 1 - fromaddr += totalsize + # save the field hdr.tid in the array tid_backup + p_tid_backup[0] = self.get_type_id(obj) + p_tid_backup = lltype.direct_ptradd(p_tid_backup, 1) + # compute forward_offset, the offset to the future copy + # of this object + forward_offset = toaddr - base_forwarding_addr + # copy the first two gc flags in forward_offset + ll_assert(forward_offset & 3 == 0, "misalignment!") + forward_offset |= (hdr.tid >> first_gcflag_bit) & 3 + hdr.tid = forward_offset + # done + toaddr += totaldstsize + fromaddr += totalsrcsize + if not we_are_translated(): + assert toaddr <= fromaddr # now update references self.root_walker.walk_roots( @@ -480,16 +463,9 @@ return self.get_header_forwarded_addr(obj) def set_null_forwarding_address(self, obj, num): - self.backup_typeid(num, obj) hdr = self.header(obj) hdr.tid = -1 # make the object forwarded to NULL - def set_forwarding_address(self, obj, newobjhdr, num): - self.backup_typeid(num, obj) - forward_offset = newobjhdr - self.base_forwarding_addr - hdr = self.header(obj) - hdr.tid = forward_offset # make the object forwarded to newobj - def restore_normal_header(self, obj, num): # Reverse of set_forwarding_address(). typeid16 = self.get_typeid_from_backup(num) @@ -504,13 +480,11 @@ def surviving(self, obj): return self._is_external(obj) or self.header_forwarded(obj).tid != -1 - def backup_typeid(self, num, obj): - self.tid_backup[num] = self.get_type_id(obj) - def get_typeid_from_backup(self, num): return self.tid_backup[num] def get_size_from_backup(self, obj, num): + # does not count the hash field, if any typeid = self.get_typeid_from_backup(num) size = self.fixed_size(typeid) if self.is_varsize(typeid): @@ -518,9 +492,6 @@ length = lenaddr.signed[0] size += length * self.varsize_item_sizes(typeid) size = llarena.round_up_for_allocation(size) - # XXX maybe we should parametrize round_up_for_allocation() - # per GC; if we do, we also need to fix the call in - # gctypelayout.encode_type_shape() return size def compact(self, resizing): @@ -640,3 +611,11 @@ # hdr.tid |= GCFLAG_HASHTAKEN return llmemory.cast_adr_to_int(obj) # direct case + +# ____________________________________________________________ + +class CannotAllocateGCArena(Exception): + pass + +def max_size_from_env(): + return read_from_env('PYPY_MARKCOMPACTGC_MAX') Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/test/test_direct.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/test/test_direct.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/test/test_direct.py Thu Aug 26 17:28:09 2010 @@ -96,7 +96,7 @@ p[index] = newvalue def malloc(self, TYPE, n=None): - addr = self.gc.malloc(self.get_type_id(TYPE), n) + addr = self.gc.malloc(self.get_type_id(TYPE), n, zero=True) return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(TYPE)) def test_simple(self): From jcreigh at codespeak.net Thu Aug 26 18:10:22 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Thu, 26 Aug 2010 18:10:22 +0200 (CEST) Subject: [pypy-svn] r76749 - in pypy/branch/asmgcc-64/pypy/jit/backend/llsupport: . test Message-ID: <20100826161022.140B9282B9C@codespeak.net> Author: jcreigh Date: Thu Aug 26 18:10:19 2010 New Revision: 76749 Modified: pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/gc.py pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/test/test_gc.py Log: remove add_(ebx|esi|edi|ebp) in favor of using add_callee_save_reg Modified: pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/gc.py Thu Aug 26 18:10:19 2010 @@ -289,20 +289,9 @@ self._encode_num(shape, num) def add_callee_save_reg(self, shape, reg_index): + assert reg_index > 0 shape.append(chr(self.LOC_REG | (reg_index << 2))) - def add_ebx(self, shape): - shape.append(chr(self.LOC_REG | 4)) - - def add_esi(self, shape): - shape.append(chr(self.LOC_REG | 8)) - - def add_edi(self, shape): - shape.append(chr(self.LOC_REG | 12)) - - def add_ebp(self, shape): - shape.append(chr(self.LOC_REG | 16)) - def compress_callshape(self, shape): # Similar to compress_callshape() in trackgcroot.py. # XXX so far, we always allocate a new small array (we could regroup Modified: pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/test/test_gc.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/test/test_gc.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/llsupport/test/test_gc.py Thu Aug 26 18:10:19 2010 @@ -73,16 +73,16 @@ gcrootmap.add_ebp_offset(shape, num1) gcrootmap.add_ebp_offset(shape, num2) assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a]) - gcrootmap.add_ebx(shape) + gcrootmap.add_callee_save_reg(shape, 1) assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, 4]) - gcrootmap.add_esi(shape) + gcrootmap.add_callee_save_reg(shape, 2) assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, 4, 8]) - gcrootmap.add_edi(shape) + gcrootmap.add_callee_save_reg(shape, 3) assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, 4, 8, 12]) - gcrootmap.add_ebp(shape) + gcrootmap.add_callee_save_reg(shape, 4) assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, 4, 8, 12, 16]) # From jcreigh at codespeak.net Fri Aug 27 18:08:35 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Fri, 27 Aug 2010 18:08:35 +0200 (CEST) Subject: [pypy-svn] r76750 - in pypy/trunk/pypy/jit/backend: test x86 x86/test Message-ID: <20100827160835.DC981282BEF@codespeak.net> Author: jcreigh Date: Fri Aug 27 18:08:32 2010 New Revision: 76750 Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/test/test_runner.py Log: 64-bit JIT: support 32-bit wide types in (get|set)(field|arrayitem) Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/runner_test.py (original) +++ pypy/trunk/pypy/jit/backend/test/runner_test.py Fri Aug 27 18:08:32 2010 @@ -785,6 +785,20 @@ 'float', descr=arraydescr) assert r.value == 4.5 + # For platforms where sizeof(INT) != sizeof(Signed) (ie, x86-64) + a_box, A = self.alloc_array_of(rffi.INT, 342) + arraydescr = self.cpu.arraydescrof(A) + assert not arraydescr.is_array_of_pointers() + r = self.execute_operation(rop.ARRAYLEN_GC, [a_box], + 'int', descr=arraydescr) + assert r.value == 342 + r = self.execute_operation(rop.SETARRAYITEM_GC, [a_box, BoxInt(310), + BoxInt(7441)], + 'void', descr=arraydescr) + assert r is None + r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(310)], + 'int', descr=arraydescr) + assert r.value == 7441 def test_string_basic(self): s_box = self.alloc_string("hello\xfe") Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Fri Aug 27 18:08:32 2010 @@ -1046,7 +1046,10 @@ self.mc.MOVZX8(resloc, source_addr) elif size == 2: self.mc.MOVZX16(resloc, source_addr) - elif size == WORD: + elif size == 4: + # MOV32 is zero-extending on 64-bit, so this is okay + self.mc.MOV32(resloc, source_addr) + elif IS_X86_64 and size == 8: self.mc.MOV(resloc, source_addr) else: raise NotImplementedError("getfield size = %d" % size) @@ -1059,19 +1062,18 @@ base_loc, ofs_loc, scale, ofs = arglocs assert isinstance(ofs, ImmedLoc) assert isinstance(scale, ImmedLoc) + src_addr = addr_add(base_loc, ofs_loc, ofs.value, scale.value) if op.result.type == FLOAT: - self.mc.MOVSD(resloc, addr_add(base_loc, ofs_loc, ofs.value, - scale.value)) + self.mc.MOVSD(resloc, src_addr) else: if scale.value == 0: - self.mc.MOVZX8(resloc, addr_add(base_loc, ofs_loc, ofs.value, - scale.value)) + self.mc.MOVZX8(resloc, src_addr) elif scale.value == 1: - self.mc.MOVZX16(resloc, addr_add(base_loc, ofs_loc, ofs.value, - scale.value)) - elif (1 << scale.value) == WORD: - self.mc.MOV(resloc, addr_add(base_loc, ofs_loc, ofs.value, - scale.value)) + self.mc.MOVZX16(resloc, src_addr) + elif scale.value == 2: + self.mc.MOV32(resloc, src_addr) + elif IS_X86_64 and scale.value == 3: + self.mc.MOV(resloc, src_addr) else: print "[asmgen]getarrayitem unsupported size: %d" % scale.value raise NotImplementedError() @@ -1086,8 +1088,10 @@ dest_addr = AddressLoc(base_loc, ofs_loc) if isinstance(value_loc, RegLoc) and value_loc.is_xmm: self.mc.MOVSD(dest_addr, value_loc) - elif size == WORD: + elif IS_X86_64 and size == 8: self.mc.MOV(dest_addr, value_loc) + elif size == 4: + self.mc.MOV32(dest_addr, value_loc) elif size == 2: self.mc.MOV16(dest_addr, value_loc) elif size == 1: @@ -1104,8 +1108,10 @@ if op.args[2].type == FLOAT: self.mc.MOVSD(dest_addr, value_loc) else: - if (1 << scale_loc.value) == WORD: + if IS_X86_64 and scale_loc.value == 3: self.mc.MOV(dest_addr, value_loc) + elif scale_loc.value == 2: + self.mc.MOV32(dest_addr, value_loc) elif scale_loc.value == 1: self.mc.MOV16(dest_addr, value_loc) elif scale_loc.value == 0: Modified: pypy/trunk/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_runner.py Fri Aug 27 18:08:32 2010 @@ -193,6 +193,7 @@ def test_getfield_setfield(self): TP = lltype.GcStruct('x', ('s', lltype.Signed), + ('i', rffi.INT), ('f', lltype.Float), ('u', rffi.USHORT), ('c1', lltype.Char), @@ -201,6 +202,7 @@ res = self.execute_operation(rop.NEW, [], 'ref', self.cpu.sizeof(TP)) ofs_s = self.cpu.fielddescrof(TP, 's') + ofs_i = self.cpu.fielddescrof(TP, 'i') #ofs_f = self.cpu.fielddescrof(TP, 'f') ofs_u = self.cpu.fielddescrof(TP, 'u') ofsc1 = self.cpu.fielddescrof(TP, 'c1') @@ -218,6 +220,11 @@ ofs_s) s = self.execute_operation(rop.GETFIELD_GC, [res], 'int', ofs_s) assert s.value == 3 + + self.execute_operation(rop.SETFIELD_GC, [res, BoxInt(1234)], 'void', ofs_i) + i = self.execute_operation(rop.GETFIELD_GC, [res], 'int', ofs_i) + assert i.value == 1234 + #u = self.execute_operation(rop.GETFIELD_GC, [res, ofs_u], 'int') #assert u.value == 5 self.execute_operation(rop.SETFIELD_GC, [res, ConstInt(1)], 'void', From jcreigh at codespeak.net Fri Aug 27 18:20:45 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Fri, 27 Aug 2010 18:20:45 +0200 (CEST) Subject: [pypy-svn] r76751 - in pypy/branch/asmgcc-64: . pypy/jit/backend/test pypy/jit/backend/x86 pypy/jit/backend/x86/test pypy/module/array/benchmark pypy/module/array/test pypy/rpython/memory/gc pypy/translator/c/test pypy/translator/goal pypy/translator/goal/test2 pypy/translator/platform pypy/translator/platform/test Message-ID: <20100827162045.D7B68282BDC@codespeak.net> Author: jcreigh Date: Fri Aug 27 18:20:43 2010 New Revision: 76751 Modified: pypy/branch/asmgcc-64/ (props changed) pypy/branch/asmgcc-64/pypy/jit/backend/test/runner_test.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/assembler.py pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_runner.py pypy/branch/asmgcc-64/pypy/module/array/benchmark/Makefile (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/intimg.c (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/intimgtst.c (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/intimgtst.py (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/loop.c (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/sum.c (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/sumtst.c (props changed) pypy/branch/asmgcc-64/pypy/module/array/benchmark/sumtst.py (props changed) pypy/branch/asmgcc-64/pypy/module/array/test/test_array_old.py (props changed) pypy/branch/asmgcc-64/pypy/rpython/memory/gc/markcompact.py pypy/branch/asmgcc-64/pypy/translator/c/test/test_standalone.py pypy/branch/asmgcc-64/pypy/translator/goal/app_main.py pypy/branch/asmgcc-64/pypy/translator/goal/test2/test_app_main.py pypy/branch/asmgcc-64/pypy/translator/platform/__init__.py pypy/branch/asmgcc-64/pypy/translator/platform/test/test_platform.py Log: merged changes from trunk through r76750 Modified: pypy/branch/asmgcc-64/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/test/runner_test.py Fri Aug 27 18:20:43 2010 @@ -785,6 +785,20 @@ 'float', descr=arraydescr) assert r.value == 4.5 + # For platforms where sizeof(INT) != sizeof(Signed) (ie, x86-64) + a_box, A = self.alloc_array_of(rffi.INT, 342) + arraydescr = self.cpu.arraydescrof(A) + assert not arraydescr.is_array_of_pointers() + r = self.execute_operation(rop.ARRAYLEN_GC, [a_box], + 'int', descr=arraydescr) + assert r.value == 342 + r = self.execute_operation(rop.SETARRAYITEM_GC, [a_box, BoxInt(310), + BoxInt(7441)], + 'void', descr=arraydescr) + assert r is None + r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(310)], + 'int', descr=arraydescr) + assert r.value == 7441 def test_string_basic(self): s_box = self.alloc_string("hello\xfe") Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/assembler.py Fri Aug 27 18:20:43 2010 @@ -1047,7 +1047,10 @@ self.mc.MOVZX8(resloc, source_addr) elif size == 2: self.mc.MOVZX16(resloc, source_addr) - elif size == WORD: + elif size == 4: + # MOV32 is zero-extending on 64-bit, so this is okay + self.mc.MOV32(resloc, source_addr) + elif IS_X86_64 and size == 8: self.mc.MOV(resloc, source_addr) else: raise NotImplementedError("getfield size = %d" % size) @@ -1060,19 +1063,18 @@ base_loc, ofs_loc, scale, ofs = arglocs assert isinstance(ofs, ImmedLoc) assert isinstance(scale, ImmedLoc) + src_addr = addr_add(base_loc, ofs_loc, ofs.value, scale.value) if op.result.type == FLOAT: - self.mc.MOVSD(resloc, addr_add(base_loc, ofs_loc, ofs.value, - scale.value)) + self.mc.MOVSD(resloc, src_addr) else: if scale.value == 0: - self.mc.MOVZX8(resloc, addr_add(base_loc, ofs_loc, ofs.value, - scale.value)) + self.mc.MOVZX8(resloc, src_addr) elif scale.value == 1: - self.mc.MOVZX16(resloc, addr_add(base_loc, ofs_loc, ofs.value, - scale.value)) - elif (1 << scale.value) == WORD: - self.mc.MOV(resloc, addr_add(base_loc, ofs_loc, ofs.value, - scale.value)) + self.mc.MOVZX16(resloc, src_addr) + elif scale.value == 2: + self.mc.MOV32(resloc, src_addr) + elif IS_X86_64 and scale.value == 3: + self.mc.MOV(resloc, src_addr) else: print "[asmgen]getarrayitem unsupported size: %d" % scale.value raise NotImplementedError() @@ -1087,8 +1089,10 @@ dest_addr = AddressLoc(base_loc, ofs_loc) if isinstance(value_loc, RegLoc) and value_loc.is_xmm: self.mc.MOVSD(dest_addr, value_loc) - elif size == WORD: + elif IS_X86_64 and size == 8: self.mc.MOV(dest_addr, value_loc) + elif size == 4: + self.mc.MOV32(dest_addr, value_loc) elif size == 2: self.mc.MOV16(dest_addr, value_loc) elif size == 1: @@ -1105,8 +1109,10 @@ if op.args[2].type == FLOAT: self.mc.MOVSD(dest_addr, value_loc) else: - if (1 << scale_loc.value) == WORD: + if IS_X86_64 and scale_loc.value == 3: self.mc.MOV(dest_addr, value_loc) + elif scale_loc.value == 2: + self.mc.MOV32(dest_addr, value_loc) elif scale_loc.value == 1: self.mc.MOV16(dest_addr, value_loc) elif scale_loc.value == 0: Modified: pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/branch/asmgcc-64/pypy/jit/backend/x86/test/test_runner.py Fri Aug 27 18:20:43 2010 @@ -193,6 +193,7 @@ def test_getfield_setfield(self): TP = lltype.GcStruct('x', ('s', lltype.Signed), + ('i', rffi.INT), ('f', lltype.Float), ('u', rffi.USHORT), ('c1', lltype.Char), @@ -201,6 +202,7 @@ res = self.execute_operation(rop.NEW, [], 'ref', self.cpu.sizeof(TP)) ofs_s = self.cpu.fielddescrof(TP, 's') + ofs_i = self.cpu.fielddescrof(TP, 'i') #ofs_f = self.cpu.fielddescrof(TP, 'f') ofs_u = self.cpu.fielddescrof(TP, 'u') ofsc1 = self.cpu.fielddescrof(TP, 'c1') @@ -218,6 +220,11 @@ ofs_s) s = self.execute_operation(rop.GETFIELD_GC, [res], 'int', ofs_s) assert s.value == 3 + + self.execute_operation(rop.SETFIELD_GC, [res, BoxInt(1234)], 'void', ofs_i) + i = self.execute_operation(rop.GETFIELD_GC, [res], 'int', ofs_i) + assert i.value == 1234 + #u = self.execute_operation(rop.GETFIELD_GC, [res, ofs_u], 'int') #assert u.value == 5 self.execute_operation(rop.SETFIELD_GC, [res, ConstInt(1)], 'void', Modified: pypy/branch/asmgcc-64/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/asmgcc-64/pypy/rpython/memory/gc/markcompact.py Fri Aug 27 18:20:43 2010 @@ -3,7 +3,8 @@ from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup from pypy.rpython.memory.gc.base import MovingGCBase -from pypy.rlib.debug import ll_assert +from pypy.rlib.debug import ll_assert, have_debug_prints +from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE from pypy.rpython.memory.support import get_address_stack, get_address_deque from pypy.rpython.memory.support import AddressDict @@ -98,8 +99,7 @@ self.space_size = self.param_space_size self.next_collect_after = self.param_space_size/2 # whatever... - if self.config.gcconfig.debugprint: - self.program_start_time = time.time() + self.program_start_time = time.time() self.space = llarena.arena_malloc(self.space_size, True) ll_assert(bool(self.space), "couldn't allocate arena") self.free = self.space @@ -289,15 +289,16 @@ return weakref_offsets def debug_collect_start(self): - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void) - llop.debug_print(lltype.Void, - ".----------- Full collection ------------------") + if have_debug_prints(): + debug_start("gc-collect") + debug_print() + debug_print(".----------- Full collection ------------------") start_time = time.time() - return start_time + return start_time + return -1 def debug_collect_finish(self, start_time): - if self.config.gcconfig.debugprint: + if start_time != -1: end_time = time.time() elapsed_time = end_time - start_time self.total_collection_time += elapsed_time @@ -305,20 +306,16 @@ total_program_time = end_time - self.program_start_time ct = self.total_collection_time cc = self.total_collection_count - llop.debug_print(lltype.Void, - "| number of collections so far ", - cc) - llop.debug_print(lltype.Void, - "| total collections per second: ", - cc / total_program_time) - llop.debug_print(lltype.Void, - "| total time in markcompact-collect: ", - ct, "seconds") - llop.debug_print(lltype.Void, - "| percentage collection<->total time:", - ct * 100.0 / total_program_time, "%") - llop.debug_print(lltype.Void, - "`----------------------------------------------") + debug_print("| number of collections so far ", + cc) + debug_print("| total collections per second: ", + cc / total_program_time) + debug_print("| total time in markcompact-collect: ", + ct, "seconds") + debug_print("| percentage collection<->total time:", + ct * 100.0 / total_program_time, "%") + debug_print("`----------------------------------------------") + debug_stop("gc-collect") def update_run_finalizers(self): Modified: pypy/branch/asmgcc-64/pypy/translator/c/test/test_standalone.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/c/test/test_standalone.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/c/test/test_standalone.py Fri Aug 27 18:20:43 2010 @@ -604,6 +604,33 @@ out, err = cbuilder.cmdexec("a b") assert out == "3" + def test_gcc_options(self): + # check that the env var CC is correctly interpreted, even if + # it contains the compiler name followed by some options. + if sys.platform == 'win32': + py.test.skip("only for gcc") + + from pypy.rpython.lltypesystem import lltype, rffi + dir = udir.ensure('test_gcc_options', dir=1) + dir.join('someextraheader.h').write('#define someextrafunc() 42\n') + eci = ExternalCompilationInfo(includes=['someextraheader.h']) + someextrafunc = rffi.llexternal('someextrafunc', [], lltype.Signed, + compilation_info=eci) + + def entry_point(argv): + return someextrafunc() + + old_cc = os.environ.get('CC') + try: + os.environ['CC'] = 'gcc -I%s' % dir + t, cbuilder = self.compile(entry_point) + finally: + if old_cc is None: + del os.environ['CC'] + else: + os.environ['CC'] = old_cc + + class TestMaemo(TestStandalone): def setup_class(cls): py.test.skip("TestMaemo: tests skipped for now") Modified: pypy/branch/asmgcc-64/pypy/translator/goal/app_main.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/goal/app_main.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/goal/app_main.py Fri Aug 27 18:20:43 2010 @@ -292,8 +292,8 @@ else: raise CommandLineError('unrecognized option %r' % (arg,)) i += 1 - sys.argv = argv[i:] - if not sys.argv: + sys.argv[:] = argv[i:] # don't change the list that sys.argv is bound to + if not sys.argv: # (relevant in case of "reload(sys)") sys.argv.append('') run_stdin = True return locals() @@ -478,6 +478,10 @@ reset.append(('PYTHONINSPECT', os.environ.get('PYTHONINSPECT', ''))) os.environ['PYTHONINSPECT'] = os.environ['PYTHONINSPECT_'] + # no one should change to which lists sys.argv and sys.path are bound + old_argv = sys.argv + old_path = sys.path + from pypy.module.sys.version import PYPY_VERSION sys.pypy_version_info = PYPY_VERSION sys.pypy_initial_path = pypy_initial_path @@ -490,3 +494,5 @@ sys.ps1 = '>>> ' # restore the normal ones, in case sys.ps2 = '... ' # we are dropping to CPython's prompt import os; os.environ.update(reset) + assert old_argv is sys.argv + assert old_path is sys.path Modified: pypy/branch/asmgcc-64/pypy/translator/goal/test2/test_app_main.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/goal/test2/test_app_main.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/goal/test2/test_app_main.py Fri Aug 27 18:20:43 2010 @@ -1,6 +1,7 @@ """ Tests for the entry point of pypy-c, app_main.py. """ +from __future__ import with_statement import py import sys, os, re import autopath Modified: pypy/branch/asmgcc-64/pypy/translator/platform/__init__.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/platform/__init__.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/platform/__init__.py Fri Aug 27 18:20:43 2010 @@ -99,15 +99,20 @@ self.__dict__ == other.__dict__) def key(self): - bits = [self.__class__.__name__, 'cc=%s' % self.cc] + bits = [self.__class__.__name__, 'cc=%r' % self.cc] for varname in self.relevant_environ: - bits.append('%s=%s' % (varname, os.environ.get(varname))) + bits.append('%s=%r' % (varname, os.environ.get(varname))) return ' '.join(bits) # some helpers which seem to be cross-platform enough def _execute_c_compiler(self, cc, args, outname, cwd=None): log.execute(cc + ' ' + ' '.join(args)) + # 'cc' can also contain some options for the C compiler; + # e.g. it can be "gcc -m32". We handle it by splitting on ' '. + cclist = cc.split() + cc = cclist[0] + args = cclist[1:] + args returncode, stdout, stderr = _run_subprocess(cc, args, self.c_environ, cwd) self._handle_error(returncode, stderr, stdout, outname) Modified: pypy/branch/asmgcc-64/pypy/translator/platform/test/test_platform.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/platform/test/test_platform.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/platform/test/test_platform.py Fri Aug 27 18:20:43 2010 @@ -131,7 +131,7 @@ self.cc = 'xcc' x = XPlatform() res = x.key() - assert res.startswith('XPlatform cc=xcc CPATH=') + assert res.startswith("XPlatform cc='xcc' CPATH=") def test_equality(): class X(Platform): From arigo at codespeak.net Fri Aug 27 18:30:19 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 27 Aug 2010 18:30:19 +0200 (CEST) Subject: [pypy-svn] r76752 - in pypy/branch/markcompact/pypy/rpython/memory/gc: . test Message-ID: <20100827163019.34EC1282BE7@codespeak.net> Author: arigo Date: Fri Aug 27 18:30:17 2010 New Revision: 76752 Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/base.py pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py pypy/branch/markcompact/pypy/rpython/memory/gc/test/test_direct.py Log: Start again to pass some tests. Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/base.py Fri Aug 27 18:30:17 2010 @@ -86,8 +86,7 @@ addr -= self.gcheaderbuilder.size_gc_header return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - def get_size(self, obj): - typeid = self.get_type_id(obj) + def _get_size_for_typeid(self, obj, typeid): size = self.fixed_size(typeid) if self.is_varsize(typeid): lenaddr = obj + self.varsize_offset_to_length(typeid) @@ -99,6 +98,9 @@ # gctypelayout.encode_type_shape() return size + def get_size(self, obj): + return self._get_size_for_typeid(obj, self.get_type_id(obj)) + def malloc(self, typeid, length=0, zero=False): """For testing. The interface used by the gctransformer is the four malloc_[fixed,var]size[_clear]() functions. Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Fri Aug 27 18:30:17 2010 @@ -9,7 +9,7 @@ from pypy.rpython.memory.support import get_address_stack, get_address_deque from pypy.rpython.memory.support import AddressDict from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage -from pypy.rlib.rarithmetic import ovfcheck, LONG_BIT +from pypy.rlib.rarithmetic import ovfcheck, LONG_BIT, intmask from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.objectmodel import we_are_translated from pypy.rpython.lltypesystem import rffi @@ -51,13 +51,17 @@ first_gcflag = 1 << first_gcflag_bit GCFLAG_HASHTAKEN = first_gcflag << 0 # someone already asked for the hash GCFLAG_HASHFIELD = first_gcflag << 1 # we have an extra hash field -GCFLAG_MARKBIT = first_gcflag << 2 # note that only the first 2 bits are preserved during a collection! +GCFLAG_MARKBIT = intmask(first_gcflag << (LONG_BIT//2-1)) +assert GCFLAG_MARKBIT < 0 # should be 0x80000000 + +GCFLAG_SAVED_HASHTAKEN = GCFLAG_HASHTAKEN >> first_gcflag_bit +GCFLAG_SAVED_HASHFIELD = GCFLAG_HASHFIELD >> first_gcflag_bit TID_TYPE = llgroup.HALFWORD BYTES_PER_TID = rffi.sizeof(TID_TYPE) -TID_BACKUP = lltype.FixedSizeArray(TID_TYPE, 1) +TID_BACKUP = rffi.CArray(TID_TYPE) class MarkCompactGC(MovingGCBase): @@ -71,7 +75,8 @@ # The default space size is 1.9375 GB, i.e. almost 2 GB, allocated as # a big mmap. The process does not actually consume that space until # needed, of course. - TRANSLATION_PARAMS = {'space_size': int((1 + 15.0/16)*1024*1024*1024)} + TRANSLATION_PARAMS = {'space_size': int((1 + 15.0/16)*1024*1024*1024), + 'min_next_collect_after': 4*1024*1024} # 4MB malloc_zero_filled = False inline_simple_malloc = True @@ -79,17 +84,27 @@ total_collection_time = 0.0 total_collection_count = 0 - def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE, space_size=4096): + def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE, space_size=4096, + min_next_collect_after=64): MovingGCBase.__init__(self, config, chunk_size) self.space_size = space_size + self.min_next_collect_after = min_next_collect_after + + def next_collection(self, used_space, num_objects_so_far): + used_space += BYTES_PER_TID * num_objects_so_far + next = (used_space // 3) * 2 + if next < self.min_next_collect_after: + next = self.min_next_collect_after + if used_space + next > self.space_size: + next = self.space_size - used_space + return next def setup(self): envsize = max_size_from_env() if envsize >= 4096: self.space_size = envsize & ~4095 - self.next_collect_after = min(self.space_size, - 4*1024*1024) # 4MB initially + self.next_collect_after = self.next_collection(0, 0) self.program_start_time = time.time() self.space = llarena.arena_malloc(self.space_size, False) @@ -172,17 +187,14 @@ return self._setup_object(result, typeid16, False, False) def obtain_free_space(self, requested_size): - self.markcompactcollect() - self.next_collect_after -= requested_size - if self.next_collect_after < 0: - raise MemoryError + self.markcompactcollect(requested_size) obtain_free_space._dont_inline_ = True def collect(self, gen=0): self.markcompactcollect() - def markcompactcollect(self): - start_time = self.debug_collect_start() + def markcompactcollect(self, requested_size=0): + start_time = self.debug_collect_start(requested_size) self.debug_check_consistency() # # Mark alive objects @@ -198,43 +210,56 @@ # Prepare new views on the same memory # toaddr = llarena.arena_new_view(self.space) - llarena.arena_reserve(self.free, self.space_size - self.free) - self.tid_backup = llmemory.cast_adr_to_ptr( - self.free, - lltype.Ptr(self.TID_BACKUP)) + maxnum = self.space_size - (self.free - self.space) + maxnum /= BYTES_PER_TID + llarena.arena_reserve(self.free, llmemory.sizeof(TID_BACKUP, maxnum)) + self.tid_backup = llmemory.cast_adr_to_ptr(self.free, + lltype.Ptr(TID_BACKUP)) # # Walk all objects and assign forward pointers in the same order, # also updating all references # - self.next_collect_after = self.space_size #weakref_offsets = self.collect_weakref_offsets() - finaladdr = self.update_forward_pointers(toaddr) - if (self.run_finalizers.non_empty() or - self.objects_with_finalizers.non_empty()): - self.update_run_finalizers() - if self.objects_with_weakrefs.non_empty(): - self.invalidate_weakrefs(weakref_offsets) + finaladdr = self.update_forward_pointers(toaddr, maxnum) + #if (self.run_finalizers.non_empty() or + # self.objects_with_finalizers.non_empty()): + # self.update_run_finalizers() + #if self.objects_with_weakrefs.non_empty(): + # self.invalidate_weakrefs(weakref_offsets) + self.update_objects_with_id() - self.compact(resizing) - if not resizing: - size = toaddr + self.space_size - finaladdr - llarena.arena_reset(finaladdr, size, True) - else: - if 1: # XXX? we_are_translated(): - # because we free stuff already in raw_memmove, we - # would get double free here. Let's free it anyway - llarena.arena_free(self.space) - llarena.arena_reset(toaddr + size_of_alive_objs, tid_backup_size, - True) - self.space = toaddr - self.free = finaladdr - self.top_of_space = toaddr + self.next_collect_after + self.compact() + # + self.tid_backup = lltype.nullptr(TID_BACKUP) + self.free = finaladdr + remaining_size = (toaddr + self.space_size) - finaladdr + llarena.arena_reset(finaladdr, remaining_size, False) + + if requested_size > remaining_size: + raise MemoryError + self.next_collect_after = self.next_collection( + (finaladdr - toaddr) + requested_size, + self.num_alive_objs + 1) + # + if not we_are_translated(): + llarena.arena_free(self.space) + self.space = toaddr + # self.debug_check_consistency() - self.tid_backup = lltype.nullptr(self.TID_BACKUP) + self.debug_collect_finish(start_time) + if self.next_collect_after < 0: + raise MemoryError + ll_assert((self.free - self.space) + # used space + requested_size + # requested extra size + BYTES_PER_TID * self.num_alive_objs + # space for TIDs + BYTES_PER_TID + # space for extra TID + self.next_collect_after # size we can allocate later + <= self.space_size, + "miscomputation of next_collect_after") + # if self.run_finalizers.non_empty(): self.execute_finalizers() - self.debug_collect_finish(start_time) - + def collect_weakref_offsets(self): weakrefs = self.objects_with_weakrefs new_weakrefs = self.AddressStack() @@ -251,11 +276,13 @@ weakrefs.delete() return weakref_offsets - def debug_collect_start(self): + def debug_collect_start(self, requested_size): if have_debug_prints(): debug_start("gc-collect") debug_print() - debug_print(".----------- Full collection ------------------") + debug_print(".----------- Full collection -------------------") + debug_print("| requested size ", + requested_size) start_time = time.time() return start_time return -1 @@ -271,6 +298,14 @@ cc = self.total_collection_count debug_print("| number of collections so far ", cc) + debug_print("| total space size ", + self.space_size) + debug_print("| number of objects alive ", + self.num_alive_objs) + debug_print("| used space size ", + self.free - self.space) + debug_print("| next collection after ", + self.next_collect_after) debug_print("| total collections per second: ", cc / total_program_time) debug_print("| total time in markcompact-collect: ", @@ -343,83 +378,114 @@ self.header(obj).tid |= GCFLAG_MARKBIT def marked(self, obj): - return self.header(obj).tid & GCFLAG_MARKBIT + # should work both if tid contains a CombinedSymbolic (for dying + # objects, at this point), or a plain integer. + return MovingGCBase.header(self, obj).tid & GCFLAG_MARKBIT - def update_forward_pointers(self, toaddr): - base_forwarding_addr = toaddr + def update_forward_pointers(self, toaddr, maxnum): + self.base_forwarding_addr = base_forwarding_addr = toaddr fromaddr = self.space size_gc_header = self.gcheaderbuilder.size_gc_header - p_tid_backup = self.tid_backup + num = 0 while fromaddr < self.free: hdr = llmemory.cast_adr_to_ptr(fromaddr, lltype.Ptr(self.HDR)) obj = fromaddr + size_gc_header - # compute the original and the new object size, including the + # compute the original object size, including the # optional hash field - baseobjsize = self.get_size(obj) - totalsrcsize = size_gc_header + baseobjsize + totalsrcsize = size_gc_header + self.get_size(obj) if hdr.tid & GCFLAG_HASHFIELD: # already a hash field, copy it too totalsrcsize += llmemory.sizeof(lltype.Signed) - totaldstsize = totalsrcsize - elif hdr.tid & GCFLAG_HASHTAKEN: - # grow a new hash field -- with the exception: if the object - # actually doesn't move, don't (otherwise, toaddr > fromaddr) - if toaddr < fromaddr: - totaldstsize += llmemory.sizeof(lltype.Signed) # - if not self.marked(obj): - hdr.tid = -1 # mark the object as dying - else: - llarena.arena_reserve(toaddr, totalsize) + if self.marked(obj): + # the object is marked as suriving. Compute the new object + # size + totaldstsize = totalsrcsize + if (hdr.tid & (GCFLAG_HASHTAKEN|GCFLAG_HASHFIELD) == + GCFLAG_HASHTAKEN): + # grow a new hash field -- with the exception: if + # the object actually doesn't move, don't + # (otherwise, we get a bogus toaddr > fromaddr) + if toaddr - base_forwarding_addr < fromaddr - self.space: + totaldstsize += llmemory.sizeof(lltype.Signed) + llarena.arena_reserve(toaddr, totaldstsize) + # # save the field hdr.tid in the array tid_backup - p_tid_backup[0] = self.get_type_id(obj) - p_tid_backup = lltype.direct_ptradd(p_tid_backup, 1) + ll_assert(num < maxnum, "overflow of the tid_backup table") + self.tid_backup[num] = self.get_type_id(obj) + num += 1 # compute forward_offset, the offset to the future copy # of this object forward_offset = toaddr - base_forwarding_addr # copy the first two gc flags in forward_offset ll_assert(forward_offset & 3 == 0, "misalignment!") forward_offset |= (hdr.tid >> first_gcflag_bit) & 3 - hdr.tid = forward_offset + hdr.tid = forward_offset | GCFLAG_MARKBIT + ll_assert(self.marked(obj), "re-marking object failed!") # done toaddr += totaldstsize + # fromaddr += totalsrcsize if not we_are_translated(): - assert toaddr <= fromaddr + assert toaddr - base_forwarding_addr <= fromaddr - self.space + self.num_alive_objs = num # now update references self.root_walker.walk_roots( - MarkCompactGC._update_root, # stack roots - MarkCompactGC._update_root, # static in prebuilt non-gc structures - MarkCompactGC._update_root) # static in prebuilt gc objects + MarkCompactGC._update_ref, # stack roots + MarkCompactGC._update_ref, # static in prebuilt non-gc structures + MarkCompactGC._update_ref) # static in prebuilt gc objects + self.walk_marked_objects(MarkCompactGC.trace_and_update_ref) + return toaddr + + def walk_marked_objects(self, callback): + num = 0 + size_gc_header = self.gcheaderbuilder.size_gc_header fromaddr = self.space - i = 0 + toaddr = self.base_forwarding_addr while fromaddr < self.free: hdr = llmemory.cast_adr_to_ptr(fromaddr, lltype.Ptr(self.HDR)) obj = fromaddr + size_gc_header - objsize = self.get_size_from_backup(obj, i) - totalsize = size_gc_header + objsize - if not self.surviving(obj): - pass + survives = self.marked(obj) + if survives: + typeid = self.get_typeid_from_backup(num) + num += 1 else: - self.trace_with_backup(obj, self._update_ref, i) - fromaddr += totalsize - i += 1 - return toaddr + typeid = self.get_type_id(obj) + baseobjsize = self._get_size_for_typeid(obj, typeid) + totalsrcsize = size_gc_header + baseobjsize + # + if survives: + grow_hash_field = False + if hdr.tid & GCFLAG_SAVED_HASHFIELD: + totalsrcsize += llmemory.sizeof(lltype.Signed) + totaldstsize = totalsrcsize + if (hdr.tid & (GCFLAG_SAVED_HASHTAKEN|GCFLAG_SAVED_HASHFIELD) + == GCFLAG_SAVED_HASHTAKEN): + if toaddr - base_forwarding_addr < fromaddr - self.space: + grow_hash_field = True + totaldstsize += llmemory.sizeof(lltype.Signed) + callback(self, obj, typeid, totalsrcsize, + toaddr, grow_hash_field) + toaddr += totaldstsize + else: + if hdr.tid & GCFLAG_HASHFIELD: + totalsrcsize += llmemory.sizeof(lltype.Signed) + # + fromaddr += totalsrcsize + walk_marked_objects._annspecialcase_ = 'specialize:arg(2)' - def trace_with_backup(self, obj, callback, arg): + def trace_and_update_ref(self, obj, typeid, _1, _2, _3): """Enumerate the locations inside the given obj that can contain GC pointers. For each such location, callback(pointer, arg) is called, where 'pointer' is an address inside the object. Typically, 'callback' is a bound method and 'arg' can be None. """ - typeid = self.get_typeid_from_backup(arg) if self.is_gcarrayofgcptr(typeid): # a performance shortcut for GcArray(gcptr) length = (obj + llmemory.gcarrayofptr_lengthoffset).signed[0] item = obj + llmemory.gcarrayofptr_itemsoffset while length > 0: - if self.points_to_valid_gc_object(item): - callback(item, arg) + self._update_ref(item) item += llmemory.gcarrayofptr_singleitemoffset length -= 1 return @@ -427,8 +493,7 @@ i = 0 while i < len(offsets): item = obj + offsets[i] - if self.points_to_valid_gc_object(item): - callback(item, arg) + self._update_ref(item) i += 1 if self.has_gcptr_in_varsize(typeid): item = obj + self.varsize_offset_to_variable_part(typeid) @@ -439,42 +504,28 @@ j = 0 while j < len(offsets): itemobj = item + offsets[j] - if self.points_to_valid_gc_object(itemobj): - callback(itemobj, arg) + self._update_ref(itemobj) j += 1 item += itemlength length -= 1 - trace_with_backup._annspecialcase_ = 'specialize:arg(2)' - def _update_root(self, pointer): - if pointer.address[0] != NULL: - pointer.address[0] = self.get_forwarding_address(pointer.address[0]) - - def _update_ref(self, pointer, ignore): - if pointer.address[0] != NULL: - pointer.address[0] = self.get_forwarding_address(pointer.address[0]) + def _update_ref(self, pointer): + if self.points_to_valid_gc_object(pointer): + pointer.address[0] = self.get_forwarding_address( + pointer.address[0]) def _is_external(self, obj): - return not (self.space <= obj < self.top_of_space) + return not (self.space <= obj < self.free) def get_forwarding_address(self, obj): if self._is_external(obj): return obj return self.get_header_forwarded_addr(obj) - def set_null_forwarding_address(self, obj, num): - hdr = self.header(obj) - hdr.tid = -1 # make the object forwarded to NULL - - def restore_normal_header(self, obj, num): - # Reverse of set_forwarding_address(). - typeid16 = self.get_typeid_from_backup(num) - hdr = self.header_forwarded(obj) - hdr.tid = self.combine(typeid16, 0) # restore the normal header - def get_header_forwarded_addr(self, obj): + GCFLAG_MASK = ~(GCFLAG_MARKBIT | 3) return (self.base_forwarding_addr + - self.header_forwarded(obj).tid + + (self.header_forwarded(obj).tid & GCFLAG_MASK) + self.gcheaderbuilder.size_gc_header) def surviving(self, obj): @@ -483,55 +534,24 @@ def get_typeid_from_backup(self, num): return self.tid_backup[num] - def get_size_from_backup(self, obj, num): - # does not count the hash field, if any - typeid = self.get_typeid_from_backup(num) - size = self.fixed_size(typeid) - if self.is_varsize(typeid): - lenaddr = obj + self.varsize_offset_to_length(typeid) - length = lenaddr.signed[0] - size += length * self.varsize_item_sizes(typeid) - size = llarena.round_up_for_allocation(size) - return size + def compact(self): + self.walk_marked_objects(MarkCompactGC.copy_and_compact) - def compact(self, resizing): - fromaddr = self.space - size_gc_header = self.gcheaderbuilder.size_gc_header - start = fromaddr - end = fromaddr - num = 0 - while fromaddr < self.free: - obj = fromaddr + size_gc_header - objsize = self.get_size_from_backup(obj, num) - totalsize = size_gc_header + objsize - if not self.surviving(obj): - # this object dies. Following line is a noop in C, - # we clear it to make debugging easier - llarena.arena_reset(fromaddr, totalsize, False) - else: - if resizing: - end = fromaddr - forward_obj = self.get_header_forwarded_addr(obj) - self.restore_normal_header(obj, num) - if obj != forward_obj: - #llop.debug_print(lltype.Void, "Copying from to", - # fromaddr, forward_ptr, totalsize) - llmemory.raw_memmove(fromaddr, - forward_obj - size_gc_header, - totalsize) - if resizing and end - start > GC_CLEARANCE: - diff = end - start - #llop.debug_print(lltype.Void, "Cleaning", start, diff) - diff = (diff / GC_CLEARANCE) * GC_CLEARANCE - #llop.debug_print(lltype.Void, "Cleaning", start, diff) - end = start + diff - if we_are_translated(): - # XXX wuaaaaa.... those objects are freed incorrectly - # here in case of test_gc - llarena.arena_reset(start, diff, True) - start += diff - num += 1 - fromaddr += totalsize + def copy_and_compact(self, obj, typeid, totalsrcsize, + toaddr, grow_hash_field): + # restore the normal header + hdr = self.header_forwarded(obj) + gcflags = hdr.tid & 3 + if grow_hash_field: + gcflags |= GCFLAG_SAVED_HASHFIELD + save_hash_field_now + hdr.tid = self.combine(typeid, gcflags << first_gcflag_bit) + # + fromaddr = obj - self.gcheaderbuilder.size_gc_header + if we_are_translated(): + llmemory.raw_memmove(fromaddr, toaddr, totalsrcsize) + else: + llmemory.raw_memcopy(fromaddr, toaddr, totalsrcsize) def debug_check_object(self, obj): # not sure what to check here Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/test/test_direct.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/test/test_direct.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/test/test_direct.py Fri Aug 27 18:30:17 2010 @@ -67,7 +67,10 @@ from pypy.config.pypyoption import get_pypy_config config = get_pypy_config(translating=True).translation self.stackroots = [] - self.gc = self.GCClass(config, **self.GC_PARAMS) + GC_PARAMS = self.GC_PARAMS.copy() + if hasattr(meth, 'GC_PARAMS'): + GC_PARAMS.update(meth.GC_PARAMS) + self.gc = self.GCClass(config, **GC_PARAMS) self.gc.DEBUG = True self.rootwalker = DirectRootWalker(self) self.gc.set_root_walker(self.rootwalker) @@ -431,3 +434,6 @@ class TestMarkCompactGC(DirectGCTest): from pypy.rpython.memory.gc.markcompact import MarkCompactGC as GCClass + def test_many_objects(self): + DirectGCTest.test_many_objects(self) + test_many_objects.GC_PARAMS = {'space_size': 3 * 4096} From arigo at codespeak.net Fri Aug 27 18:55:52 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 27 Aug 2010 18:55:52 +0200 (CEST) Subject: [pypy-svn] r76753 - in pypy/branch/markcompact/pypy/rpython/memory/gc: . test Message-ID: <20100827165552.2E458282BDC@codespeak.net> Author: arigo Date: Fri Aug 27 18:55:50 2010 New Revision: 76753 Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py pypy/branch/markcompact/pypy/rpython/memory/gc/test/test_direct.py Log: Some more passing tests. Refactored the computation of next_collect_after. Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Fri Aug 27 18:55:50 2010 @@ -85,18 +85,25 @@ total_collection_count = 0 def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE, space_size=4096, - min_next_collect_after=64): + min_next_collect_after=128): MovingGCBase.__init__(self, config, chunk_size) self.space_size = space_size self.min_next_collect_after = min_next_collect_after - def next_collection(self, used_space, num_objects_so_far): + def next_collection(self, used_space, num_objects_so_far, requested_size): used_space += BYTES_PER_TID * num_objects_so_far - next = (used_space // 3) * 2 + ll_assert(used_space <= self.space_size, + "used_space + num_objects_so_far overflow") + try: + next = ((used_space + requested_size) // 3) * 2 + except OverflowError: + next = self.space_size if next < self.min_next_collect_after: next = self.min_next_collect_after - if used_space + next > self.space_size: + if next > self.space_size - used_space: next = self.space_size - used_space + # the value we return guarantees that used_space + next <= space_size, + # with 'BYTES_PER_TID*num_objects_so_far' included in used_space return next def setup(self): @@ -104,7 +111,7 @@ if envsize >= 4096: self.space_size = envsize & ~4095 - self.next_collect_after = self.next_collection(0, 0) + self.next_collect_after = self.next_collection(0, 0, 0) self.program_start_time = time.time() self.space = llarena.arena_malloc(self.space_size, False) @@ -180,14 +187,23 @@ offset_to_length, can_collect): size_gc_header = self.gcheaderbuilder.size_gc_header nonvarsize = size_gc_header + size - totalsize = self._get_totalsize_var(nonvarsize, itemize, length) + totalsize = self._get_totalsize_var(nonvarsize, itemsize, length) result = self._get_memory(totalsize) llmemory.raw_memclear(result, totalsize) (result + size_gc_header + offset_to_length).signed[0] = length return self._setup_object(result, typeid16, False, False) def obtain_free_space(self, requested_size): - self.markcompactcollect(requested_size) + while True: + executed_some_finalizers = self.markcompactcollect(requested_size) + self.next_collect_after -= requested_size + if self.next_collect_after >= 0: + break # ok + else: + if executed_some_finalizers: + pass # try again to do a collection + else: + raise MemoryError obtain_free_space._dont_inline_ = True def collect(self, gen=0): @@ -234,12 +250,9 @@ self.free = finaladdr remaining_size = (toaddr + self.space_size) - finaladdr llarena.arena_reset(finaladdr, remaining_size, False) - - if requested_size > remaining_size: - raise MemoryError - self.next_collect_after = self.next_collection( - (finaladdr - toaddr) + requested_size, - self.num_alive_objs + 1) + self.next_collect_after = self.next_collection(finaladdr - toaddr, + self.num_alive_objs, + requested_size) # if not we_are_translated(): llarena.arena_free(self.space) @@ -249,16 +262,12 @@ self.debug_collect_finish(start_time) if self.next_collect_after < 0: raise MemoryError - ll_assert((self.free - self.space) + # used space - requested_size + # requested extra size - BYTES_PER_TID * self.num_alive_objs + # space for TIDs - BYTES_PER_TID + # space for extra TID - self.next_collect_after # size we can allocate later - <= self.space_size, - "miscomputation of next_collect_after") # if self.run_finalizers.non_empty(): self.execute_finalizers() + return True # executed some finalizers + else: + return False # no finalizer executed def collect_weakref_offsets(self): weakrefs = self.objects_with_weakrefs Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/test/test_direct.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/test/test_direct.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/test/test_direct.py Fri Aug 27 18:55:50 2010 @@ -437,3 +437,11 @@ def test_many_objects(self): DirectGCTest.test_many_objects(self) test_many_objects.GC_PARAMS = {'space_size': 3 * 4096} + + def test_varsized_from_stack(self): + DirectGCTest.test_varsized_from_stack(self) + test_varsized_from_stack.GC_PARAMS = {'space_size': 2 * 4096} + + def test_varsized_from_prebuilt_gc(self): + DirectGCTest.test_varsized_from_prebuilt_gc(self) + test_varsized_from_prebuilt_gc.GC_PARAMS = {'space_size': 3 * 4096} From arigo at codespeak.net Fri Aug 27 19:03:53 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 27 Aug 2010 19:03:53 +0200 (CEST) Subject: [pypy-svn] r76754 - pypy/branch/markcompact/pypy/rpython/memory/gc Message-ID: <20100827170353.E9600282BDC@codespeak.net> Author: arigo Date: Fri Aug 27 19:03:52 2010 New Revision: 76754 Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Log: Pass more tests. Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Fri Aug 27 19:03:52 2010 @@ -270,6 +270,7 @@ return False # no finalizer executed def collect_weakref_offsets(self): + xxx weakrefs = self.objects_with_weakrefs new_weakrefs = self.AddressStack() weakref_offsets = lltype.malloc(self.WEAKREF_OFFSETS, @@ -290,7 +291,7 @@ debug_start("gc-collect") debug_print() debug_print(".----------- Full collection -------------------") - debug_print("| requested size ", + debug_print("| requested size:", requested_size) start_time = time.time() return start_time @@ -326,6 +327,7 @@ def update_run_finalizers(self): + xxx run_finalizers = self.AddressDeque() while self.run_finalizers.non_empty(): obj = self.run_finalizers.popleft() @@ -567,6 +569,7 @@ pass def mark_objects_with_finalizers(self): + xxx new_with_finalizers = self.AddressDeque() run_finalizers = self.run_finalizers new_run_finalizers = self.AddressDeque() @@ -592,6 +595,7 @@ # walk over list of objects that contain weakrefs # if the object it references survives then update the weakref # otherwise invalidate the weakref + xxx new_with_weakref = self.AddressStack() i = 0 while self.objects_with_weakrefs.non_empty(): @@ -614,13 +618,6 @@ self.objects_with_weakrefs = new_with_weakref lltype.free(weakref_offsets, flavor='raw') - def get_size_incl_hash(self, obj): - size = self.get_size(obj) - hdr = self.header(obj) - if hdr.tid & GCFLAG_HASHFIELD: - size += llmemory.sizeof(lltype.Signed) - return size - def identityhash(self, gcobj): # Unlike SemiSpaceGC.identityhash(), this function does not have # to care about reducing top_of_space. The reason is as @@ -639,7 +636,14 @@ return obj.signed[0] # hdr.tid |= GCFLAG_HASHTAKEN - return llmemory.cast_adr_to_int(obj) # direct case + if we_are_translated(): + return llmemory.cast_adr_to_int(obj) # direct case + else: + try: + adr = llarena.getfakearenaaddress(obj) # -> arena address + except RuntimeError: + return llmemory.cast_adr_to_int(obj) # not in an arena... + return adr - self.space # ____________________________________________________________ From arigo at codespeak.net Fri Aug 27 19:18:36 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 27 Aug 2010 19:18:36 +0200 (CEST) Subject: [pypy-svn] r76755 - pypy/branch/markcompact/pypy/rpython/memory/gc Message-ID: <20100827171836.DC488282B9C@codespeak.net> Author: arigo Date: Fri Aug 27 19:18:35 2010 New Revision: 76755 Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Log: Fix finalizers. Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Fri Aug 27 19:18:35 2010 @@ -217,10 +217,6 @@ # self.to_see = self.AddressDeque() self.trace_from_roots() - #if (self.objects_with_finalizers.non_empty() or - # self.run_finalizers.non_empty()): - # self.mark_objects_with_finalizers() - # self._trace_and_mark() self.to_see.delete() # # Prepare new views on the same memory @@ -237,9 +233,9 @@ # #weakref_offsets = self.collect_weakref_offsets() finaladdr = self.update_forward_pointers(toaddr, maxnum) - #if (self.run_finalizers.non_empty() or - # self.objects_with_finalizers.non_empty()): - # self.update_run_finalizers() + if (self.run_finalizers.non_empty() or + self.objects_with_finalizers.non_empty()): + self.update_run_finalizers() #if self.objects_with_weakrefs.non_empty(): # self.invalidate_weakrefs(weakref_offsets) @@ -327,13 +323,14 @@ def update_run_finalizers(self): - xxx - run_finalizers = self.AddressDeque() - while self.run_finalizers.non_empty(): - obj = self.run_finalizers.popleft() - run_finalizers.append(self.get_forwarding_address(obj)) - self.run_finalizers.delete() - self.run_finalizers = run_finalizers + if self.run_finalizers.non_empty(): # uncommon case + run_finalizers = self.AddressDeque() + while self.run_finalizers.non_empty(): + obj = self.run_finalizers.popleft() + run_finalizers.append(self.get_forwarding_address(obj)) + self.run_finalizers.delete() + self.run_finalizers = run_finalizers + # objects_with_finalizers = self.AddressDeque() while self.objects_with_finalizers.non_empty(): obj = self.objects_with_finalizers.popleft() @@ -367,6 +364,9 @@ MarkCompactGC._mark_root, # stack roots MarkCompactGC._mark_root, # static in prebuilt non-gc structures MarkCompactGC._mark_root) # static in prebuilt gc objects + if (self.objects_with_finalizers.non_empty() or + self.run_finalizers.non_empty()): + self.trace_from_objects_with_finalizers() self._trace_and_mark() def _trace_and_mark(self): @@ -375,18 +375,15 @@ self.trace(obj, self._mark_obj, None) def _mark_obj(self, pointer, ignored): - obj = pointer.address[0] - if self.marked(obj): - return - self.mark(obj) - self.to_see.append(obj) + self.mark(pointer.address[0]) def _mark_root(self, root): self.mark(root.address[0]) - self.to_see.append(root.address[0]) def mark(self, obj): - self.header(obj).tid |= GCFLAG_MARKBIT + if not self.marked(obj): + self.header(obj).tid |= GCFLAG_MARKBIT + self.to_see.append(obj) def marked(self, obj): # should work both if tid contains a CombinedSymbolic (for dying @@ -568,26 +565,27 @@ # not sure what to check here pass - def mark_objects_with_finalizers(self): - xxx + def trace_from_objects_with_finalizers(self): + if self.run_finalizers.non_empty(): # uncommon case + new_run_finalizers = self.AddressDeque() + while self.run_finalizers.non_empty(): + x = self.run_finalizers.popleft() + self.mark(x) + new_run_finalizers.append(x) + self.run_finalizers.delete() + self.run_finalizers = new_run_finalizers + # + # xxx we get to run the finalizers in a random order + self._trace_and_mark() new_with_finalizers = self.AddressDeque() - run_finalizers = self.run_finalizers - new_run_finalizers = self.AddressDeque() - while run_finalizers.non_empty(): - x = run_finalizers.popleft() - self.mark(x) - self.to_see.append(x) - new_run_finalizers.append(x) - run_finalizers.delete() - self.run_finalizers = new_run_finalizers while self.objects_with_finalizers.non_empty(): x = self.objects_with_finalizers.popleft() if self.marked(x): new_with_finalizers.append(x) else: - new_run_finalizers.append(x) + self.run_finalizers.append(x) self.mark(x) - self.to_see.append(x) + self._trace_and_mark() self.objects_with_finalizers.delete() self.objects_with_finalizers = new_with_finalizers From arigo at codespeak.net Fri Aug 27 19:26:22 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 27 Aug 2010 19:26:22 +0200 (CEST) Subject: [pypy-svn] r76756 - in pypy/branch/markcompact/pypy/rpython/memory: . gc test Message-ID: <20100827172622.9619F282B9C@codespeak.net> Author: arigo Date: Fri Aug 27 19:26:20 2010 New Revision: 76756 Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py pypy/branch/markcompact/pypy/rpython/memory/gcwrapper.py pypy/branch/markcompact/pypy/rpython/memory/test/test_gc.py Log: Fix tests, update. Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Fri Aug 27 19:26:20 2010 @@ -95,15 +95,17 @@ ll_assert(used_space <= self.space_size, "used_space + num_objects_so_far overflow") try: - next = ((used_space + requested_size) // 3) * 2 + next = (used_space // 3) * 2 + requested_size except OverflowError: next = self.space_size if next < self.min_next_collect_after: next = self.min_next_collect_after if next > self.space_size - used_space: next = self.space_size - used_space - # the value we return guarantees that used_space + next <= space_size, - # with 'BYTES_PER_TID*num_objects_so_far' included in used_space + # The value we return guarantees that used_space + next <= space_size, + # with 'BYTES_PER_TID*num_objects_so_far' included in used_space. + # Normally, the value we return should also be at least requested_size + # unless we are out of memory. return next def setup(self): Modified: pypy/branch/markcompact/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gcwrapper.py Fri Aug 27 19:26:20 2010 @@ -119,6 +119,9 @@ else: return True + def pyobjectptr(self, klass): + raise NotImplementedError(klass) + # ____________________________________________________________ class LLInterpRootWalker: Modified: pypy/branch/markcompact/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/test/test_gc.py Fri Aug 27 19:26:20 2010 @@ -639,13 +639,11 @@ class TestMarkCompactGC(TestSemiSpaceGC): from pypy.rpython.memory.gc.markcompact import MarkCompactGC as GCClass + GC_PARAMS = {'space_size': 16384} def test_finalizer_order(self): py.test.skip("Not implemented yet") -class TestMarkCompactGCGrowing(TestMarkCompactGC): - GC_PARAMS = {'space_size': 16*WORD} - class TestHybridGC(TestGenerationalGC): from pypy.rpython.memory.gc.hybrid import HybridGC as GCClass GC_CANNOT_MALLOC_NONMOVABLE = False From arigo at codespeak.net Fri Aug 27 19:40:52 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 27 Aug 2010 19:40:52 +0200 (CEST) Subject: [pypy-svn] r76757 - pypy/branch/markcompact/pypy/rpython/lltypesystem Message-ID: <20100827174052.26FBA282B9C@codespeak.net> Author: arigo Date: Fri Aug 27 19:40:50 2010 New Revision: 76757 Modified: pypy/branch/markcompact/pypy/rpython/lltypesystem/llgroup.py Log: Forgot to check this in. Modified: pypy/branch/markcompact/pypy/rpython/lltypesystem/llgroup.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/lltypesystem/llgroup.py (original) +++ pypy/branch/markcompact/pypy/rpython/lltypesystem/llgroup.py Fri Aug 27 19:40:50 2010 @@ -135,6 +135,10 @@ assert (other & CombinedSymbolic.MASK) == 0 return CombinedSymbolic(self.lowpart, self.rest - other) + def __rshift__(self, other): + assert other >= HALFSHIFT + return self.rest >> (other - HALFSHIFT) + def __eq__(self, other): if (isinstance(other, CombinedSymbolic) and self.lowpart is other.lowpart): From arigo at codespeak.net Fri Aug 27 19:41:04 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 27 Aug 2010 19:41:04 +0200 (CEST) Subject: [pypy-svn] r76758 - pypy/branch/markcompact/pypy/rpython/memory/gc Message-ID: <20100827174104.DB0D7282B9C@codespeak.net> Author: arigo Date: Fri Aug 27 19:41:03 2010 New Revision: 76758 Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Log: Actually, weakrefs are rather easy in this GC. Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Fri Aug 27 19:41:03 2010 @@ -70,7 +70,6 @@ withhash_flag_is_in_field = 'tid', GCFLAG_HASHFIELD # ^^^ all prebuilt objects have GCFLAG_HASHTAKEN, but only some have # GCFLAG_HASHFIELD (and then they are one word longer). - WEAKREF_OFFSETS = rffi.CArray(lltype.Signed) # The default space size is 1.9375 GB, i.e. almost 2 GB, allocated as # a big mmap. The process does not actually consume that space until @@ -122,7 +121,6 @@ self.free = self.space MovingGCBase.setup(self) self.objects_with_finalizers = self.AddressDeque() - self.objects_with_weakrefs = self.AddressStack() self.tid_backup = lltype.nullptr(TID_BACKUP) def init_gc_object(self, addr, typeid16, flags=0): @@ -163,8 +161,6 @@ self.init_gc_object(result, typeid16) if has_finalizer: self.objects_with_finalizers.append(result + size_gc_header) - if contains_weakptr: - self.objects_with_weakrefs.append(result + size_gc_header) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) _setup_object._always_inline_ = True @@ -233,13 +229,10 @@ # Walk all objects and assign forward pointers in the same order, # also updating all references # - #weakref_offsets = self.collect_weakref_offsets() finaladdr = self.update_forward_pointers(toaddr, maxnum) if (self.run_finalizers.non_empty() or self.objects_with_finalizers.non_empty()): self.update_run_finalizers() - #if self.objects_with_weakrefs.non_empty(): - # self.invalidate_weakrefs(weakref_offsets) self.update_objects_with_id() self.compact() @@ -267,23 +260,6 @@ else: return False # no finalizer executed - def collect_weakref_offsets(self): - xxx - weakrefs = self.objects_with_weakrefs - new_weakrefs = self.AddressStack() - weakref_offsets = lltype.malloc(self.WEAKREF_OFFSETS, - weakrefs.length(), flavor='raw') - i = 0 - while weakrefs.non_empty(): - obj = weakrefs.pop() - offset = self.weakpointer_offset(self.get_type_id(obj)) - weakref_offsets[i] = offset - new_weakrefs.append(obj) - i += 1 - self.objects_with_weakrefs = new_weakrefs - weakrefs.delete() - return weakref_offsets - def debug_collect_start(self, requested_size): if have_debug_prints(): debug_start("gc-collect") @@ -518,12 +494,26 @@ j += 1 item += itemlength length -= 1 + else: + weakofs = self.weakpointer_offset(typeid) + if weakofs >= 0: + self._update_weakref(obj + weakofs) def _update_ref(self, pointer): if self.points_to_valid_gc_object(pointer): pointer.address[0] = self.get_forwarding_address( pointer.address[0]) + def _update_weakref(self, pointer): + # either update the weak pointer's destination, or + # if it dies, write a NULL + if self.points_to_valid_gc_object(pointer): + if self.marked(pointer.address[0]): + pointer.address[0] = self.get_forwarding_address( + pointer.address[0]) + else: + pointer.address[0] = NULL + def _is_external(self, obj): return not (self.space <= obj < self.free) @@ -591,33 +581,6 @@ self.objects_with_finalizers.delete() self.objects_with_finalizers = new_with_finalizers - def invalidate_weakrefs(self, weakref_offsets): - # walk over list of objects that contain weakrefs - # if the object it references survives then update the weakref - # otherwise invalidate the weakref - xxx - new_with_weakref = self.AddressStack() - i = 0 - while self.objects_with_weakrefs.non_empty(): - obj = self.objects_with_weakrefs.pop() - if not self.surviving(obj): - continue # weakref itself dies - newobj = self.get_forwarding_address(obj) - offset = weakref_offsets[i] - pointing_to = (obj + offset).address[0] - # XXX I think that pointing_to cannot be NULL here - if pointing_to: - if self.surviving(pointing_to): - (obj + offset).address[0] = self.get_forwarding_address( - pointing_to) - new_with_weakref.append(newobj) - else: - (obj + offset).address[0] = NULL - i += 1 - self.objects_with_weakrefs.delete() - self.objects_with_weakrefs = new_with_weakref - lltype.free(weakref_offsets, flavor='raw') - def identityhash(self, gcobj): # Unlike SemiSpaceGC.identityhash(), this function does not have # to care about reducing top_of_space. The reason is as From arigo at codespeak.net Fri Aug 27 19:43:47 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 27 Aug 2010 19:43:47 +0200 (CEST) Subject: [pypy-svn] r76759 - pypy/branch/markcompact/pypy/rpython/memory/gc Message-ID: <20100827174347.48461282B9C@codespeak.net> Author: arigo Date: Fri Aug 27 19:43:45 2010 New Revision: 76759 Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Log: Remove unused argument. Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Fri Aug 27 19:43:45 2010 @@ -156,7 +156,7 @@ return totalsize _get_totalsize_var._always_inline_ = True - def _setup_object(self, result, typeid16, has_finalizer, contains_weakptr): + def _setup_object(self, result, typeid16, has_finalizer): size_gc_header = self.gcheaderbuilder.size_gc_header self.init_gc_object(result, typeid16) if has_finalizer: @@ -169,8 +169,7 @@ size_gc_header = self.gcheaderbuilder.size_gc_header totalsize = size_gc_header + size result = self._get_memory(totalsize) - return self._setup_object(result, typeid16, has_finalizer, - contains_weakptr) + return self._setup_object(result, typeid16, has_finalizer) def malloc_fixedsize_clear(self, typeid16, size, can_collect, has_finalizer=False, contains_weakptr=False): @@ -178,8 +177,7 @@ totalsize = size_gc_header + size result = self._get_memory(totalsize) llmemory.raw_memclear(result, totalsize) - return self._setup_object(result, typeid16, has_finalizer, - contains_weakptr) + return self._setup_object(result, typeid16, has_finalizer) def malloc_varsize_clear(self, typeid16, length, size, itemsize, offset_to_length, can_collect): @@ -189,7 +187,7 @@ result = self._get_memory(totalsize) llmemory.raw_memclear(result, totalsize) (result + size_gc_header + offset_to_length).signed[0] = length - return self._setup_object(result, typeid16, False, False) + return self._setup_object(result, typeid16, False) def obtain_free_space(self, requested_size): while True: From arigo at codespeak.net Fri Aug 27 19:52:20 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 27 Aug 2010 19:52:20 +0200 (CEST) Subject: [pypy-svn] r76760 - in pypy/branch/markcompact/pypy/rpython/memory: gc test Message-ID: <20100827175220.8A391282B9C@codespeak.net> Author: arigo Date: Fri Aug 27 19:52:19 2010 New Revision: 76760 Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py pypy/branch/markcompact/pypy/rpython/memory/test/test_transformed_gc.py Log: Some RPython fixes. Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Fri Aug 27 19:52:19 2010 @@ -220,7 +220,9 @@ toaddr = llarena.arena_new_view(self.space) maxnum = self.space_size - (self.free - self.space) maxnum /= BYTES_PER_TID - llarena.arena_reserve(self.free, llmemory.sizeof(TID_BACKUP, maxnum)) + if not we_are_translated(): + llarena.arena_reserve(self.free, + llmemory.sizeof(TID_BACKUP, maxnum)) self.tid_backup = llmemory.cast_adr_to_ptr(self.free, lltype.Ptr(TID_BACKUP)) # @@ -366,6 +368,14 @@ # objects, at this point), or a plain integer. return MovingGCBase.header(self, obj).tid & GCFLAG_MARKBIT + def toaddr_smaller_than_fromaddr(self, toaddr, fromaddr): + if we_are_translated(): + return toaddr < fromaddr + else: + # convert the addresses to integers, because they are + # theoretically not from the same arena + return toaddr - self.base_forwarding_addr < fromaddr - self.space + def update_forward_pointers(self, toaddr, maxnum): self.base_forwarding_addr = base_forwarding_addr = toaddr fromaddr = self.space @@ -389,7 +399,7 @@ # grow a new hash field -- with the exception: if # the object actually doesn't move, don't # (otherwise, we get a bogus toaddr > fromaddr) - if toaddr - base_forwarding_addr < fromaddr - self.space: + if self.toaddr_smaller_than_fromaddr(toaddr, fromaddr): totaldstsize += llmemory.sizeof(lltype.Signed) llarena.arena_reserve(toaddr, totaldstsize) # @@ -445,7 +455,7 @@ totaldstsize = totalsrcsize if (hdr.tid & (GCFLAG_SAVED_HASHTAKEN|GCFLAG_SAVED_HASHFIELD) == GCFLAG_SAVED_HASHTAKEN): - if toaddr - base_forwarding_addr < fromaddr - self.space: + if self.toaddr_smaller_than_fromaddr(toaddr, fromaddr): grow_hash_field = True totaldstsize += llmemory.sizeof(lltype.Signed) callback(self, obj, typeid, totalsrcsize, @@ -456,7 +466,7 @@ totalsrcsize += llmemory.sizeof(lltype.Signed) # fromaddr += totalsrcsize - walk_marked_objects._annspecialcase_ = 'specialize:arg(2)' + walk_marked_objects._annspecialcase_ = 'specialize:arg(1)' def trace_and_update_ref(self, obj, typeid, _1, _2, _3): """Enumerate the locations inside the given obj that can contain Modified: pypy/branch/markcompact/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/test/test_transformed_gc.py Fri Aug 27 19:52:19 2010 @@ -1138,13 +1138,10 @@ class TestMarkCompactGC(GenericMovingGCTests): gcname = 'markcompact' - def setup_class(cls): - py.test.skip("Disabled for now, sorry") - class gcpolicy(gc.FrameworkGcPolicy): class transformerclass(framework.FrameworkGCTransformer): from pypy.rpython.memory.gc.markcompact import MarkCompactGC as GCClass - GC_PARAMS = {'space_size': 512*WORD} + GC_PARAMS = {'space_size': 4096*WORD} root_stack_depth = 200 class TestGenerationGC(GenericMovingGCTests): From arigo at codespeak.net Fri Aug 27 20:03:47 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 27 Aug 2010 20:03:47 +0200 (CEST) Subject: [pypy-svn] r76761 - pypy/branch/markcompact/pypy/rpython/memory/test Message-ID: <20100827180347.F1CCA282B9C@codespeak.net> Author: arigo Date: Fri Aug 27 20:03:43 2010 New Revision: 76761 Modified: pypy/branch/markcompact/pypy/rpython/memory/test/test_gc.py Log: Give more space size for some tests, and skip an irrelevant test. Modified: pypy/branch/markcompact/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/test/test_gc.py Fri Aug 27 20:03:43 2010 @@ -639,10 +639,14 @@ class TestMarkCompactGC(TestSemiSpaceGC): from pypy.rpython.memory.gc.markcompact import MarkCompactGC as GCClass - GC_PARAMS = {'space_size': 16384} + GC_PARAMS = {'space_size': 65536+16384} + GC_CAN_SHRINK_ARRAY = False def test_finalizer_order(self): py.test.skip("Not implemented yet") + def test_writebarrier_before_copy(self): + py.test.skip("Not relevant, and crashes because llarena does not " + "support empty GcStructs") class TestHybridGC(TestGenerationalGC): from pypy.rpython.memory.gc.hybrid import HybridGC as GCClass From arigo at codespeak.net Fri Aug 27 20:22:59 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 27 Aug 2010 20:22:59 +0200 (CEST) Subject: [pypy-svn] r76762 - in pypy/branch/markcompact/pypy/rpython/lltypesystem: . test Message-ID: <20100827182259.AE996282B9C@codespeak.net> Author: arigo Date: Fri Aug 27 20:22:51 2010 New Revision: 76762 Modified: pypy/branch/markcompact/pypy/rpython/lltypesystem/llgroup.py pypy/branch/markcompact/pypy/rpython/lltypesystem/opimpl.py pypy/branch/markcompact/pypy/rpython/lltypesystem/test/test_llgroup.py Log: Test and fixes. Modified: pypy/branch/markcompact/pypy/rpython/lltypesystem/llgroup.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/lltypesystem/llgroup.py (original) +++ pypy/branch/markcompact/pypy/rpython/lltypesystem/llgroup.py Fri Aug 27 20:22:51 2010 @@ -137,7 +137,7 @@ def __rshift__(self, other): assert other >= HALFSHIFT - return self.rest >> (other - HALFSHIFT) + return self.rest >> other def __eq__(self, other): if (isinstance(other, CombinedSymbolic) and Modified: pypy/branch/markcompact/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/markcompact/pypy/rpython/lltypesystem/opimpl.py Fri Aug 27 20:22:51 2010 @@ -222,6 +222,13 @@ assert isinstance(y, (int, llmemory.AddressOffset)) return intmask(x * y) +def op_int_rshift(x, y): + if not isinstance(x, int): + from pypy.rpython.lltypesystem import llgroup + assert isinstance(x, llgroup.CombinedSymbolic) + assert isinstance(y, int) + return x >> y + def op_int_floordiv(x, y): assert isinstance(x, (int, llmemory.AddressOffset)) assert isinstance(y, (int, llmemory.AddressOffset)) Modified: pypy/branch/markcompact/pypy/rpython/lltypesystem/test/test_llgroup.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/lltypesystem/test/test_llgroup.py (original) +++ pypy/branch/markcompact/pypy/rpython/lltypesystem/test/test_llgroup.py Fri Aug 27 20:22:51 2010 @@ -105,6 +105,8 @@ assert p == test.p1b assert cslist[0] & ~MASK == 0x45 << HALFSHIFT assert cslist[1] & ~MASK == 0x41 << HALFSHIFT + assert cslist[0] >> HALFSHIFT == 0x45 + assert cslist[1] >> (HALFSHIFT+1) == 0x41 >> 1 # return 42 return f From arigo at codespeak.net Fri Aug 27 20:30:40 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 27 Aug 2010 20:30:40 +0200 (CEST) Subject: [pypy-svn] r76763 - in pypy/branch/markcompact/pypy/rpython/memory/gc: . test Message-ID: <20100827183040.EDC5A282B9C@codespeak.net> Author: arigo Date: Fri Aug 27 20:30:36 2010 New Revision: 76763 Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py pypy/branch/markcompact/pypy/rpython/memory/gc/test/test_direct.py Log: Test and fix for identityhash(). Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Fri Aug 27 20:30:36 2010 @@ -386,7 +386,8 @@ obj = fromaddr + size_gc_header # compute the original object size, including the # optional hash field - totalsrcsize = size_gc_header + self.get_size(obj) + basesize = size_gc_header + self.get_size(obj) + totalsrcsize = basesize if hdr.tid & GCFLAG_HASHFIELD: # already a hash field, copy it too totalsrcsize += llmemory.sizeof(lltype.Signed) # @@ -401,7 +402,13 @@ # (otherwise, we get a bogus toaddr > fromaddr) if self.toaddr_smaller_than_fromaddr(toaddr, fromaddr): totaldstsize += llmemory.sizeof(lltype.Signed) - llarena.arena_reserve(toaddr, totaldstsize) + # + if not we_are_translated(): + llarena.arena_reserve(toaddr, basesize) + if (raw_malloc_usage(totaldstsize) > + raw_malloc_usage(basesize)): + llarena.arena_reserve(toaddr + basesize, + llmemory.sizeof(lltype.Signed)) # # save the field hdr.tid in the array tid_backup ll_assert(num < maxnum, "overflow of the tid_backup table") @@ -446,7 +453,8 @@ else: typeid = self.get_type_id(obj) baseobjsize = self._get_size_for_typeid(obj, typeid) - totalsrcsize = size_gc_header + baseobjsize + basesize = size_gc_header + baseobjsize + totalsrcsize = basesize # if survives: grow_hash_field = False @@ -458,8 +466,7 @@ if self.toaddr_smaller_than_fromaddr(toaddr, fromaddr): grow_hash_field = True totaldstsize += llmemory.sizeof(lltype.Signed) - callback(self, obj, typeid, totalsrcsize, - toaddr, grow_hash_field) + callback(self, obj, typeid, basesize, toaddr, grow_hash_field) toaddr += totaldstsize else: if hdr.tid & GCFLAG_HASHFIELD: @@ -545,21 +552,26 @@ def compact(self): self.walk_marked_objects(MarkCompactGC.copy_and_compact) - def copy_and_compact(self, obj, typeid, totalsrcsize, - toaddr, grow_hash_field): + def copy_and_compact(self, obj, typeid, basesize, toaddr, grow_hash_field): + # 'basesize' is the size without any hash field # restore the normal header hdr = self.header_forwarded(obj) gcflags = hdr.tid & 3 if grow_hash_field: gcflags |= GCFLAG_SAVED_HASHFIELD - save_hash_field_now + hashvalue = self.get_identityhash_from_addr(obj) + (toaddr + basesize).signed[0] = hashvalue + elif gcflags & GCFLAG_SAVED_HASHFIELD: + hashvalue = (fromaddr + basesize).signed[0] + (toaddr + basesize).signed[0] = hashvalue + # hdr.tid = self.combine(typeid, gcflags << first_gcflag_bit) # fromaddr = obj - self.gcheaderbuilder.size_gc_header if we_are_translated(): - llmemory.raw_memmove(fromaddr, toaddr, totalsrcsize) + llmemory.raw_memmove(fromaddr, toaddr, basesize) else: - llmemory.raw_memcopy(fromaddr, toaddr, totalsrcsize) + llmemory.raw_memcopy(fromaddr, toaddr, basesize) def debug_check_object(self, obj): # not sure what to check here @@ -603,10 +615,13 @@ hdr = self.header(obj) # if hdr.tid & GCFLAG_HASHFIELD: # the hash is in a field at the end - obj += self.get_size(obj) + obj = llarena.getfakearenaaddress(obj) + self.get_size(obj) return obj.signed[0] # hdr.tid |= GCFLAG_HASHTAKEN + return self.get_identityhash_from_addr(obj) + + def get_identityhash_from_addr(self, obj): if we_are_translated(): return llmemory.cast_adr_to_int(obj) # direct case else: Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/test/test_direct.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/test/test_direct.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/test/test_direct.py Fri Aug 27 20:30:36 2010 @@ -314,7 +314,16 @@ print hash assert isinstance(hash, (int, long)) assert hash == self.gc.identityhash(p_const) - + # (5) p is actually moving (for the markcompact gc) + p0 = self.malloc(S) + self.stackroots.append(p0) + p = self.malloc(S) + self.stackroots.append(p) + hash = self.gc.identityhash(p) + self.stackroots.pop(-2) + self.gc.collect() # p0 goes away, p shifts left + assert hash == self.gc.identityhash(self.stackroots[-1]) + self.stackroots.pop() class TestSemiSpaceGC(DirectGCTest): from pypy.rpython.memory.gc.semispace import SemiSpaceGC as GCClass From arigo at codespeak.net Fri Aug 27 20:33:42 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 27 Aug 2010 20:33:42 +0200 (CEST) Subject: [pypy-svn] r76764 - in pypy/branch/markcompact/pypy/rpython/memory/gc: . test Message-ID: <20100827183342.C279E282B9C@codespeak.net> Author: arigo Date: Fri Aug 27 20:33:40 2010 New Revision: 76764 Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py pypy/branch/markcompact/pypy/rpython/memory/gc/test/test_direct.py Log: Test and fix. Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Fri Aug 27 20:33:40 2010 @@ -562,6 +562,8 @@ hashvalue = self.get_identityhash_from_addr(obj) (toaddr + basesize).signed[0] = hashvalue elif gcflags & GCFLAG_SAVED_HASHFIELD: + fromaddr = llarena.getfakearenaaddress(obj) + fromaddr -= self.gcheaderbuilder.size_gc_header hashvalue = (fromaddr + basesize).signed[0] (toaddr + basesize).signed[0] = hashvalue # Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/test/test_direct.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/test/test_direct.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/test/test_direct.py Fri Aug 27 20:33:40 2010 @@ -323,6 +323,8 @@ self.stackroots.pop(-2) self.gc.collect() # p0 goes away, p shifts left assert hash == self.gc.identityhash(self.stackroots[-1]) + self.gc.collect() + assert hash == self.gc.identityhash(self.stackroots[-1]) self.stackroots.pop() class TestSemiSpaceGC(DirectGCTest): From arigo at codespeak.net Fri Aug 27 21:07:15 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 27 Aug 2010 21:07:15 +0200 (CEST) Subject: [pypy-svn] r76765 - in pypy/branch/markcompact/pypy/rpython: lltypesystem memory/gc Message-ID: <20100827190715.86299282B9C@codespeak.net> Author: arigo Date: Fri Aug 27 21:06:57 2010 New Revision: 76765 Modified: pypy/branch/markcompact/pypy/rpython/lltypesystem/rffi.py pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Log: Tweak tweak tweak... Modified: pypy/branch/markcompact/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/markcompact/pypy/rpython/lltypesystem/rffi.py Fri Aug 27 21:06:57 2010 @@ -593,9 +593,12 @@ """ str -> char* """ array = lltype.malloc(TYPEP.TO, len(s) + 1, flavor='raw') - for i in range(len(s)): + i = len(s) + array[i] = lastchar + i -= 1 + while i >= 0: array[i] = s[i] - array[len(s)] = lastchar + i -= 1 return array str2charp._annenforceargs_ = [strtype] Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Fri Aug 27 21:06:57 2010 @@ -1,6 +1,3 @@ - -import time - from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup from pypy.rpython.memory.gc.base import MovingGCBase, read_from_env from pypy.rlib.debug import ll_assert, have_debug_prints @@ -80,8 +77,11 @@ malloc_zero_filled = False inline_simple_malloc = True inline_simple_malloc_varsize = True - total_collection_time = 0.0 - total_collection_count = 0 + #total_collection_time = 0.0 + #total_collection_count = 0 + + free = NULL + next_collect_after = -1 def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE, space_size=4096, min_next_collect_after=128): @@ -112,9 +112,7 @@ if envsize >= 4096: self.space_size = envsize & ~4095 - self.next_collect_after = self.next_collection(0, 0, 0) - - self.program_start_time = time.time() + #self.program_start_time = time.time() self.space = llarena.arena_malloc(self.space_size, False) if not self.space: raise CannotAllocateGCArena @@ -122,6 +120,7 @@ MovingGCBase.setup(self) self.objects_with_finalizers = self.AddressDeque() self.tid_backup = lltype.nullptr(TID_BACKUP) + self.next_collect_after = self.next_collection(0, 0, 0) def init_gc_object(self, addr, typeid16, flags=0): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) @@ -138,8 +137,9 @@ requested_size = raw_malloc_usage(totalsize) + BYTES_PER_TID self.next_collect_after -= requested_size if self.next_collect_after < 0: - self.obtain_free_space(requested_size) - result = self.free + result = self.obtain_free_space(requested_size) + else: + result = self.free self.free += totalsize llarena.arena_reserve(result, totalsize) return result @@ -190,6 +190,8 @@ return self._setup_object(result, typeid16, False) def obtain_free_space(self, requested_size): + if self.free == NULL: + return self._emergency_initial_block(requested_size) while True: executed_some_finalizers = self.markcompactcollect(requested_size) self.next_collect_after -= requested_size @@ -200,13 +202,23 @@ pass # try again to do a collection else: raise MemoryError + return self.free obtain_free_space._dont_inline_ = True + def _emergency_initial_block(self, requested_size): + # xxx before the GC is fully setup, we might get there. Hopefully + # we will only allocate a couple of strings, e.g. in read_from_env(). + # Just allocate them raw and leak them. + debug_start("gc-initial-block") + debug_print("leaking", requested_size, "bytes") + debug_stop("gc-initial-block") + return llmemory.raw_malloc(requested_size) + def collect(self, gen=0): self.markcompactcollect() def markcompactcollect(self, requested_size=0): - start_time = self.debug_collect_start(requested_size) + self.debug_collect_start(requested_size) self.debug_check_consistency() # # Mark alive objects @@ -220,9 +232,9 @@ toaddr = llarena.arena_new_view(self.space) maxnum = self.space_size - (self.free - self.space) maxnum /= BYTES_PER_TID - if not we_are_translated(): - llarena.arena_reserve(self.free, - llmemory.sizeof(TID_BACKUP, maxnum)) + llarena.arena_reserve(self.free, + llmemory.itemoffsetof(TID_BACKUP) + + llmemory.sizeof(TID_BACKUP.OF) * maxnum) self.tid_backup = llmemory.cast_adr_to_ptr(self.free, lltype.Ptr(TID_BACKUP)) # @@ -250,7 +262,7 @@ self.space = toaddr # self.debug_check_consistency() - self.debug_collect_finish(start_time) + self.debug_collect_finish() if self.next_collect_after < 0: raise MemoryError # @@ -261,27 +273,27 @@ return False # no finalizer executed def debug_collect_start(self, requested_size): - if have_debug_prints(): + if 1:# have_debug_prints(): debug_start("gc-collect") debug_print() debug_print(".----------- Full collection -------------------") debug_print("| requested size:", requested_size) - start_time = time.time() - return start_time - return -1 - - def debug_collect_finish(self, start_time): - if start_time != -1: - end_time = time.time() - elapsed_time = end_time - start_time - self.total_collection_time += elapsed_time - self.total_collection_count += 1 - total_program_time = end_time - self.program_start_time - ct = self.total_collection_time - cc = self.total_collection_count - debug_print("| number of collections so far ", - cc) + #start_time = time.time() + #return start_time + #return -1 + + def debug_collect_finish(self): + if 1:# start_time != -1: + #end_time = time.time() + #elapsed_time = end_time - start_time + #self.total_collection_time += elapsed_time + #self.total_collection_count += 1 + #total_program_time = end_time - self.program_start_time + #ct = self.total_collection_time + #cc = self.total_collection_count + #debug_print("| number of collections so far ", + # cc) debug_print("| total space size ", self.space_size) debug_print("| number of objects alive ", @@ -290,12 +302,12 @@ self.free - self.space) debug_print("| next collection after ", self.next_collect_after) - debug_print("| total collections per second: ", - cc / total_program_time) - debug_print("| total time in markcompact-collect: ", - ct, "seconds") - debug_print("| percentage collection<->total time:", - ct * 100.0 / total_program_time, "%") + #debug_print("| total collections per second: ", + # cc / total_program_time) + #debug_print("| total time in markcompact-collect: ", + # ct, "seconds") + #debug_print("| percentage collection<->total time:", + # ct * 100.0 / total_program_time, "%") debug_print("`----------------------------------------------") debug_stop("gc-collect") @@ -570,6 +582,7 @@ hdr.tid = self.combine(typeid, gcflags << first_gcflag_bit) # fromaddr = obj - self.gcheaderbuilder.size_gc_header + llarena.arena_reserve(toaddr, basesize) if we_are_translated(): llmemory.raw_memmove(fromaddr, toaddr, basesize) else: From jcreigh at codespeak.net Fri Aug 27 21:18:21 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Fri, 27 Aug 2010 21:18:21 +0200 (CEST) Subject: [pypy-svn] r76766 - pypy/branch/asmgcc-64/pypy/module/pypyjit/test Message-ID: <20100827191821.E1BEB282B9C@codespeak.net> Author: jcreigh Date: Fri Aug 27 21:18:17 2010 New Revision: 76766 Modified: pypy/branch/asmgcc-64/pypy/module/pypyjit/test/test_pypy_c.py Log: fix factorial tests by bumping up to a number that is a "long" on 64-bits as well Modified: pypy/branch/asmgcc-64/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/asmgcc-64/pypy/module/pypyjit/test/test_pypy_c.py Fri Aug 27 21:18:17 2010 @@ -189,7 +189,7 @@ return r ''', 28, ([5], 120), - ([20], 2432902008176640000L)) + ([25], 15511210043330985984000000L)) def test_factorialrec(self): self.run_source(''' @@ -200,7 +200,7 @@ return 1 ''', 0, ([5], 120), - ([20], 2432902008176640000L)) + ([25], 15511210043330985984000000L)) def test_richards(self): self.run_source(''' From arigo at codespeak.net Sat Aug 28 17:44:36 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 28 Aug 2010 17:44:36 +0200 (CEST) Subject: [pypy-svn] r76767 - in pypy/trunk/pypy/interpreter: . test Message-ID: <20100828154436.E8193282BEC@codespeak.net> Author: arigo Date: Sat Aug 28 17:44:32 2010 New Revision: 76767 Modified: pypy/trunk/pypy/interpreter/generator.py pypy/trunk/pypy/interpreter/pyframe.py pypy/trunk/pypy/interpreter/test/test_generator.py Log: Test and fix about generator.throw(). Modified: pypy/trunk/pypy/interpreter/generator.py ============================================================================== --- pypy/trunk/pypy/interpreter/generator.py (original) +++ pypy/trunk/pypy/interpreter/generator.py Sat Aug 28 17:44:32 2010 @@ -1,7 +1,6 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.gateway import NoneNotWrapped -from pypy.rlib.rarithmetic import intmask from pypy.rlib import jit from pypy.interpreter.pyopcode import LoopBlock @@ -37,7 +36,7 @@ return next yielded value or raise StopIteration.""" return self.send_ex(w_arg) - def send_ex(self, w_arg, exc=False): + def send_ex(self, w_arg, operr=None): space = self.space if self.running: raise OperationError(space.w_ValueError, @@ -57,7 +56,7 @@ self.running = True try: try: - w_result = self.frame.execute_generator_frame(w_arg, exc) + w_result = self.frame.execute_generator_frame(w_arg, operr) except OperationError: # errors finish a frame self.frame.frame_finished_execution = True @@ -89,12 +88,7 @@ operr = OperationError(w_type, w_val, tb) operr.normalize_exception(space) - - ec = space.getexecutioncontext() - next_instr = self.frame.handle_operation_error(ec, operr) - self.frame.last_instr = intmask(next_instr - 1) - - return self.send_ex(space.w_None, True) + return self.send_ex(space.w_None, operr) def descr_next(self): """next() -> the next value, or raise StopIteration""" Modified: pypy/trunk/pypy/interpreter/pyframe.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyframe.py (original) +++ pypy/trunk/pypy/interpreter/pyframe.py Sat Aug 28 17:44:32 2010 @@ -10,6 +10,7 @@ from pypy.rlib.objectmodel import we_are_translated, instantiate from pypy.rlib.jit import hint from pypy.rlib.debug import make_sure_not_resized +from pypy.rlib.rarithmetic import intmask from pypy.rlib import jit, rstack from pypy.tool import stdlib_opcode @@ -125,8 +126,12 @@ else: return self.execute_frame() - def execute_generator_frame(self, w_inputvalue, ex=False): - if self.last_instr != -1 and not ex: + def execute_generator_frame(self, w_inputvalue, operr=None): + if operr is not None: + ec = self.space.getexecutioncontext() + next_instr = self.handle_operation_error(ec, operr) + self.last_instr = intmask(next_instr - 1) + elif self.last_instr != -1: self.pushvalue(w_inputvalue) return self.execute_frame() Modified: pypy/trunk/pypy/interpreter/test/test_generator.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_generator.py (original) +++ pypy/trunk/pypy/interpreter/test/test_generator.py Sat Aug 28 17:44:32 2010 @@ -126,6 +126,16 @@ raises(ValueError, g.throw, ValueError) assert g.gi_frame is None + def test_throw_bug(self): + def f(): + try: + x.throw(IndexError) # => "generator already executing" + except ValueError: + yield 1 + x = f() + res = list(x) + assert res == [1] + def test_close(self): def f(): yield 1 From agaynor at codespeak.net Sat Aug 28 18:59:09 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sat, 28 Aug 2010 18:59:09 +0200 (CEST) Subject: [pypy-svn] r76770 - in pypy/trunk/pypy/module/_socket: . test Message-ID: <20100828165909.B41A0282BEC@codespeak.net> Author: agaynor Date: Sat Aug 28 18:59:08 2010 New Revision: 76770 Modified: pypy/trunk/pypy/module/_socket/interp_socket.py pypy/trunk/pypy/module/_socket/test/test_sock_app.py Log: Prevent a crash from an unhandled exception in _socket.socket.connect_ex when a bad address is provided. Modified: pypy/trunk/pypy/module/_socket/interp_socket.py ============================================================================== --- pypy/trunk/pypy/module/_socket/interp_socket.py (original) +++ pypy/trunk/pypy/module/_socket/interp_socket.py Sat Aug 28 18:59:08 2010 @@ -74,7 +74,11 @@ This is like connect(address), but returns an error code (the errno value) instead of raising an exception when an error occurs. """ - error = self.connect_ex(self.addr_from_object(space, w_addr)) + try: + addr = self.addr_from_object(space, w_addr) + except SocketError, e: + raise converted_error(space, e) + error = self.connect_ex(addr) return space.wrap(error) connect_ex_w.unwrap_spec = ['self', ObjSpace, W_Root] Modified: pypy/trunk/pypy/module/_socket/test/test_sock_app.py ============================================================================== --- pypy/trunk/pypy/module/_socket/test/test_sock_app.py (original) +++ pypy/trunk/pypy/module/_socket/test/test_sock_app.py Sat Aug 28 18:59:08 2010 @@ -339,6 +339,13 @@ name = s.getpeername() # Will raise socket.error if not connected assert name[1] == 80 s.close() + + def test_socket_connect_ex(self): + import _socket + s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0) + # Make sure we get an app-level error, not an interp one. + raises(_socket.gaierror, s.connect_ex, ("wrong.invalid", 80)) + s.close() def test_socket_connect_typeerrors(self): tests = [ From arigo at codespeak.net Sat Aug 28 19:19:13 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 28 Aug 2010 19:19:13 +0200 (CEST) Subject: [pypy-svn] r76771 - in pypy/branch/markcompact/pypy/rpython: lltypesystem memory/gc Message-ID: <20100828171913.CC219282BEC@codespeak.net> Author: arigo Date: Sat Aug 28 19:19:12 2010 New Revision: 76771 Modified: pypy/branch/markcompact/pypy/rpython/lltypesystem/llmemory.py pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Log: Fixes. Modified: pypy/branch/markcompact/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/markcompact/pypy/rpython/lltypesystem/llmemory.py Sat Aug 28 19:19:12 2010 @@ -361,19 +361,27 @@ # ____________________________________________________________ +def _sizeof_none(TYPE): + assert not TYPE._is_varsize() + return ItemOffset(TYPE) +_sizeof_none._annspecialcase_ = 'specialize:memo' + +def _sizeof_int(TYPE, n): + "NOT_RPYTHON" + if isinstance(TYPE, lltype.Struct): + return FieldOffset(TYPE, TYPE._arrayfld) + \ + itemoffsetof(TYPE._flds[TYPE._arrayfld], n) + else: + raise Exception("don't know how to take the size of a %r"%TYPE) + def sizeof(TYPE, n=None): if n is None: - assert not TYPE._is_varsize() - return ItemOffset(TYPE) + return _sizeof_none(TYPE) + elif isinstance(TYPE, lltype.Array): + return itemoffsetof(TYPE) + _sizeof_none(TYPE.OF) * n else: - if isinstance(TYPE, lltype.Array): - return itemoffsetof(TYPE, n) - elif isinstance(TYPE, lltype.Struct): - return FieldOffset(TYPE, TYPE._arrayfld) + \ - itemoffsetof(TYPE._flds[TYPE._arrayfld], n) - else: - raise Exception("don't know how to take the size of a %r"%TYPE) -sizeof._annspecialcase_ = 'specialize:memo' # only for n == None + return _sizeof_int(TYPE, n) +sizeof._annspecialcase_ = 'specialize:arg(0)' def offsetof(TYPE, fldname): assert fldname in TYPE._flds Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Sat Aug 28 19:19:12 2010 @@ -8,7 +8,7 @@ from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage from pypy.rlib.rarithmetic import ovfcheck, LONG_BIT, intmask from pypy.rpython.lltypesystem.lloperation import llop -from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.objectmodel import we_are_translated, running_on_llinterp from pypy.rpython.lltypesystem import rffi from pypy.rpython.memory.gcheader import GCHeaderBuilder @@ -60,6 +60,9 @@ BYTES_PER_TID = rffi.sizeof(TID_TYPE) TID_BACKUP = rffi.CArray(TID_TYPE) +def translated_to_c(): + return we_are_translated() and not running_on_llinterp + class MarkCompactGC(MovingGCBase): HDR = lltype.Struct('header', ('tid', lltype.Signed)) @@ -232,9 +235,7 @@ toaddr = llarena.arena_new_view(self.space) maxnum = self.space_size - (self.free - self.space) maxnum /= BYTES_PER_TID - llarena.arena_reserve(self.free, - llmemory.itemoffsetof(TID_BACKUP) + - llmemory.sizeof(TID_BACKUP.OF) * maxnum) + llarena.arena_reserve(self.free, llmemory.sizeof(TID_BACKUP, maxnum)) self.tid_backup = llmemory.cast_adr_to_ptr(self.free, lltype.Ptr(TID_BACKUP)) # @@ -251,13 +252,13 @@ # self.tid_backup = lltype.nullptr(TID_BACKUP) self.free = finaladdr - remaining_size = (toaddr + self.space_size) - finaladdr - llarena.arena_reset(finaladdr, remaining_size, False) self.next_collect_after = self.next_collection(finaladdr - toaddr, self.num_alive_objs, requested_size) # - if not we_are_translated(): + if not translated_to_c(): + remaining_size = (toaddr + self.space_size) - finaladdr + llarena.arena_reset(finaladdr, remaining_size, False) llarena.arena_free(self.space) self.space = toaddr # @@ -381,7 +382,7 @@ return MovingGCBase.header(self, obj).tid & GCFLAG_MARKBIT def toaddr_smaller_than_fromaddr(self, toaddr, fromaddr): - if we_are_translated(): + if translated_to_c(): return toaddr < fromaddr else: # convert the addresses to integers, because they are @@ -415,7 +416,7 @@ if self.toaddr_smaller_than_fromaddr(toaddr, fromaddr): totaldstsize += llmemory.sizeof(lltype.Signed) # - if not we_are_translated(): + if not translated_to_c(): llarena.arena_reserve(toaddr, basesize) if (raw_malloc_usage(totaldstsize) > raw_malloc_usage(basesize)): @@ -438,7 +439,7 @@ toaddr += totaldstsize # fromaddr += totalsrcsize - if not we_are_translated(): + if not translated_to_c(): assert toaddr - base_forwarding_addr <= fromaddr - self.space self.num_alive_objs = num @@ -582,8 +583,7 @@ hdr.tid = self.combine(typeid, gcflags << first_gcflag_bit) # fromaddr = obj - self.gcheaderbuilder.size_gc_header - llarena.arena_reserve(toaddr, basesize) - if we_are_translated(): + if translated_to_c(): llmemory.raw_memmove(fromaddr, toaddr, basesize) else: llmemory.raw_memcopy(fromaddr, toaddr, basesize) @@ -637,7 +637,7 @@ return self.get_identityhash_from_addr(obj) def get_identityhash_from_addr(self, obj): - if we_are_translated(): + if translated_to_c(): return llmemory.cast_adr_to_int(obj) # direct case else: try: From arigo at codespeak.net Sat Aug 28 19:24:07 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 28 Aug 2010 19:24:07 +0200 (CEST) Subject: [pypy-svn] r76772 - pypy/branch/markcompact/pypy/rpython/memory/gc/test Message-ID: <20100828172407.A4C58282BEC@codespeak.net> Author: arigo Date: Sat Aug 28 19:24:06 2010 New Revision: 76772 Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/test/test_direct.py Log: Fix on 64-bit. Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/test/test_direct.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/test/test_direct.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/test/test_direct.py Sat Aug 28 19:24:06 2010 @@ -447,12 +447,12 @@ def test_many_objects(self): DirectGCTest.test_many_objects(self) - test_many_objects.GC_PARAMS = {'space_size': 3 * 4096} + test_many_objects.GC_PARAMS = {'space_size': 3 * 1024 * WORD} def test_varsized_from_stack(self): DirectGCTest.test_varsized_from_stack(self) - test_varsized_from_stack.GC_PARAMS = {'space_size': 2 * 4096} + test_varsized_from_stack.GC_PARAMS = {'space_size': 2 * 1024 * WORD} def test_varsized_from_prebuilt_gc(self): DirectGCTest.test_varsized_from_prebuilt_gc(self) - test_varsized_from_prebuilt_gc.GC_PARAMS = {'space_size': 3 * 4096} + test_varsized_from_prebuilt_gc.GC_PARAMS = {'space_size': 3 * 1024 * WORD} From arigo at codespeak.net Sat Aug 28 19:31:59 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 28 Aug 2010 19:31:59 +0200 (CEST) Subject: [pypy-svn] r76773 - pypy/trunk/lib-python/modified-2.5.2/test Message-ID: <20100828173159.ED89B282BEC@codespeak.net> Author: arigo Date: Sat Aug 28 19:31:58 2010 New Revision: 76773 Removed: pypy/trunk/lib-python/modified-2.5.2/test/test_re.py Log: Remove this test from "modified". I think the original test should now fully pass. From arigo at codespeak.net Sat Aug 28 19:42:18 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 28 Aug 2010 19:42:18 +0200 (CEST) Subject: [pypy-svn] r76774 - pypy/branch/markcompact/pypy/rpython/memory/gc Message-ID: <20100828174218.D1DAD282BEC@codespeak.net> Author: arigo Date: Sat Aug 28 19:42:17 2010 New Revision: 76774 Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Log: Use a better way to detect overflows. Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Sat Aug 28 19:42:17 2010 @@ -153,10 +153,13 @@ varsize = ovfcheck(itemsize * length) except OverflowError: raise MemoryError - totalsize = llarena.round_up_for_allocation(nonvarsize + varsize) - if totalsize < 0: # if wrapped around + # Careful to detect overflows. The following works even if varsize + # is almost equal to sys.maxint; morever, self.space_size is known + # to be at least 4095 bytes smaller than sys.maxint, so this function + # always raises instead of returning an integer >= sys.maxint-4095. + if varsize > self.space_size - nonvarsize: raise MemoryError - return totalsize + return llarena.round_up_for_allocation(nonvarsize + varsize) _get_totalsize_var._always_inline_ = True def _setup_object(self, result, typeid16, has_finalizer): From arigo at codespeak.net Sat Aug 28 19:45:34 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 28 Aug 2010 19:45:34 +0200 (CEST) Subject: [pypy-svn] r76775 - pypy/branch/markcompact/pypy/rpython/memory/gc Message-ID: <20100828174534.2BF3B282BEC@codespeak.net> Author: arigo Date: Sat Aug 28 19:45:31 2010 New Revision: 76775 Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Log: Oups. Fix. Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Sat Aug 28 19:45:31 2010 @@ -157,7 +157,8 @@ # is almost equal to sys.maxint; morever, self.space_size is known # to be at least 4095 bytes smaller than sys.maxint, so this function # always raises instead of returning an integer >= sys.maxint-4095. - if varsize > self.space_size - nonvarsize: + if (raw_malloc_usage(varsize) > self.space_size - + raw_malloc_usage(nonvarsize)): raise MemoryError return llarena.round_up_for_allocation(nonvarsize + varsize) _get_totalsize_var._always_inline_ = True From arigo at codespeak.net Sat Aug 28 19:50:21 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 28 Aug 2010 19:50:21 +0200 (CEST) Subject: [pypy-svn] r76776 - pypy/branch/markcompact/pypy/rpython/lltypesystem Message-ID: <20100828175021.03084282BEC@codespeak.net> Author: arigo Date: Sat Aug 28 19:50:20 2010 New Revision: 76776 Modified: pypy/branch/markcompact/pypy/rpython/lltypesystem/opimpl.py Log: Fix missing feature in llinterp'ed mode. Modified: pypy/branch/markcompact/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/markcompact/pypy/rpython/lltypesystem/opimpl.py Sat Aug 28 19:50:20 2010 @@ -197,6 +197,18 @@ assert isinstance(y, int) return intmask(x - y) +def op_int_ge(x, y): + # special case for 'AddressOffset >= 0' + assert isinstance(x, (int, llmemory.AddressOffset)) + assert isinstance(y, int) + return x >= y + +def op_int_lt(x, y): + # special case for 'AddressOffset < 0' + assert isinstance(x, (int, llmemory.AddressOffset)) + assert isinstance(y, int) + return x < y + def op_int_between(a, b, c): assert lltype.typeOf(a) is lltype.Signed assert lltype.typeOf(b) is lltype.Signed From cfbolz at codespeak.net Sun Aug 29 09:42:40 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 29 Aug 2010 09:42:40 +0200 (CEST) Subject: [pypy-svn] r76777 - in pypy/branch/better-map-instances/pypy: interpreter objspace/std objspace/std/test Message-ID: <20100829074240.76B76282BEB@codespeak.net> Author: cfbolz Date: Sun Aug 29 09:42:36 2010 New Revision: 76777 Modified: pypy/branch/better-map-instances/pypy/interpreter/baseobjspace.py pypy/branch/better-map-instances/pypy/interpreter/typedef.py pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py Log: only use the slot index in mapdict, makes it possible to revert 76692 and kill an XXX Modified: pypy/branch/better-map-instances/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/better-map-instances/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/better-map-instances/pypy/interpreter/baseobjspace.py Sun Aug 29 09:42:36 2010 @@ -105,10 +105,10 @@ return space.wrap("<%s at 0x%s%s>" % (info, addrstring, moreinfo)) - def getslotvalue(self, member): + def getslotvalue(self, index): raise NotImplementedError - def setslotvalue(self, member, w_val): + def setslotvalue(self, index, w_val): raise NotImplementedError def descr_call_mismatch(self, space, opname, RequiredClass, args): Modified: pypy/branch/better-map-instances/pypy/interpreter/typedef.py ============================================================================== --- pypy/branch/better-map-instances/pypy/interpreter/typedef.py (original) +++ pypy/branch/better-map-instances/pypy/interpreter/typedef.py Sun Aug 29 09:42:36 2010 @@ -255,10 +255,10 @@ def user_setup_slots(self, nslots): if nslots > 0: self.slots_w = [None] * nslots - def setslotvalue(self, member, w_value): - self.slots_w[member.index] = w_value - def getslotvalue(self, member): - return self.slots_w[member.index] + def setslotvalue(self, index, w_value): + self.slots_w[index] = w_value + def getslotvalue(self, index): + return self.slots_w[index] add(Proto) wantdict = "dict" in features @@ -537,7 +537,7 @@ else: self = member self.typecheck(space, w_obj) - w_result = w_obj.getslotvalue(self) + w_result = w_obj.getslotvalue(self.index) if w_result is None: raise OperationError(space.w_AttributeError, space.wrap(self.name)) # XXX better message @@ -548,14 +548,14 @@ Write into the slot 'member' of the given 'obj'.""" self = member self.typecheck(space, w_obj) - w_obj.setslotvalue(self, w_value) + w_obj.setslotvalue(self.index, w_value) def descr_member_del(space, member, w_obj): """member.__delete__(obj) Delete the value of the slot 'member' from the given 'obj'.""" self = member self.typecheck(space, w_obj) - w_obj.setslotvalue(self, None) + w_obj.setslotvalue(self.index, None) Member.typedef = TypeDef( "member_descriptor", Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py Sun Aug 29 09:42:36 2010 @@ -291,14 +291,12 @@ assert not self.typedef.hasdict self._init_empty(w_subtype.terminator) - def getslotvalue(self, member): - # XXX we don't need member here, it's enough to have member.index - # XXX and revert the change to the signature of getslotvalue(), maybe. - key = (member.name, SLOTS_STARTING_FROM + member.index) + def getslotvalue(self, index): + key = ("slot", SLOTS_STARTING_FROM + index) return self.map.read(self, key) - def setslotvalue(self, member, w_value): - key = (member.name, SLOTS_STARTING_FROM + member.index) + def setslotvalue(self, index, w_value): + key = ("slot", SLOTS_STARTING_FROM + index) self.map.write(self, key, w_value) # used by _weakref implemenation Modified: pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py Sun Aug 29 09:42:36 2010 @@ -4,11 +4,6 @@ space = FakeSpace() -class FakeMember(object): - def __init__(self, name, index=0): - self.name = name - self.index = index - class Class(object): def __init__(self, hasdict=True): self.hasdict = True @@ -162,9 +157,9 @@ def test_slots(): cls = Class() obj = cls.instantiate() - a = FakeMember("a") - b = FakeMember("b") - c = FakeMember("c") + a = 0 + b = 1 + c = 2 obj.setslotvalue(a, 50) obj.setslotvalue(b, 60) obj.setslotvalue(c, 70) @@ -194,27 +189,13 @@ assert obj2.storage == [501, 601, 701, 51, 61, 71] assert obj.map is obj2.map -def test_slots_same_name(): - cls = Class() - obj = cls.instantiate() - a = FakeMember("a") - b = FakeMember("a", 1) - c = FakeMember("a", 2) - obj.setslotvalue(a, 50) - obj.setslotvalue(b, 60) - obj.setslotvalue(c, 70) - assert obj.getslotvalue(a) == 50 - assert obj.getslotvalue(b) == 60 - assert obj.getslotvalue(c) == 70 - assert obj.storage == [50, 60, 70] - def test_slots_no_dict(): cls = Class(hasdict=False) obj = cls.instantiate() - a = FakeMember("a") - b = FakeMember("b") - c = FakeMember("c") + a = 0 + b = 1 + c = 2 obj.setslotvalue(a, 50) obj.setslotvalue(b, 60) assert obj.getslotvalue(a) == 50 @@ -235,9 +216,9 @@ def test_materialize_r_dict(): cls = Class() obj = cls.instantiate() - a = FakeMember("a") - b = FakeMember("b") - c = FakeMember("c") + a = 0 + b = 1 + c = 2 obj.setslotvalue(a, 50) obj.setslotvalue(b, 60) obj.setslotvalue(c, 70) @@ -305,9 +286,9 @@ def test_get_setdictvalue_after_devolve(): cls = Class() obj = cls.instantiate() - a = FakeMember("a") - b = FakeMember("b") - c = FakeMember("c") + a = 0 + b = 1 + c = 2 obj.setslotvalue(a, 50) obj.setslotvalue(b, 60) obj.setslotvalue(c, 70) From arigo at codespeak.net Sun Aug 29 11:55:32 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 29 Aug 2010 11:55:32 +0200 (CEST) Subject: [pypy-svn] r76778 - in pypy/branch/markcompact/pypy: rpython/memory/test translator translator/c/test Message-ID: <20100829095532.0B06F282B9E@codespeak.net> Author: arigo Date: Sun Aug 29 11:55:30 2010 New Revision: 76778 Modified: pypy/branch/markcompact/pypy/rpython/memory/test/test_transformed_gc.py pypy/branch/markcompact/pypy/translator/c/test/test_newgc.py pypy/branch/markcompact/pypy/translator/exceptiontransform.py Log: * Finish to port mark&compact. * Add tests about tagged pointers in test_transformed_gc. Modified: pypy/branch/markcompact/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/test/test_transformed_gc.py Sun Aug 29 11:55:30 2010 @@ -1144,6 +1144,10 @@ GC_PARAMS = {'space_size': 4096*WORD} root_stack_depth = 200 + def test_writebarrier_before_copy(self): + py.test.skip("Not relevant, and crashes because llarena does not " + "support empty GcStructs") + class TestGenerationGC(GenericMovingGCTests): gcname = "generation" GC_CAN_SHRINK_ARRAY = True @@ -1533,3 +1537,12 @@ GC_PARAMS = {'space_size': 512*WORD, 'nursery_size': 32*WORD} root_stack_depth = 200 + +class TestMarkCompactTaggedpointerGC(TaggedPointerGCTests): + gcname = 'markcompact' + + class gcpolicy(gc.FrameworkGcPolicy): + class transformerclass(framework.FrameworkGCTransformer): + from pypy.rpython.memory.gc.markcompact import MarkCompactGC as GCClass + GC_PARAMS = {'space_size': 4096*WORD} + root_stack_depth = 200 Modified: pypy/branch/markcompact/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/markcompact/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/markcompact/pypy/translator/c/test/test_newgc.py Sun Aug 29 11:55:30 2010 @@ -67,9 +67,8 @@ if not fullname.startswith('define'): continue keyword = conftest.option.keyword - if keyword: - if keyword.startswith('test_'): - keyword = keyword[len('test_'):] + if keyword.startswith('test_'): + keyword = keyword[len('test_'):] if keyword not in fullname: continue prefix, name = fullname.split('_', 1) @@ -1072,12 +1071,12 @@ should_be_moving = True GC_CAN_SHRINK_ARRAY = False - def setup_class(cls): - py.test.skip("Disabled for now") - def test_gc_set_max_heap_size(self): py.test.skip("not implemented") + def test_gc_heap_stats(self): + py.test.skip("not implemented") + def test_finalizer_order(self): py.test.skip("not implemented") Modified: pypy/branch/markcompact/pypy/translator/exceptiontransform.py ============================================================================== --- pypy/branch/markcompact/pypy/translator/exceptiontransform.py (original) +++ pypy/branch/markcompact/pypy/translator/exceptiontransform.py Sun Aug 29 11:55:30 2010 @@ -197,7 +197,7 @@ for graph in self.translator.graphs: self.create_exception_handling(graph) - def create_exception_handling(self, graph, always_exc_clear=False): + def create_exception_handling(self, graph): """After an exception in a direct_call (or indirect_call), that is not caught by an explicit except statement, we need to reraise the exception. So after this @@ -212,7 +212,6 @@ self.raise_analyzer.analyze_direct_call(graph) graph.exceptiontransformed = self.exc_data_ptr - self.always_exc_clear = always_exc_clear join_blocks(graph) # collect the blocks before changing them n_need_exc_matching_blocks = 0 @@ -455,13 +454,18 @@ block.recloseblock(l0, l) insert_zeroing_op = False - # XXX this is not right. it also inserts zero_gc_pointers_inside - # XXX on a path that malloc_nonmovable returns null, but does not raise - # XXX which might end up with a segfault. But we don't have such gc now - if spaceop.opname == 'malloc' or spaceop.opname == 'malloc_nonmovable': + if spaceop.opname == 'malloc': flavor = spaceop.args[1].value['flavor'] if flavor == 'gc': insert_zeroing_op = True + elif spaceop.opname == 'malloc_nonmovable': + # xxx we cannot insert zero_gc_pointers_inside after + # malloc_nonmovable, because it can return null. For now + # we simply always force the zero=True flag on + # malloc_nonmovable. + c_flags = spaceop.args[1] + c_flags.value = c_flags.value.copy() + spaceop.args[1].value['zero'] = True if insert_zeroing_op: if normalafterblock is None: @@ -479,16 +483,6 @@ [v_result_after], varoftype(lltype.Void))) - if self.always_exc_clear: - # insert code that clears the exception even in the non-exceptional - # case... this is a hint for the JIT, but pointless otherwise - if normalafterblock is None: - normalafterblock = insert_empty_block(None, l0) - llops = rtyper.LowLevelOpList(None) - self.gen_setfield('exc_value', self.c_null_evalue, llops) - self.gen_setfield('exc_type', self.c_null_etype, llops) - normalafterblock.operations[:0] = llops - class LLTypeExceptionTransformer(BaseExceptionTransformer): From arigo at codespeak.net Sun Aug 29 12:19:33 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 29 Aug 2010 12:19:33 +0200 (CEST) Subject: [pypy-svn] r76779 - pypy/branch/markcompact/pypy/translator/c/test Message-ID: <20100829101933.86D91282B9E@codespeak.net> Author: arigo Date: Sun Aug 29 12:19:29 2010 New Revision: 76779 Modified: pypy/branch/markcompact/pypy/translator/c/test/test_newgc.py Log: Add a test using the MarkCompactGC, tagged pointers, and removetypeptr. Modified: pypy/branch/markcompact/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/markcompact/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/markcompact/pypy/translator/c/test/test_newgc.py Sun Aug 29 12:19:29 2010 @@ -1082,10 +1082,9 @@ # ____________________________________________________________________ -class TestHybridTaggedPointers(TestHybridGC): +class TaggedPointersTest(object): taggedpointers = True - def define_tagged(cls): class Unrelated(object): pass @@ -1128,3 +1127,10 @@ __slots__ = 'smallint' def meth(self, x): return self.smallint + x + 3 + + +class TestHybridTaggedPointers(TaggedPointersTest, TestHybridGC): + pass + +class TestMarkCompactGCMostCompact(TaggedPointersTest, TestMarkCompactGC): + removetypeptr = True From arigo at codespeak.net Sun Aug 29 12:24:55 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 29 Aug 2010 12:24:55 +0200 (CEST) Subject: [pypy-svn] r76780 - in pypy/branch/better-map-instances/pypy: rpython/lltypesystem rpython/lltypesystem/test rpython/memory rpython/memory/gc rpython/memory/gc/test rpython/memory/test translator translator/c/test Message-ID: <20100829102455.DDBC5282B9E@codespeak.net> Author: arigo Date: Sun Aug 29 12:24:52 2010 New Revision: 76780 Modified: pypy/branch/better-map-instances/pypy/rpython/lltypesystem/llgroup.py pypy/branch/better-map-instances/pypy/rpython/lltypesystem/llmemory.py pypy/branch/better-map-instances/pypy/rpython/lltypesystem/opimpl.py pypy/branch/better-map-instances/pypy/rpython/lltypesystem/rffi.py pypy/branch/better-map-instances/pypy/rpython/lltypesystem/test/test_llgroup.py pypy/branch/better-map-instances/pypy/rpython/memory/gc/base.py pypy/branch/better-map-instances/pypy/rpython/memory/gc/generation.py pypy/branch/better-map-instances/pypy/rpython/memory/gc/markcompact.py pypy/branch/better-map-instances/pypy/rpython/memory/gc/test/test_direct.py pypy/branch/better-map-instances/pypy/rpython/memory/gcwrapper.py pypy/branch/better-map-instances/pypy/rpython/memory/test/test_gc.py pypy/branch/better-map-instances/pypy/rpython/memory/test/test_transformed_gc.py pypy/branch/better-map-instances/pypy/translator/c/test/test_newgc.py pypy/branch/better-map-instances/pypy/translator/exceptiontransform.py Log: Merge branch/markcompact (up to r76779) in that branch. Modified: pypy/branch/better-map-instances/pypy/rpython/lltypesystem/llgroup.py ============================================================================== --- pypy/branch/better-map-instances/pypy/rpython/lltypesystem/llgroup.py (original) +++ pypy/branch/better-map-instances/pypy/rpython/lltypesystem/llgroup.py Sun Aug 29 12:24:52 2010 @@ -135,6 +135,10 @@ assert (other & CombinedSymbolic.MASK) == 0 return CombinedSymbolic(self.lowpart, self.rest - other) + def __rshift__(self, other): + assert other >= HALFSHIFT + return self.rest >> other + def __eq__(self, other): if (isinstance(other, CombinedSymbolic) and self.lowpart is other.lowpart): Modified: pypy/branch/better-map-instances/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/better-map-instances/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/better-map-instances/pypy/rpython/lltypesystem/llmemory.py Sun Aug 29 12:24:52 2010 @@ -361,19 +361,27 @@ # ____________________________________________________________ +def _sizeof_none(TYPE): + assert not TYPE._is_varsize() + return ItemOffset(TYPE) +_sizeof_none._annspecialcase_ = 'specialize:memo' + +def _sizeof_int(TYPE, n): + "NOT_RPYTHON" + if isinstance(TYPE, lltype.Struct): + return FieldOffset(TYPE, TYPE._arrayfld) + \ + itemoffsetof(TYPE._flds[TYPE._arrayfld], n) + else: + raise Exception("don't know how to take the size of a %r"%TYPE) + def sizeof(TYPE, n=None): if n is None: - assert not TYPE._is_varsize() - return ItemOffset(TYPE) + return _sizeof_none(TYPE) + elif isinstance(TYPE, lltype.Array): + return itemoffsetof(TYPE) + _sizeof_none(TYPE.OF) * n else: - if isinstance(TYPE, lltype.Array): - return itemoffsetof(TYPE, n) - elif isinstance(TYPE, lltype.Struct): - return FieldOffset(TYPE, TYPE._arrayfld) + \ - itemoffsetof(TYPE._flds[TYPE._arrayfld], n) - else: - raise Exception("don't know how to take the size of a %r"%TYPE) -sizeof._annspecialcase_ = 'specialize:memo' # only for n == None + return _sizeof_int(TYPE, n) +sizeof._annspecialcase_ = 'specialize:arg(0)' def offsetof(TYPE, fldname): assert fldname in TYPE._flds Modified: pypy/branch/better-map-instances/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/better-map-instances/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/better-map-instances/pypy/rpython/lltypesystem/opimpl.py Sun Aug 29 12:24:52 2010 @@ -197,6 +197,18 @@ assert isinstance(y, int) return intmask(x - y) +def op_int_ge(x, y): + # special case for 'AddressOffset >= 0' + assert isinstance(x, (int, llmemory.AddressOffset)) + assert isinstance(y, int) + return x >= y + +def op_int_lt(x, y): + # special case for 'AddressOffset < 0' + assert isinstance(x, (int, llmemory.AddressOffset)) + assert isinstance(y, int) + return x < y + def op_int_between(a, b, c): assert lltype.typeOf(a) is lltype.Signed assert lltype.typeOf(b) is lltype.Signed @@ -222,6 +234,13 @@ assert isinstance(y, (int, llmemory.AddressOffset)) return intmask(x * y) +def op_int_rshift(x, y): + if not isinstance(x, int): + from pypy.rpython.lltypesystem import llgroup + assert isinstance(x, llgroup.CombinedSymbolic) + assert isinstance(y, int) + return x >> y + def op_int_floordiv(x, y): assert isinstance(x, (int, llmemory.AddressOffset)) assert isinstance(y, (int, llmemory.AddressOffset)) Modified: pypy/branch/better-map-instances/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/better-map-instances/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/better-map-instances/pypy/rpython/lltypesystem/rffi.py Sun Aug 29 12:24:52 2010 @@ -593,9 +593,12 @@ """ str -> char* """ array = lltype.malloc(TYPEP.TO, len(s) + 1, flavor='raw') - for i in range(len(s)): + i = len(s) + array[i] = lastchar + i -= 1 + while i >= 0: array[i] = s[i] - array[len(s)] = lastchar + i -= 1 return array str2charp._annenforceargs_ = [strtype] Modified: pypy/branch/better-map-instances/pypy/rpython/lltypesystem/test/test_llgroup.py ============================================================================== --- pypy/branch/better-map-instances/pypy/rpython/lltypesystem/test/test_llgroup.py (original) +++ pypy/branch/better-map-instances/pypy/rpython/lltypesystem/test/test_llgroup.py Sun Aug 29 12:24:52 2010 @@ -105,6 +105,8 @@ assert p == test.p1b assert cslist[0] & ~MASK == 0x45 << HALFSHIFT assert cslist[1] & ~MASK == 0x41 << HALFSHIFT + assert cslist[0] >> HALFSHIFT == 0x45 + assert cslist[1] >> (HALFSHIFT+1) == 0x41 >> 1 # return 42 return f Modified: pypy/branch/better-map-instances/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/better-map-instances/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/better-map-instances/pypy/rpython/memory/gc/base.py Sun Aug 29 12:24:52 2010 @@ -86,8 +86,7 @@ addr -= self.gcheaderbuilder.size_gc_header return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - def get_size(self, obj): - typeid = self.get_type_id(obj) + def _get_size_for_typeid(self, obj, typeid): size = self.fixed_size(typeid) if self.is_varsize(typeid): lenaddr = obj + self.varsize_offset_to_length(typeid) @@ -99,6 +98,9 @@ # gctypelayout.encode_type_shape() return size + def get_size(self, obj): + return self._get_size_for_typeid(obj, self.get_type_id(obj)) + def malloc(self, typeid, length=0, zero=False): """For testing. The interface used by the gctransformer is the four malloc_[fixed,var]size[_clear]() functions. @@ -348,3 +350,23 @@ globals(), locals(), [classname]) GCClass = getattr(module, classname) return GCClass, GCClass.TRANSLATION_PARAMS + +def read_from_env(varname): + import os + value = os.environ.get(varname) + if value: + realvalue = value[:-1] + if value[-1] in 'kK': + factor = 1024 + elif value[-1] in 'mM': + factor = 1024*1024 + elif value[-1] in 'gG': + factor = 1024*1024*1024 + else: + factor = 1 + realvalue = value + try: + return int(float(realvalue) * factor) + except ValueError: + pass + return -1 Modified: pypy/branch/better-map-instances/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/better-map-instances/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/better-map-instances/pypy/rpython/memory/gc/generation.py Sun Aug 29 12:24:52 2010 @@ -2,6 +2,7 @@ from pypy.rpython.memory.gc.semispace import SemiSpaceGC from pypy.rpython.memory.gc.semispace import GCFLAG_EXTERNAL, GCFLAG_FORWARDED from pypy.rpython.memory.gc.semispace import GC_HASH_TAKEN_ADDR +from pypy.rpython.memory.gc.base import read_from_env from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE @@ -625,18 +626,7 @@ import os def nursery_size_from_env(): - value = os.environ.get('PYPY_GENERATIONGC_NURSERY') - if value: - if value[-1] in 'kK': - factor = 1024 - value = value[:-1] - else: - factor = 1 - try: - return int(value) * factor - except ValueError: - pass - return -1 + return read_from_env('PYPY_GENERATIONGC_NURSERY') def best_nursery_size_for_L2cache(L2cache): # Heuristically, the best nursery size to choose is about half Modified: pypy/branch/better-map-instances/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/better-map-instances/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/better-map-instances/pypy/rpython/memory/gc/markcompact.py Sun Aug 29 12:24:52 2010 @@ -1,26 +1,17 @@ - -import time - from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup -from pypy.rpython.memory.gc.base import MovingGCBase -from pypy.rlib.debug import ll_assert +from pypy.rpython.memory.gc.base import MovingGCBase, read_from_env +from pypy.rlib.debug import ll_assert, have_debug_prints +from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE from pypy.rpython.memory.support import get_address_stack, get_address_deque from pypy.rpython.memory.support import AddressDict from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage -from pypy.rlib.rarithmetic import ovfcheck +from pypy.rlib.rarithmetic import ovfcheck, LONG_BIT, intmask from pypy.rpython.lltypesystem.lloperation import llop -from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.objectmodel import we_are_translated, running_on_llinterp from pypy.rpython.lltypesystem import rffi from pypy.rpython.memory.gcheader import GCHeaderBuilder -first_gcflag = 1 << 16 -GCFLAG_MARKBIT = first_gcflag << 0 -GCFLAG_HASHTAKEN = first_gcflag << 1 # someone already asked for the hash -GCFLAG_HASHFIELD = first_gcflag << 2 # we have an extra hash field - -memoryError = MemoryError() - # Mark'n'compact garbage collector # # main point of this GC is to save as much memory as possible @@ -33,41 +24,44 @@ # this gc works more or less like semispace, but has some essential # differencies. The main difference is that we have separate phases of # marking and assigning pointers, hence order of objects is preserved. -# This means we can reuse the same space if it did not grow enough. -# More importantly, in case we need to resize space we can copy it bit by -# bit, hence avoiding double memory consumption at peak times +# This means we can reuse the same space, overwriting it as we collect. -# so the algorithm itself is performed in 3 stages (module weakrefs and -# finalizers) +# so the algorithm itself is performed in 3 stages (modulo weakrefs and +# finalizers): # 1. We mark alive objects # 2. We walk all objects and assign forward pointers in the same order, # also updating all references -# 3. We compact the space by moving. In case we move to the same space, -# we use arena_new_view trick, which looks like new space to tests, -# but compiles to the same pointer. Also we use raw_memmove in case -# objects overlap. - -# Exact algorithm for space resizing: we keep allocated more space than needed -# (2x, can be even more), but it's full of zeroes. After each collection, -# we bump next_collect_after which is a marker where to start each collection. -# It should be exponential (but less than 2) from the size occupied by objects +# 3. We compact the space by moving. We use 'arena_new_view' trick, which +# looks like new space to tests, but compiles to the same pointer. +# Also we use raw_memmove in case the object overlaps with its destination. + +# After each collection, we bump 'next_collect_after' which is a marker +# where to start each collection. It should be exponential (but less +# than 2) from the size occupied by objects so far. # field optimization - we don't need forward pointer and flags at the same -# time. Instead we copy list of tids when we know how many objects are alive -# and store forward pointer there. +# time. Instead we copy the TIDs in a list when we know how many objects are +# alive, and store the forward pointer in the old object header. +first_gcflag_bit = LONG_BIT//2 +first_gcflag = 1 << first_gcflag_bit +GCFLAG_HASHTAKEN = first_gcflag << 0 # someone already asked for the hash +GCFLAG_HASHFIELD = first_gcflag << 1 # we have an extra hash field +# note that only the first 2 bits are preserved during a collection! +GCFLAG_MARKBIT = intmask(first_gcflag << (LONG_BIT//2-1)) +assert GCFLAG_MARKBIT < 0 # should be 0x80000000 + +GCFLAG_SAVED_HASHTAKEN = GCFLAG_HASHTAKEN >> first_gcflag_bit +GCFLAG_SAVED_HASHFIELD = GCFLAG_HASHFIELD >> first_gcflag_bit -# in case we need to grow space, we use -# current_space_size * FREE_SPACE_MULTIPLIER / FREE_SPACE_DIVIDER + needed -FREE_SPACE_MULTIPLIER = 3 -FREE_SPACE_DIVIDER = 2 -FREE_SPACE_ADD = 256 -# XXX adjust -GC_CLEARANCE = 32*1024 TID_TYPE = llgroup.HALFWORD BYTES_PER_TID = rffi.sizeof(TID_TYPE) +TID_BACKUP = rffi.CArray(TID_TYPE) + +def translated_to_c(): + return we_are_translated() and not running_on_llinterp class MarkCompactGC(MovingGCBase): @@ -76,38 +70,60 @@ withhash_flag_is_in_field = 'tid', GCFLAG_HASHFIELD # ^^^ all prebuilt objects have GCFLAG_HASHTAKEN, but only some have # GCFLAG_HASHFIELD (and then they are one word longer). - TID_BACKUP = lltype.Array(TID_TYPE, hints={'nolength':True}) - WEAKREF_OFFSETS = lltype.Array(lltype.Signed) + # The default space size is 1.9375 GB, i.e. almost 2 GB, allocated as + # a big mmap. The process does not actually consume that space until + # needed, of course. + TRANSLATION_PARAMS = {'space_size': int((1 + 15.0/16)*1024*1024*1024), + 'min_next_collect_after': 4*1024*1024} # 4MB - TRANSLATION_PARAMS = {'space_size': 8*1024*1024} # XXX adjust - - malloc_zero_filled = True + malloc_zero_filled = False inline_simple_malloc = True inline_simple_malloc_varsize = True - first_unused_gcflag = first_gcflag << 3 - total_collection_time = 0.0 - total_collection_count = 0 - - def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE, space_size=4096): - import py; py.test.skip("Disabled for now, sorry") - self.param_space_size = space_size + #total_collection_time = 0.0 + #total_collection_count = 0 + + free = NULL + next_collect_after = -1 + + def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE, space_size=4096, + min_next_collect_after=128): MovingGCBase.__init__(self, config, chunk_size) + self.space_size = space_size + self.min_next_collect_after = min_next_collect_after - def setup(self): - self.space_size = self.param_space_size - self.next_collect_after = self.param_space_size/2 # whatever... + def next_collection(self, used_space, num_objects_so_far, requested_size): + used_space += BYTES_PER_TID * num_objects_so_far + ll_assert(used_space <= self.space_size, + "used_space + num_objects_so_far overflow") + try: + next = (used_space // 3) * 2 + requested_size + except OverflowError: + next = self.space_size + if next < self.min_next_collect_after: + next = self.min_next_collect_after + if next > self.space_size - used_space: + next = self.space_size - used_space + # The value we return guarantees that used_space + next <= space_size, + # with 'BYTES_PER_TID*num_objects_so_far' included in used_space. + # Normally, the value we return should also be at least requested_size + # unless we are out of memory. + return next - if self.config.gcconfig.debugprint: - self.program_start_time = time.time() - self.space = llarena.arena_malloc(self.space_size, True) - ll_assert(bool(self.space), "couldn't allocate arena") + def setup(self): + envsize = max_size_from_env() + if envsize >= 4096: + self.space_size = envsize & ~4095 + + #self.program_start_time = time.time() + self.space = llarena.arena_malloc(self.space_size, False) + if not self.space: + raise CannotAllocateGCArena self.free = self.space - self.top_of_space = self.space + self.next_collect_after MovingGCBase.setup(self) self.objects_with_finalizers = self.AddressDeque() - self.objects_with_weakrefs = self.AddressStack() - self.tid_backup = lltype.nullptr(self.TID_BACKUP) + self.tid_backup = lltype.nullptr(TID_BACKUP) + self.next_collect_after = self.next_collection(0, 0, 0) def init_gc_object(self, addr, typeid16, flags=0): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) @@ -117,217 +133,199 @@ hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) flags |= GCFLAG_HASHTAKEN hdr.tid = self.combine(typeid16, flags) - # XXX we can store forward_ptr to itself, if we fix C backend - # so that get_forwarding_address(obj) returns - # obj itself if obj is a prebuilt object - def malloc_fixedsize_clear(self, typeid16, size, can_collect, - has_finalizer=False, contains_weakptr=False): - size_gc_header = self.gcheaderbuilder.size_gc_header - totalsize = size_gc_header + size - result = self.free - if raw_malloc_usage(totalsize) > self.top_of_space - result: - result = self.obtain_free_space(totalsize) + def _get_memory(self, totalsize): + # also counts the space that will be needed during the following + # collection to store the TID + requested_size = raw_malloc_usage(totalsize) + BYTES_PER_TID + self.next_collect_after -= requested_size + if self.next_collect_after < 0: + result = self.obtain_free_space(requested_size) + else: + result = self.free + self.free += totalsize llarena.arena_reserve(result, totalsize) + return result + _get_memory._always_inline_ = True + + def _get_totalsize_var(self, nonvarsize, itemsize, length): + try: + varsize = ovfcheck(itemsize * length) + except OverflowError: + raise MemoryError + # Careful to detect overflows. The following works even if varsize + # is almost equal to sys.maxint; morever, self.space_size is known + # to be at least 4095 bytes smaller than sys.maxint, so this function + # always raises instead of returning an integer >= sys.maxint-4095. + if (raw_malloc_usage(varsize) > self.space_size - + raw_malloc_usage(nonvarsize)): + raise MemoryError + return llarena.round_up_for_allocation(nonvarsize + varsize) + _get_totalsize_var._always_inline_ = True + + def _setup_object(self, result, typeid16, has_finalizer): + size_gc_header = self.gcheaderbuilder.size_gc_header self.init_gc_object(result, typeid16) - self.free += totalsize if has_finalizer: self.objects_with_finalizers.append(result + size_gc_header) - if contains_weakptr: - self.objects_with_weakrefs.append(result + size_gc_header) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) - + _setup_object._always_inline_ = True + + def malloc_fixedsize(self, typeid16, size, can_collect, + has_finalizer=False, contains_weakptr=False): + size_gc_header = self.gcheaderbuilder.size_gc_header + totalsize = size_gc_header + size + result = self._get_memory(totalsize) + return self._setup_object(result, typeid16, has_finalizer) + + def malloc_fixedsize_clear(self, typeid16, size, can_collect, + has_finalizer=False, contains_weakptr=False): + size_gc_header = self.gcheaderbuilder.size_gc_header + totalsize = size_gc_header + size + result = self._get_memory(totalsize) + llmemory.raw_memclear(result, totalsize) + return self._setup_object(result, typeid16, has_finalizer) + def malloc_varsize_clear(self, typeid16, length, size, itemsize, offset_to_length, can_collect): size_gc_header = self.gcheaderbuilder.size_gc_header nonvarsize = size_gc_header + size - try: - varsize = ovfcheck(itemsize * length) - totalsize = ovfcheck(nonvarsize + varsize) - except OverflowError: - raise memoryError - result = self.free - if raw_malloc_usage(totalsize) > self.top_of_space - result: - result = self.obtain_free_space(totalsize) - llarena.arena_reserve(result, totalsize) - self.init_gc_object(result, typeid16) + totalsize = self._get_totalsize_var(nonvarsize, itemsize, length) + result = self._get_memory(totalsize) + llmemory.raw_memclear(result, totalsize) (result + size_gc_header + offset_to_length).signed[0] = length - self.free = result + llarena.round_up_for_allocation(totalsize) - return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) + return self._setup_object(result, typeid16, False) - def obtain_free_space(self, totalsize): - # a bit of tweaking to maximize the performance and minimize the - # amount of code in an inlined version of malloc_fixedsize_clear() - if not self.try_obtain_free_space(totalsize): - raise memoryError + def obtain_free_space(self, requested_size): + if self.free == NULL: + return self._emergency_initial_block(requested_size) + while True: + executed_some_finalizers = self.markcompactcollect(requested_size) + self.next_collect_after -= requested_size + if self.next_collect_after >= 0: + break # ok + else: + if executed_some_finalizers: + pass # try again to do a collection + else: + raise MemoryError return self.free obtain_free_space._dont_inline_ = True - def try_obtain_free_space(self, needed): - needed = raw_malloc_usage(needed) - while 1: - self.markcompactcollect(needed) - missing = needed - (self.top_of_space - self.free) - if missing < 0: - return True - - def new_space_size(self, occupied, needed): - res = (occupied * FREE_SPACE_MULTIPLIER / - FREE_SPACE_DIVIDER + FREE_SPACE_ADD + needed) - # align it to 4096, which is somewhat around page size - return ((res/4096) + 1) * 4096 - - def double_space_size(self, minimal_size): - while self.space_size <= minimal_size: - self.space_size *= 2 - toaddr = llarena.arena_malloc(self.space_size, True) - return toaddr - - def compute_alive_objects(self): - fromaddr = self.space - addraftercollect = self.space - num = 1 - while fromaddr < self.free: - size_gc_header = self.gcheaderbuilder.size_gc_header - tid = llmemory.cast_adr_to_ptr(fromaddr, lltype.Ptr(self.HDR)).tid - obj = fromaddr + size_gc_header - objsize = self.get_size(obj) - objtotalsize = size_gc_header + objsize - if self.marked(obj): - copy_has_hash_field = ((tid & GCFLAG_HASHFIELD) != 0 or - ((tid & GCFLAG_HASHTAKEN) != 0 and - addraftercollect < fromaddr)) - addraftercollect += raw_malloc_usage(objtotalsize) - if copy_has_hash_field: - addraftercollect += llmemory.sizeof(lltype.Signed) - num += 1 - fromaddr += objtotalsize - if tid & GCFLAG_HASHFIELD: - fromaddr += llmemory.sizeof(lltype.Signed) - ll_assert(addraftercollect <= fromaddr, - "markcompactcollect() is trying to increase memory usage") - self.totalsize_of_objs = addraftercollect - self.space - return num + def _emergency_initial_block(self, requested_size): + # xxx before the GC is fully setup, we might get there. Hopefully + # we will only allocate a couple of strings, e.g. in read_from_env(). + # Just allocate them raw and leak them. + debug_start("gc-initial-block") + debug_print("leaking", requested_size, "bytes") + debug_stop("gc-initial-block") + return llmemory.raw_malloc(requested_size) def collect(self, gen=0): self.markcompactcollect() - - def markcompactcollect(self, needed=0): - start_time = self.debug_collect_start() + + def markcompactcollect(self, requested_size=0): + self.debug_collect_start(requested_size) self.debug_check_consistency() - self.to_see = self.AddressStack() - self.mark_roots_recursively() - if (self.objects_with_finalizers.non_empty() or - self.run_finalizers.non_empty()): - self.mark_objects_with_finalizers() - self._trace_and_mark() + # + # Mark alive objects + # + self.to_see = self.AddressDeque() + self.trace_from_roots() self.to_see.delete() - num_of_alive_objs = self.compute_alive_objects() - size_of_alive_objs = self.totalsize_of_objs - totalsize = self.new_space_size(size_of_alive_objs, needed + - num_of_alive_objs * BYTES_PER_TID) - tid_backup_size = (llmemory.sizeof(self.TID_BACKUP, 0) + - llmemory.sizeof(TID_TYPE) * num_of_alive_objs) - used_space_now = self.next_collect_after + raw_malloc_usage(tid_backup_size) - if totalsize >= self.space_size or used_space_now >= self.space_size: - toaddr = self.double_space_size(totalsize) - llarena.arena_reserve(toaddr + size_of_alive_objs, tid_backup_size) - self.tid_backup = llmemory.cast_adr_to_ptr( - toaddr + size_of_alive_objs, - lltype.Ptr(self.TID_BACKUP)) - resizing = True - else: - toaddr = llarena.arena_new_view(self.space) - llarena.arena_reserve(self.top_of_space, tid_backup_size) - self.tid_backup = llmemory.cast_adr_to_ptr( - self.top_of_space, - lltype.Ptr(self.TID_BACKUP)) - resizing = False - self.next_collect_after = totalsize - weakref_offsets = self.collect_weakref_offsets() - finaladdr = self.update_forward_pointers(toaddr, num_of_alive_objs) + # + # Prepare new views on the same memory + # + toaddr = llarena.arena_new_view(self.space) + maxnum = self.space_size - (self.free - self.space) + maxnum /= BYTES_PER_TID + llarena.arena_reserve(self.free, llmemory.sizeof(TID_BACKUP, maxnum)) + self.tid_backup = llmemory.cast_adr_to_ptr(self.free, + lltype.Ptr(TID_BACKUP)) + # + # Walk all objects and assign forward pointers in the same order, + # also updating all references + # + finaladdr = self.update_forward_pointers(toaddr, maxnum) if (self.run_finalizers.non_empty() or self.objects_with_finalizers.non_empty()): self.update_run_finalizers() - if self.objects_with_weakrefs.non_empty(): - self.invalidate_weakrefs(weakref_offsets) + self.update_objects_with_id() - self.compact(resizing) - if not resizing: - size = toaddr + self.space_size - finaladdr - llarena.arena_reset(finaladdr, size, True) - else: - if we_are_translated(): - # because we free stuff already in raw_memmove, we - # would get double free here. Let's free it anyway - llarena.arena_free(self.space) - llarena.arena_reset(toaddr + size_of_alive_objs, tid_backup_size, - True) - self.space = toaddr - self.free = finaladdr - self.top_of_space = toaddr + self.next_collect_after + self.compact() + # + self.tid_backup = lltype.nullptr(TID_BACKUP) + self.free = finaladdr + self.next_collect_after = self.next_collection(finaladdr - toaddr, + self.num_alive_objs, + requested_size) + # + if not translated_to_c(): + remaining_size = (toaddr + self.space_size) - finaladdr + llarena.arena_reset(finaladdr, remaining_size, False) + llarena.arena_free(self.space) + self.space = toaddr + # self.debug_check_consistency() - self.tid_backup = lltype.nullptr(self.TID_BACKUP) + self.debug_collect_finish() + if self.next_collect_after < 0: + raise MemoryError + # if self.run_finalizers.non_empty(): self.execute_finalizers() - self.debug_collect_finish(start_time) - - def collect_weakref_offsets(self): - weakrefs = self.objects_with_weakrefs - new_weakrefs = self.AddressStack() - weakref_offsets = lltype.malloc(self.WEAKREF_OFFSETS, - weakrefs.length(), flavor='raw') - i = 0 - while weakrefs.non_empty(): - obj = weakrefs.pop() - offset = self.weakpointer_offset(self.get_type_id(obj)) - weakref_offsets[i] = offset - new_weakrefs.append(obj) - i += 1 - self.objects_with_weakrefs = new_weakrefs - weakrefs.delete() - return weakref_offsets - - def debug_collect_start(self): - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void) - llop.debug_print(lltype.Void, - ".----------- Full collection ------------------") - start_time = time.time() - return start_time - - def debug_collect_finish(self, start_time): - if self.config.gcconfig.debugprint: - end_time = time.time() - elapsed_time = end_time - start_time - self.total_collection_time += elapsed_time - self.total_collection_count += 1 - total_program_time = end_time - self.program_start_time - ct = self.total_collection_time - cc = self.total_collection_count - llop.debug_print(lltype.Void, - "| number of collections so far ", - cc) - llop.debug_print(lltype.Void, - "| total collections per second: ", - cc / total_program_time) - llop.debug_print(lltype.Void, - "| total time in markcompact-collect: ", - ct, "seconds") - llop.debug_print(lltype.Void, - "| percentage collection<->total time:", - ct * 100.0 / total_program_time, "%") - llop.debug_print(lltype.Void, - "`----------------------------------------------") + return True # executed some finalizers + else: + return False # no finalizer executed + + def debug_collect_start(self, requested_size): + if 1:# have_debug_prints(): + debug_start("gc-collect") + debug_print() + debug_print(".----------- Full collection -------------------") + debug_print("| requested size:", + requested_size) + #start_time = time.time() + #return start_time + #return -1 + + def debug_collect_finish(self): + if 1:# start_time != -1: + #end_time = time.time() + #elapsed_time = end_time - start_time + #self.total_collection_time += elapsed_time + #self.total_collection_count += 1 + #total_program_time = end_time - self.program_start_time + #ct = self.total_collection_time + #cc = self.total_collection_count + #debug_print("| number of collections so far ", + # cc) + debug_print("| total space size ", + self.space_size) + debug_print("| number of objects alive ", + self.num_alive_objs) + debug_print("| used space size ", + self.free - self.space) + debug_print("| next collection after ", + self.next_collect_after) + #debug_print("| total collections per second: ", + # cc / total_program_time) + #debug_print("| total time in markcompact-collect: ", + # ct, "seconds") + #debug_print("| percentage collection<->total time:", + # ct * 100.0 / total_program_time, "%") + debug_print("`----------------------------------------------") + debug_stop("gc-collect") def update_run_finalizers(self): - run_finalizers = self.AddressDeque() - while self.run_finalizers.non_empty(): - obj = self.run_finalizers.popleft() - run_finalizers.append(self.get_forwarding_address(obj)) - self.run_finalizers.delete() - self.run_finalizers = run_finalizers + if self.run_finalizers.non_empty(): # uncommon case + run_finalizers = self.AddressDeque() + while self.run_finalizers.non_empty(): + obj = self.run_finalizers.popleft() + run_finalizers.append(self.get_forwarding_address(obj)) + self.run_finalizers.delete() + self.run_finalizers = run_finalizers + # objects_with_finalizers = self.AddressDeque() while self.objects_with_finalizers.non_empty(): obj = self.objects_with_finalizers.popleft() @@ -356,90 +354,156 @@ tid = self.header(addr).tid return llop.extract_ushort(llgroup.HALFWORD, tid) - def mark_roots_recursively(self): + def trace_from_roots(self): self.root_walker.walk_roots( - MarkCompactGC._mark_root_recursively, # stack roots - MarkCompactGC._mark_root_recursively, # static in prebuilt non-gc structures - MarkCompactGC._mark_root_recursively) # static in prebuilt gc objects + MarkCompactGC._mark_root, # stack roots + MarkCompactGC._mark_root, # static in prebuilt non-gc structures + MarkCompactGC._mark_root) # static in prebuilt gc objects + if (self.objects_with_finalizers.non_empty() or + self.run_finalizers.non_empty()): + self.trace_from_objects_with_finalizers() self._trace_and_mark() def _trace_and_mark(self): - # XXX depth-first tracing... it can consume a lot of rawmalloced - # memory for very long stacks in some cases while self.to_see.non_empty(): - obj = self.to_see.pop() + obj = self.to_see.popleft() self.trace(obj, self._mark_obj, None) def _mark_obj(self, pointer, ignored): - obj = pointer.address[0] - if self.marked(obj): - return - self.mark(obj) - self.to_see.append(obj) + self.mark(pointer.address[0]) - def _mark_root_recursively(self, root): + def _mark_root(self, root): self.mark(root.address[0]) - self.to_see.append(root.address[0]) def mark(self, obj): - self.header(obj).tid |= GCFLAG_MARKBIT + if not self.marked(obj): + self.header(obj).tid |= GCFLAG_MARKBIT + self.to_see.append(obj) def marked(self, obj): - return self.header(obj).tid & GCFLAG_MARKBIT + # should work both if tid contains a CombinedSymbolic (for dying + # objects, at this point), or a plain integer. + return MovingGCBase.header(self, obj).tid & GCFLAG_MARKBIT + + def toaddr_smaller_than_fromaddr(self, toaddr, fromaddr): + if translated_to_c(): + return toaddr < fromaddr + else: + # convert the addresses to integers, because they are + # theoretically not from the same arena + return toaddr - self.base_forwarding_addr < fromaddr - self.space - def update_forward_pointers(self, toaddr, num_of_alive_objs): - self.base_forwarding_addr = toaddr + def update_forward_pointers(self, toaddr, maxnum): + self.base_forwarding_addr = base_forwarding_addr = toaddr fromaddr = self.space size_gc_header = self.gcheaderbuilder.size_gc_header - i = 0 + num = 0 while fromaddr < self.free: hdr = llmemory.cast_adr_to_ptr(fromaddr, lltype.Ptr(self.HDR)) obj = fromaddr + size_gc_header - objsize = self.get_size(obj) - totalsize = size_gc_header + objsize - if not self.marked(obj): - self.set_null_forwarding_address(obj, i) - else: - llarena.arena_reserve(toaddr, totalsize) - self.set_forwarding_address(obj, toaddr, i) - toaddr += totalsize - i += 1 - fromaddr += totalsize + # compute the original object size, including the + # optional hash field + basesize = size_gc_header + self.get_size(obj) + totalsrcsize = basesize + if hdr.tid & GCFLAG_HASHFIELD: # already a hash field, copy it too + totalsrcsize += llmemory.sizeof(lltype.Signed) + # + if self.marked(obj): + # the object is marked as suriving. Compute the new object + # size + totaldstsize = totalsrcsize + if (hdr.tid & (GCFLAG_HASHTAKEN|GCFLAG_HASHFIELD) == + GCFLAG_HASHTAKEN): + # grow a new hash field -- with the exception: if + # the object actually doesn't move, don't + # (otherwise, we get a bogus toaddr > fromaddr) + if self.toaddr_smaller_than_fromaddr(toaddr, fromaddr): + totaldstsize += llmemory.sizeof(lltype.Signed) + # + if not translated_to_c(): + llarena.arena_reserve(toaddr, basesize) + if (raw_malloc_usage(totaldstsize) > + raw_malloc_usage(basesize)): + llarena.arena_reserve(toaddr + basesize, + llmemory.sizeof(lltype.Signed)) + # + # save the field hdr.tid in the array tid_backup + ll_assert(num < maxnum, "overflow of the tid_backup table") + self.tid_backup[num] = self.get_type_id(obj) + num += 1 + # compute forward_offset, the offset to the future copy + # of this object + forward_offset = toaddr - base_forwarding_addr + # copy the first two gc flags in forward_offset + ll_assert(forward_offset & 3 == 0, "misalignment!") + forward_offset |= (hdr.tid >> first_gcflag_bit) & 3 + hdr.tid = forward_offset | GCFLAG_MARKBIT + ll_assert(self.marked(obj), "re-marking object failed!") + # done + toaddr += totaldstsize + # + fromaddr += totalsrcsize + if not translated_to_c(): + assert toaddr - base_forwarding_addr <= fromaddr - self.space + self.num_alive_objs = num # now update references self.root_walker.walk_roots( - MarkCompactGC._update_root, # stack roots - MarkCompactGC._update_root, # static in prebuilt non-gc structures - MarkCompactGC._update_root) # static in prebuilt gc objects + MarkCompactGC._update_ref, # stack roots + MarkCompactGC._update_ref, # static in prebuilt non-gc structures + MarkCompactGC._update_ref) # static in prebuilt gc objects + self.walk_marked_objects(MarkCompactGC.trace_and_update_ref) + return toaddr + + def walk_marked_objects(self, callback): + num = 0 + size_gc_header = self.gcheaderbuilder.size_gc_header fromaddr = self.space - i = 0 + toaddr = self.base_forwarding_addr while fromaddr < self.free: hdr = llmemory.cast_adr_to_ptr(fromaddr, lltype.Ptr(self.HDR)) obj = fromaddr + size_gc_header - objsize = self.get_size_from_backup(obj, i) - totalsize = size_gc_header + objsize - if not self.surviving(obj): - pass + survives = self.marked(obj) + if survives: + typeid = self.get_typeid_from_backup(num) + num += 1 else: - self.trace_with_backup(obj, self._update_ref, i) - fromaddr += totalsize - i += 1 - return toaddr + typeid = self.get_type_id(obj) + baseobjsize = self._get_size_for_typeid(obj, typeid) + basesize = size_gc_header + baseobjsize + totalsrcsize = basesize + # + if survives: + grow_hash_field = False + if hdr.tid & GCFLAG_SAVED_HASHFIELD: + totalsrcsize += llmemory.sizeof(lltype.Signed) + totaldstsize = totalsrcsize + if (hdr.tid & (GCFLAG_SAVED_HASHTAKEN|GCFLAG_SAVED_HASHFIELD) + == GCFLAG_SAVED_HASHTAKEN): + if self.toaddr_smaller_than_fromaddr(toaddr, fromaddr): + grow_hash_field = True + totaldstsize += llmemory.sizeof(lltype.Signed) + callback(self, obj, typeid, basesize, toaddr, grow_hash_field) + toaddr += totaldstsize + else: + if hdr.tid & GCFLAG_HASHFIELD: + totalsrcsize += llmemory.sizeof(lltype.Signed) + # + fromaddr += totalsrcsize + walk_marked_objects._annspecialcase_ = 'specialize:arg(1)' - def trace_with_backup(self, obj, callback, arg): + def trace_and_update_ref(self, obj, typeid, _1, _2, _3): """Enumerate the locations inside the given obj that can contain GC pointers. For each such location, callback(pointer, arg) is called, where 'pointer' is an address inside the object. Typically, 'callback' is a bound method and 'arg' can be None. """ - typeid = self.get_typeid_from_backup(arg) if self.is_gcarrayofgcptr(typeid): # a performance shortcut for GcArray(gcptr) length = (obj + llmemory.gcarrayofptr_lengthoffset).signed[0] item = obj + llmemory.gcarrayofptr_itemsoffset while length > 0: - if self.points_to_valid_gc_object(item): - callback(item, arg) + self._update_ref(item) item += llmemory.gcarrayofptr_singleitemoffset length -= 1 return @@ -447,8 +511,7 @@ i = 0 while i < len(offsets): item = obj + offsets[i] - if self.points_to_valid_gc_object(item): - callback(item, arg) + self._update_ref(item) i += 1 if self.has_gcptr_in_varsize(typeid): item = obj + self.varsize_offset_to_variable_part(typeid) @@ -459,171 +522,104 @@ j = 0 while j < len(offsets): itemobj = item + offsets[j] - if self.points_to_valid_gc_object(itemobj): - callback(itemobj, arg) + self._update_ref(itemobj) j += 1 item += itemlength length -= 1 - trace_with_backup._annspecialcase_ = 'specialize:arg(2)' - - def _update_root(self, pointer): - if pointer.address[0] != NULL: - pointer.address[0] = self.get_forwarding_address(pointer.address[0]) - - def _update_ref(self, pointer, ignore): - if pointer.address[0] != NULL: - pointer.address[0] = self.get_forwarding_address(pointer.address[0]) + else: + weakofs = self.weakpointer_offset(typeid) + if weakofs >= 0: + self._update_weakref(obj + weakofs) + + def _update_ref(self, pointer): + if self.points_to_valid_gc_object(pointer): + pointer.address[0] = self.get_forwarding_address( + pointer.address[0]) + + def _update_weakref(self, pointer): + # either update the weak pointer's destination, or + # if it dies, write a NULL + if self.points_to_valid_gc_object(pointer): + if self.marked(pointer.address[0]): + pointer.address[0] = self.get_forwarding_address( + pointer.address[0]) + else: + pointer.address[0] = NULL def _is_external(self, obj): - return not (self.space <= obj < self.top_of_space) + return not (self.space <= obj < self.free) def get_forwarding_address(self, obj): if self._is_external(obj): return obj return self.get_header_forwarded_addr(obj) - def set_null_forwarding_address(self, obj, num): - self.backup_typeid(num, obj) - hdr = self.header(obj) - hdr.tid = -1 # make the object forwarded to NULL - - def set_forwarding_address(self, obj, newobjhdr, num): - self.backup_typeid(num, obj) - forward_offset = newobjhdr - self.base_forwarding_addr - hdr = self.header(obj) - hdr.tid = forward_offset # make the object forwarded to newobj - - def restore_normal_header(self, obj, num): - # Reverse of set_forwarding_address(). - typeid16 = self.get_typeid_from_backup(num) - hdr = self.header_forwarded(obj) - hdr.tid = self.combine(typeid16, 0) # restore the normal header - def get_header_forwarded_addr(self, obj): + GCFLAG_MASK = ~(GCFLAG_MARKBIT | 3) return (self.base_forwarding_addr + - self.header_forwarded(obj).tid + + (self.header_forwarded(obj).tid & GCFLAG_MASK) + self.gcheaderbuilder.size_gc_header) def surviving(self, obj): return self._is_external(obj) or self.header_forwarded(obj).tid != -1 - def backup_typeid(self, num, obj): - self.tid_backup[num] = self.get_type_id(obj) - def get_typeid_from_backup(self, num): return self.tid_backup[num] - def get_size_from_backup(self, obj, num): - typeid = self.get_typeid_from_backup(num) - size = self.fixed_size(typeid) - if self.is_varsize(typeid): - lenaddr = obj + self.varsize_offset_to_length(typeid) - length = lenaddr.signed[0] - size += length * self.varsize_item_sizes(typeid) - size = llarena.round_up_for_allocation(size) - # XXX maybe we should parametrize round_up_for_allocation() - # per GC; if we do, we also need to fix the call in - # gctypelayout.encode_type_shape() - return size + def compact(self): + self.walk_marked_objects(MarkCompactGC.copy_and_compact) - def compact(self, resizing): - fromaddr = self.space - size_gc_header = self.gcheaderbuilder.size_gc_header - start = fromaddr - end = fromaddr - num = 0 - while fromaddr < self.free: - obj = fromaddr + size_gc_header - objsize = self.get_size_from_backup(obj, num) - totalsize = size_gc_header + objsize - if not self.surviving(obj): - # this object dies. Following line is a noop in C, - # we clear it to make debugging easier - llarena.arena_reset(fromaddr, totalsize, False) - else: - if resizing: - end = fromaddr - forward_obj = self.get_header_forwarded_addr(obj) - self.restore_normal_header(obj, num) - if obj != forward_obj: - #llop.debug_print(lltype.Void, "Copying from to", - # fromaddr, forward_ptr, totalsize) - llmemory.raw_memmove(fromaddr, - forward_obj - size_gc_header, - totalsize) - if resizing and end - start > GC_CLEARANCE: - diff = end - start - #llop.debug_print(lltype.Void, "Cleaning", start, diff) - diff = (diff / GC_CLEARANCE) * GC_CLEARANCE - #llop.debug_print(lltype.Void, "Cleaning", start, diff) - end = start + diff - if we_are_translated(): - # XXX wuaaaaa.... those objects are freed incorrectly - # here in case of test_gc - llarena.arena_reset(start, diff, True) - start += diff - num += 1 - fromaddr += totalsize + def copy_and_compact(self, obj, typeid, basesize, toaddr, grow_hash_field): + # 'basesize' is the size without any hash field + # restore the normal header + hdr = self.header_forwarded(obj) + gcflags = hdr.tid & 3 + if grow_hash_field: + gcflags |= GCFLAG_SAVED_HASHFIELD + hashvalue = self.get_identityhash_from_addr(obj) + (toaddr + basesize).signed[0] = hashvalue + elif gcflags & GCFLAG_SAVED_HASHFIELD: + fromaddr = llarena.getfakearenaaddress(obj) + fromaddr -= self.gcheaderbuilder.size_gc_header + hashvalue = (fromaddr + basesize).signed[0] + (toaddr + basesize).signed[0] = hashvalue + # + hdr.tid = self.combine(typeid, gcflags << first_gcflag_bit) + # + fromaddr = obj - self.gcheaderbuilder.size_gc_header + if translated_to_c(): + llmemory.raw_memmove(fromaddr, toaddr, basesize) + else: + llmemory.raw_memcopy(fromaddr, toaddr, basesize) def debug_check_object(self, obj): # not sure what to check here pass - def mark_objects_with_finalizers(self): + def trace_from_objects_with_finalizers(self): + if self.run_finalizers.non_empty(): # uncommon case + new_run_finalizers = self.AddressDeque() + while self.run_finalizers.non_empty(): + x = self.run_finalizers.popleft() + self.mark(x) + new_run_finalizers.append(x) + self.run_finalizers.delete() + self.run_finalizers = new_run_finalizers + # + # xxx we get to run the finalizers in a random order + self._trace_and_mark() new_with_finalizers = self.AddressDeque() - run_finalizers = self.run_finalizers - new_run_finalizers = self.AddressDeque() - while run_finalizers.non_empty(): - x = run_finalizers.popleft() - self.mark(x) - self.to_see.append(x) - new_run_finalizers.append(x) - run_finalizers.delete() - self.run_finalizers = new_run_finalizers while self.objects_with_finalizers.non_empty(): x = self.objects_with_finalizers.popleft() if self.marked(x): new_with_finalizers.append(x) else: - new_run_finalizers.append(x) + self.run_finalizers.append(x) self.mark(x) - self.to_see.append(x) + self._trace_and_mark() self.objects_with_finalizers.delete() self.objects_with_finalizers = new_with_finalizers - def invalidate_weakrefs(self, weakref_offsets): - # walk over list of objects that contain weakrefs - # if the object it references survives then update the weakref - # otherwise invalidate the weakref - new_with_weakref = self.AddressStack() - i = 0 - while self.objects_with_weakrefs.non_empty(): - obj = self.objects_with_weakrefs.pop() - if not self.surviving(obj): - continue # weakref itself dies - newobj = self.get_forwarding_address(obj) - offset = weakref_offsets[i] - pointing_to = (obj + offset).address[0] - # XXX I think that pointing_to cannot be NULL here - if pointing_to: - if self.surviving(pointing_to): - (obj + offset).address[0] = self.get_forwarding_address( - pointing_to) - new_with_weakref.append(newobj) - else: - (obj + offset).address[0] = NULL - i += 1 - self.objects_with_weakrefs.delete() - self.objects_with_weakrefs = new_with_weakref - lltype.free(weakref_offsets, flavor='raw') - - def get_size_incl_hash(self, obj): - size = self.get_size(obj) - hdr = self.header(obj) - if hdr.tid & GCFLAG_HASHFIELD: - size += llmemory.sizeof(lltype.Signed) - return size - def identityhash(self, gcobj): # Unlike SemiSpaceGC.identityhash(), this function does not have # to care about reducing top_of_space. The reason is as @@ -638,8 +634,26 @@ hdr = self.header(obj) # if hdr.tid & GCFLAG_HASHFIELD: # the hash is in a field at the end - obj += self.get_size(obj) + obj = llarena.getfakearenaaddress(obj) + self.get_size(obj) return obj.signed[0] # hdr.tid |= GCFLAG_HASHTAKEN - return llmemory.cast_adr_to_int(obj) # direct case + return self.get_identityhash_from_addr(obj) + + def get_identityhash_from_addr(self, obj): + if translated_to_c(): + return llmemory.cast_adr_to_int(obj) # direct case + else: + try: + adr = llarena.getfakearenaaddress(obj) # -> arena address + except RuntimeError: + return llmemory.cast_adr_to_int(obj) # not in an arena... + return adr - self.space + +# ____________________________________________________________ + +class CannotAllocateGCArena(Exception): + pass + +def max_size_from_env(): + return read_from_env('PYPY_MARKCOMPACTGC_MAX') Modified: pypy/branch/better-map-instances/pypy/rpython/memory/gc/test/test_direct.py ============================================================================== --- pypy/branch/better-map-instances/pypy/rpython/memory/gc/test/test_direct.py (original) +++ pypy/branch/better-map-instances/pypy/rpython/memory/gc/test/test_direct.py Sun Aug 29 12:24:52 2010 @@ -67,7 +67,10 @@ from pypy.config.pypyoption import get_pypy_config config = get_pypy_config(translating=True).translation self.stackroots = [] - self.gc = self.GCClass(config, **self.GC_PARAMS) + GC_PARAMS = self.GC_PARAMS.copy() + if hasattr(meth, 'GC_PARAMS'): + GC_PARAMS.update(meth.GC_PARAMS) + self.gc = self.GCClass(config, **GC_PARAMS) self.gc.DEBUG = True self.rootwalker = DirectRootWalker(self) self.gc.set_root_walker(self.rootwalker) @@ -96,7 +99,7 @@ p[index] = newvalue def malloc(self, TYPE, n=None): - addr = self.gc.malloc(self.get_type_id(TYPE), n) + addr = self.gc.malloc(self.get_type_id(TYPE), n, zero=True) return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(TYPE)) def test_simple(self): @@ -311,7 +314,18 @@ print hash assert isinstance(hash, (int, long)) assert hash == self.gc.identityhash(p_const) - + # (5) p is actually moving (for the markcompact gc) + p0 = self.malloc(S) + self.stackroots.append(p0) + p = self.malloc(S) + self.stackroots.append(p) + hash = self.gc.identityhash(p) + self.stackroots.pop(-2) + self.gc.collect() # p0 goes away, p shifts left + assert hash == self.gc.identityhash(self.stackroots[-1]) + self.gc.collect() + assert hash == self.gc.identityhash(self.stackroots[-1]) + self.stackroots.pop() class TestSemiSpaceGC(DirectGCTest): from pypy.rpython.memory.gc.semispace import SemiSpaceGC as GCClass @@ -431,3 +445,14 @@ class TestMarkCompactGC(DirectGCTest): from pypy.rpython.memory.gc.markcompact import MarkCompactGC as GCClass + def test_many_objects(self): + DirectGCTest.test_many_objects(self) + test_many_objects.GC_PARAMS = {'space_size': 3 * 1024 * WORD} + + def test_varsized_from_stack(self): + DirectGCTest.test_varsized_from_stack(self) + test_varsized_from_stack.GC_PARAMS = {'space_size': 2 * 1024 * WORD} + + def test_varsized_from_prebuilt_gc(self): + DirectGCTest.test_varsized_from_prebuilt_gc(self) + test_varsized_from_prebuilt_gc.GC_PARAMS = {'space_size': 3 * 1024 * WORD} Modified: pypy/branch/better-map-instances/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/branch/better-map-instances/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/branch/better-map-instances/pypy/rpython/memory/gcwrapper.py Sun Aug 29 12:24:52 2010 @@ -119,6 +119,9 @@ else: return True + def pyobjectptr(self, klass): + raise NotImplementedError(klass) + # ____________________________________________________________ class LLInterpRootWalker: Modified: pypy/branch/better-map-instances/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/branch/better-map-instances/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/branch/better-map-instances/pypy/rpython/memory/test/test_gc.py Sun Aug 29 12:24:52 2010 @@ -639,12 +639,14 @@ class TestMarkCompactGC(TestSemiSpaceGC): from pypy.rpython.memory.gc.markcompact import MarkCompactGC as GCClass + GC_PARAMS = {'space_size': 65536+16384} + GC_CAN_SHRINK_ARRAY = False def test_finalizer_order(self): py.test.skip("Not implemented yet") - -class TestMarkCompactGCGrowing(TestMarkCompactGC): - GC_PARAMS = {'space_size': 16*WORD} + def test_writebarrier_before_copy(self): + py.test.skip("Not relevant, and crashes because llarena does not " + "support empty GcStructs") class TestHybridGC(TestGenerationalGC): from pypy.rpython.memory.gc.hybrid import HybridGC as GCClass Modified: pypy/branch/better-map-instances/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/better-map-instances/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/better-map-instances/pypy/rpython/memory/test/test_transformed_gc.py Sun Aug 29 12:24:52 2010 @@ -1138,15 +1138,16 @@ class TestMarkCompactGC(GenericMovingGCTests): gcname = 'markcompact' - def setup_class(cls): - py.test.skip("Disabled for now, sorry") - class gcpolicy(gc.FrameworkGcPolicy): class transformerclass(framework.FrameworkGCTransformer): from pypy.rpython.memory.gc.markcompact import MarkCompactGC as GCClass - GC_PARAMS = {'space_size': 512*WORD} + GC_PARAMS = {'space_size': 4096*WORD} root_stack_depth = 200 + def test_writebarrier_before_copy(self): + py.test.skip("Not relevant, and crashes because llarena does not " + "support empty GcStructs") + class TestGenerationGC(GenericMovingGCTests): gcname = "generation" GC_CAN_SHRINK_ARRAY = True @@ -1536,3 +1537,12 @@ GC_PARAMS = {'space_size': 512*WORD, 'nursery_size': 32*WORD} root_stack_depth = 200 + +class TestMarkCompactTaggedpointerGC(TaggedPointerGCTests): + gcname = 'markcompact' + + class gcpolicy(gc.FrameworkGcPolicy): + class transformerclass(framework.FrameworkGCTransformer): + from pypy.rpython.memory.gc.markcompact import MarkCompactGC as GCClass + GC_PARAMS = {'space_size': 4096*WORD} + root_stack_depth = 200 Modified: pypy/branch/better-map-instances/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/better-map-instances/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/better-map-instances/pypy/translator/c/test/test_newgc.py Sun Aug 29 12:24:52 2010 @@ -67,9 +67,8 @@ if not fullname.startswith('define'): continue keyword = conftest.option.keyword - if keyword: - if keyword.startswith('test_'): - keyword = keyword[len('test_'):] + if keyword.startswith('test_'): + keyword = keyword[len('test_'):] if keyword not in fullname: continue prefix, name = fullname.split('_', 1) @@ -1072,21 +1071,20 @@ should_be_moving = True GC_CAN_SHRINK_ARRAY = False - def setup_class(cls): - py.test.skip("Disabled for now") - def test_gc_set_max_heap_size(self): py.test.skip("not implemented") + def test_gc_heap_stats(self): + py.test.skip("not implemented") + def test_finalizer_order(self): py.test.skip("not implemented") # ____________________________________________________________________ -class TestHybridTaggedPointers(TestHybridGC): +class TaggedPointersTest(object): taggedpointers = True - def define_tagged(cls): class Unrelated(object): pass @@ -1129,3 +1127,10 @@ __slots__ = 'smallint' def meth(self, x): return self.smallint + x + 3 + + +class TestHybridTaggedPointers(TaggedPointersTest, TestHybridGC): + pass + +class TestMarkCompactGCMostCompact(TaggedPointersTest, TestMarkCompactGC): + removetypeptr = True Modified: pypy/branch/better-map-instances/pypy/translator/exceptiontransform.py ============================================================================== --- pypy/branch/better-map-instances/pypy/translator/exceptiontransform.py (original) +++ pypy/branch/better-map-instances/pypy/translator/exceptiontransform.py Sun Aug 29 12:24:52 2010 @@ -197,7 +197,7 @@ for graph in self.translator.graphs: self.create_exception_handling(graph) - def create_exception_handling(self, graph, always_exc_clear=False): + def create_exception_handling(self, graph): """After an exception in a direct_call (or indirect_call), that is not caught by an explicit except statement, we need to reraise the exception. So after this @@ -212,7 +212,6 @@ self.raise_analyzer.analyze_direct_call(graph) graph.exceptiontransformed = self.exc_data_ptr - self.always_exc_clear = always_exc_clear join_blocks(graph) # collect the blocks before changing them n_need_exc_matching_blocks = 0 @@ -455,13 +454,18 @@ block.recloseblock(l0, l) insert_zeroing_op = False - # XXX this is not right. it also inserts zero_gc_pointers_inside - # XXX on a path that malloc_nonmovable returns null, but does not raise - # XXX which might end up with a segfault. But we don't have such gc now - if spaceop.opname == 'malloc' or spaceop.opname == 'malloc_nonmovable': + if spaceop.opname == 'malloc': flavor = spaceop.args[1].value['flavor'] if flavor == 'gc': insert_zeroing_op = True + elif spaceop.opname == 'malloc_nonmovable': + # xxx we cannot insert zero_gc_pointers_inside after + # malloc_nonmovable, because it can return null. For now + # we simply always force the zero=True flag on + # malloc_nonmovable. + c_flags = spaceop.args[1] + c_flags.value = c_flags.value.copy() + spaceop.args[1].value['zero'] = True if insert_zeroing_op: if normalafterblock is None: @@ -479,16 +483,6 @@ [v_result_after], varoftype(lltype.Void))) - if self.always_exc_clear: - # insert code that clears the exception even in the non-exceptional - # case... this is a hint for the JIT, but pointless otherwise - if normalafterblock is None: - normalafterblock = insert_empty_block(None, l0) - llops = rtyper.LowLevelOpList(None) - self.gen_setfield('exc_value', self.c_null_evalue, llops) - self.gen_setfield('exc_type', self.c_null_etype, llops) - normalafterblock.operations[:0] = llops - class LLTypeExceptionTransformer(BaseExceptionTransformer): From arigo at codespeak.net Sun Aug 29 12:27:28 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 29 Aug 2010 12:27:28 +0200 (CEST) Subject: [pypy-svn] r76781 - pypy/branch/better-map-instances/pypy/config Message-ID: <20100829102728.4DAC8282B9E@codespeak.net> Author: arigo Date: Sun Aug 29 12:27:26 2010 New Revision: 76781 Modified: pypy/branch/better-map-instances/pypy/config/pypyoption.py Log: Fix -Omem when translating pypy: enable mapdict, not inlineddict. Modified: pypy/branch/better-map-instances/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/better-map-instances/pypy/config/pypyoption.py (original) +++ pypy/branch/better-map-instances/pypy/config/pypyoption.py Sun Aug 29 12:27:26 2010 @@ -351,7 +351,7 @@ config.objspace.std.suggest(withprebuiltint=True) config.objspace.std.suggest(withrangelist=True) config.objspace.std.suggest(withprebuiltchar=True) - config.objspace.std.suggest(withinlineddict=True) + config.objspace.std.suggest(withmapdict=True) config.objspace.std.suggest(withstrslice=True) config.objspace.std.suggest(withstrjoin=True) # xxx other options? ropes maybe? From arigo at codespeak.net Sun Aug 29 12:41:38 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 29 Aug 2010 12:41:38 +0200 (CEST) Subject: [pypy-svn] r76782 - in pypy/trunk/pypy/interpreter: . test Message-ID: <20100829104138.1137F282B9E@codespeak.net> Author: arigo Date: Sun Aug 29 12:41:34 2010 New Revision: 76782 Modified: pypy/trunk/pypy/interpreter/generator.py pypy/trunk/pypy/interpreter/test/test_generator.py Log: Test and fix. Modified: pypy/trunk/pypy/interpreter/generator.py ============================================================================== --- pypy/trunk/pypy/interpreter/generator.py (original) +++ pypy/trunk/pypy/interpreter/generator.py Sun Aug 29 12:41:34 2010 @@ -42,7 +42,11 @@ raise OperationError(space.w_ValueError, space.wrap('generator already executing')) if self.frame.frame_finished_execution: - raise OperationError(space.w_StopIteration, space.w_None) + # xxx a bit ad-hoc, but we don't want to go inside + # execute_generator_frame() if the frame is actually finished + if operr is None: + operr = OperationError(space.w_StopIteration, space.w_None) + raise operr # XXX it's not clear that last_instr should be promoted at all # but as long as it is necessary for call_assembler, let's do it early last_instr = jit.hint(self.frame.last_instr, promote=True) Modified: pypy/trunk/pypy/interpreter/test/test_generator.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_generator.py (original) +++ pypy/trunk/pypy/interpreter/test/test_generator.py Sun Aug 29 12:41:34 2010 @@ -136,6 +136,15 @@ res = list(x) assert res == [1] + def test_throw_on_finished_generator(self): + def f(): + yield 1 + g = f() + res = g.next() + assert res == 1 + raises(StopIteration, g.next) + raises(NameError, g.throw, NameError) + def test_close(self): def f(): yield 1 From cfbolz at codespeak.net Sun Aug 29 13:02:07 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 29 Aug 2010 13:02:07 +0200 (CEST) Subject: [pypy-svn] r76783 - pypy/branch/better-map-instances/pypy/objspace/std Message-ID: <20100829110207.F02FF282B9E@codespeak.net> Author: cfbolz Date: Sun Aug 29 13:02:05 2010 New Revision: 76783 Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py pypy/branch/better-map-instances/pypy/objspace/std/typeobject.py Log: add some JIT hints Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py Sun Aug 29 13:02:05 2010 @@ -1,3 +1,4 @@ +from pypy.rlib import jit # ____________________________________________________________ # attribute shapes @@ -28,13 +29,16 @@ def set_terminator(self, obj, terminator): raise NotImplementedError("abstract base class") + @jit.purefunction def size_estimate(self): return self._size_estimate >> NUM_DIGITS def search(self, attrtype): return None - def add_attr(self, obj, selector, w_value): + @jit.purefunction + def _get_new_attr(self, name, index): + selector = name, index cache = self.cache_attrs if cache is None: cache = self.cache_attrs = {} @@ -42,8 +46,15 @@ if attr is None: attr = PlainAttribute(selector, self) cache[selector] = attr + return attr + + @jit.unroll_safe + def add_attr(self, obj, selector, w_value): + # grumble, jit needs this + attr = self._get_new_attr(selector[0], selector[1]) oldattr = obj._get_mapdict_map() - oldattr._size_estimate += attr.size_estimate() - oldattr.size_estimate() + if not jit.we_are_jitted(): + oldattr._size_estimate += attr.size_estimate() - oldattr.size_estimate() if attr.length() > len(obj._get_mapdict_storage()): # note that attr.size_estimate() is always at least attr.length() new_storage = [None] * attr.size_estimate() @@ -62,6 +73,7 @@ class Terminator(AbstractAttribute): + _immutable_ = True def __init__(self, w_cls, space): self.w_cls = w_cls self.space = space @@ -95,6 +107,7 @@ return self.copy(obj) class DictTerminator(Terminator): + _immutable_ = True def __init__(self, w_cls, space): Terminator.__init__(self, w_cls, space) self.devolved_dict_terminator = DevolvedDictTerminator(w_cls, space) @@ -146,6 +159,7 @@ assert 0, "should be unreachable" class PlainAttribute(AbstractAttribute): + _immutable_ = True def __init__(self, selector, back): self.selector = selector self.position = back.length() @@ -228,14 +242,15 @@ class Object(W_Root): # slightly evil to make it inherit from W_Root def _init_empty(self, map): + from pypy.rlib.debug import make_sure_not_resized self.map = map - self.storage = [None] * map.size_estimate() + self.storage = make_sure_not_resized([None] * map.size_estimate()) def _become(self, new_obj): self.map = new_obj.map self.storage = new_obj.storage def _get_mapdict_map(self): - return self.map + return jit.hint(self.map, promote=True) def _get_mapdict_storage(self): return self.storage def _set_mapdict_map(self, map): @@ -247,26 +262,26 @@ # objspace interface def getdictvalue(self, space, attrname): - return self.map.read(self, (attrname, DICT)) + return self._get_mapdict_map().read(self, (attrname, DICT)) def setdictvalue(self, space, attrname, w_value, shadows_type=True): - return self.map.write(self, (attrname, DICT), w_value) + return self._get_mapdict_map().write(self, (attrname, DICT), w_value) def deldictvalue(self, space, w_name): attrname = space.str_w(w_name) - new_obj = self.map.delete(self, (attrname, DICT)) + new_obj = self._get_mapdict_map().delete(self, (attrname, DICT)) if new_obj is None: return False self._become(new_obj) return True def getdict(self): - w_dict = self.map.read(self, ("dict", SPECIAL)) + w_dict = self._get_mapdict_map().read(self, ("dict", SPECIAL)) if w_dict is not None: assert isinstance(w_dict, W_DictMultiObject) return w_dict w_dict = MapDictImplementation(self.space, self) - flag = self.map.write(self, ("dict", SPECIAL), w_dict) + flag = self._get_mapdict_map().write(self, ("dict", SPECIAL), w_dict) assert flag return w_dict @@ -276,14 +291,14 @@ w_olddict = self.getdict() assert isinstance(w_dict, W_DictMultiObject) w_olddict._as_rdict() - flag = self.map.write(self, ("dict", SPECIAL), w_dict) + flag = self._get_mapdict_map().write(self, ("dict", SPECIAL), w_dict) assert flag def getclass(self, space): - return self.map.get_terminator().w_cls + return self._get_mapdict_map().get_terminator().w_cls def setclass(self, space, w_cls): - new_obj = self.map.set_terminator(self, w_cls.terminator) + new_obj = self._get_mapdict_map().set_terminator(self, w_cls.terminator) self._become(new_obj) def user_setup(self, space, w_subtype): @@ -293,17 +308,17 @@ def getslotvalue(self, index): key = ("slot", SLOTS_STARTING_FROM + index) - return self.map.read(self, key) + return self._get_mapdict_map().read(self, key) def setslotvalue(self, index, w_value): key = ("slot", SLOTS_STARTING_FROM + index) - self.map.write(self, key, w_value) + self._get_mapdict_map().write(self, key, w_value) # used by _weakref implemenation def getweakref(self): from pypy.module._weakref.interp__weakref import WeakrefLifeline - lifeline = self.map.read(self, ("weakref", SPECIAL)) + lifeline = self._get_mapdict_map().read(self, ("weakref", SPECIAL)) if lifeline is None: return None assert isinstance(lifeline, WeakrefLifeline) @@ -312,7 +327,7 @@ def setweakref(self, space, weakreflifeline): from pypy.module._weakref.interp__weakref import WeakrefLifeline assert isinstance(weakreflifeline, WeakrefLifeline) - self.map.write(self, ("weakref", SPECIAL), weakreflifeline) + self._get_mapdict_map().write(self, ("weakref", SPECIAL), weakreflifeline) # ____________________________________________________________ Modified: pypy/branch/better-map-instances/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/typeobject.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/typeobject.py Sun Aug 29 13:02:05 2010 @@ -75,7 +75,9 @@ 'weakrefable', 'hasdict', 'nslots', - 'instancetypedef'] + 'instancetypedef', + 'terminator', + ] # for config.objspace.std.getattributeshortcut # (False is a conservative default, fixed during real usage) From arigo at codespeak.net Sun Aug 29 13:53:25 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 29 Aug 2010 13:53:25 +0200 (CEST) Subject: [pypy-svn] r76784 - pypy/branch/markcompact/pypy/rpython/memory Message-ID: <20100829115325.DCAAE282B9E@codespeak.net> Author: arigo Date: Sun Aug 29 13:53:22 2010 New Revision: 76784 Modified: pypy/branch/markcompact/pypy/rpython/memory/gctypelayout.py Log: Improve the detection of buggy type_ids. Modified: pypy/branch/markcompact/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gctypelayout.py Sun Aug 29 13:53:22 2010 @@ -44,16 +44,18 @@ self.type_info_group_ptr = type_info_group._as_ptr() def get(self, typeid): - _check_typeid(typeid) - return llop.get_group_member(GCData.TYPE_INFO_PTR, - self.type_info_group_ptr, - typeid) + res = llop.get_group_member(GCData.TYPE_INFO_PTR, + self.type_info_group_ptr, + typeid) + _check_valid_type_info(res) + return res def get_varsize(self, typeid): - _check_typeid(typeid) - return llop.get_group_member(GCData.VARSIZE_TYPE_INFO_PTR, - self.type_info_group_ptr, - typeid) + res = llop.get_group_member(GCData.VARSIZE_TYPE_INFO_PTR, + self.type_info_group_ptr, + typeid) + _check_valid_type_info_varsize(res) + return res def q_is_varsize(self, typeid): infobits = self.get(typeid).infobits @@ -115,15 +117,21 @@ # the lowest 16bits are used to store group member index -T_MEMBER_INDEX = 0xffff +T_MEMBER_INDEX = 0xffff T_IS_VARSIZE = 0x10000 T_HAS_GCPTR_IN_VARSIZE = 0x20000 T_IS_GCARRAY_OF_GCPTR = 0x40000 T_IS_WEAKREF = 0x80000 +T_KEY_MASK = 0xFF000000 +T_KEY_VALUE = 0x7A000000 # bug detection only -def _check_typeid(typeid): - ll_assert(llop.is_group_member_nonzero(lltype.Bool, typeid), - "invalid type_id") +def _check_valid_type_info(p): + ll_assert(p.infobits & T_KEY_MASK == T_KEY_VALUE, "invalid type_id") + +def _check_valid_type_info_varsize(p): + ll_assert(p.header.infobits & (T_KEY_MASK | T_IS_VARSIZE) == + (T_KEY_VALUE | T_IS_VARSIZE), + "invalid varsize type_id") def encode_type_shape(builder, info, TYPE, index): @@ -167,7 +175,7 @@ varinfo.varitemsize = llmemory.sizeof(ARRAY.OF) if TYPE == WEAKREF: infobits |= T_IS_WEAKREF - info.infobits = infobits + info.infobits = infobits | T_KEY_VALUE # ____________________________________________________________ @@ -250,14 +258,18 @@ _, TYPE = TYPE._first_struct() def get_info(self, type_id): - return llop.get_group_member(GCData.TYPE_INFO_PTR, - self.type_info_group._as_ptr(), - type_id) + res = llop.get_group_member(GCData.TYPE_INFO_PTR, + self.type_info_group._as_ptr(), + type_id) + _check_valid_type_info(res) + return res def get_info_varsize(self, type_id): - return llop.get_group_member(GCData.VARSIZE_TYPE_INFO_PTR, - self.type_info_group._as_ptr(), - type_id) + res = llop.get_group_member(GCData.VARSIZE_TYPE_INFO_PTR, + self.type_info_group._as_ptr(), + type_id) + _check_valid_type_info_varsize(res) + return res def is_weakref(self, type_id): return self.get_info(type_id).infobits & T_IS_WEAKREF From cfbolz at codespeak.net Sun Aug 29 14:28:26 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 29 Aug 2010 14:28:26 +0200 (CEST) Subject: [pypy-svn] r76785 - in pypy/branch/better-map-instances/pypy/objspace/std: . test Message-ID: <20100829122826.88502282B9E@codespeak.net> Author: cfbolz Date: Sun Aug 29 14:28:24 2010 New Revision: 76785 Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py Log: some tests about changing the class of an instance. revealed a bug. Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py Sun Aug 29 14:28:24 2010 @@ -158,6 +158,12 @@ def remove_dict_entries(self, obj): assert 0, "should be unreachable" + def set_terminator(self, obj, terminator): + if not isinstance(terminator, DevolvedDictTerminator): + assert isinstance(terminator, DictTerminator) + terminator = terminator.devolved_dict_terminator + return Terminator.set_terminator(self, obj, terminator) + class PlainAttribute(AbstractAttribute): _immutable_ = True def __init__(self, selector, back): Modified: pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py Sun Aug 29 14:28:24 2010 @@ -399,3 +399,48 @@ A.slot1.__set__(x, 'parent') # using A.slot1 assert x.slot1 == 'child' # B.slot1 should still have its old value assert A.slot1.__get__(x) == 'parent' + + def test_change_class(self): + class A(object): + pass + class B(object): + pass + a = A() + a.x = 1 + a.y = 2 + assert a.x == 1 + assert a.y == 2 + a.__class__ = B + assert a.x == 1 + assert a.y == 2 + assert isinstance(a, B) + + # dict accessed: + a = A() + a.x = 1 + a.y = 2 + assert a.x == 1 + assert a.y == 2 + d = a.__dict__ + assert d == {"x": 1, "y": 2} + a.__class__ = B + assert a.x == 1 + assert a.y == 2 + assert a.__dict__ is d + assert d == {"x": 1, "y": 2} + assert isinstance(a, B) + + # dict devolved: + a = A() + a.x = 1 + a.y = 2 + assert a.x == 1 + assert a.y == 2 + d = a.__dict__ + d[1] = 3 + assert d == {"x": 1, "y": 2, 1:3} + a.__class__ = B + assert a.x == 1 + assert a.y == 2 + assert a.__dict__ is d + assert isinstance(a, B) From cfbolz at codespeak.net Sun Aug 29 14:32:56 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 29 Aug 2010 14:32:56 +0200 (CEST) Subject: [pypy-svn] r76786 - pypy/branch/better-map-instances/pypy/doc/config Message-ID: <20100829123256.5C2CC282B9E@codespeak.net> Author: cfbolz Date: Sun Aug 29 14:32:55 2010 New Revision: 76786 Added: pypy/branch/better-map-instances/pypy/doc/config/objspace.std.withmapdict.txt (contents, props changed) Log: add some docs Added: pypy/branch/better-map-instances/pypy/doc/config/objspace.std.withmapdict.txt ============================================================================== --- (empty file) +++ pypy/branch/better-map-instances/pypy/doc/config/objspace.std.withmapdict.txt Sun Aug 29 14:32:55 2010 @@ -0,0 +1,5 @@ +Enable the new version of "sharing dictionaries". + +See the section in `Standard Interpreter Optimizations`_ for more details. + +.. _`Standard Interpreter Optimizations`: ../interpreter-optimizations.html#sharing-dicts From cfbolz at codespeak.net Sun Aug 29 14:58:23 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 29 Aug 2010 14:58:23 +0200 (CEST) Subject: [pypy-svn] r76787 - pypy/branch/better-map-instances/pypy/objspace/std/test Message-ID: <20100829125823.E42B1282B9E@codespeak.net> Author: cfbolz Date: Sun Aug 29 14:58:21 2010 New Revision: 76787 Modified: pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py Log: two skipped tests that pypy doesn't seem to support in general so far Modified: pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/test/test_mapdict.py Sun Aug 29 14:58:21 2010 @@ -444,3 +444,57 @@ assert a.y == 2 assert a.__dict__ is d assert isinstance(a, B) + + def test_change_class_slots(self): + skip("not supported by pypy yet") + class A(object): + __slots__ = ["x", "y"] + + class B(object): + __slots__ = ["x", "y"] + + a = A() + a.x = 1 + a.y = 2 + assert a.x == 1 + assert a.y == 2 + a.__class__ = B + assert a.x == 1 + assert a.y == 2 + assert isinstance(a, B) + + def test_change_class_slots_dict(self): + skip("not supported by pypy yet") + class A(object): + __slots__ = ["x", "__dict__"] + class B(object): + __slots__ = ["x", "__dict__"] + # dict accessed: + a = A() + a.x = 1 + a.y = 2 + assert a.x == 1 + assert a.y == 2 + d = a.__dict__ + assert d == {"y": 2} + a.__class__ = B + assert a.x == 1 + assert a.y == 2 + assert a.__dict__ is d + assert d == {"y": 2} + assert isinstance(a, B) + + # dict devolved: + a = A() + a.x = 1 + a.y = 2 + assert a.x == 1 + assert a.y == 2 + d = a.__dict__ + d[1] = 3 + assert d == {"x": 1, "y": 2, 1:3} + a.__class__ = B + assert a.x == 1 + assert a.y == 2 + assert a.__dict__ is d + assert isinstance(a, B) From arigo at codespeak.net Sun Aug 29 16:27:59 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 29 Aug 2010 16:27:59 +0200 (CEST) Subject: [pypy-svn] r76788 - pypy/branch/markcompact/pypy/rpython/memory/gc Message-ID: <20100829142759.B1B02282B9E@codespeak.net> Author: arigo Date: Sun Aug 29 16:27:57 2010 New Revision: 76788 Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Log: Write basic tests here. Found a bug. Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Sun Aug 29 16:27:57 2010 @@ -593,8 +593,13 @@ llmemory.raw_memcopy(fromaddr, toaddr, basesize) def debug_check_object(self, obj): - # not sure what to check here - pass + # Test that GCFLAG_MARKBIT is not set. It should not be set at the + # very start or at the very end of a collection -- only temporarily + # during the collection. + tid = self.header(obj).tid + assert tid & GCFLAG_MARKBIT == 0 + type_id = self.get_type_id(obj) + self.has_gcptr_in_varsize(type_id) # checks that the type_id is valid def trace_from_objects_with_finalizers(self): if self.run_finalizers.non_empty(): # uncommon case From arigo at codespeak.net Sun Aug 29 16:43:42 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 29 Aug 2010 16:43:42 +0200 (CEST) Subject: [pypy-svn] r76789 - pypy/branch/markcompact/pypy/rpython/memory/gc Message-ID: <20100829144342.58015282B9E@codespeak.net> Author: arigo Date: Sun Aug 29 16:43:39 2010 New Revision: 76789 Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Log: Fix for r76788. Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Sun Aug 29 16:43:39 2010 @@ -131,7 +131,10 @@ def init_gc_object_immortal(self, addr, typeid16, flags=0): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - flags |= GCFLAG_HASHTAKEN + flags |= GCFLAG_HASHTAKEN | GCFLAG_MARKBIT + # All prebuilt GC objects have the GCFLAG_MARKBIT always set. + # That's convenient to make the GC always think that they + # survive the current collection. hdr.tid = self.combine(typeid16, flags) def _get_memory(self, totalsize): @@ -561,7 +564,7 @@ self.gcheaderbuilder.size_gc_header) def surviving(self, obj): - return self._is_external(obj) or self.header_forwarded(obj).tid != -1 + return self.marked(obj) def get_typeid_from_backup(self, num): return self.tid_backup[num] @@ -593,11 +596,18 @@ llmemory.raw_memcopy(fromaddr, toaddr, basesize) def debug_check_object(self, obj): - # Test that GCFLAG_MARKBIT is not set. It should not be set at the - # very start or at the very end of a collection -- only temporarily - # during the collection. tid = self.header(obj).tid - assert tid & GCFLAG_MARKBIT == 0 + if self._is_external(obj): + # All external objects have GCFLAG_MARKBIT and GCFLAG_HASHTAKEN + # set. + assert tid & GCFLAG_MARKBIT + assert tid & GCFLAG_HASHTAKEN + else: + # Non-external objects have GCFLAG_MARKBIT that should not be set + # at the very start or at the very end of a collection -- only + # temporarily during the collection. + assert tid & GCFLAG_MARKBIT == 0 + # type_id = self.get_type_id(obj) self.has_gcptr_in_varsize(type_id) # checks that the type_id is valid From cfbolz at codespeak.net Sun Aug 29 17:16:03 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 29 Aug 2010 17:16:03 +0200 (CEST) Subject: [pypy-svn] r76790 - pypy/branch/better-map-instances/pypy/config Message-ID: <20100829151603.6A950282B9E@codespeak.net> Author: cfbolz Date: Sun Aug 29 17:16:01 2010 New Revision: 76790 Modified: pypy/branch/better-map-instances/pypy/config/pypyoption.py Log: make mapdict no longer the default, but -Ojit enables it Modified: pypy/branch/better-map-instances/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/better-map-instances/pypy/config/pypyoption.py (original) +++ pypy/branch/better-map-instances/pypy/config/pypyoption.py Sun Aug 29 17:16:01 2010 @@ -240,7 +240,7 @@ BoolOption("withmapdict", "make instances really small but slow without the JIT", - default=True, + default=False, requires=[("objspace.std.withshadowtracking", False), ("objspace.std.withinlineddict", False), ("objspace.std.withsharingdict", False), @@ -327,6 +327,9 @@ # all the good optimizations for PyPy should be listed here if level in ['2', '3']: config.objspace.opcodes.suggest(CALL_LIKELY_BUILTIN=True) + config.objspace.std.suggest(withinlineddict=True) + if type_system != 'ootype': + config.objspace.std.suggest(withsharingdict=True) if level in ['2', '3', 'jit']: config.objspace.opcodes.suggest(CALL_METHOD=True) config.objspace.std.suggest(withrangelist=True) @@ -336,9 +339,6 @@ config.objspace.std.suggest(optimized_list_getitem=True) config.objspace.std.suggest(getattributeshortcut=True) config.objspace.std.suggest(newshortcut=True) - if type_system != 'ootype': - config.objspace.std.suggest(withsharingdict=True) - config.objspace.std.suggest(withinlineddict=True) # extra costly optimizations only go in level 3 if level == '3': @@ -367,6 +367,7 @@ # extra optimizations with the JIT if level == 'jit': config.objspace.std.suggest(withcelldict=True) + config.objspace.std.suggest(withmapdict=True) def enable_allworkingmodules(config): From cfbolz at codespeak.net Sun Aug 29 17:51:03 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 29 Aug 2010 17:51:03 +0200 (CEST) Subject: [pypy-svn] r76791 - pypy/branch/better-map-instances/pypy/module/pypyjit/test Message-ID: <20100829155103.2C757282B9E@codespeak.net> Author: cfbolz Date: Sun Aug 29 17:51:00 2010 New Revision: 76791 Modified: pypy/branch/better-map-instances/pypy/module/pypyjit/test/test_pypy_c.py Log: make tests more detailed Modified: pypy/branch/better-map-instances/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/better-map-instances/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/better-map-instances/pypy/module/pypyjit/test/test_pypy_c.py Sun Aug 29 17:51:00 2010 @@ -270,7 +270,7 @@ assert len(ops) == 2 assert not ops[0].get_opnames("call") assert not ops[0].get_opnames("new") - assert len(ops[0].get_opnames("guard")) <= 7 + assert len(ops[0].get_opnames("guard")) <= 2 assert not ops[1] # second LOOKUP_METHOD folded away ops = self.get_by_bytecode("CALL_METHOD") @@ -281,7 +281,7 @@ else: assert not bytecode.get_opnames("call") assert not bytecode.get_opnames("new") - assert len(bytecode.get_opnames("guard")) <= 9 + assert len(bytecode.get_opnames("guard")) <= 6 assert len(ops[1]) < len(ops[0]) ops = self.get_by_bytecode("LOAD_ATTR") @@ -315,8 +315,8 @@ assert len(ops) == 2 assert not ops[0].get_opnames("call") assert not ops[0].get_opnames("new") - assert len(ops[0].get_opnames("guard")) <= 7 - assert len(ops[0].get_opnames("getfield")) < 6 + assert len(ops[0].get_opnames("guard")) <= 2 + assert len(ops[0].get_opnames("getfield")) < 5 assert not ops[1] # second LOOKUP_METHOD folded away def test_default_and_kw(self): @@ -380,7 +380,7 @@ a.x = 2 i = i + a.x return i - ''', 67, + ''', 69, ([20], 20), ([31], 32)) @@ -388,7 +388,7 @@ self.get_by_bytecode("CALL_FUNCTION")) assert not callA.get_opnames("call") assert not callA.get_opnames("new") - assert len(callA.get_opnames("guard")) <= 8 + assert len(callA.get_opnames("guard")) <= 2 assert not callisinstance1.get_opnames("call") assert not callisinstance1.get_opnames("new") assert len(callisinstance1.get_opnames("guard")) <= 2 @@ -740,6 +740,8 @@ '''%(op1, float(a)/4.0, float(b)/4.0, op2), 109, ([], res)) def test_boolrewrite_ptr(self): + # XXX this test is way too imprecise in what it is actually testing + # it should count the number of guards instead compares = ('a == b', 'b == a', 'a != b', 'b != a', 'a == c', 'c != b') for e1 in compares: for e2 in compares: @@ -761,7 +763,7 @@ n = 215 self.run_source(''' - class tst: + class tst(object): pass def main(): a = tst() @@ -825,6 +827,8 @@ ''', 65, ([], 122880)) def test_array_intimg(self): + # XXX this test is way too imprecise in what it is actually testing + # it should count the number of guards instead for tc, maxops in zip('ilILd', (67, 67, 69, 69, 61)): res = 73574560 if tc in 'IL': From cfbolz at codespeak.net Sun Aug 29 17:51:13 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 29 Aug 2010 17:51:13 +0200 (CEST) Subject: [pypy-svn] r76792 - pypy/extradoc/planning Message-ID: <20100829155113.62E22282BD6@codespeak.net> Author: cfbolz Date: Sun Aug 29 17:51:11 2010 New Revision: 76792 Modified: pypy/extradoc/planning/jit.txt Log: update status Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Sun Aug 29 17:51:11 2010 @@ -82,6 +82,7 @@ - pypy/objspace/std/inlinedict: put the class into the structure to get only one promote when using an instance, instead of two: the promotion of the '.w__class__' and the promotion of the '.structure' + DONE on the better-map-instances branch - guard_true(frame.is_being_profiled) all over the place From arigo at codespeak.net Sun Aug 29 19:14:22 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 29 Aug 2010 19:14:22 +0200 (CEST) Subject: [pypy-svn] r76793 - pypy/branch/markcompact/pypy/rpython/memory/gc Message-ID: <20100829171422.B0330282B9E@codespeak.net> Author: arigo Date: Sun Aug 29 19:14:20 2010 New Revision: 76793 Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/base.py Log: Move the call to self.debug_check_object(obj) at a place where it is easier to debug from (because we have the source object too). Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/base.py Sun Aug 29 19:14:20 2010 @@ -220,7 +220,6 @@ pending = self._debug_pending while pending.non_empty(): obj = pending.pop() - self.debug_check_object(obj) self.trace(obj, self._debug_callback2, None) self._debug_seen.delete() self._debug_pending.delete() @@ -229,6 +228,7 @@ seen = self._debug_seen if not seen.contains(obj): seen.add(obj) + self.debug_check_object(obj) self._debug_pending.append(obj) def _debug_callback(self, root): obj = root.address[0] From arigo at codespeak.net Sun Aug 29 19:14:47 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 29 Aug 2010 19:14:47 +0200 (CEST) Subject: [pypy-svn] r76794 - pypy/branch/markcompact/pypy/rpython/memory/gc Message-ID: <20100829171447.A2C98282B9E@codespeak.net> Author: arigo Date: Sun Aug 29 19:14:46 2010 New Revision: 76794 Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Log: Assert that we don't call get_forwarding_address() on a target that dies. Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Sun Aug 29 19:14:46 2010 @@ -558,9 +558,10 @@ return self.get_header_forwarded_addr(obj) def get_header_forwarded_addr(self, obj): + tid = self.header_forwarded(obj).tid + ll_assert(tid & GCFLAG_MARKBIT != 0, "dying object is not forwarded") GCFLAG_MASK = ~(GCFLAG_MARKBIT | 3) - return (self.base_forwarding_addr + - (self.header_forwarded(obj).tid & GCFLAG_MASK) + + return (self.base_forwarding_addr + (tid & GCFLAG_MASK) + self.gcheaderbuilder.size_gc_header) def surviving(self, obj): From arigo at codespeak.net Sun Aug 29 19:39:16 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 29 Aug 2010 19:39:16 +0200 (CEST) Subject: [pypy-svn] r76795 - pypy/branch/markcompact/pypy/rpython/memory/gc Message-ID: <20100829173916.77FA8282B9E@codespeak.net> Author: arigo Date: Sun Aug 29 19:39:14 2010 New Revision: 76795 Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Log: More debugging checks. Useless so far. Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Sun Aug 29 19:39:14 2010 @@ -249,7 +249,7 @@ # Walk all objects and assign forward pointers in the same order, # also updating all references # - finaladdr = self.update_forward_pointers(toaddr, maxnum) + self.update_forward_pointers(toaddr, maxnum) if (self.run_finalizers.non_empty() or self.objects_with_finalizers.non_empty()): self.update_run_finalizers() @@ -258,14 +258,14 @@ self.compact() # self.tid_backup = lltype.nullptr(TID_BACKUP) - self.free = finaladdr - self.next_collect_after = self.next_collection(finaladdr - toaddr, + self.free = self.finaladdr + self.next_collect_after = self.next_collection(self.finaladdr - toaddr, self.num_alive_objs, requested_size) # if not translated_to_c(): - remaining_size = (toaddr + self.space_size) - finaladdr - llarena.arena_reset(finaladdr, remaining_size, False) + remaining_size = (toaddr + self.space_size) - self.finaladdr + llarena.arena_reset(self.finaladdr, remaining_size, False) llarena.arena_free(self.space) self.space = toaddr # @@ -449,6 +449,7 @@ if not translated_to_c(): assert toaddr - base_forwarding_addr <= fromaddr - self.space self.num_alive_objs = num + self.finaladdr = toaddr # now update references self.root_walker.walk_roots( @@ -456,7 +457,6 @@ MarkCompactGC._update_ref, # static in prebuilt non-gc structures MarkCompactGC._update_ref) # static in prebuilt gc objects self.walk_marked_objects(MarkCompactGC.trace_and_update_ref) - return toaddr def walk_marked_objects(self, callback): num = 0 @@ -561,8 +561,10 @@ tid = self.header_forwarded(obj).tid ll_assert(tid & GCFLAG_MARKBIT != 0, "dying object is not forwarded") GCFLAG_MASK = ~(GCFLAG_MARKBIT | 3) - return (self.base_forwarding_addr + (tid & GCFLAG_MASK) + - self.gcheaderbuilder.size_gc_header) + res = (self.base_forwarding_addr + (tid & GCFLAG_MASK) + + self.gcheaderbuilder.size_gc_header) + ll_assert(res < self.finaladdr, "forwarded address >= self.finaladdr") + return res def surviving(self, obj): return self.marked(obj) From arigo at codespeak.net Mon Aug 30 09:49:23 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 30 Aug 2010 09:49:23 +0200 (CEST) Subject: [pypy-svn] r76798 - in pypy/branch/markcompact/pypy: rpython/memory/gc translator/c/test Message-ID: <20100830074923.CA4F5282B90@codespeak.net> Author: arigo Date: Mon Aug 30 09:49:21 2010 New Revision: 76798 Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py pypy/branch/markcompact/pypy/translator/c/test/test_newgc.py Log: A direct test for the bug, and a fix. Argh. Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Mon Aug 30 09:49:21 2010 @@ -583,12 +583,12 @@ if grow_hash_field: gcflags |= GCFLAG_SAVED_HASHFIELD hashvalue = self.get_identityhash_from_addr(obj) - (toaddr + basesize).signed[0] = hashvalue elif gcflags & GCFLAG_SAVED_HASHFIELD: fromaddr = llarena.getfakearenaaddress(obj) fromaddr -= self.gcheaderbuilder.size_gc_header hashvalue = (fromaddr + basesize).signed[0] - (toaddr + basesize).signed[0] = hashvalue + else: + hashvalue = 0 # not used # hdr.tid = self.combine(typeid, gcflags << first_gcflag_bit) # @@ -597,6 +597,9 @@ llmemory.raw_memmove(fromaddr, toaddr, basesize) else: llmemory.raw_memcopy(fromaddr, toaddr, basesize) + # + if gcflags & GCFLAG_SAVED_HASHFIELD: + (toaddr + basesize).signed[0] = hashvalue def debug_check_object(self, obj): tid = self.header(obj).tid Modified: pypy/branch/markcompact/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/markcompact/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/markcompact/pypy/translator/c/test/test_newgc.py Mon Aug 30 09:49:21 2010 @@ -1080,6 +1080,52 @@ def test_finalizer_order(self): py.test.skip("not implemented") + def define_adding_a_hash(cls): + from pypy.rlib.objectmodel import compute_identity_hash + S1 = lltype.GcStruct('S1', ('x', lltype.Signed)) + S2 = lltype.GcStruct('S2', ('p1', lltype.Ptr(S1)), + ('p2', lltype.Ptr(S1)), + ('p3', lltype.Ptr(S1)), + ('p4', lltype.Ptr(S1)), + ('p5', lltype.Ptr(S1)), + ('p6', lltype.Ptr(S1)), + ('p7', lltype.Ptr(S1)), + ('p8', lltype.Ptr(S1)), + ('p9', lltype.Ptr(S1))) + def g(): + lltype.malloc(S1) # forgotten, will be shifted over + s2 = lltype.malloc(S2) # a big object, overlaps its old position + s2.p1 = lltype.malloc(S1); s2.p1.x = 1010 + s2.p2 = lltype.malloc(S1); s2.p2.x = 1020 + s2.p3 = lltype.malloc(S1); s2.p3.x = 1030 + s2.p4 = lltype.malloc(S1); s2.p4.x = 1040 + s2.p5 = lltype.malloc(S1); s2.p5.x = 1050 + s2.p6 = lltype.malloc(S1); s2.p6.x = 1060 + s2.p7 = lltype.malloc(S1); s2.p7.x = 1070 + s2.p8 = lltype.malloc(S1); s2.p8.x = 1080 + s2.p9 = lltype.malloc(S1); s2.p9.x = 1090 + return s2 + def f(): + rgc.collect() + s2 = g() + h2 = compute_identity_hash(s2) + rgc.collect() # shift s2 to the left, but add a hash field + assert s2.p1.x == 1010 + assert s2.p2.x == 1020 + assert s2.p3.x == 1030 + assert s2.p4.x == 1040 + assert s2.p5.x == 1050 + assert s2.p6.x == 1060 + assert s2.p7.x == 1070 + assert s2.p8.x == 1080 + assert s2.p9.x == 1090 + return h2 - compute_identity_hash(s2) + return f + + def test_adding_a_hash(self): + res = self.run("adding_a_hash") + assert res == 0 + # ____________________________________________________________________ class TaggedPointersTest(object): From arigo at codespeak.net Mon Aug 30 10:40:19 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 30 Aug 2010 10:40:19 +0200 (CEST) Subject: [pypy-svn] r76799 - pypy/branch/markcompact/pypy/rpython/memory/gc Message-ID: <20100830084019.8EE43282B90@codespeak.net> Author: arigo Date: Mon Aug 30 10:40:16 2010 New Revision: 76799 Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Log: Move test. Modified: pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/markcompact/pypy/rpython/memory/gc/markcompact.py Mon Aug 30 10:40:16 2010 @@ -602,6 +602,9 @@ (toaddr + basesize).signed[0] = hashvalue def debug_check_object(self, obj): + type_id = self.get_type_id(obj) + self.has_gcptr_in_varsize(type_id) # checks that the type_id is valid + # tid = self.header(obj).tid if self._is_external(obj): # All external objects have GCFLAG_MARKBIT and GCFLAG_HASHTAKEN @@ -613,9 +616,6 @@ # at the very start or at the very end of a collection -- only # temporarily during the collection. assert tid & GCFLAG_MARKBIT == 0 - # - type_id = self.get_type_id(obj) - self.has_gcptr_in_varsize(type_id) # checks that the type_id is valid def trace_from_objects_with_finalizers(self): if self.run_finalizers.non_empty(): # uncommon case From arigo at codespeak.net Mon Aug 30 12:55:46 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 30 Aug 2010 12:55:46 +0200 (CEST) Subject: [pypy-svn] r76800 - pypy/branch/better-map-instances/pypy/objspace/std Message-ID: <20100830105546.96B3A282B90@codespeak.net> Author: arigo Date: Mon Aug 30 12:55:44 2010 New Revision: 76800 Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py Log: Don't use _immutable_. It gives really obscure GC bugs to have _immutable_ set on a subclass, but not on the base class. Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py Mon Aug 30 12:55:44 2010 @@ -73,7 +73,7 @@ class Terminator(AbstractAttribute): - _immutable_ = True + _immutable_fields_ = ['w_cls'] def __init__(self, w_cls, space): self.w_cls = w_cls self.space = space @@ -107,7 +107,7 @@ return self.copy(obj) class DictTerminator(Terminator): - _immutable_ = True + _immutable_fields_ = ['devolved_dict_terminator'] def __init__(self, w_cls, space): Terminator.__init__(self, w_cls, space) self.devolved_dict_terminator = DevolvedDictTerminator(w_cls, space) @@ -165,7 +165,7 @@ return Terminator.set_terminator(self, obj, terminator) class PlainAttribute(AbstractAttribute): - _immutable_ = True + _immutable_fields_ = ['selector', 'position', 'back'] def __init__(self, selector, back): self.selector = selector self.position = back.length() From arigo at codespeak.net Mon Aug 30 15:37:26 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 30 Aug 2010 15:37:26 +0200 (CEST) Subject: [pypy-svn] r76802 - pypy/branch/no-_immutable_ Message-ID: <20100830133726.3C5FF282B9E@codespeak.net> Author: arigo Date: Mon Aug 30 15:37:24 2010 New Revision: 76802 Added: pypy/branch/no-_immutable_/ - copied from r76801, pypy/trunk/ Log: A branch in which to kill "_immutable_=True" and replace it with _immutable_fields_ only. From arigo at codespeak.net Mon Aug 30 15:56:32 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 30 Aug 2010 15:56:32 +0200 (CEST) Subject: [pypy-svn] r76803 - in pypy/branch/no-_immutable_/pypy/rpython: . test Message-ID: <20100830135632.0CEE3282B9E@codespeak.net> Author: arigo Date: Mon Aug 30 15:56:30 2010 New Revision: 76803 Modified: pypy/branch/no-_immutable_/pypy/rpython/rclass.py pypy/branch/no-_immutable_/pypy/rpython/test/test_rclass.py Log: Check that a field doesn't end up higher in the class hierarchy than where it was originally defined in _immutable_fields_. Modified: pypy/branch/no-_immutable_/pypy/rpython/rclass.py ============================================================================== --- pypy/branch/no-_immutable_/pypy/rpython/rclass.py (original) +++ pypy/branch/no-_immutable_/pypy/rpython/rclass.py Mon Aug 30 15:56:30 2010 @@ -18,6 +18,10 @@ def _freeze_(self): return True +class ImmutableConflictError(Exception): + """Raised when an attribute 'x' is found on some parent class, + but defined in a subclass to be in _immutable_fields_.""" + def getclassrepr(rtyper, classdef): try: @@ -154,8 +158,9 @@ def _check_for_immutable_hints(self, hints): if '_immutable_' in self.classdef.classdesc.classdict: - hints = hints.copy() - hints['immutable'] = True + raise TyperError( + "%r: the _immutable_ hint is not supported any more.\n" + "Use _immutable_fields_ instead." % (self,)) self.immutable_field_list = [] # unless overwritten below if self.classdef.classdesc.lookup('_immutable_fields_') is not None: hints = hints.copy() @@ -189,18 +194,32 @@ rbase = self while rbase.classdef is not None: immutable_fields.update( - dict.fromkeys(rbase.immutable_field_list)) + dict.fromkeys(rbase.immutable_field_list, rbase)) rbase = rbase.rbase self._parse_field_list(immutable_fields, accessor) def _parse_field_list(self, fields, accessor): with_suffix = {} - for name in fields: + for name, rbase in fields.items(): if name.endswith('[*]'): name = name[:-3] suffix = '[*]' else: suffix = '' + # + # check that the field is not higher up the class hierarchy + # than where it was originally defined in _immutable_fields_ + if rbase.rbase.classdef is not None: + try: + rbase.rbase._get_field(name) + except KeyError: + pass + else: + raise ImmutableConflictError( + "the field %r is declared in _immutable_fields_ in " + "class %r, but actually ends up in parent class" + % (name, rbase)) + # try: mangled_name, r = self._get_field(name) except KeyError: Modified: pypy/branch/no-_immutable_/pypy/rpython/test/test_rclass.py ============================================================================== --- pypy/branch/no-_immutable_/pypy/rpython/test/test_rclass.py (original) +++ pypy/branch/no-_immutable_/pypy/rpython/test/test_rclass.py Mon Aug 30 15:56:30 2010 @@ -709,7 +709,7 @@ def test_immutable(self): class I(object): - _immutable_ = True + _immutable_fields_ = ["v"] def __init__(self, v): self.v = v @@ -796,27 +796,17 @@ assert accessor.fields == {"inst_y" : ""} or \ accessor.fields == {"oy" : ""} # for ootype - def test_immutable_inheritance(self): - class I(object): - def __init__(self, v): - self.v = v - - class J(I): - _immutable_ = True - def __init__(self, v, w): - self.w = w - I.__init__(self, v) - - j = J(3, 4) - def f(): - j.v = j.v * 1 # make the annotator think it is mutated - j.w = j.w * 1 # make the annotator think it is mutated - return j.v + j.w - - t, typer, graph = self.gengraph(f, [], backendopt=True) - f_summary = summary(graph) - assert f_summary == {"setfield": 2} or \ - f_summary == {"oosetfield": 2} # for ootype + def test_immutable_forbidden_inheritance(self): + from pypy.rpython.rclass import ImmutableConflictError + class A(object): + pass + class B(A): + _immutable_fields_ = ['v'] + def f(): + A().v = 123 + B() # crash: class B says 'v' is immutable, + # but it is defined on parent class I + py.test.raises(ImmutableConflictError, self.gengraph, f, []) class TestLLtype(BaseTestRclass, LLRtypeMixin): From cfbolz at codespeak.net Mon Aug 30 17:16:38 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 30 Aug 2010 17:16:38 +0200 (CEST) Subject: [pypy-svn] r76807 - pypy/branch/better-map-instances/pypy/objspace/std Message-ID: <20100830151638.9F9DA282BD4@codespeak.net> Author: cfbolz Date: Mon Aug 30 17:16:36 2010 New Revision: 76807 Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py Log: move imports too top Modified: pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py ============================================================================== --- pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py (original) +++ pypy/branch/better-map-instances/pypy/objspace/std/mapdict.py Mon Aug 30 17:16:36 2010 @@ -1,4 +1,10 @@ from pypy.rlib import jit + +from pypy.interpreter.baseobjspace import W_Root +from pypy.objspace.std.dictmultiobject import W_DictMultiObject +from pypy.objspace.std.dictmultiobject import IteratorImplementation +from pypy.objspace.std.dictmultiobject import _is_sane_hash + # ____________________________________________________________ # attribute shapes @@ -244,7 +250,6 @@ SPECIAL = 1 SLOTS_STARTING_FROM = 2 -from pypy.interpreter.baseobjspace import W_Root class Object(W_Root): # slightly evil to make it inherit from W_Root def _init_empty(self, map): @@ -339,9 +344,6 @@ # ____________________________________________________________ # dict implementation -from pypy.objspace.std.dictmultiobject import W_DictMultiObject -from pypy.objspace.std.dictmultiobject import IteratorImplementation -from pypy.objspace.std.dictmultiobject import _is_sane_hash class MapDictImplementation(W_DictMultiObject): def __init__(self, space, w_obj): From fijal at codespeak.net Mon Aug 30 19:25:29 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 30 Aug 2010 19:25:29 +0200 (CEST) Subject: [pypy-svn] r76808 - in pypy/trunk/pypy/jit/tool: . test Message-ID: <20100830172529.8E1FB282B9E@codespeak.net> Author: fijal Date: Mon Aug 30 19:25:27 2010 New Revision: 76808 Modified: pypy/trunk/pypy/jit/tool/test/test_traceviewer.py pypy/trunk/pypy/jit/tool/traceviewer.py Log: Add to traceviewer an option (--use-threshold) that will display only top 20 traces as colored Modified: pypy/trunk/pypy/jit/tool/test/test_traceviewer.py ============================================================================== --- pypy/trunk/pypy/jit/tool/test/test_traceviewer.py (original) +++ pypy/trunk/pypy/jit/tool/test/test_traceviewer.py Mon Aug 30 19:25:27 2010 @@ -52,10 +52,10 @@ def test_postparse(self): real_loops = [FinalBlock("debug_merge_point(' #40 POP_TOP')", None)] - postprocess(real_loops, real_loops[:]) + postprocess(real_loops, real_loops[:], {}) assert real_loops[0].header.startswith("_runCallbacks, file '/tmp/x/twisted-trunk/twisted/internet/defer.py', line 357") def test_load_actual(self): fname = py.path.local(__file__).join('..', 'data.log.bz2') - main(str(fname), view=False) + main(str(fname), False, view=False) # assert did not explode Modified: pypy/trunk/pypy/jit/tool/traceviewer.py ============================================================================== --- pypy/trunk/pypy/jit/tool/traceviewer.py (original) +++ pypy/trunk/pypy/jit/tool/traceviewer.py Mon Aug 30 19:25:27 2010 @@ -1,11 +1,12 @@ #!/usr/bin/env python -""" Usage: traceviewer.py loopfile +""" Usage: traceviewer.py [--use-threshold] loopfile """ import optparse import sys import re import math +import py import autopath from pypy.translator.tool.graphpage import GraphPage @@ -33,6 +34,7 @@ dotgen.emit_edge('node%d' % (counter - 1), 'node%d' % counter) counter += 1 lines_so_far = [] + print line lines_so_far.append(line) dotgen.emit_node('node%d' % counter, shape="box", label="\n".join(lines_so_far)) @@ -40,13 +42,13 @@ self.source = dotgen.generate(target=None) class Page(GraphPage): - def compute(self, graphs): + def compute(self, graphs, counts): dotgen = DotGen('trace') self.loops = graphs self.links = {} self.cache = {} for loop in self.loops: - loop.generate(dotgen) + loop.generate(dotgen, counts) loop.getlinks(self.links) self.cache["loop" + str(loop.no)] = loop self.source = dotgen.generate(target=None) @@ -71,9 +73,14 @@ def getlinks(self, links): links[self.linksource] = self.name() - def generate(self, dotgen): + def generate(self, dotgen, counts): + val = counts.get(self.key, 0) + if val > counts.threshold: + fillcolor = get_gradient_color(self.ratio) + else: + fillcolor = "white" dotgen.emit_node(self.name(), label=self.header, - shape='box', fillcolor=get_gradient_color(self.ratio)) + shape='box', fillcolor=fillcolor) def get_content(self): return self._content @@ -113,11 +120,11 @@ self.target = target BasicBlock.__init__(self, content) - def postprocess(self, loops, memo): - postprocess_loop(self.target, loops, memo) + def postprocess(self, loops, memo, counts): + postprocess_loop(self.target, loops, memo, counts) - def generate(self, dotgen): - BasicBlock.generate(self, dotgen) + def generate(self, dotgen, counts): + BasicBlock.generate(self, dotgen, counts) if self.target is not None: dotgen.emit_edge(self.name(), self.target.name()) @@ -127,12 +134,12 @@ self.right = right BasicBlock.__init__(self, content) - def postprocess(self, loops, memo): - postprocess_loop(self.left, loops, memo) - postprocess_loop(self.right, loops, memo) + def postprocess(self, loops, memo, counts): + postprocess_loop(self.left, loops, memo, counts) + postprocess_loop(self.right, loops, memo, counts) - def generate(self, dotgen): - BasicBlock.generate(self, dotgen) + def generate(self, dotgen, counts): + BasicBlock.generate(self, dotgen, counts) dotgen.emit_edge(self.name(), self.left.name()) dotgen.emit_edge(self.name(), self.right.name()) @@ -176,13 +183,11 @@ real_loops = [] counter = 1 bar = progressbar.ProgressBar(color='blue') - single_percent = len(loops) / 100 allloops = [] - for i, loop in enumerate(loops): + for i, loop in enumerate(loops): if i > MAX_LOOPS: return real_loops, allloops - if single_percent and i % single_percent == 0: - bar.render(i / single_percent) + bar.render((i * 100) / len(loops)) firstline = loop[:loop.find("\n")] m = re.match('# Loop (\d+)', firstline) if m: @@ -202,17 +207,19 @@ counter += loop.count("\n") + 2 return real_loops, allloops -def postprocess_loop(loop, loops, memo): +def postprocess_loop(loop, loops, memo, counts): if loop in memo: return memo.add(loop) if loop is None: return - m = re.search("debug_merge_point\(' (.*?)'", loop.content) + m = re.search("debug_merge_point\('( (.*?))'", loop.content) if m is None: name = '?' + loop.key = '?' else: - name = m.group(1) + " " + m.group(2) + name = m.group(2) + " " + m.group(3) + loop.key = m.group(1) opsno = loop.content.count("\n") lastline = loop.content[loop.content.rfind("\n", 0, len(loop.content) - 2):] m = re.search('descr= 20 and options.use_threshold: + counts.threshold = l[-20] + else: + counts.threshold = 0 + for_print = [(v, k) for k, v in counts.iteritems()] + for_print.sort() + else: + counts = {} log = logparser.parse_log_file(loopfile) loops = logparser.extract_category(log, "jit-log-opt-") real_loops, allloops = splitloops(loops) - postprocess(real_loops, allloops) + postprocess(real_loops, allloops, counts) if view: - Page(allloops).display() + Page(allloops, counts).display() if __name__ == '__main__': parser = optparse.OptionParser(usage=__doc__) + parser.add_option('--use-threshold', dest='use_threshold', + action="store_true") options, args = parser.parse_args(sys.argv) if len(args) != 2: print __doc__ sys.exit(1) - main(args[1]) + main(args[1], options.use_threshold) From fijal at codespeak.net Mon Aug 30 19:27:05 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 30 Aug 2010 19:27:05 +0200 (CEST) Subject: [pypy-svn] r76809 - pypy/trunk/pypy/jit/tool Message-ID: <20100830172705.4A7B4282B9E@codespeak.net> Author: fijal Date: Mon Aug 30 19:27:03 2010 New Revision: 76809 Modified: pypy/trunk/pypy/jit/tool/traceviewer.py Log: remove a debugging print Modified: pypy/trunk/pypy/jit/tool/traceviewer.py ============================================================================== --- pypy/trunk/pypy/jit/tool/traceviewer.py (original) +++ pypy/trunk/pypy/jit/tool/traceviewer.py Mon Aug 30 19:27:03 2010 @@ -34,7 +34,6 @@ dotgen.emit_edge('node%d' % (counter - 1), 'node%d' % counter) counter += 1 lines_so_far = [] - print line lines_so_far.append(line) dotgen.emit_node('node%d' % counter, shape="box", label="\n".join(lines_so_far)) From fijal at codespeak.net Tue Aug 31 12:38:33 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 31 Aug 2010 12:38:33 +0200 (CEST) Subject: [pypy-svn] r76810 - pypy/build/bot2/codespeak-html Message-ID: <20100831103833.96983282BD4@codespeak.net> Author: fijal Date: Tue Aug 31 12:38:31 2010 New Revision: 76810 Modified: pypy/build/bot2/codespeak-html/index.html Log: Update to speed.pypy.org (should have been done ages ago) Modified: pypy/build/bot2/codespeak-html/index.html ============================================================================== --- pypy/build/bot2/codespeak-html/index.html (original) +++ pypy/build/bot2/codespeak-html/index.html Tue Aug 31 12:38:31 2010 @@ -9,7 +9,7 @@

Welcome to codespeak's PyPy Buildbot!