From fijal at codespeak.net Sat May 1 02:52:48 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 1 May 2010 02:52:48 +0200 (CEST) Subject: [pypy-svn] r74303 - pypy/trunk/pypy/module/cpyext/test Message-ID: <20100501005248.6C14F282BF5@codespeak.net> Author: fijal Date: Sat May 1 02:52:45 2010 New Revision: 74303 Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Log: Split test_typeobject in 5 pieces. Just because it's nicer (also because running a single test becomes nicer) Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Sat May 1 02:52:45 2010 @@ -18,6 +18,10 @@ assert type(obj) is module.fooType print "type of obj has type", type(type(obj)) print "type of type of obj has type", type(type(type(obj))) + + def test_typeobject2(self): + module = self.import_module(name='foo') + obj = module.new() obj2 = obj.copy() assert module.new().name == "Foo Example" c = module.fooType.copy @@ -32,6 +36,10 @@ print obj.foo assert obj.foo == 42 assert obj.int_member == obj.foo + + def test_typeobject3(self): + module = self.import_module(name='foo') + obj = module.new() obj.int_member = 23 assert obj.int_member == 23 obj.int_member = 42 @@ -44,6 +52,10 @@ raises(SystemError, "obj.broken_member = 42") assert module.fooType.broken_member.__doc__ is None assert module.fooType.object_member.__doc__ == "A Python object." + + def test_typeobject4(self): + module = self.import_module(name='foo') + obj = module.new() assert obj.object_member is None obj.object_member = "hello" assert obj.object_member == "hello" @@ -58,6 +70,9 @@ del obj.object_member_ex raises(AttributeError, "del obj.object_member_ex") + def test_typeobject5(self): + module = self.import_module(name='foo') + obj = module.new() assert obj.string_member == "Hello from PyPy" raises(TypeError, "obj.string_member = 42") raises(TypeError, "del obj.string_member") From fijal at codespeak.net Sat May 1 21:48:04 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 1 May 2010 21:48:04 +0200 (CEST) Subject: [pypy-svn] r74307 - pypy/trunk/pypy/doc Message-ID: <20100501194804.9ACAA282B9E@codespeak.net> Author: fijal Date: Sat May 1 21:47:59 2010 New Revision: 74307 Removed: pypy/trunk/pypy/doc/README.Debian Log: This is at the very least extremely innacurate, remove From fijal at codespeak.net Sat May 1 21:48:36 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 1 May 2010 21:48:36 +0200 (CEST) Subject: [pypy-svn] r74308 - pypy/trunk/pypy/doc Message-ID: <20100501194836.843DA282B9E@codespeak.net> Author: fijal Date: Sat May 1 21:48:35 2010 New Revision: 74308 Removed: pypy/trunk/pypy/doc/README.compiling Log: inacurate and misleading From fijal at codespeak.net Sat May 1 22:42:00 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 1 May 2010 22:42:00 +0200 (CEST) Subject: [pypy-svn] r74309 - pypy/trunk/pypy/doc Message-ID: <20100501204200.A9EA8282B9E@codespeak.net> Author: fijal Date: Sat May 1 22:41:59 2010 New Revision: 74309 Modified: pypy/trunk/pypy/doc/project-ideas.txt Log: Update a bit (?) Modified: pypy/trunk/pypy/doc/project-ideas.txt ============================================================================== --- pypy/trunk/pypy/doc/project-ideas.txt (original) +++ pypy/trunk/pypy/doc/project-ideas.txt Sat May 1 22:41:59 2010 @@ -21,22 +21,9 @@ -------------------------------- PyPy's Just-In-Time compiler relies on backends for actual code -generation. We have so far a 32-bit Intel backend, and a CLI one. Open -ideas are to write a backend for **Intel 64** (AMD64); or a backend for -Java; or trying again to use LLVM-JIT (which I do not really recommend). - -CTypes ------- - -Support ctypes on more backends. Right now ctypes is supported only -when compiling PyPy to C, and there is a bit of unfinished work to -support it on **Intel 64.** A nice project would be to support it when -compiling to .NET or the JVM. That's not too hard, the only thing needed -is to port a small module that does the actual invocation of external -libraries (a related project is to port this module to Jython or -IronPython to get support for ctypes there, which is something that was -tried but not finished as far as I know). - +generation. We have so far a 32-bit Intel backend, and a CLI one. There is +Summer of Code project for 64bit (x86_64) backend, but other options +(ARM, llvm) remain open. .. _distribution: .. _persistence: From fijal at codespeak.net Sat May 1 22:49:12 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 1 May 2010 22:49:12 +0200 (CEST) Subject: [pypy-svn] r74310 - pypy/trunk/pypy/module/cpyext Message-ID: <20100501204912.2B171282B9E@codespeak.net> Author: fijal Date: Sat May 1 22:49:10 2010 New Revision: 74310 Modified: pypy/trunk/pypy/module/cpyext/methodobject.py Log: Keep up to date Modified: pypy/trunk/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/methodobject.py (original) +++ pypy/trunk/pypy/module/cpyext/methodobject.py Sat May 1 22:49:10 2010 @@ -217,7 +217,7 @@ __doc__ = interp_attrproperty('doc', cls=W_PyCWrapperObject), __objclass__ = interp_attrproperty_w('w_objclass', cls=W_PyCWrapperObject), __repr__ = interp2app(W_PyCWrapperObject.descr_method_repr), - # XXX missing: __getattribute__, __repr__ + # XXX missing: __getattribute__ ) W_PyCWrapperObject.typedef.acceptable_as_base_class = False From afa at codespeak.net Sat May 1 22:59:49 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 1 May 2010 22:59:49 +0200 (CEST) Subject: [pypy-svn] r74311 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100501205949.A92C2282B9E@codespeak.net> Author: afa Date: Sat May 1 22:59:48 2010 New Revision: 74311 Modified: pypy/trunk/pypy/module/cpyext/api.py pypy/trunk/pypy/module/cpyext/methodobject.py pypy/trunk/pypy/module/cpyext/test/test_methodobject.py Log: Fix W_PyCMethodObject.__repr__, and write a considerable test to run it. Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Sat May 1 22:59:48 2010 @@ -104,7 +104,8 @@ # class ApiFunction: - def __init__(self, argtypes, restype, callable, borrowed, error): + def __init__(self, argtypes, restype, callable, + borrowed=False, error=_NOT_SPECIFIED): self.argtypes = argtypes self.restype = restype self.functype = lltype.Ptr(lltype.FuncType(argtypes, restype)) Modified: pypy/trunk/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/methodobject.py (original) +++ pypy/trunk/pypy/module/cpyext/methodobject.py Sat May 1 22:59:48 2010 @@ -112,7 +112,7 @@ self.w_objclass = from_ref(space, pyo) def __repr__(self): - self.space.unwrap(self.descr_method_repr()) + return self.space.unwrap(self.descr_method_repr()) def descr_method_repr(self): return self.getrepr(self.space, "built-in method '%s' of '%s' object" % (self.name, self.w_objclass.getname(self.space, '?'))) Modified: pypy/trunk/pypy/module/cpyext/test/test_methodobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_methodobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_methodobject.py Sat May 1 22:59:48 2010 @@ -1,5 +1,10 @@ from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase +from pypy.module.cpyext.methodobject import PyMethodDef +from pypy.module.cpyext.api import ApiFunction +from pypy.module.cpyext.pyobject import PyObject, make_ref, Py_DecRef +from pypy.module.cpyext.methodobject import PyDescr_NewMethod +from pypy.rpython.lltypesystem import rffi, lltype class AppTestMethodObject(AppTestCpythonExtensionBase): def test_call_METH(self): @@ -59,3 +64,28 @@ assert mod.getarg_OLD(1, 2) == (1, 2) assert mod.isCFunction(mod.getarg_O) == "getarg_O" + +class TestPyCMethodObject(BaseApiTest): + def test_repr(self, space): + """ + W_PyCMethodObject has a repr string which describes it as a method + and gives its name and the name of its class. + """ + def func(space, w_self, w_args): + return space.w_None + c_func = ApiFunction([PyObject, PyObject], PyObject, func) + func.api_func = c_func + ml = lltype.malloc(PyMethodDef, flavor='raw', zero=True) + namebuf = rffi.str2charp('func') + ml.c_ml_name = namebuf + ml.c_ml_meth = c_func.get_llhelper(space) + + pto = make_ref(space, space.w_str) + method = PyDescr_NewMethod(space, pto, ml) + + assert repr(method).startswith( + " Author: exarkun Date: Sat May 1 23:28:14 2010 New Revision: 74312 Modified: pypy/branch/py-packagecontext/pypy/module/cpyext/modsupport.py pypy/branch/py-packagecontext/pypy/module/cpyext/state.py pypy/branch/py-packagecontext/pypy/module/cpyext/test/test_cpyext.py Log: write some docstrings; update the package_context comment. Modified: pypy/branch/py-packagecontext/pypy/module/cpyext/modsupport.py ============================================================================== --- pypy/branch/py-packagecontext/pypy/module/cpyext/modsupport.py (original) +++ pypy/branch/py-packagecontext/pypy/module/cpyext/modsupport.py Sat May 1 23:28:14 2010 @@ -8,7 +8,17 @@ from pypy.module.cpyext.state import State from pypy.interpreter.error import OperationError +#@cpython_api([rffi.CCHARP], PyObject, borrowed=True) def PyImport_AddModule(space, name): + """Return the module object corresponding to a module name. The name argument + may be of the form package.module. First check the modules dictionary if + there's one there, and if not, create a new one and insert it in the modules + dictionary. + + This function does not load or import the module; if the module wasn't already + loaded, you will get an empty module object. Use PyImport_ImportModule() + or one of its variants to import a module. Package structures implied by a + dotted name for name are not created if not already present.""" w_name = space.wrap(name) w_modules = space.sys.get('modules') @@ -22,6 +32,20 @@ @cpython_api([CONST_STRING, lltype.Ptr(PyMethodDef), CONST_STRING, PyObject, rffi.INT_real], PyObject, borrowed=False) # we cannot borrow here def Py_InitModule4(space, name, methods, doc, w_self, apiver): + """ + Create a new module object based on a name and table of functions, returning + the new module object. If doc is non-NULL, it will be used to define the + docstring for the module. If self is non-NULL, it will passed to the + functions of the module as their (otherwise NULL) first parameter. (This was + added as an experimental feature, and there are no known uses in the current + version of Python.) For apiver, the only value which should be passed is + defined by the constant PYTHON_API_VERSION. + + Note that the name parameter is actually ignored, and the module name is + taken from the package_context attribute of the cpyext.State in the space + cache. CPython includes some extra checking here to make sure the module + being initialized lines up with what's expected, but we don't. + """ from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr modname = rffi.charp2str(name) state = space.fromcache(State) Modified: pypy/branch/py-packagecontext/pypy/module/cpyext/state.py ============================================================================== --- pypy/branch/py-packagecontext/pypy/module/cpyext/state.py (original) +++ pypy/branch/py-packagecontext/pypy/module/cpyext/state.py Sat May 1 23:28:14 2010 @@ -25,11 +25,10 @@ # When importing a package, use this to keep track of its name. This is # necessary because an extension module in a package might not supply # its own fully qualified name to Py_InitModule. If it doesn't, we need - # to be able to figure out what module is being initialized. When a - # package import is in progress, this is set to the name of the package. - # The rest of the time, it's None. Packages may be imported - # recursively, in which case the outer state is preserved somewhere in - # the stack and then restored when the inner import is complete. + # to be able to figure out what module is being initialized. Recursive + # imports will clobber this value, which might be confusing, but it + # doesn't hurt anything because the code that cares about it will have + # already read it by that time. self.package_context = None Modified: pypy/branch/py-packagecontext/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/py-packagecontext/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/py-packagecontext/pypy/module/cpyext/test/test_cpyext.py Sat May 1 23:28:14 2010 @@ -50,6 +50,16 @@ raises(ImportError, cpyext.load_module, self.libc, "invalid.function") def compile_module(modname, **kwds): + """ + Build an extension module and return the filename of the resulting native + code file. + + modname is the name of the module, possibly including dots if it is a module + inside a package. + + Any extra keyword arguments are passed on to ExternalCompilationInfo to + build the module (so specify your source with one of those). + """ modname = modname.split('.')[-1] eci = ExternalCompilationInfo( export_symbols=['init%s' % (modname,)], @@ -137,6 +147,9 @@ importhook(cls.space, "os") # warm up reference counts def compile_module(self, name, **kwds): + """ + Build an extension module linked against the cpyext api library. + """ state = self.space.fromcache(State) api_library = state.api_lib if sys.platform == 'win32': @@ -212,12 +225,19 @@ return self.import_module(name=modname, init=init, body=body) def record_imported_module(self, name): + """ + Record a module imported in a test so that it can be cleaned up in + teardown before the check for leaks is done. + + name gives the name of the module in the space's sys.modules. + """ self.imported_module_names.append(name) def setup_method(self, func): # A list of modules which the test caused to be imported (in # self.space). These will be cleaned up automatically in teardown. self.imported_module_names = [] + self.w_import_module = self.space.wrap(self.import_module) self.w_import_extension = self.space.wrap(self.import_extension) self.w_compile_module = self.space.wrap(self.compile_module) @@ -234,6 +254,10 @@ #self.check_and_print_leaks() def unimport_module(self, name): + """ + Remove the named module from the space's sys.modules and discard the + reference (owned by "the test") to it. + """ w_modules = self.space.sys.get('modules') w_name = self.space.wrap(name) w_mod = self.space.getitem(w_modules, w_name) From exarkun at codespeak.net Sat May 1 23:35:10 2010 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Sat, 1 May 2010 23:35:10 +0200 (CEST) Subject: [pypy-svn] r74313 - in pypy/trunk: . pypy/module/cpyext pypy/module/cpyext/test Message-ID: <20100501213510.52DF0282B9C@codespeak.net> Author: exarkun Date: Sat May 1 23:35:08 2010 New Revision: 74313 Added: pypy/trunk/pypy/module/cpyext/test/date.c - copied unchanged from r74312, pypy/branch/py-packagecontext/pypy/module/cpyext/test/date.c Modified: pypy/trunk/ (props changed) pypy/trunk/pypy/module/cpyext/api.py pypy/trunk/pypy/module/cpyext/modsupport.py pypy/trunk/pypy/module/cpyext/state.py pypy/trunk/pypy/module/cpyext/test/test_cpyext.py pypy/trunk/pypy/module/cpyext/test/test_import.py Log: (fijal, afa, exarkun) Handle extensions which init their module object with a relative name by keeping track of what we think the name of the module being imported is and using that instead of the name supplied by the extension. Also (somewhat unrelated) skip a test which was reporting leaked references. Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Sat May 1 23:35:08 2010 @@ -747,24 +747,28 @@ @unwrap_spec(ObjSpace, str, str) def load_extension_module(space, path, name): state = space.fromcache(State) - from pypy.rlib import libffi + state.package_context = name try: - dll = libffi.CDLL(path, False) - except libffi.DLOpenError, e: - raise operationerrfmt( - space.w_ImportError, - "unable to load extension module '%s': %s", - path, e.msg) - try: - initptr = libffi.dlsym(dll.lib, 'init%s' % (name.split('.')[-1],)) - except KeyError: - raise operationerrfmt( - space.w_ImportError, - "function init%s not found in library %s", - name, path) - initfunc = rffi.cast(initfunctype, initptr) - generic_cpy_call(space, initfunc) - state.check_and_raise_exception() + from pypy.rlib import libffi + try: + dll = libffi.CDLL(path, False) + except libffi.DLOpenError, e: + raise operationerrfmt( + space.w_ImportError, + "unable to load extension module '%s': %s", + path, e.msg) + try: + initptr = libffi.dlsym(dll.lib, 'init%s' % (name.split('.')[-1],)) + except KeyError: + raise operationerrfmt( + space.w_ImportError, + "function init%s not found in library %s", + name, path) + initfunc = rffi.cast(initfunctype, initptr) + generic_cpy_call(space, initfunc) + state.check_and_raise_exception() + finally: + state.package_context = None @specialize.ll() def generic_cpy_call(space, func, *args): Modified: pypy/trunk/pypy/module/cpyext/modsupport.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/modsupport.py (original) +++ pypy/trunk/pypy/module/cpyext/modsupport.py Sat May 1 23:35:08 2010 @@ -5,22 +5,52 @@ from pypy.interpreter.module import Module from pypy.module.cpyext.methodobject import W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod, PyMethodDef, PyCFunction from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall +from pypy.module.cpyext.state import State from pypy.interpreter.error import OperationError +#@cpython_api([rffi.CCHARP], PyObject, borrowed=True) def PyImport_AddModule(space, name): + """Return the module object corresponding to a module name. The name argument + may be of the form package.module. First check the modules dictionary if + there's one there, and if not, create a new one and insert it in the modules + dictionary. + + This function does not load or import the module; if the module wasn't already + loaded, you will get an empty module object. Use PyImport_ImportModule() + or one of its variants to import a module. Package structures implied by a + dotted name for name are not created if not already present.""" w_name = space.wrap(name) - w_mod = space.wrap(Module(space, w_name)) - w_modules = space.sys.get('modules') - space.setitem(w_modules, w_name, w_mod) + + w_mod = space.finditem_str(w_modules, name) + if w_mod is None: + w_mod = space.wrap(Module(space, w_name)) + space.setitem(w_modules, w_name, w_mod) + return w_mod @cpython_api([CONST_STRING, lltype.Ptr(PyMethodDef), CONST_STRING, PyObject, rffi.INT_real], PyObject, borrowed=False) # we cannot borrow here def Py_InitModule4(space, name, methods, doc, w_self, apiver): + """ + Create a new module object based on a name and table of functions, returning + the new module object. If doc is non-NULL, it will be used to define the + docstring for the module. If self is non-NULL, it will passed to the + functions of the module as their (otherwise NULL) first parameter. (This was + added as an experimental feature, and there are no known uses in the current + version of Python.) For apiver, the only value which should be passed is + defined by the constant PYTHON_API_VERSION. + + Note that the name parameter is actually ignored, and the module name is + taken from the package_context attribute of the cpyext.State in the space + cache. CPython includes some extra checking here to make sure the module + being initialized lines up with what's expected, but we don't. + """ from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr modname = rffi.charp2str(name) - w_mod = PyImport_AddModule(space, modname) + state = space.fromcache(State) + w_mod = PyImport_AddModule(space, state.package_context) + dict_w = {} convert_method_defs(space, dict_w, methods, lltype.nullptr(PyTypeObjectPtr.TO), w_self) for key, w_value in dict_w.items(): Modified: pypy/trunk/pypy/module/cpyext/state.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/state.py (original) +++ pypy/trunk/pypy/module/cpyext/state.py Sat May 1 23:35:08 2010 @@ -22,6 +22,16 @@ self.exc_value = None self.new_method_def = lltype.nullptr(PyMethodDef) + # When importing a package, use this to keep track of its name. This is + # necessary because an extension module in a package might not supply + # its own fully qualified name to Py_InitModule. If it doesn't, we need + # to be able to figure out what module is being initialized. Recursive + # imports will clobber this value, which might be confusing, but it + # doesn't hurt anything because the code that cares about it will have + # already read it by that time. + self.package_context = None + + def _freeze_(self): assert not self.borrowed_objects and not self.borrow_mapping self.py_objects_r2w.clear() # is not valid anymore after translation Modified: pypy/trunk/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Sat May 1 23:35:08 2010 @@ -10,7 +10,7 @@ from pypy.translator import platform from pypy.translator.gensupp import uniquemodulename from pypy.tool.udir import udir -from pypy.module.cpyext import api +from pypy.module.cpyext import api, typeobject from pypy.module.cpyext.state import State from pypy.module.cpyext.pyobject import Py_DecRef, InvalidPointerException from pypy.translator.goal import autopath @@ -41,8 +41,19 @@ raises(ImportError, cpyext.load_module, self.libc, "invalid.function") def compile_module(modname, **kwds): + """ + Build an extension module and return the filename of the resulting native + code file. + + modname is the name of the module, possibly including dots if it is a module + inside a package. + + Any extra keyword arguments are passed on to ExternalCompilationInfo to + build the module (so specify your source with one of those). + """ + modname = modname.split('.')[-1] eci = ExternalCompilationInfo( - export_symbols=['init%s' % (modname.split('.')[-1],)], + export_symbols=['init%s' % (modname,)], include_dirs=api.include_dirs, **kwds ) @@ -87,7 +98,7 @@ if delta != 0: leaking = True print >>sys.stderr, "Leaking %r: %i references" % (w_obj, delta) - lifeline = api.lifeline_dict.get(w_obj) + lifeline = typeobject.lifeline_dict.get(w_obj) if lifeline is not None: refcnt = lifeline.pyo.c_ob_refcnt if refcnt > 0: @@ -123,6 +134,24 @@ def setup_class(cls): cls.space = gettestobjspace(usemodules=['cpyext', 'thread']) cls.space.getbuiltinmodule("cpyext") + from pypy.module.imp.importing import importhook + importhook(cls.space, "os") # warm up reference counts + + def compile_module(self, name, **kwds): + """ + Build an extension module linked against the cpyext api library. + """ + state = self.space.fromcache(State) + api_library = state.api_lib + if sys.platform == 'win32': + kwds["libraries"] = [api_library] + # '%s' undefined; assuming extern returning int + kwds["compile_extra"] = ["/we4013"] + else: + kwds["link_files"] = [str(api_library + '.so')] + kwds["compile_extra"] = ["-Werror=implicit-function-declaration"] + return compile_module(name, **kwds) + def import_module(self, name, init=None, body='', load_it=True, filename=None): """ @@ -151,20 +180,11 @@ / 'cpyext'/ 'test' / (filename + ".c") kwds = dict(separate_module_files=[filename]) - state = self.space.fromcache(State) - api_library = state.api_lib - if sys.platform == 'win32': - kwds["libraries"] = [api_library] - # '%s' undefined; assuming extern returning int - kwds["compile_extra"] = ["/we4013"] - else: - kwds["link_files"] = [str(api_library + '.so')] - kwds["compile_extra"] = ["-Werror=implicit-function-declaration"] - mod = compile_module(name, **kwds) + mod = self.compile_module(name, **kwds) - self.name = name if load_it: api.load_extension_module(self.space, mod, name) + self.imported_module_names.append(name) return self.space.getitem( self.space.sys.get('modules'), self.space.wrap(name)) @@ -195,9 +215,28 @@ init = """Py_InitModule("%s", methods);""" % (modname,) return self.import_module(name=modname, init=init, body=body) + def record_imported_module(self, name): + """ + Record a module imported in a test so that it can be cleaned up in + teardown before the check for leaks is done. + + name gives the name of the module in the space's sys.modules. + """ + self.imported_module_names.append(name) + def setup_method(self, func): + # A list of modules which the test caused to be imported (in + # self.space). These will be cleaned up automatically in teardown. + self.imported_module_names = [] + self.w_import_module = self.space.wrap(self.import_module) self.w_import_extension = self.space.wrap(self.import_extension) + self.w_compile_module = self.space.wrap(self.compile_module) + self.w_record_imported_module = self.space.wrap( + self.record_imported_module) + self.w_here = self.space.wrap( + str(py.path.local(autopath.pypydir)) + '/module/cpyext/test/') + # create the file lock before we count allocations self.space.call_method(self.space.sys.get("stdout"), "flush") @@ -205,27 +244,30 @@ freeze_refcnts(self) #self.check_and_print_leaks() + def unimport_module(self, name): + """ + Remove the named module from the space's sys.modules and discard the + reference (owned by "the test") to it. + """ + w_modules = self.space.sys.get('modules') + w_name = self.space.wrap(name) + w_mod = self.space.getitem(w_modules, w_name) + self.space.delitem(w_modules, w_name) + Py_DecRef(self.space, w_mod) + def teardown_method(self, func): - try: - w_mod = self.space.getitem(self.space.sys.get('modules'), - self.space.wrap(self.name)) - self.space.delitem(self.space.sys.get('modules'), - self.space.wrap(self.name)) - Py_DecRef(self.space, w_mod) - state = self.space.fromcache(State) - for w_obj in state.non_heaptypes: - Py_DecRef(self.space, w_obj) - state.non_heaptypes[:] = [] - while state.borrowed_objects: - addr = state.borrowed_objects.keys()[0] - w_obj = state.py_objects_r2w[addr] - Py_DecRef(self.space, w_obj) - state.borrowed_objects = {} - state.borrow_mapping = {} - except OperationError: - pass - except AttributeError: - pass + for name in self.imported_module_names: + self.unimport_module(name) + state = self.space.fromcache(State) + for w_obj in state.non_heaptypes: + Py_DecRef(self.space, w_obj) + state.non_heaptypes[:] = [] + while state.borrowed_objects: + addr = state.borrowed_objects.keys()[0] + w_obj = state.py_objects_r2w[addr] + Py_DecRef(self.space, w_obj) + state.borrowed_objects = {} + state.borrow_mapping = {} if self.check_and_print_leaks(): assert False, "Test leaks or loses object(s)." @@ -313,7 +355,7 @@ assert module.return_cookie() == 3.14 - def test_InitModule4Dotted(self): + def test_InitModule4_dotted(self): """ If the module name passed to Py_InitModule4 includes a package, only the module name (the part after the last dot) is considered when @@ -324,17 +366,45 @@ assert module.__name__ == expected_name - def test_InitModuleInPackage(self): + def test_InitModule4_in_package(self): """ If `apple.banana` is an extension module which calls Py_InitModule4 with only "banana" as a name, the resulting module nevertheless is stored at `sys.modules["apple.banana"]`. """ - skip("About to implement this") module = self.import_module(name="apple.banana", filename="banana") assert module.__name__ == "apple.banana" + def test_recursive_package_import(self): + """ + If `cherry.date` is an extension module which imports `apple.banana`, + the latter is added to `sys.modules` for the `"apple.banana"` key. + """ + # Build the extensions. + banana = self.compile_module( + "apple.banana", separate_module_files=[self.here + 'banana.c']) + self.record_imported_module("apple.banana") + date = self.compile_module( + "cherry.date", separate_module_files=[self.here + 'date.c']) + self.record_imported_module("cherry.date") + + # Set up some package state so that the extensions can actually be + # imported. + import sys, types, os + cherry = sys.modules['cherry'] = types.ModuleType('cherry') + cherry.__path__ = [os.path.dirname(date)] + + apple = sys.modules['apple'] = types.ModuleType('apple') + apple.__path__ = [os.path.dirname(banana)] + + import cherry.date + import apple.banana + + assert sys.modules['apple.banana'].__name__ == 'apple.banana' + assert sys.modules['cherry.date'].__name__ == 'cherry.date' + + def test_modinit_func(self): """ A module can use the PyMODINIT_FUNC macro to declare or define its Modified: pypy/trunk/pypy/module/cpyext/test/test_import.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_import.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_import.py Sat May 1 23:35:08 2010 @@ -14,6 +14,7 @@ class AppTestImportLogic(AppTestCpythonExtensionBase): def test_import_logic(self): + skip("leak?") path = self.import_module(name='test_import_module', load_it=False) import sys sys.path.append(path) From exarkun at codespeak.net Sat May 1 23:35:30 2010 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Sat, 1 May 2010 23:35:30 +0200 (CEST) Subject: [pypy-svn] r74314 - pypy/branch/py-packagecontext Message-ID: <20100501213530.484D9282B9C@codespeak.net> Author: exarkun Date: Sat May 1 23:35:28 2010 New Revision: 74314 Removed: pypy/branch/py-packagecontext/ Log: merged From afa at codespeak.net Sun May 2 00:04:21 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sun, 2 May 2010 00:04:21 +0200 (CEST) Subject: [pypy-svn] r74315 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100501220421.4C252282B9C@codespeak.net> Author: afa Date: Sun May 2 00:04:19 2010 New Revision: 74315 Modified: pypy/trunk/pypy/module/cpyext/methodobject.py pypy/trunk/pypy/module/cpyext/modsupport.py pypy/trunk/pypy/module/cpyext/test/test_methodobject.py pypy/trunk/pypy/module/cpyext/typeobject.py Log: Store the real pypy type in descriptors, instead of the PyTypeObject. This feels more correct (cpyext is not comfortable with borrowed references) and even simplifies some parts. Modified: pypy/trunk/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/methodobject.py (original) +++ pypy/trunk/pypy/module/cpyext/methodobject.py Sun May 2 00:04:19 2010 @@ -104,12 +104,11 @@ class W_PyCMethodObject(W_PyCFunctionObject): w_self = None - def __init__(self, space, ml, pto): + def __init__(self, space, ml, w_type): self.space = space self.ml = ml self.name = rffi.charp2str(ml.c_ml_name) - pyo = rffi.cast(PyObject, pto) - self.w_objclass = from_ref(space, pyo) + self.w_objclass = w_type def __repr__(self): return self.space.unwrap(self.descr_method_repr()) @@ -226,8 +225,8 @@ return space.wrap(W_PyCFunctionObject(space, ml, w_self)) -def PyDescr_NewMethod(space, pto, method): - return space.wrap(W_PyCMethodObject(space, method, pto)) +def PyDescr_NewMethod(space, w_type, method): + return space.wrap(W_PyCMethodObject(space, method, w_type)) def PyDescr_NewWrapper(space, pto, method_name, wrapper_func, doc, flags, func): # not exactly the API sig Modified: pypy/trunk/pypy/module/cpyext/modsupport.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/modsupport.py (original) +++ pypy/trunk/pypy/module/cpyext/modsupport.py Sun May 2 00:04:19 2010 @@ -52,7 +52,7 @@ w_mod = PyImport_AddModule(space, state.package_context) dict_w = {} - convert_method_defs(space, dict_w, methods, lltype.nullptr(PyTypeObjectPtr.TO), w_self) + convert_method_defs(space, dict_w, methods, None, w_self) for key, w_value in dict_w.items(): space.setattr(w_mod, space.wrap(key), w_value) if doc: @@ -61,7 +61,7 @@ return w_mod -def convert_method_defs(space, dict_w, methods, pto, w_self=None): +def convert_method_defs(space, dict_w, methods, w_type, w_self=None): methods = rffi.cast(rffi.CArrayPtr(PyMethodDef), methods) if methods: i = -1 @@ -77,7 +77,7 @@ else: doc = None - if not pto: + if w_type is None: if flags & METH_CLASS or flags & METH_STATIC: raise OperationError(space.w_ValueError, space.wrap("module functions cannot set METH_CLASS or METH_STATIC")) @@ -89,14 +89,14 @@ if flags & METH_STATIC: raise OperationError(space.w_ValueError, space.wrap("method cannot be both class and static")) - #w_obj = PyDescr_NewClassMethod(pto, method) + #w_obj = PyDescr_NewClassMethod(space, w_type, method) w_obj = space.w_Ellipsis # XXX elif flags & METH_STATIC: w_func = PyCFunction_NewEx(space, method, None) w_obj = space.w_Ellipsis # XXX #w_obj = PyStaticMethod_New(space, w_func) else: - w_obj = PyDescr_NewMethod(space, pto, method) + w_obj = PyDescr_NewMethod(space, w_type, method) dict_w[methodname] = w_obj Modified: pypy/trunk/pypy/module/cpyext/test/test_methodobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_methodobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_methodobject.py Sun May 2 00:04:19 2010 @@ -80,12 +80,9 @@ ml.c_ml_name = namebuf ml.c_ml_meth = c_func.get_llhelper(space) - pto = make_ref(space, space.w_str) - method = PyDescr_NewMethod(space, pto, ml) - + method = PyDescr_NewMethod(space, space.w_str, ml) assert repr(method).startswith( " Author: afa Date: Sun May 2 00:10:58 2010 New Revision: 74316 Modified: pypy/trunk/pypy/module/cpyext/test/test_thread.py Log: fix (or xfix?) these tests: a "xfail" test must stop on an assertion, not raise a random AttributeError. Modified: pypy/trunk/pypy/module/cpyext/test/test_thread.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_thread.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_thread.py Sun May 2 00:10:58 2010 @@ -26,12 +26,14 @@ @py.test.mark.xfail def test_acquire_lock(self, space, api): + assert hasattr(api, 'PyThread_acquire_lock') lock = allocate_ll_lock() assert api.PyThread_acquire_lock(lock, space.w_int(0)) == 1 assert api.PyThread_acquire_lock(lock, space.w_int(1)) == 0 @py.test.mark.xfail def test_release_lock(self, space, api): + assert hasattr(api, 'PyThread_acquire_lock') lock = allocate_ll_lock() api.PyThread_acquire_lock(lock, space.w_int(0)) api.PyThread_release_lock(lock) From benjamin at codespeak.net Sun May 2 03:23:05 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 2 May 2010 03:23:05 +0200 (CEST) Subject: [pypy-svn] r74317 - pypy/trunk/pypy/interpreter Message-ID: <20100502012305.B77D1282B9C@codespeak.net> Author: benjamin Date: Sun May 2 03:23:02 2010 New Revision: 74317 Modified: pypy/trunk/pypy/interpreter/pyopcode.py Log: kill whitespace Modified: pypy/trunk/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyopcode.py (original) +++ pypy/trunk/pypy/interpreter/pyopcode.py Sun May 2 03:23:02 2010 @@ -1014,7 +1014,7 @@ (ofs, ord(c), name) ) STOP_CODE = MISSING_OPCODE - + def BUILD_MAP(self, itemcount, next_instr): if itemcount != 0: raise BytecodeCorruption From benjamin at codespeak.net Sun May 2 03:26:53 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 2 May 2010 03:26:53 +0200 (CEST) Subject: [pypy-svn] r74318 - pypy/trunk/pypy/interpreter Message-ID: <20100502012653.1870F282B9C@codespeak.net> Author: benjamin Date: Sun May 2 03:26:51 2010 New Revision: 74318 Modified: pypy/trunk/pypy/interpreter/pyopcode.py Log: remove function import Modified: pypy/trunk/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyopcode.py (original) +++ pypy/trunk/pypy/interpreter/pyopcode.py Sun May 2 03:26:51 2010 @@ -886,8 +886,6 @@ @jit.unroll_safe def call_function(self, oparg, w_star=None, w_starstar=None): - from pypy.interpreter.function import is_builtin_code - n_arguments = oparg & 0xff n_keywords = (oparg>>8) & 0xff if n_keywords: @@ -909,7 +907,7 @@ args = self.argument_factory(arguments, keywords, keywords_w, w_star, w_starstar) w_function = self.popvalue() - if self.is_being_profiled and is_builtin_code(w_function): + if self.is_being_profiled and function.is_builtin_code(w_function): w_result = self.space.call_args_and_c_profile(self, w_function, args) else: From benjamin at codespeak.net Sun May 2 03:34:18 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 2 May 2010 03:34:18 +0200 (CEST) Subject: [pypy-svn] r74319 - pypy/trunk/pypy/interpreter Message-ID: <20100502013418.99CE8282B9C@codespeak.net> Author: benjamin Date: Sun May 2 03:34:15 2010 New Revision: 74319 Modified: pypy/trunk/pypy/interpreter/main.py Log: remove extra imports Modified: pypy/trunk/pypy/interpreter/main.py ============================================================================== --- pypy/trunk/pypy/interpreter/main.py (original) +++ pypy/trunk/pypy/interpreter/main.py Sun May 2 03:34:15 2010 @@ -1,7 +1,7 @@ -from pypy.interpreter import module, eval +import sys +from pypy.interpreter import eval, module from pypy.interpreter.error import OperationError -from pypy.interpreter.pycode import PyCode -import sys, types + def ensure__main__(space): w_main = space.wrap('__main__') From benjamin at codespeak.net Sun May 2 03:41:51 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 2 May 2010 03:41:51 +0200 (CEST) Subject: [pypy-svn] r74320 - pypy/trunk/pypy/interpreter Message-ID: <20100502014151.DC4F5282B9C@codespeak.net> Author: benjamin Date: Sun May 2 03:41:49 2010 New Revision: 74320 Modified: pypy/trunk/pypy/interpreter/executioncontext.py Log: use prebuilt recursion error Modified: pypy/trunk/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/trunk/pypy/interpreter/executioncontext.py (original) +++ pypy/trunk/pypy/interpreter/executioncontext.py Sun May 2 03:41:49 2010 @@ -50,8 +50,7 @@ def enter(self, frame): if self.framestackdepth > self.space.sys.recursionlimit: - raise OperationError(self.space.w_RuntimeError, - self.space.wrap("maximum recursion depth exceeded")) + raise self.space.prebuilt_recursion_error self.framestackdepth += 1 frame.f_backref = self.topframeref self.topframeref = jit.virtual_ref(frame) From benjamin at codespeak.net Sun May 2 03:54:31 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 2 May 2010 03:54:31 +0200 (CEST) Subject: [pypy-svn] r74321 - pypy/trunk/pypy/interpreter Message-ID: <20100502015431.0D8C0282B9C@codespeak.net> Author: benjamin Date: Sun May 2 03:54:28 2010 New Revision: 74321 Modified: pypy/trunk/pypy/interpreter/baseobjspace.py pypy/trunk/pypy/interpreter/pycompiler.py Log: move compiler imports to toplevel Modified: pypy/trunk/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/trunk/pypy/interpreter/baseobjspace.py (original) +++ pypy/trunk/pypy/interpreter/baseobjspace.py Sun May 2 03:54:28 2010 @@ -4,7 +4,6 @@ from pypy.interpreter.executioncontext import UserDelAction, FrameTraceAction from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.argument import Arguments -from pypy.interpreter.pycompiler import PythonAstCompiler from pypy.interpreter.miscutils import ThreadLocals from pypy.tool.cache import Cache from pypy.tool.uid import HUGEVAL_BYTES @@ -579,6 +578,7 @@ try: return self.default_compiler except AttributeError: + from pypy.interpreter.pycompiler import PythonAstCompiler compiler = PythonAstCompiler(self) self.default_compiler = compiler return compiler Modified: pypy/trunk/pypy/interpreter/pycompiler.py ============================================================================== --- pypy/trunk/pypy/interpreter/pycompiler.py (original) +++ pypy/trunk/pypy/interpreter/pycompiler.py Sun May 2 03:54:28 2010 @@ -3,10 +3,13 @@ Compiler instances are stored into 'space.getexecutioncontext().compiler'. """ -import sys -from pypy.interpreter.astcompiler.consts import PyCF_DONT_IMPLY_DEDENT +from pypy.interpreter import pycode +from pypy.interpreter.pyparser import future, pyparse, error as parseerror +from pypy.interpreter.astcompiler import (astbuilder, codegen, consts, misc, + optimize) from pypy.interpreter.error import OperationError + class AbstractCompiler(object): """Abstract base class for a bytecode compiler.""" @@ -33,7 +36,7 @@ # Hackish default implementation based on the stdlib 'codeop' module. # See comments over there. space = self.space - flags |= PyCF_DONT_IMPLY_DEDENT + flags |= consts.PyCF_DONT_IMPLY_DEDENT # Check for source consisting of only blank lines and comments if mode != "eval": in_comment = False @@ -82,8 +85,7 @@ def getcodeflags(self, code): """Return the __future__ compiler flags that were used to compile the given code object.""" - from pypy.interpreter.pycode import PyCode - if isinstance(code, PyCode): + if isinstance(code, pycode.PyCode): return code.co_flags & self.compiler_flags else: return 0 @@ -98,62 +100,49 @@ the whole source after having only added a new '\n') """ def __init__(self, space, override_version=None): - - from pypy.interpreter.pyparser.pyparse import PythonParser - from pypy.interpreter.pyparser import future PyCodeCompiler.__init__(self, space) - self.parser = PythonParser(space) + self.parser = pyparse.PythonParser(space) self.additional_rules = {} self.future_flags = future.futureFlags_2_5 self.compiler_flags = self.future_flags.allowed_flags def compile_ast(self, node, filename, mode, flags): - from pypy.interpreter.pyparser.pyparse import CompileInfo - from pypy.interpreter.astcompiler.misc import parse_future - info = CompileInfo(filename, mode, flags, parse_future(node)) + future_features = misc.parse_future(node) + info = pyparse.CompileInfo(filename, mode, flags, future_flags) return self._compile_ast(node, info) def _compile_ast(self, node, info): - from pypy.interpreter.astcompiler import optimize - from pypy.interpreter.astcompiler.codegen import compile_ast - from pypy.interpreter.pyparser.error import SyntaxError space = self.space try: mod = optimize.optimize_ast(space, node, info) - code = compile_ast(space, mod, info) - except SyntaxError, e: + code = codegen.compile_ast(space, mod, info) + except parseerror.SyntaxError, e: raise OperationError(space.w_SyntaxError, e.wrap_info(space)) return code def compile_to_ast(self, source, filename, mode, flags): - from pypy.interpreter.pyparser.pyparse import CompileInfo - info = CompileInfo(filename, mode, flags) + info = pyparse.CompileInfo(filename, mode, flags) return self._compile_to_ast(source, info) def _compile_to_ast(self, source, info): - from pypy.interpreter.pyparser.future import get_futures - from pypy.interpreter.pyparser.error import (SyntaxError, - IndentationError) - from pypy.interpreter.astcompiler.astbuilder import ast_from_node space = self.space try: - f_flags, future_info = get_futures(self.future_flags, source) + f_flags, future_info = future.get_futures(self.future_flags, source) info.last_future_import = future_info info.flags |= f_flags parse_tree = self.parser.parse_source(source, info) - mod = ast_from_node(space, parse_tree, info) - except IndentationError, e: + mod = astbuilder.ast_from_node(space, parse_tree, info) + except parseerror.IndentationError, e: raise OperationError(space.w_IndentationError, e.wrap_info(space)) - except SyntaxError, e: + except parseerror.SyntaxError, e: raise OperationError(space.w_SyntaxError, e.wrap_info(space)) return mod def compile(self, source, filename, mode, flags, hidden_applevel=False): - from pypy.interpreter.pyparser.pyparse import CompileInfo - info = CompileInfo(filename, mode, flags, - hidden_applevel=hidden_applevel) + info = pyparse.CompileInfo(filename, mode, flags, + hidden_applevel=hidden_applevel) mod = self._compile_to_ast(source, info) return self._compile_ast(mod, info) From afa at codespeak.net Sun May 2 15:53:59 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sun, 2 May 2010 15:53:59 +0200 (CEST) Subject: [pypy-svn] r74322 - pypy/trunk/pypy/interpreter Message-ID: <20100502135359.0419D282BD4@codespeak.net> Author: afa Date: Sun May 2 15:53:57 2010 New Revision: 74322 Modified: pypy/trunk/pypy/interpreter/pycompiler.py Log: Typo, fix tests in pypy.module._ast Modified: pypy/trunk/pypy/interpreter/pycompiler.py ============================================================================== --- pypy/trunk/pypy/interpreter/pycompiler.py (original) +++ pypy/trunk/pypy/interpreter/pycompiler.py Sun May 2 15:53:57 2010 @@ -107,8 +107,8 @@ self.compiler_flags = self.future_flags.allowed_flags def compile_ast(self, node, filename, mode, flags): - future_features = misc.parse_future(node) - info = pyparse.CompileInfo(filename, mode, flags, future_flags) + future_pos = misc.parse_future(node) + info = pyparse.CompileInfo(filename, mode, flags, future_pos) return self._compile_ast(node, info) def _compile_ast(self, node, info): From hpk at codespeak.net Sun May 2 16:39:34 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sun, 2 May 2010 16:39:34 +0200 (CEST) Subject: [pypy-svn] r74323 - in pypy/branch/py12: py py/_plugin py/_test pypy pypy/translator/benchmark Message-ID: <20100502143934.AEE5E282BD4@codespeak.net> Author: hpk Date: Sun May 2 16:39:33 2010 New Revision: 74323 Modified: pypy/branch/py12/py/__init__.py pypy/branch/py12/py/_plugin/hookspec.py pypy/branch/py12/py/_plugin/pytest_default.py pypy/branch/py12/py/_test/cmdline.py pypy/branch/py12/py/_test/collect.py pypy/branch/py12/py/_test/pluginmanager.py pypy/branch/py12/pypy/conftest.py pypy/branch/py12/pypy/translator/benchmark/conftest.py Log: copy again from py-trunk, now to be released as 1.3.0 because of the new hooks that were introduced to ease pypy porting Modified: pypy/branch/py12/py/__init__.py ============================================================================== --- pypy/branch/py12/py/__init__.py (original) +++ pypy/branch/py12/py/__init__.py Sun May 2 16:39:33 2010 @@ -8,7 +8,7 @@ (c) Holger Krekel and others, 2004-2010 """ -__version__ = version = "1.2.2" +__version__ = version = "1.3.0" import py.apipkg Modified: pypy/branch/py12/py/_plugin/hookspec.py ============================================================================== --- pypy/branch/py12/py/_plugin/hookspec.py (original) +++ pypy/branch/py12/py/_plugin/hookspec.py Sun May 2 16:39:33 2010 @@ -6,14 +6,14 @@ # Command line and configuration # ------------------------------------------------------------------------- -def pytest_addoption(parser): - """ called before commandline parsing. """ +def pytest_namespace(): + "return dict of name->object which will get stored at py.test. namespace" -def pytest_registerhooks(pluginmanager): - """ called after commandline parsing before pytest_configure. """ +def pytest_addoption(parser): + "add optparse-style options via parser.addoption." -def pytest_namespace(): - """ return dict of name->object which will get stored at py.test. namespace""" +def pytest_addhooks(pluginmanager): + "add hooks via pluginmanager.registerhooks(module)" def pytest_configure(config): """ called after command line options have been parsed. @@ -27,12 +27,12 @@ # collection hooks # ------------------------------------------------------------------------- -def pytest_ignore_collect_path(path, config): +def pytest_ignore_collect(path, config): """ return true value to prevent considering this path for collection. This hook is consulted for all files and directories prior to considering collection hooks. """ -pytest_ignore_collect_path.firstresult = True +pytest_ignore_collect.firstresult = True def pytest_collect_directory(path, parent): """ return Collection node or None for the given path. """ Modified: pypy/branch/py12/py/_plugin/pytest_default.py ============================================================================== --- pypy/branch/py12/py/_plugin/pytest_default.py (original) +++ pypy/branch/py12/py/_plugin/pytest_default.py Sun May 2 16:39:33 2010 @@ -28,7 +28,7 @@ """ the pytest config object with access to command line opts.""" return request.config -def pytest_ignore_collect_path(path, config): +def pytest_ignore_collect(path, config): ignore_paths = config.getconftest_pathlist("collect_ignore", path=path) ignore_paths = ignore_paths or [] excludeopt = config.getvalue("ignore") Modified: pypy/branch/py12/py/_test/cmdline.py ============================================================================== --- pypy/branch/py12/py/_test/cmdline.py (original) +++ pypy/branch/py12/py/_test/cmdline.py Sun May 2 16:39:33 2010 @@ -21,4 +21,3 @@ e = sys.exc_info()[1] sys.stderr.write("ERROR: %s\n" %(e.args[0],)) raise SystemExit(3) - Modified: pypy/branch/py12/py/_test/collect.py ============================================================================== --- pypy/branch/py12/py/_test/collect.py (original) +++ pypy/branch/py12/py/_test/collect.py Sun May 2 16:39:33 2010 @@ -297,7 +297,7 @@ return l def consider(self, path): - if self.ihook.pytest_ignore_collect_path(path=path, config=self.config): + if self.ihook.pytest_ignore_collect(path=path, config=self.config): return if path.check(file=1): res = self.consider_file(path) Modified: pypy/branch/py12/py/_test/pluginmanager.py ============================================================================== --- pypy/branch/py12/py/_test/pluginmanager.py (original) +++ pypy/branch/py12/py/_test/pluginmanager.py Sun May 2 16:39:33 2010 @@ -39,8 +39,7 @@ if name in self._name2plugin: return False self._name2plugin[name] = plugin - self.call_plugin(plugin, "pytest_registerhooks", - {'pluginmanager': self}) + self.call_plugin(plugin, "pytest_addhooks", {'pluginmanager': self}) self.hook.pytest_plugin_registered(manager=self, plugin=plugin) self.registry.register(plugin) return True @@ -59,8 +58,8 @@ if plugin == val: return True - def registerhooks(self, spec): - self.hook._registerhooks(spec) + def addhooks(self, spec): + self.hook._addhooks(spec, prefix="pytest_") def getplugins(self): return list(self.registry) @@ -304,22 +303,31 @@ return l class HookRelay: - def __init__(self, hookspecs, registry): + def __init__(self, hookspecs, registry, prefix="pytest_"): if not isinstance(hookspecs, list): hookspecs = [hookspecs] self._hookspecs = [] self._registry = registry for hookspec in hookspecs: - self._registerhooks(hookspec) + self._addhooks(hookspec, prefix) - def _registerhooks(self, hookspecs): + def _addhooks(self, hookspecs, prefix): self._hookspecs.append(hookspecs) + added = False for name, method in vars(hookspecs).items(): - if name[:1] != "_": + if name.startswith(prefix): + if not method.__doc__: + raise ValueError("docstring required for hook %r, in %r" + % (method, hookspecs)) firstresult = getattr(method, 'firstresult', False) hc = HookCaller(self, name, firstresult=firstresult) setattr(self, name, hc) + added = True #print ("setting new hook", name) + if not added: + raise ValueError("did not find new %r hooks in %r" %( + prefix, hookspecs,)) + def _performcall(self, name, multicall): return multicall.execute() Modified: pypy/branch/py12/pypy/conftest.py ============================================================================== --- pypy/branch/py12/pypy/conftest.py (original) +++ pypy/branch/py12/pypy/conftest.py Sun May 2 16:39:33 2010 @@ -515,5 +515,5 @@ py.test.skip("pexpect not found") -def pytest_ignore_collect_path(path): +def pytest_ignore_collect(path): return path.check(link=1) Modified: pypy/branch/py12/pypy/translator/benchmark/conftest.py ============================================================================== --- pypy/branch/py12/pypy/translator/benchmark/conftest.py (original) +++ pypy/branch/py12/pypy/translator/benchmark/conftest.py Sun May 2 16:39:33 2010 @@ -1,4 +1,4 @@ import py -def pytest_ignore_collect_path(path): +def pytest_ignore_collect(path): return path.basename == "test" From hpk at codespeak.net Sun May 2 18:05:14 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sun, 2 May 2010 18:05:14 +0200 (CEST) Subject: [pypy-svn] r74324 - in pypy/branch/py12: . lib-python/modified-2.5.2/distutils lib-python/modified-2.5.2/distutils/command pypy pypy/_interfaces pypy/annotation pypy/annotation/test pypy/config pypy/doc pypy/doc/config pypy/interpreter pypy/interpreter/test pypy/jit/backend/test pypy/jit/backend/x86/test pypy/jit/metainterp pypy/lib pypy/module/cpyext pypy/module/cpyext/include pypy/module/cpyext/src pypy/module/cpyext/test pypy/module/imp pypy/module/thread pypy/module/thread/test pypy/objspace/std pypy/objspace/std/test pypy/rlib pypy/rlib/test pypy/rpython pypy/rpython/lltypesystem pypy/rpython/lltypesystem/test pypy/rpython/test pypy/tool pypy/translator pypy/translator/c pypy/translator/c/gcc pypy/translator/c/gcc/test pypy/translator/c/src pypy/translator/c/test pypy/translator/goal pypy/translator/platform pypy/translator/test pypy/translator/tool pypy/translator/tool/test Message-ID: <20100502160514.8D137282BD4@codespeak.net> Author: hpk Date: Sun May 2 18:05:10 2010 New Revision: 74324 Added: pypy/branch/py12/pypy/_interfaces/ (props changed) - copied from r74323, pypy/trunk/pypy/_interfaces/ pypy/branch/py12/pypy/doc/config/objspace.usemodules.cpyext.txt - copied unchanged from r74323, pypy/trunk/pypy/doc/config/objspace.usemodules.cpyext.txt pypy/branch/py12/pypy/doc/config/translation.secondaryentrypoints.txt - copied unchanged from r74323, pypy/trunk/pypy/doc/config/translation.secondaryentrypoints.txt pypy/branch/py12/pypy/doc/config/translation.shared.txt - copied unchanged from r74323, pypy/trunk/pypy/doc/config/translation.shared.txt pypy/branch/py12/pypy/module/cpyext/ (props changed) - copied from r74323, pypy/trunk/pypy/module/cpyext/ pypy/branch/py12/pypy/rlib/_rweakkeydict.py - copied unchanged from r74323, pypy/trunk/pypy/rlib/_rweakkeydict.py pypy/branch/py12/pypy/rlib/_rweakvaldict.py - copied unchanged from r74323, pypy/trunk/pypy/rlib/_rweakvaldict.py pypy/branch/py12/pypy/rlib/entrypoint.py - copied unchanged from r74323, pypy/trunk/pypy/rlib/entrypoint.py pypy/branch/py12/pypy/rlib/exports.py - copied unchanged from r74323, pypy/trunk/pypy/rlib/exports.py pypy/branch/py12/pypy/rlib/test/test_rweakkeydict.py - copied unchanged from r74323, pypy/trunk/pypy/rlib/test/test_rweakkeydict.py pypy/branch/py12/pypy/rlib/test/test_rweakvaldict.py - copied unchanged from r74323, pypy/trunk/pypy/rlib/test/test_rweakvaldict.py Removed: pypy/branch/py12/pypy/doc/README.Debian pypy/branch/py12/pypy/doc/README.compiling pypy/branch/py12/pypy/rlib/rweakrefimpl.py pypy/branch/py12/pypy/rlib/test/test_rweakref.py Modified: pypy/branch/py12/ (props changed) pypy/branch/py12/lib-python/modified-2.5.2/distutils/command/build_ext.py pypy/branch/py12/lib-python/modified-2.5.2/distutils/msvccompiler.py pypy/branch/py12/lib-python/modified-2.5.2/distutils/spawn.py pypy/branch/py12/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py pypy/branch/py12/pypy/annotation/annrpython.py pypy/branch/py12/pypy/annotation/description.py pypy/branch/py12/pypy/annotation/test/test_annrpython.py pypy/branch/py12/pypy/config/pypyoption.py pypy/branch/py12/pypy/config/translationoption.py pypy/branch/py12/pypy/conftest.py pypy/branch/py12/pypy/doc/project-ideas.txt pypy/branch/py12/pypy/interpreter/argument.py pypy/branch/py12/pypy/interpreter/baseobjspace.py pypy/branch/py12/pypy/interpreter/executioncontext.py pypy/branch/py12/pypy/interpreter/main.py pypy/branch/py12/pypy/interpreter/pycompiler.py pypy/branch/py12/pypy/interpreter/pyopcode.py pypy/branch/py12/pypy/interpreter/test/test_argument.py pypy/branch/py12/pypy/interpreter/test/test_typedef.py pypy/branch/py12/pypy/interpreter/typedef.py pypy/branch/py12/pypy/jit/backend/test/support.py pypy/branch/py12/pypy/jit/backend/x86/test/test_zrpy_gc.py pypy/branch/py12/pypy/jit/metainterp/support.py pypy/branch/py12/pypy/jit/metainterp/warmspot.py pypy/branch/py12/pypy/lib/identity_dict.py pypy/branch/py12/pypy/module/cpyext/include/ (props changed) pypy/branch/py12/pypy/module/cpyext/src/ (props changed) pypy/branch/py12/pypy/module/cpyext/test/ (props changed) pypy/branch/py12/pypy/module/imp/importing.py pypy/branch/py12/pypy/module/thread/ll_thread.py pypy/branch/py12/pypy/module/thread/test/test_ll_thread.py pypy/branch/py12/pypy/objspace/std/fake.py pypy/branch/py12/pypy/objspace/std/objspace.py pypy/branch/py12/pypy/objspace/std/test/test_setobject.py pypy/branch/py12/pypy/objspace/std/typeobject.py pypy/branch/py12/pypy/rlib/rarithmetic.py pypy/branch/py12/pypy/rlib/rstring.py pypy/branch/py12/pypy/rlib/rweakref.py pypy/branch/py12/pypy/rlib/test/test_rarithmetic.py pypy/branch/py12/pypy/rlib/test/test_rbigint.py pypy/branch/py12/pypy/rlib/test/test_rstring.py pypy/branch/py12/pypy/rpython/annlowlevel.py pypy/branch/py12/pypy/rpython/lltypesystem/ll2ctypes.py pypy/branch/py12/pypy/rpython/lltypesystem/lltype.py pypy/branch/py12/pypy/rpython/lltypesystem/rdict.py pypy/branch/py12/pypy/rpython/lltypesystem/rffi.py pypy/branch/py12/pypy/rpython/lltypesystem/test/test_ll2ctypes.py pypy/branch/py12/pypy/rpython/lltypesystem/test/test_lltype.py pypy/branch/py12/pypy/rpython/test/test_llinterp.py pypy/branch/py12/pypy/tool/fixeol pypy/branch/py12/pypy/translator/c/gcc/test/test_asmgcroot.py pypy/branch/py12/pypy/translator/c/gcc/trackgcroot.py pypy/branch/py12/pypy/translator/c/genc.py pypy/branch/py12/pypy/translator/c/node.py pypy/branch/py12/pypy/translator/c/src/commondefs.h pypy/branch/py12/pypy/translator/c/src/main.h pypy/branch/py12/pypy/translator/c/src/obmalloc.c pypy/branch/py12/pypy/translator/c/test/test_genc.py pypy/branch/py12/pypy/translator/c/test/test_newgc.py pypy/branch/py12/pypy/translator/c/test/test_standalone.py pypy/branch/py12/pypy/translator/driver.py pypy/branch/py12/pypy/translator/goal/ann_override.py pypy/branch/py12/pypy/translator/platform/__init__.py pypy/branch/py12/pypy/translator/platform/darwin.py pypy/branch/py12/pypy/translator/platform/linux.py pypy/branch/py12/pypy/translator/platform/posix.py pypy/branch/py12/pypy/translator/platform/windows.py pypy/branch/py12/pypy/translator/test/test_unsimplify.py pypy/branch/py12/pypy/translator/tool/cbuild.py pypy/branch/py12/pypy/translator/tool/test/test_cbuild.py pypy/branch/py12/pypy/translator/unsimplify.py Log: merge trunk Modified: pypy/branch/py12/lib-python/modified-2.5.2/distutils/command/build_ext.py ============================================================================== --- pypy/branch/py12/lib-python/modified-2.5.2/distutils/command/build_ext.py (original) +++ pypy/branch/py12/lib-python/modified-2.5.2/distutils/command/build_ext.py Sun May 2 18:05:10 2010 @@ -167,7 +167,7 @@ # for Release and Debug builds. # also Python's library directory must be appended to library_dirs if os.name == 'nt': - self.library_dirs.append(os.path.join(sys.exec_prefix, 'libs')) + self.library_dirs.append(os.path.join(sys.pypy_prefix, 'pypy', '_interfaces')) if self.debug: self.build_temp = os.path.join(self.build_temp, "Debug") else: @@ -175,8 +175,8 @@ # Append the source distribution include and library directories, # this allows distutils on windows to work in the source tree - self.include_dirs.append(os.path.join(sys.exec_prefix, 'PC')) - self.library_dirs.append(os.path.join(sys.exec_prefix, 'PCBuild')) + #self.include_dirs.append(os.path.join(sys.exec_prefix, 'PC')) + #self.library_dirs.append(os.path.join(sys.exec_prefix, 'PCBuild')) # OS/2 (EMX) doesn't support Debug vs Release builds, but has the # import libraries in its "Config" subdirectory @@ -645,24 +645,12 @@ shared extension. On most platforms, this is just 'ext.libraries'; on Windows and OS/2, we add the Python library (eg. python20.dll). """ - # The python library is always needed on Windows. For MSVC, this - # is redundant, since the library is mentioned in a pragma in - # pyconfig.h that MSVC groks. The other Windows compilers all seem - # to need it mentioned explicitly, though, so that's what we do. - # Append '_d' to the python import library on debug builds. + # The python library is always needed on Windows. if sys.platform == "win32": - from distutils.msvccompiler import MSVCCompiler - if not isinstance(self.compiler, MSVCCompiler): - template = "python%d%d" - if self.debug: - template = template + '_d' - pythonlib = (template % - (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff)) - # don't extend ext.libraries, it may be shared with other - # extensions, it is a reference to the original list - return ext.libraries + [pythonlib] - else: - return ext.libraries + pythonlib = 'libpypy-c.exe' + # don't extend ext.libraries, it may be shared with other + # extensions, it is a reference to the original list + return ext.libraries + [pythonlib] elif sys.platform == "os2emx": # EMX/GCC requires the python library explicitly, and I # believe VACPP does as well (though not confirmed) - AIM Apr01 Modified: pypy/branch/py12/lib-python/modified-2.5.2/distutils/msvccompiler.py ============================================================================== --- pypy/branch/py12/lib-python/modified-2.5.2/distutils/msvccompiler.py (original) +++ pypy/branch/py12/lib-python/modified-2.5.2/distutils/msvccompiler.py Sun May 2 18:05:10 2010 @@ -1,154 +1,162 @@ -"""distutils.msvccompiler +"""distutils.msvc9compiler Contains MSVCCompiler, an implementation of the abstract CCompiler class -for the Microsoft Visual Studio. +for the Microsoft Visual Studio 2008. + +The module is compatible with VS 2005 and VS 2008. You can find legacy support +for older versions of VS in distutils.msvccompiler. """ # Written by Perry Stoll # hacked by Robin Becker and Thomas Heller to do a better job of # finding DevStudio (through the registry) +# ported to VS2005 and VS 2008 by Christian Heimes -# This module should be kept compatible with Python 2.1. - -__revision__ = "$Id: msvccompiler.py 54645 2007-04-01 18:29:47Z neal.norwitz $" +__revision__ = "$Id: msvc9compiler.py 78713 2010-03-06 02:17:28Z tarek.ziade $" -import sys, os, string -from distutils.errors import \ - DistutilsExecError, DistutilsPlatformError, \ - CompileError, LibError, LinkError -from distutils.ccompiler import \ - CCompiler, gen_preprocess_options, gen_lib_options +import os +import subprocess +import sys +import re + +from distutils.errors import (DistutilsExecError, DistutilsPlatformError, + CompileError, LibError, LinkError) +from distutils.ccompiler import CCompiler, gen_lib_options from distutils import log +from distutils.util import get_platform -_can_read_reg = 0 -try: - import _winreg - - _can_read_reg = 1 - hkey_mod = _winreg - - RegOpenKeyEx = _winreg.OpenKeyEx - RegEnumKey = _winreg.EnumKey - RegEnumValue = _winreg.EnumValue - RegError = _winreg.error +import _winreg -except ImportError: - try: - import win32api - import win32con - _can_read_reg = 1 - hkey_mod = win32con - - RegOpenKeyEx = win32api.RegOpenKeyEx - RegEnumKey = win32api.RegEnumKey - RegEnumValue = win32api.RegEnumValue - RegError = win32api.error - - except ImportError: - log.info("Warning: Can't read registry to find the " - "necessary compiler setting\n" - "Make sure that Python modules _winreg, " - "win32api or win32con are installed.") - pass - -if _can_read_reg: - HKEYS = (hkey_mod.HKEY_USERS, - hkey_mod.HKEY_CURRENT_USER, - hkey_mod.HKEY_LOCAL_MACHINE, - hkey_mod.HKEY_CLASSES_ROOT) +RegOpenKeyEx = _winreg.OpenKeyEx +RegEnumKey = _winreg.EnumKey +RegEnumValue = _winreg.EnumValue +RegError = _winreg.error + +HKEYS = (_winreg.HKEY_USERS, + _winreg.HKEY_CURRENT_USER, + _winreg.HKEY_LOCAL_MACHINE, + _winreg.HKEY_CLASSES_ROOT) + +VS_BASE = r"Software\Microsoft\VisualStudio\%0.1f" +VSEXPRESS_BASE = r"Software\Microsoft\VCExpress\%0.1f" +WINSDK_BASE = r"Software\Microsoft\Microsoft SDKs\Windows" +NET_BASE = r"Software\Microsoft\.NETFramework" + +# A map keyed by get_platform() return values to values accepted by +# 'vcvarsall.bat'. Note a cross-compile may combine these (eg, 'x86_amd64' is +# the param to cross-compile on x86 targetting amd64.) +PLAT_TO_VCVARS = { + 'win32' : 'x86', + 'win-amd64' : 'amd64', + 'win-ia64' : 'ia64', +} -def read_keys(base, key): - """Return list of registry keys.""" +class Reg: + """Helper class to read values from the registry + """ - try: - handle = RegOpenKeyEx(base, key) - except RegError: - return None - L = [] - i = 0 - while 1: + def get_value(cls, path, key): + for base in HKEYS: + d = cls.read_values(base, path) + if d and key in d: + return d[key] + raise KeyError(key) + get_value = classmethod(get_value) + + def read_keys(cls, base, key): + """Return list of registry keys.""" try: - k = RegEnumKey(handle, i) + handle = RegOpenKeyEx(base, key) except RegError: - break - L.append(k) - i = i + 1 - return L + return None + L = [] + i = 0 + while True: + try: + k = RegEnumKey(handle, i) + except RegError: + break + L.append(k) + i += 1 + return L + read_keys = classmethod(read_keys) -def read_values(base, key): - """Return dict of registry keys and values. + def read_values(cls, base, key): + """Return dict of registry keys and values. - All names are converted to lowercase. - """ - try: - handle = RegOpenKeyEx(base, key) - except RegError: - return None - d = {} - i = 0 - while 1: + All names are converted to lowercase. + """ try: - name, value, type = RegEnumValue(handle, i) + handle = RegOpenKeyEx(base, key) except RegError: - break - name = name.lower() - d[convert_mbcs(name)] = convert_mbcs(value) - i = i + 1 - return d - -def convert_mbcs(s): - enc = getattr(s, "encode", None) - if enc is not None: - try: - s = enc("mbcs") - except UnicodeError: - pass - return s + return None + d = {} + i = 0 + while True: + try: + name, value, type = RegEnumValue(handle, i) + except RegError: + break + name = name.lower() + d[cls.convert_mbcs(name)] = cls.convert_mbcs(value) + i += 1 + return d + read_values = classmethod(read_values) + + def convert_mbcs(s): + dec = getattr(s, "decode", None) + if dec is not None: + try: + s = dec("mbcs") + except UnicodeError: + pass + return s + convert_mbcs = staticmethod(convert_mbcs) class MacroExpander: def __init__(self, version): self.macros = {} + self.vsbase = VS_BASE % version self.load_macros(version) def set_macro(self, macro, path, key): - for base in HKEYS: - d = read_values(base, path) - if d: - self.macros["$(%s)" % macro] = d[key] - break + self.macros["$(%s)" % macro] = Reg.get_value(path, key) def load_macros(self, version): - vsbase = r"Software\Microsoft\VisualStudio\%0.1f" % version - self.set_macro("VCInstallDir", vsbase + r"\Setup\VC", "productdir") - self.set_macro("VSInstallDir", vsbase + r"\Setup\VS", "productdir") - net = r"Software\Microsoft\.NETFramework" - self.set_macro("FrameworkDir", net, "installroot") + self.set_macro("VCInstallDir", self.vsbase + r"\Setup\VC", "productdir") + self.set_macro("VSInstallDir", self.vsbase + r"\Setup\VS", "productdir") + self.set_macro("FrameworkDir", NET_BASE, "installroot") try: - if version > 7.0: - self.set_macro("FrameworkSDKDir", net, "sdkinstallrootv1.1") + if version >= 8.0: + self.set_macro("FrameworkSDKDir", NET_BASE, + "sdkinstallrootv2.0") else: - self.set_macro("FrameworkSDKDir", net, "sdkinstallroot") - except KeyError, exc: # - raise DistutilsPlatformError, \ - ("""Python was built with Visual Studio 2003; + raise KeyError("sdkinstallrootv2.0") + except KeyError: + raise DistutilsPlatformError( + """Python was built with Visual Studio 2008; extensions must be built with a compiler than can generate compatible binaries. -Visual Studio 2003 was not found on this system. If you have Cygwin installed, +Visual Studio 2008 was not found on this system. If you have Cygwin installed, you can try compiling with MingW32, by passing "-c mingw32" to setup.py.""") - p = r"Software\Microsoft\NET Framework Setup\Product" - for base in HKEYS: - try: - h = RegOpenKeyEx(base, p) - except RegError: - continue - key = RegEnumKey(h, 0) - d = read_values(base, r"%s\%s" % (p, key)) - self.macros["$(FrameworkVersion)"] = d["version"] + if version >= 9.0: + self.set_macro("FrameworkVersion", self.vsbase, "clr version") + self.set_macro("WindowsSdkDir", WINSDK_BASE, "currentinstallfolder") + else: + p = r"Software\Microsoft\NET Framework Setup\Product" + for base in HKEYS: + try: + h = RegOpenKeyEx(base, p) + except RegError: + continue + key = RegEnumKey(h, 0) + d = Reg.get_value(base, r"%s\%s" % (p, key)) + self.macros["$(FrameworkVersion)"] = d["version"] def sub(self, s): for k, v in self.macros.items(): - s = string.replace(s, k, v) + s = s.replace(k, v) return s def get_build_version(): @@ -157,35 +165,7 @@ For Python 2.3 and up, the version number is included in sys.version. For earlier versions, assume the compiler is MSVC 6. """ - - prefix = "MSC v." - i = string.find(sys.version, prefix) - if i == -1: - return 6 - i = i + len(prefix) - s, rest = sys.version[i:].split(" ", 1) - majorVersion = int(s[:-2]) - 6 - minorVersion = int(s[2:3]) / 10.0 - # I don't think paths are affected by minor version in version 6 - if majorVersion == 6: - minorVersion = 0 - if majorVersion >= 6: - return majorVersion + minorVersion - # else we don't know what version of the compiler this is - return None - -def get_build_architecture(): - """Return the processor architecture. - - Possible results are "Intel", "Itanium", or "AMD64". - """ - - prefix = " bit (" - i = string.find(sys.version, prefix) - if i == -1: - return "Intel" - j = string.find(sys.version, ")", i) - return sys.version[i+len(prefix):j] + return 8.0 def normalize_and_reduce_paths(paths): """Return a list of normalized paths with duplicates removed. @@ -201,8 +181,104 @@ reduced_paths.append(np) return reduced_paths +def removeDuplicates(variable): + """Remove duplicate values of an environment variable. + """ + oldList = variable.split(os.pathsep) + newList = [] + for i in oldList: + if i not in newList: + newList.append(i) + newVariable = os.pathsep.join(newList) + return newVariable + +def find_vcvarsall(version): + """Find the vcvarsall.bat file + + At first it tries to find the productdir of VS 2008 in the registry. If + that fails it falls back to the VS90COMNTOOLS env var. + """ + vsbase = VS_BASE % version + try: + productdir = Reg.get_value(r"%s\Setup\VC" % vsbase, + "productdir") + except KeyError: + productdir = None + + # trying Express edition + if productdir is None: + vsbase = VSEXPRESS_BASE % version + try: + productdir = Reg.get_value(r"%s\Setup\VC" % vsbase, + "productdir") + except KeyError: + productdir = None + log.debug("Unable to find productdir in registry") + + if not productdir or not os.path.isdir(productdir): + toolskey = "VS%0.f0COMNTOOLS" % version + toolsdir = os.environ.get(toolskey, None) + + if toolsdir and os.path.isdir(toolsdir): + productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC") + productdir = os.path.abspath(productdir) + if not os.path.isdir(productdir): + log.debug("%s is not a valid directory" % productdir) + return None + else: + log.debug("Env var %s is not set or invalid" % toolskey) + if not productdir: + log.debug("No productdir found") + return None + vcvarsall = os.path.join(productdir, "vcvarsall.bat") + if os.path.isfile(vcvarsall): + return vcvarsall + log.debug("Unable to find vcvarsall.bat") + return None + +def query_vcvarsall(version, arch="x86"): + """Launch vcvarsall.bat and read the settings from its environment + """ + vcvarsall = find_vcvarsall(version) + interesting = set(("include", "lib", "libpath", "path")) + result = {} + + if vcvarsall is None: + raise DistutilsPlatformError("Unable to find vcvarsall.bat") + log.debug("Calling 'vcvarsall.bat %s' (version=%s)", arch, version) + popen = subprocess.Popen('"%s" %s & set' % (vcvarsall, arch), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + stdout, stderr = popen.communicate() + if popen.wait() != 0: + raise DistutilsPlatformError(stderr.decode("mbcs")) + + stdout = stdout.decode("mbcs") + for line in stdout.split("\n"): + line = Reg.convert_mbcs(line) + if '=' not in line: + continue + line = line.strip() + key, value = line.split('=', 1) + key = key.lower() + if key in interesting: + if value.endswith(os.pathsep): + value = value[:-1] + result[key] = removeDuplicates(value) + + if len(result) != len(interesting): + raise ValueError(str(list(result.keys()))) + + return result + +# More globals +VERSION = get_build_version() +if VERSION < 8.0: + raise DistutilsPlatformError("VC %0.1f is not supported by this module" % VERSION) +# MACROS = MacroExpander(VERSION) -class MSVCCompiler (CCompiler) : +class MSVCCompiler(CCompiler) : """Concrete class that implements an interface to Microsoft Visual C++, as defined by the CCompiler abstract class.""" @@ -232,27 +308,29 @@ static_lib_format = shared_lib_format = '%s%s' exe_extension = '.exe' - def __init__ (self, verbose=0, dry_run=0, force=0): + def __init__(self, verbose=0, dry_run=0, force=0): CCompiler.__init__ (self, verbose, dry_run, force) - self.__version = get_build_version() - self.__arch = get_build_architecture() - if self.__arch == "Intel": - # x86 - if self.__version >= 7: - self.__root = r"Software\Microsoft\VisualStudio" - self.__macros = MacroExpander(self.__version) - else: - self.__root = r"Software\Microsoft\Devstudio" - self.__product = "Visual Studio version %s" % self.__version - else: - # Win64. Assume this was built with the platform SDK - self.__product = "Microsoft SDK compiler %s" % (self.__version + 6) - + self.__version = VERSION + self.__root = r"Software\Microsoft\VisualStudio" + # self.__macros = MACROS + self.__paths = [] + # target platform (.plat_name is consistent with 'bdist') + self.plat_name = None + self.__arch = None # deprecated name self.initialized = False - def initialize(self): - self.__paths = [] - if os.environ.has_key("DISTUTILS_USE_SDK") and os.environ.has_key("MSSdk") and self.find_exe("cl.exe"): + def initialize(self, plat_name=None): + # multi-init means we would need to check platform same each time... + assert not self.initialized, "don't init multiple times" + if plat_name is None: + plat_name = get_platform() + # sanity check for platforms to prevent obscure errors later. + ok_plats = 'win32', 'win-amd64', 'win-ia64' + if plat_name not in ok_plats: + raise DistutilsPlatformError("--plat-name must be one of %s" % + (ok_plats,)) + + if "DISTUTILS_USE_SDK" in os.environ and "MSSdk" in os.environ and self.find_exe("cl.exe"): # Assume that the SDK set up everything alright; don't try to be # smarter self.cc = "cl.exe" @@ -261,36 +339,54 @@ self.rc = "rc.exe" self.mc = "mc.exe" else: - self.__paths = self.get_msvc_paths("path") + # On x86, 'vcvars32.bat amd64' creates an env that doesn't work; + # to cross compile, you use 'x86_amd64'. + # On AMD64, 'vcvars32.bat amd64' is a native build env; to cross + # compile use 'x86' (ie, it runs the x86 compiler directly) + # No idea how itanium handles this, if at all. + if plat_name == get_platform() or plat_name == 'win32': + # native build or cross-compile to win32 + plat_spec = PLAT_TO_VCVARS[plat_name] + else: + # cross compile from win32 -> some 64bit + plat_spec = PLAT_TO_VCVARS[get_platform()] + '_' + \ + PLAT_TO_VCVARS[plat_name] + + vc_env = query_vcvarsall(VERSION, plat_spec) + + # take care to only use strings in the environment. + self.__paths = vc_env['path'].encode('mbcs').split(os.pathsep) + os.environ['lib'] = vc_env['lib'].encode('mbcs') + os.environ['include'] = vc_env['include'].encode('mbcs') - if len (self.__paths) == 0: - raise DistutilsPlatformError, \ - ("Python was built with %s, " + if len(self.__paths) == 0: + raise DistutilsPlatformError("Python was built with %s, " "and extensions need to be built with the same " - "version of the compiler, but it isn't installed." % self.__product) + "version of the compiler, but it isn't installed." + % self.__product) self.cc = self.find_exe("cl.exe") self.linker = self.find_exe("link.exe") self.lib = self.find_exe("lib.exe") self.rc = self.find_exe("rc.exe") # resource compiler self.mc = self.find_exe("mc.exe") # message compiler - self.set_path_env_var('lib') - self.set_path_env_var('include') + #self.set_path_env_var('lib') + #self.set_path_env_var('include') # extend the MSVC path with the current path try: - for p in string.split(os.environ['path'], ';'): + for p in os.environ['path'].split(';'): self.__paths.append(p) except KeyError: pass self.__paths = normalize_and_reduce_paths(self.__paths) - os.environ['path'] = string.join(self.__paths, ';') + os.environ['path'] = ";".join(self.__paths) self.preprocess_options = None - if self.__arch == "Intel": - self.compile_options = [ '/nologo', '/Ox', '/MD', '/W3', '/GX' , + if self.__arch == "x86": + self.compile_options = [ '/nologo', '/Ox', '/MD', '/W3', '/DNDEBUG'] - self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', '/GX', + self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', '/Z7', '/D_DEBUG'] else: # Win64 @@ -304,20 +400,16 @@ self.ldflags_shared_debug = [ '/DLL', '/nologo', '/INCREMENTAL:no', '/DEBUG' ] - else: - self.ldflags_shared_debug = [ - '/DLL', '/nologo', '/INCREMENTAL:no', '/pdb:None', '/DEBUG' - ] self.ldflags_static = [ '/nologo'] self.initialized = True # -- Worker methods ------------------------------------------------ - def object_filenames (self, - source_filenames, - strip_dir=0, - output_dir=''): + def object_filenames(self, + source_filenames, + strip_dir=0, + output_dir=''): # Copied from ccompiler.py, extended to return .res as 'object'-file # for .rc input file if output_dir is None: output_dir = '' @@ -344,17 +436,16 @@ base + self.obj_extension)) return obj_names - # object_filenames () - def compile(self, sources, output_dir=None, macros=None, include_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, depends=None): - if not self.initialized: self.initialize() - macros, objects, extra_postargs, pp_opts, build = \ - self._setup_compile(output_dir, macros, include_dirs, sources, - depends, extra_postargs) + if not self.initialized: + self.initialize() + compile_info = self._setup_compile(output_dir, macros, include_dirs, + sources, depends, extra_postargs) + macros, objects, extra_postargs, pp_opts, build = compile_info compile_opts = extra_preargs or [] compile_opts.append ('/c') @@ -383,13 +474,12 @@ input_opt = src output_opt = "/fo" + obj try: - self.spawn ([self.rc] + pp_opts + - [output_opt] + [input_opt]) + self.spawn([self.rc] + pp_opts + + [output_opt] + [input_opt]) except DistutilsExecError, msg: - raise CompileError, msg + raise CompileError(msg) continue elif ext in self._mc_extensions: - # Compile .MC to .RC file to .RES file. # * '-h dir' specifies the directory for the # generated include file @@ -401,99 +491,95 @@ # we use the source-directory for the include file and # the build directory for the RC file and message # resources. This works at least for win32all. - - h_dir = os.path.dirname (src) - rc_dir = os.path.dirname (obj) + h_dir = os.path.dirname(src) + rc_dir = os.path.dirname(obj) try: # first compile .MC to .RC and .H file - self.spawn ([self.mc] + - ['-h', h_dir, '-r', rc_dir] + [src]) + self.spawn([self.mc] + + ['-h', h_dir, '-r', rc_dir] + [src]) base, _ = os.path.splitext (os.path.basename (src)) rc_file = os.path.join (rc_dir, base + '.rc') # then compile .RC to .RES file - self.spawn ([self.rc] + - ["/fo" + obj] + [rc_file]) + self.spawn([self.rc] + + ["/fo" + obj] + [rc_file]) except DistutilsExecError, msg: - raise CompileError, msg + raise CompileError(msg) continue else: # how to handle this file? - raise CompileError ( - "Don't know how to compile %s to %s" % \ - (src, obj)) + raise CompileError("Don't know how to compile %s to %s" + % (src, obj)) output_opt = "/Fo" + obj try: - self.spawn ([self.cc] + compile_opts + pp_opts + - [input_opt, output_opt] + - extra_postargs) + self.spawn([self.cc] + compile_opts + pp_opts + + [input_opt, output_opt] + + extra_postargs) except DistutilsExecError, msg: - raise CompileError, msg + raise CompileError(msg) return objects - # compile () - - def create_static_lib (self, - objects, - output_libname, - output_dir=None, - debug=0, - target_lang=None): - - if not self.initialized: self.initialize() - (objects, output_dir) = self._fix_object_args (objects, output_dir) - output_filename = \ - self.library_filename (output_libname, output_dir=output_dir) + def create_static_lib(self, + objects, + output_libname, + output_dir=None, + debug=0, + target_lang=None): + + if not self.initialized: + self.initialize() + (objects, output_dir) = self._fix_object_args(objects, output_dir) + output_filename = self.library_filename(output_libname, + output_dir=output_dir) - if self._need_link (objects, output_filename): + if self._need_link(objects, output_filename): lib_args = objects + ['/OUT:' + output_filename] if debug: - pass # XXX what goes here? + pass # XXX what goes here? try: - self.spawn ([self.lib] + lib_args) + self.spawn([self.lib] + lib_args) except DistutilsExecError, msg: - raise LibError, msg - + raise LibError(msg) else: log.debug("skipping %s (up-to-date)", output_filename) - # create_static_lib () - def link (self, - target_desc, - objects, - output_filename, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - export_symbols=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - build_temp=None, - target_lang=None): - - if not self.initialized: self.initialize() - (objects, output_dir) = self._fix_object_args (objects, output_dir) - (libraries, library_dirs, runtime_library_dirs) = \ - self._fix_lib_args (libraries, library_dirs, runtime_library_dirs) + def link(self, + target_desc, + objects, + output_filename, + output_dir=None, + libraries=None, + library_dirs=None, + runtime_library_dirs=None, + export_symbols=None, + debug=0, + extra_preargs=None, + extra_postargs=None, + build_temp=None, + target_lang=None): + + if not self.initialized: + self.initialize() + (objects, output_dir) = self._fix_object_args(objects, output_dir) + fixed_args = self._fix_lib_args(libraries, library_dirs, + runtime_library_dirs) + (libraries, library_dirs, runtime_library_dirs) = fixed_args if runtime_library_dirs: self.warn ("I don't know what to do with 'runtime_library_dirs': " + str (runtime_library_dirs)) - lib_opts = gen_lib_options (self, - library_dirs, runtime_library_dirs, - libraries) + lib_opts = gen_lib_options(self, + library_dirs, runtime_library_dirs, + libraries) if output_dir is not None: - output_filename = os.path.join (output_dir, output_filename) - - if self._need_link (objects, output_filename): + output_filename = os.path.join(output_dir, output_filename) + if self._need_link(objects, output_filename): if target_desc == CCompiler.EXECUTABLE: if debug: ldflags = self.ldflags_shared_debug[1:] @@ -517,47 +603,99 @@ # needed! Make sure they are generated in the temporary build # directory. Since they have different names for debug and release # builds, they can go into the same directory. + build_temp = os.path.dirname(objects[0]) if export_symbols is not None: (dll_name, dll_ext) = os.path.splitext( os.path.basename(output_filename)) implib_file = os.path.join( - os.path.dirname(objects[0]), + build_temp, self.library_filename(dll_name)) ld_args.append ('/IMPLIB:' + implib_file) + # Embedded manifests are recommended - see MSDN article titled + # "How to: Embed a Manifest Inside a C/C++ Application" + # (currently at http://msdn2.microsoft.com/en-us/library/ms235591(VS.80).aspx) + # Ask the linker to generate the manifest in the temp dir, so + # we can embed it later. + temp_manifest = os.path.join( + build_temp, + os.path.basename(output_filename) + ".manifest") + ld_args.append('/MANIFESTFILE:' + temp_manifest) + if extra_preargs: ld_args[:0] = extra_preargs if extra_postargs: ld_args.extend(extra_postargs) - self.mkpath (os.path.dirname (output_filename)) + self.mkpath(os.path.dirname(output_filename)) try: - self.spawn ([self.linker] + ld_args) + self.spawn([self.linker] + ld_args) except DistutilsExecError, msg: - raise LinkError, msg + raise LinkError(msg) + # embed the manifest + # XXX - this is somewhat fragile - if mt.exe fails, distutils + # will still consider the DLL up-to-date, but it will not have a + # manifest. Maybe we should link to a temp file? OTOH, that + # implies a build environment error that shouldn't go undetected. + if target_desc == CCompiler.EXECUTABLE: + mfid = 1 + else: + mfid = 2 + self._remove_visual_c_ref(temp_manifest) + out_arg = '-outputresource:%s;%s' % (output_filename, mfid) + try: + self.spawn(['mt.exe', '-nologo', '-manifest', + temp_manifest, out_arg]) + except DistutilsExecError, msg: + raise LinkError(msg) else: log.debug("skipping %s (up-to-date)", output_filename) - # link () - + def _remove_visual_c_ref(self, manifest_file): + try: + # Remove references to the Visual C runtime, so they will + # fall through to the Visual C dependency of Python.exe. + # This way, when installed for a restricted user (e.g. + # runtimes are not in WinSxS folder, but in Python's own + # folder), the runtimes do not need to be in every folder + # with .pyd's. + manifest_f = open(manifest_file) + try: + manifest_buf = manifest_f.read() + finally: + manifest_f.close() + pattern = re.compile( + r"""|)""", + re.DOTALL) + manifest_buf = re.sub(pattern, "", manifest_buf) + pattern = "\s*" + manifest_buf = re.sub(pattern, "", manifest_buf) + manifest_f = open(manifest_file, 'w') + try: + manifest_f.write(manifest_buf) + finally: + manifest_f.close() + except IOError: + pass # -- Miscellaneous methods ----------------------------------------- # These are all used by the 'gen_lib_options() function, in # ccompiler.py. - def library_dir_option (self, dir): + def library_dir_option(self, dir): return "/LIBPATH:" + dir - def runtime_library_dir_option (self, dir): - raise DistutilsPlatformError, \ - "don't know how to set runtime library search path for MSVC++" + def runtime_library_dir_option(self, dir): + raise DistutilsPlatformError( + "don't know how to set runtime library search path for MSVC++") - def library_option (self, lib): - return self.library_filename (lib) + def library_option(self, lib): + return self.library_filename(lib) - def find_library_file (self, dirs, lib, debug=0): + def find_library_file(self, dirs, lib, debug=0): # Prefer a debugging library if found (and requested), but deal # with it if we don't have one. if debug: @@ -573,8 +711,6 @@ # Oops, didn't find it in *any* of 'dirs' return None - # find_library_file () - # Helper methods for using the MSVC registry settings def find_exe(self, exe): @@ -586,67 +722,15 @@ absolute path that is known to exist. If none of them work, just return the original program name, 'exe'. """ - for p in self.__paths: fn = os.path.join(os.path.abspath(p), exe) if os.path.isfile(fn): return fn # didn't find it; try existing path - for p in string.split(os.environ['Path'],';'): + for p in os.environ['Path'].split(';'): fn = os.path.join(os.path.abspath(p),exe) if os.path.isfile(fn): return fn return exe - - def get_msvc_paths(self, path, platform='x86'): - """Get a list of devstudio directories (include, lib or path). - - Return a list of strings. The list will be empty if unable to - access the registry or appropriate registry keys not found. - """ - - if not _can_read_reg: - return [] - - path = path + " dirs" - if self.__version >= 7: - key = (r"%s\%0.1f\VC\VC_OBJECTS_PLATFORM_INFO\Win32\Directories" - % (self.__root, self.__version)) - else: - key = (r"%s\6.0\Build System\Components\Platforms" - r"\Win32 (%s)\Directories" % (self.__root, platform)) - - for base in HKEYS: - d = read_values(base, key) - if d: - if self.__version >= 7: - return string.split(self.__macros.sub(d[path]), ";") - else: - return string.split(d[path], ";") - # MSVC 6 seems to create the registry entries we need only when - # the GUI is run. - if self.__version == 6: - for base in HKEYS: - if read_values(base, r"%s\6.0" % self.__root) is not None: - self.warn("It seems you have Visual Studio 6 installed, " - "but the expected registry settings are not present.\n" - "You must at least run the Visual Studio GUI once " - "so that these entries are created.") - break - return [] - - def set_path_env_var(self, name): - """Set environment variable 'name' to an MSVC path type value. - - This is equivalent to a SET command prior to execution of spawned - commands. - """ - - if name == "lib": - p = self.get_msvc_paths("library") - else: - p = self.get_msvc_paths(name) - if p: - os.environ[name] = string.join(p, ';') Modified: pypy/branch/py12/lib-python/modified-2.5.2/distutils/spawn.py ============================================================================== --- pypy/branch/py12/lib-python/modified-2.5.2/distutils/spawn.py (original) +++ pypy/branch/py12/lib-python/modified-2.5.2/distutils/spawn.py Sun May 2 18:05:10 2010 @@ -69,7 +69,6 @@ dry_run=0): executable = cmd[0] - cmd = _nt_quote_args(cmd) if search_path: # either we find one or it stays the same executable = find_executable(executable) or executable @@ -77,7 +76,8 @@ if not dry_run: # spawn for NT requires a full path to the .exe try: - rc = os.spawnv(os.P_WAIT, executable, cmd) + import subprocess + rc = subprocess.call(cmd) except OSError, exc: # this seems to happen when the command isn't found raise DistutilsExecError, \ Modified: pypy/branch/py12/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py ============================================================================== --- pypy/branch/py12/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py (original) +++ pypy/branch/py12/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py Sun May 2 18:05:10 2010 @@ -11,6 +11,12 @@ python_build = False +def get_python_inc(plat_specific=0, prefix=None): + from os.path import join as j + if plat_specific: + return j(sys.pypy_prefix, "pypy", "_interfaces") + return j(sys.pypy_prefix, 'pypy', 'module', 'cpyext', 'include') + def get_python_version(): """Return a string containing the major and minor Python version, leaving off the patchlevel. Sample return values could be '1.5' @@ -47,6 +53,7 @@ """Initialize the module as appropriate for POSIX systems.""" g = {} g['EXE'] = "" + g['SO'] = ".so" global _config_vars _config_vars = g @@ -56,6 +63,7 @@ """Initialize the module as appropriate for NT""" g = {} g['EXE'] = ".exe" + g['SO'] = ".pyd" global _config_vars _config_vars = g Modified: pypy/branch/py12/pypy/annotation/annrpython.py ============================================================================== --- pypy/branch/py12/pypy/annotation/annrpython.py (original) +++ pypy/branch/py12/pypy/annotation/annrpython.py Sun May 2 18:05:10 2010 @@ -83,20 +83,23 @@ #___ convenience high-level interface __________________ - def build_types(self, function, input_arg_types, complete_now=True): + def build_types(self, function, input_arg_types, complete_now=True, + main_entry_point=False): """Recursively build annotations about the specific entry point.""" assert isinstance(function, types.FunctionType), "fix that!" + from pypy.annotation.policy import AnnotatorPolicy + policy = AnnotatorPolicy() # make input arguments and set their type - inputcells = [self.typeannotation(t) for t in input_arg_types] + args_s = [self.typeannotation(t) for t in input_arg_types] - desc = self.bookkeeper.getdesc(function) - desc.getcallfamily() # record this implicit call (hint for back-ends) - flowgraph = desc.specialize(inputcells) + flowgraph, inputcells = self.get_call_parameters(function, args_s, policy) if not isinstance(flowgraph, FunctionGraph): assert isinstance(flowgraph, annmodel.SomeObject) return flowgraph + if main_entry_point: + self.translator.entry_point_graph = flowgraph return self.build_graph_types(flowgraph, inputcells, complete_now=complete_now) def get_call_parameters(self, function, args_s, policy): Modified: pypy/branch/py12/pypy/annotation/description.py ============================================================================== --- pypy/branch/py12/pypy/annotation/description.py (original) +++ pypy/branch/py12/pypy/annotation/description.py Sun May 2 18:05:10 2010 @@ -209,8 +209,9 @@ if len(self._cache) != 1: raise NoStandardGraph(self) [graph] = self._cache.values() + relax_sig_check = getattr(self.pyobj, "relax_sig_check", False) if (graph.signature != self.signature or - graph.defaults != self.defaults): + graph.defaults != self.defaults) and not relax_sig_check: raise NoStandardGraph(self) return graph Modified: pypy/branch/py12/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/py12/pypy/annotation/test/test_annrpython.py (original) +++ pypy/branch/py12/pypy/annotation/test/test_annrpython.py Sun May 2 18:05:10 2010 @@ -3185,6 +3185,13 @@ assert isinstance(s, annmodel.SomeList) assert s.listdef.listitem.resized + def test_varargs(self): + def f(*args): + return args[0] + 42 + a = self.RPythonAnnotator() + s = a.build_types(f, [int, int]) + assert isinstance(s, annmodel.SomeInteger) + def test_listitem_no_mutating(self): from pypy.rlib.debug import check_annotation called = [] @@ -3304,6 +3311,15 @@ s = a.build_types(f, [int]) assert s.knowntype is int + def test_relax(self): + def f(*args): + return args[0] + args[1] + f.relax_sig_check = True + def g(x): + return f(x, x - x) + a = self.RPythonAnnotator() + s = a.build_types(g, [int]) + assert a.bookkeeper.getdesc(f).getuniquegraph() def g(n): return [0,1,2,n] Modified: pypy/branch/py12/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/py12/pypy/config/pypyoption.py (original) +++ pypy/branch/py12/pypy/config/pypyoption.py Sun May 2 18:05:10 2010 @@ -29,7 +29,8 @@ "rctime" , "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios", "zlib", "struct", "md5", "sha", "bz2", "_minimal_curses", "cStringIO", - "thread", "itertools", "pyexpat", "_ssl"] + "thread", "itertools", "pyexpat", "_ssl"] # "cpyext"] commented out until + # it stops adding _pyolifeline on W_Root )) working_oo_modules = default_modules.copy() @@ -62,11 +63,14 @@ module_dependencies = {} -module_suggests = { # the reason you want _rawffi is for ctypes, which - # itself needs the interp-level struct module - # because 'P' is missing from the app-level one - '_rawffi': [("objspace.usemodules.struct", True)], - } +module_suggests = { + # the reason you want _rawffi is for ctypes, which + # itself needs the interp-level struct module + # because 'P' is missing from the app-level one + "_rawffi": [("objspace.usemodules.struct", True)], + "cpyext": [("translation.secondaryentrypoints", "cpyext"), + ("translation.shared", sys.platform == "win32")], + } module_import_dependencies = { # no _rawffi if importing pypy.rlib.libffi raises ImportError Modified: pypy/branch/py12/pypy/config/translationoption.py ============================================================================== --- pypy/branch/py12/pypy/config/translationoption.py (original) +++ pypy/branch/py12/pypy/config/translationoption.py Sun May 2 18:05:10 2010 @@ -42,6 +42,9 @@ }, cmdline="-b --backend"), + BoolOption("shared", "Build as a shared library", + default=False, cmdline="--shared"), + BoolOption("log", "Include debug prints in the translation (PYPYLOG=...)", default=True, cmdline="--log"), @@ -138,6 +141,9 @@ ArbitraryOption("instrumentctl", "internal", default=None), StrOption("output", "Output file name", cmdline="--output"), + StrOption("secondaryentrypoints", + "Comma separated list of keys choosing secondary entrypoints", + cmdline="--entrypoints", default=""), BoolOption("dump_static_data_info", "Dump static data info", cmdline="--dump_static_data_info", Modified: pypy/branch/py12/pypy/conftest.py ============================================================================== --- pypy/branch/py12/pypy/conftest.py (original) +++ pypy/branch/py12/pypy/conftest.py Sun May 2 18:05:10 2010 @@ -59,7 +59,7 @@ try: return _SPACECACHE[key] except KeyError: - if option.runappdirect: + if getattr(option, 'runappdirect', None): if name not in (None, 'std'): myname = getattr(sys, 'pypy_objspaceclass', '') if not myname.lower().startswith(name): Modified: pypy/branch/py12/pypy/doc/project-ideas.txt ============================================================================== --- pypy/branch/py12/pypy/doc/project-ideas.txt (original) +++ pypy/branch/py12/pypy/doc/project-ideas.txt Sun May 2 18:05:10 2010 @@ -21,22 +21,9 @@ -------------------------------- PyPy's Just-In-Time compiler relies on backends for actual code -generation. We have so far a 32-bit Intel backend, and a CLI one. Open -ideas are to write a backend for **Intel 64** (AMD64); or a backend for -Java; or trying again to use LLVM-JIT (which I do not really recommend). - -CTypes ------- - -Support ctypes on more backends. Right now ctypes is supported only -when compiling PyPy to C, and there is a bit of unfinished work to -support it on **Intel 64.** A nice project would be to support it when -compiling to .NET or the JVM. That's not too hard, the only thing needed -is to port a small module that does the actual invocation of external -libraries (a related project is to port this module to Jython or -IronPython to get support for ctypes there, which is something that was -tried but not finished as far as I know). - +generation. We have so far a 32-bit Intel backend, and a CLI one. There is +Summer of Code project for 64bit (x86_64) backend, but other options +(ARM, llvm) remain open. .. _distribution: .. _persistence: Modified: pypy/branch/py12/pypy/interpreter/argument.py ============================================================================== --- pypy/branch/py12/pypy/interpreter/argument.py (original) +++ pypy/branch/py12/pypy/interpreter/argument.py Sun May 2 18:05:10 2010 @@ -128,19 +128,6 @@ kwds_w[self.keywords[i]] = self.keywords_w[i] return self.arguments_w, kwds_w - def unpack_cpy(self, starting_at=0): - assert starting_at >= 0 - space = self.space - args_w = self.arguments_w - w_kw = space.newdict() - if self.keywords: - for i in range(len(self.keywords)): - space.setitem(w_kw, space.wrap(self.keywords[i]), self.keywords_w[i]) - if starting_at != 0: - args_w = args_w[starting_at:] - args_tuple = space.newtuple([space.wrap(args_w), w_kw]) - return args_tuple - def replace_arguments(self, args_w): "Return a new Arguments with a args_w as positional arguments." return Arguments(self.space, args_w, self.keywords, self.keywords_w) Modified: pypy/branch/py12/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/py12/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/py12/pypy/interpreter/baseobjspace.py Sun May 2 18:05:10 2010 @@ -4,7 +4,6 @@ from pypy.interpreter.executioncontext import UserDelAction, FrameTraceAction from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.argument import Arguments -from pypy.interpreter.pycompiler import PythonAstCompiler from pypy.interpreter.miscutils import ThreadLocals from pypy.tool.cache import Cache from pypy.tool.uid import HUGEVAL_BYTES @@ -579,6 +578,7 @@ try: return self.default_compiler except AttributeError: + from pypy.interpreter.pycompiler import PythonAstCompiler compiler = PythonAstCompiler(self) self.default_compiler = compiler return compiler Modified: pypy/branch/py12/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/branch/py12/pypy/interpreter/executioncontext.py (original) +++ pypy/branch/py12/pypy/interpreter/executioncontext.py Sun May 2 18:05:10 2010 @@ -50,8 +50,7 @@ def enter(self, frame): if self.framestackdepth > self.space.sys.recursionlimit: - raise OperationError(self.space.w_RuntimeError, - self.space.wrap("maximum recursion depth exceeded")) + raise self.space.prebuilt_recursion_error self.framestackdepth += 1 frame.f_backref = self.topframeref self.topframeref = jit.virtual_ref(frame) Modified: pypy/branch/py12/pypy/interpreter/main.py ============================================================================== --- pypy/branch/py12/pypy/interpreter/main.py (original) +++ pypy/branch/py12/pypy/interpreter/main.py Sun May 2 18:05:10 2010 @@ -1,7 +1,7 @@ -from pypy.interpreter import module, eval +import sys +from pypy.interpreter import eval, module from pypy.interpreter.error import OperationError -from pypy.interpreter.pycode import PyCode -import sys, types + def ensure__main__(space): w_main = space.wrap('__main__') Modified: pypy/branch/py12/pypy/interpreter/pycompiler.py ============================================================================== --- pypy/branch/py12/pypy/interpreter/pycompiler.py (original) +++ pypy/branch/py12/pypy/interpreter/pycompiler.py Sun May 2 18:05:10 2010 @@ -3,10 +3,13 @@ Compiler instances are stored into 'space.getexecutioncontext().compiler'. """ -import sys -from pypy.interpreter.astcompiler.consts import PyCF_DONT_IMPLY_DEDENT +from pypy.interpreter import pycode +from pypy.interpreter.pyparser import future, pyparse, error as parseerror +from pypy.interpreter.astcompiler import (astbuilder, codegen, consts, misc, + optimize) from pypy.interpreter.error import OperationError + class AbstractCompiler(object): """Abstract base class for a bytecode compiler.""" @@ -33,7 +36,7 @@ # Hackish default implementation based on the stdlib 'codeop' module. # See comments over there. space = self.space - flags |= PyCF_DONT_IMPLY_DEDENT + flags |= consts.PyCF_DONT_IMPLY_DEDENT # Check for source consisting of only blank lines and comments if mode != "eval": in_comment = False @@ -82,8 +85,7 @@ def getcodeflags(self, code): """Return the __future__ compiler flags that were used to compile the given code object.""" - from pypy.interpreter.pycode import PyCode - if isinstance(code, PyCode): + if isinstance(code, pycode.PyCode): return code.co_flags & self.compiler_flags else: return 0 @@ -98,62 +100,49 @@ the whole source after having only added a new '\n') """ def __init__(self, space, override_version=None): - - from pypy.interpreter.pyparser.pyparse import PythonParser - from pypy.interpreter.pyparser import future PyCodeCompiler.__init__(self, space) - self.parser = PythonParser(space) + self.parser = pyparse.PythonParser(space) self.additional_rules = {} self.future_flags = future.futureFlags_2_5 self.compiler_flags = self.future_flags.allowed_flags def compile_ast(self, node, filename, mode, flags): - from pypy.interpreter.pyparser.pyparse import CompileInfo - from pypy.interpreter.astcompiler.misc import parse_future - info = CompileInfo(filename, mode, flags, parse_future(node)) + future_pos = misc.parse_future(node) + info = pyparse.CompileInfo(filename, mode, flags, future_pos) return self._compile_ast(node, info) def _compile_ast(self, node, info): - from pypy.interpreter.astcompiler import optimize - from pypy.interpreter.astcompiler.codegen import compile_ast - from pypy.interpreter.pyparser.error import SyntaxError space = self.space try: mod = optimize.optimize_ast(space, node, info) - code = compile_ast(space, mod, info) - except SyntaxError, e: + code = codegen.compile_ast(space, mod, info) + except parseerror.SyntaxError, e: raise OperationError(space.w_SyntaxError, e.wrap_info(space)) return code def compile_to_ast(self, source, filename, mode, flags): - from pypy.interpreter.pyparser.pyparse import CompileInfo - info = CompileInfo(filename, mode, flags) + info = pyparse.CompileInfo(filename, mode, flags) return self._compile_to_ast(source, info) def _compile_to_ast(self, source, info): - from pypy.interpreter.pyparser.future import get_futures - from pypy.interpreter.pyparser.error import (SyntaxError, - IndentationError) - from pypy.interpreter.astcompiler.astbuilder import ast_from_node space = self.space try: - f_flags, future_info = get_futures(self.future_flags, source) + f_flags, future_info = future.get_futures(self.future_flags, source) info.last_future_import = future_info info.flags |= f_flags parse_tree = self.parser.parse_source(source, info) - mod = ast_from_node(space, parse_tree, info) - except IndentationError, e: + mod = astbuilder.ast_from_node(space, parse_tree, info) + except parseerror.IndentationError, e: raise OperationError(space.w_IndentationError, e.wrap_info(space)) - except SyntaxError, e: + except parseerror.SyntaxError, e: raise OperationError(space.w_SyntaxError, e.wrap_info(space)) return mod def compile(self, source, filename, mode, flags, hidden_applevel=False): - from pypy.interpreter.pyparser.pyparse import CompileInfo - info = CompileInfo(filename, mode, flags, - hidden_applevel=hidden_applevel) + info = pyparse.CompileInfo(filename, mode, flags, + hidden_applevel=hidden_applevel) mod = self._compile_to_ast(source, info) return self._compile_ast(mod, info) Modified: pypy/branch/py12/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/py12/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/py12/pypy/interpreter/pyopcode.py Sun May 2 18:05:10 2010 @@ -886,8 +886,6 @@ @jit.unroll_safe def call_function(self, oparg, w_star=None, w_starstar=None): - from pypy.interpreter.function import is_builtin_code - n_arguments = oparg & 0xff n_keywords = (oparg>>8) & 0xff if n_keywords: @@ -909,7 +907,7 @@ args = self.argument_factory(arguments, keywords, keywords_w, w_star, w_starstar) w_function = self.popvalue() - if self.is_being_profiled and is_builtin_code(w_function): + if self.is_being_profiled and function.is_builtin_code(w_function): w_result = self.space.call_args_and_c_profile(self, w_function, args) else: @@ -1014,7 +1012,7 @@ (ofs, ord(c), name) ) STOP_CODE = MISSING_OPCODE - + def BUILD_MAP(self, itemcount, next_instr): if itemcount != 0: raise BytecodeCorruption Modified: pypy/branch/py12/pypy/interpreter/test/test_argument.py ============================================================================== --- pypy/branch/py12/pypy/interpreter/test/test_argument.py (original) +++ pypy/branch/py12/pypy/interpreter/test/test_argument.py Sun May 2 18:05:10 2010 @@ -135,12 +135,6 @@ assert args1.keywords is args.keywords assert args1.keywords_w is args.keywords_w - def test_unpack_cpy(self): - space = DummySpace() - args = Arguments(space, ["0"]) - assert space.eq_w(args.unpack_cpy(), space.newtuple([space.newlist([space.wrap("0")]), space.newdict()])) - assert space.eq_w(args.unpack_cpy(1), space.newtuple([space.newlist(), space.newdict()])) - def test_fixedunpacked(self): space = DummySpace() Modified: pypy/branch/py12/pypy/interpreter/test/test_typedef.py ============================================================================== --- pypy/branch/py12/pypy/interpreter/test/test_typedef.py (original) +++ pypy/branch/py12/pypy/interpreter/test/test_typedef.py Sun May 2 18:05:10 2010 @@ -1,5 +1,6 @@ from pypy.interpreter import typedef from pypy.tool.udir import udir +from pypy.interpreter.baseobjspace import Wrappable # this test isn't so much to test that the objspace interface *works* # -- it's more to test that it's *there* @@ -132,6 +133,18 @@ assert len(set) <= 6, "%s has %d subclasses:\n%r" % ( cls, len(set), [subcls.__name__ for subcls in set]) + def test_getsetproperty(self): + class W_SomeType(Wrappable): + pass + def fget(self, space, w_self): + assert self is prop + prop = typedef.GetSetProperty(fget, use_closure=True) + W_SomeType.typedef = typedef.TypeDef( + 'some_type', + x=prop) + w_obj = self.space.wrap(W_SomeType()) + assert self.space.getattr(w_obj, self.space.wrap('x')) is self.space.w_None + class AppTestTypeDef: Modified: pypy/branch/py12/pypy/interpreter/typedef.py ============================================================================== --- pypy/branch/py12/pypy/interpreter/typedef.py (original) +++ pypy/branch/py12/pypy/interpreter/typedef.py Sun May 2 18:05:10 2010 @@ -9,7 +9,7 @@ DescrMismatch from pypy.interpreter.error import OperationError, operationerrfmt from pypy.tool.sourcetools import compile2, func_with_new_name -from pypy.rlib.objectmodel import instantiate, compute_identity_hash +from pypy.rlib.objectmodel import instantiate, compute_identity_hash, specialize from pypy.rlib.jit import hint class TypeDef: @@ -301,44 +301,63 @@ # ____________________________________________________________ -def make_descr_typecheck_wrapper(func, extraargs=(), cls=None): + at specialize.arg(0) +def make_descr_typecheck_wrapper(tag, func, extraargs=(), cls=None, + use_closure=False): if func is None: return None - if cls is None: + return _make_descr_typecheck_wrapper(tag, func, extraargs, cls, use_closure) + + at specialize.memo() +def _make_descr_typecheck_wrapper(tag, func, extraargs, cls, use_closure): + # - if cls is None, the wrapped object is passed to the function + # - if cls is a class, an unwrapped instance is passed + # - if cls is a string, XXX unused? + if cls is None and use_closure: return func if hasattr(func, 'im_func'): assert func.im_class is cls func = func.im_func - + miniglobals = { func.__name__: func, 'OperationError': OperationError } if isinstance(cls, str): + assert 0, "unused?" #print " value Read the value of the property of the given obj.""" @@ -400,7 +424,7 @@ return space.wrap(property) else: try: - return property.fget(space, w_obj) + return property.fget(property, space, w_obj) except DescrMismatch, e: return w_obj.descr_call_mismatch(space, '__getattribute__',\ property.reqcls, Arguments(space, [w_obj, @@ -414,7 +438,7 @@ raise OperationError(space.w_TypeError, space.wrap("readonly attribute")) try: - fset(space, w_obj, w_value) + fset(property, space, w_obj, w_value) except DescrMismatch, e: w_obj.descr_call_mismatch(space, '__setattr__',\ property.reqcls, Arguments(space, [w_obj, @@ -428,7 +452,7 @@ raise OperationError(space.w_AttributeError, space.wrap("cannot delete attribute")) try: - fdel(space, w_obj) + fdel(property, space, w_obj) except DescrMismatch, e: w_obj.descr_call_mismatch(space, '__delattr__',\ property.reqcls, Arguments(space, [w_obj, Modified: pypy/branch/py12/pypy/jit/backend/test/support.py ============================================================================== --- pypy/branch/py12/pypy/jit/backend/test/support.py (original) +++ pypy/branch/py12/pypy/jit/backend/test/support.py Sun May 2 18:05:10 2010 @@ -2,7 +2,7 @@ import sys from pypy.rlib.debug import debug_print from pypy.rlib.jit import OPTIMIZER_FULL -from pypy.translator.translator import TranslationContext +from pypy.translator.translator import TranslationContext, graphof class BaseCompiledMixin(object): @@ -56,7 +56,8 @@ """ % (arglist,)) exec src.compile() in locals() - t.buildannotator().build_types(function, [int] * len(args)) + t.buildannotator().build_types(function, [int] * len(args), + main_entry_point=True) t.buildrtyper(type_system=self.type_system).specialize() warmrunnerdesc = WarmRunnerDesc(t, translate_support_code=True, CPUClass=self.CPUClass, Modified: pypy/branch/py12/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/branch/py12/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/branch/py12/pypy/jit/backend/x86/test/test_zrpy_gc.py Sun May 2 18:05:10 2010 @@ -82,7 +82,7 @@ for name, value in kwds.items(): setattr(t.config.translation, name, value) ann = t.buildannotator(policy=annpolicy.StrictAnnotatorPolicy()) - ann.build_types(f, [s_list_of_strings]) + ann.build_types(f, [s_list_of_strings], main_entry_point=True) t.buildrtyper().specialize() if kwds['jit']: apply_jit(t, optimizer=OPTIMIZER_SIMPLE) Modified: pypy/branch/py12/pypy/jit/metainterp/support.py ============================================================================== --- pypy/branch/py12/pypy/jit/metainterp/support.py (original) +++ pypy/branch/py12/pypy/jit/metainterp/support.py Sun May 2 18:05:10 2010 @@ -41,7 +41,7 @@ annpolicy.allow_someobjects = False a = t.buildannotator(policy=annpolicy) argtypes = getargtypes(a, values) - a.build_types(func, argtypes) + a.build_types(func, argtypes, main_entry_point=True) rtyper = t.buildrtyper(type_system = type_system) rtyper.specialize() if inline: Modified: pypy/branch/py12/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/py12/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/py12/pypy/jit/metainterp/warmspot.py Sun May 2 18:05:10 2010 @@ -231,7 +231,7 @@ def check_access_directly_sanity(self, graphs): from pypy.translator.backendopt.inline import collect_called_graphs jit_graphs = set(graphs) - for graph in collect_called_graphs(self.translator.graphs[0], + for graph in collect_called_graphs(self.translator.entry_point_graph, self.translator): if graph in jit_graphs: continue Modified: pypy/branch/py12/pypy/lib/identity_dict.py ============================================================================== --- pypy/branch/py12/pypy/lib/identity_dict.py (original) +++ pypy/branch/py12/pypy/lib/identity_dict.py Sun May 2 18:05:10 2010 @@ -30,6 +30,12 @@ def __contains__(self, arg): return id(arg) in self._dict + def copy(self): + d = type(self)() + d.update(self.iteritems()) + assert len(d) == len(self) + return d + class IdentityDictPyPy(object, DictMixin): __slots__ = ["_dict"] @@ -52,6 +58,11 @@ def __contains__(self, arg): return arg in self._dict + def copy(self): + d = type(self)() + d.update(self.iteritems()) + assert len(d) == len(self) + return d if idict is None: identity_dict = IdentityDictPurePython Modified: pypy/branch/py12/pypy/module/imp/importing.py ============================================================================== --- pypy/branch/py12/pypy/module/imp/importing.py (original) +++ pypy/branch/py12/pypy/module/imp/importing.py Sun May 2 18:05:10 2010 @@ -25,6 +25,11 @@ # PY_CODERESOURCE = 8 IMP_HOOK = 9 +if sys.platform.startswith('win'): + so_extension = ".pyd" +else: + so_extension = ".so" + def find_modtype(space, filepart): """Check which kind of module to import for the given filepart, which is a path without extension. Returns PY_SOURCE, PY_COMPILED or @@ -40,16 +45,18 @@ # look for a lone .pyc file. # The "imp" module does not respect this, and is allowed to find # lone .pyc files. - if not space.config.objspace.lonepycfiles: - return SEARCH_ERROR, None, None - # check the .pyc file - if space.config.objspace.usepycfiles: + if space.config.objspace.usepycfiles and space.config.objspace.lonepycfiles: pycfile = filepart + ".pyc" if os.path.exists(pycfile) and case_ok(pycfile): # existing .pyc file return PY_COMPILED, ".pyc", "rb" + if space.config.objspace.usemodules.cpyext: + pydfile = filepart + so_extension + if os.path.exists(pydfile) and case_ok(pydfile): + return C_EXTENSION, so_extension, "rb" + return SEARCH_ERROR, None, None if sys.platform in ['linux2', 'freebsd']: @@ -332,6 +339,9 @@ except: stream.close() raise + if modtype == C_EXTENSION: + filename = filepart + suffix + return FindInfo(modtype, filename, None, suffix, filemode) except StreamErrors: pass @@ -356,7 +366,7 @@ if find_info.modtype == C_BUILTIN: return space.getbuiltinmodule(find_info.filename, force_init=True) - if find_info.modtype in (PY_SOURCE, PY_COMPILED, PKG_DIRECTORY): + if find_info.modtype in (PY_SOURCE, PY_COMPILED, C_EXTENSION, PKG_DIRECTORY): w_mod = None if reuse: try: @@ -397,6 +407,12 @@ # fetch the module again, in case of "substitution" w_mod = check_sys_modules(space, w_modulename) return w_mod + elif find_info.modtype == C_EXTENSION and space.config.objspace.usemodules.cpyext: + # the next line is mandantory to init cpyext + space.getbuiltinmodule("cpyext") + from pypy.module.cpyext.api import load_extension_module + load_extension_module(space, find_info.filename, space.str_w(w_modulename)) + return check_sys_modules(space, w_modulename) except OperationError: w_mods = space.sys.get('modules') space.call_method(w_mods, 'pop', w_modulename, space.w_None) Modified: pypy/branch/py12/pypy/module/thread/ll_thread.py ============================================================================== --- pypy/branch/py12/pypy/module/thread/ll_thread.py (original) +++ pypy/branch/py12/pypy/module/thread/ll_thread.py Sun May 2 18:05:10 2010 @@ -35,15 +35,15 @@ ident = thread.start_new_thread(func, ()) except thread.error: ident = -1 - return rffi.cast(rffi.INT, ident) + return rffi.cast(rffi.LONG, ident) CALLBACK = lltype.Ptr(lltype.FuncType([], lltype.Void)) -c_thread_start = llexternal('RPyThreadStart', [CALLBACK], rffi.INT, +c_thread_start = llexternal('RPyThreadStart', [CALLBACK], rffi.LONG, _callable=_emulated_start_new_thread, threadsafe=True) # release the GIL, but most # importantly, reacquire it # around the callback -c_thread_get_ident = llexternal('RPyThreadGetIdent', [], rffi.INT, +c_thread_get_ident = llexternal('RPyThreadGetIdent', [], rffi.LONG, _nowrapper=True) # always call directly TLOCKP = rffi.COpaquePtr('struct RPyOpaque_ThreadLock', Modified: pypy/branch/py12/pypy/module/thread/test/test_ll_thread.py ============================================================================== --- pypy/branch/py12/pypy/module/thread/test/test_ll_thread.py (original) +++ pypy/branch/py12/pypy/module/thread/test/test_ll_thread.py Sun May 2 18:05:10 2010 @@ -47,15 +47,16 @@ state.freed_counter += 1 def bootstrap(): - state.my_thread_ident = get_ident() + state.my_thread_ident = state.z.ident = get_ident() assert state.my_thread_ident == get_ident() + assert get_ident() == state.z.ident state.seen_value = state.z.value state.z = None state.done = 1 def g(i): state.z = Z(i) - start_new_thread(bootstrap, ()) + return start_new_thread(bootstrap, ()) g._dont_inline_ = True def f(): @@ -65,7 +66,7 @@ for i in range(50): state.done = 0 state.seen_value = 0 - g(i) + ident = g(i) gc.collect() willing_to_wait_more = 1000 while not state.done: @@ -74,6 +75,7 @@ raise Exception("thread didn't start?") time.sleep(0.01) assert state.my_thread_ident != main_ident + assert state.my_thread_ident == ident assert state.seen_value == i # try to force Boehm to do some freeing for i in range(3): Modified: pypy/branch/py12/pypy/objspace/std/fake.py ============================================================================== --- pypy/branch/py12/pypy/objspace/std/fake.py (original) +++ pypy/branch/py12/pypy/objspace/std/fake.py Sun May 2 18:05:10 2010 @@ -114,6 +114,13 @@ cpy_type.__name__, base, **kw) def __init__(w_self, space, val): w_self.val = val + w_self.space = space + def getdict(w_self): + try: + d = w_self.val.__dict__ + except AttributeError: + return W_Object.getdict(w_self) + return w_self.space.wrap(d) def unwrap(w_self, space): return w_self.val W_Fake.__name__ = 'W_Fake%s'%(cpy_type.__name__.capitalize()) Modified: pypy/branch/py12/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/py12/pypy/objspace/std/objspace.py (original) +++ pypy/branch/py12/pypy/objspace/std/objspace.py Sun May 2 18:05:10 2010 @@ -7,7 +7,7 @@ from pypy.objspace.std import (builtinshortcut, stdtypedef, frame, model, transparent, callmethod, proxyobject) from pypy.objspace.descroperation import DescrOperation, raiseattrerror -from pypy.rlib.objectmodel import instantiate +from pypy.rlib.objectmodel import instantiate, r_dict from pypy.rlib.debug import make_sure_not_resized from pypy.rlib.rarithmetic import base_int from pypy.rlib.objectmodel import we_are_translated @@ -206,8 +206,11 @@ return W_ComplexObject(x.real, x.imag) if isinstance(x, set): - wrappeditems = [self.wrap(item) for item in x] - return W_SetObject(self, wrappeditems) + rdict_w = r_dict(self.eq_w, self.hash_w) + for item in x: + rdict_w[self.wrap(item)] = None + res = W_SetObject(self, rdict_w) + return res if isinstance(x, frozenset): wrappeditems = [self.wrap(item) for item in x] @@ -450,21 +453,33 @@ raiseattrerror(self, w_obj, name) def finditem_str(self, w_obj, key): - # performance shortcut to avoid creating the OperationError(KeyError) + """ Perform a getitem on w_obj with key (string). Returns found + element or None on element not found. + + performance shortcut to avoid creating the OperationError(KeyError) + and allocating W_StringObject + """ if (isinstance(w_obj, W_DictMultiObject) and not w_obj.user_overridden_class): return w_obj.getitem_str(key) return ObjSpace.finditem_str(self, w_obj, key) def finditem(self, w_obj, w_key): - # performance shortcut to avoid creating the OperationError(KeyError) + """ Perform a getitem on w_obj with w_key (any object). Returns found + element or None on element not found. + + performance shortcut to avoid creating the OperationError(KeyError). + """ if (isinstance(w_obj, W_DictMultiObject) and not w_obj.user_overridden_class): return w_obj.getitem(w_key) return ObjSpace.finditem(self, w_obj, w_key) def setitem_str(self, w_obj, key, w_value, shadows_type=True): - # performance shortcut to avoid creating the OperationError(KeyError) + """ Same as setitem, but takes string instead of any wrapped object + + XXX what shadows_type means??? + """ if (isinstance(w_obj, W_DictMultiObject) and not w_obj.user_overridden_class): w_obj.setitem_str(key, w_value, shadows_type) Modified: pypy/branch/py12/pypy/objspace/std/test/test_setobject.py ============================================================================== --- pypy/branch/py12/pypy/objspace/std/test/test_setobject.py (original) +++ pypy/branch/py12/pypy/objspace/std/test/test_setobject.py Sun May 2 18:05:10 2010 @@ -46,6 +46,8 @@ t = W_SetObject(self.space, None) _initialize_set(self.space, t, self.word) assert self.space.eq_w(s,t) + u = self.space.wrap(set('simsalabim')) + assert self.space.eq_w(s,u) class AppTestAppSetTest: def test_subtype(self): Modified: pypy/branch/py12/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/branch/py12/pypy/objspace/std/typeobject.py (original) +++ pypy/branch/py12/pypy/objspace/std/typeobject.py Sun May 2 18:05:10 2010 @@ -14,6 +14,7 @@ from pypy.rlib.rarithmetic import intmask, r_uint from copy_reg import _HEAPTYPE +_CPYTYPE = 1 # used for non-heap types defined in C # from compiler/misc.py @@ -96,7 +97,7 @@ w_self.needsdel = False w_self.weakrefable = False w_self.weak_subclasses = [] - w_self.__flags__ = 0 # or _HEAPTYPE + w_self.__flags__ = 0 # or _HEAPTYPE or _CPYTYPE w_self.instancetypedef = overridetypedef if overridetypedef is not None: @@ -353,6 +354,9 @@ def is_heaptype(w_self): return w_self.__flags__&_HEAPTYPE + def is_cpytype(w_self): + return w_self.__flags__ & _CPYTYPE + def get_module(w_self): space = w_self.space if w_self.is_heaptype() and '__module__' in w_self.dict_w: Modified: pypy/branch/py12/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/branch/py12/pypy/rlib/rarithmetic.py (original) +++ pypy/branch/py12/pypy/rlib/rarithmetic.py Sun May 2 18:05:10 2010 @@ -126,6 +126,10 @@ raise OverflowError def compute_restype(self_type, other_type): + if self_type is other_type: + if self_type is bool: + return int + return self_type if other_type in (bool, int, long): if self_type is bool: return int Modified: pypy/branch/py12/pypy/rlib/rstring.py ============================================================================== --- pypy/branch/py12/pypy/rlib/rstring.py (original) +++ pypy/branch/py12/pypy/rlib/rstring.py Sun May 2 18:05:10 2010 @@ -1,11 +1,49 @@ - -""" String builder interface +""" String builder interface and string functions """ from pypy.rpython.extregistry import ExtRegistryEntry from pypy.annotation.model import SomeObject, SomeString, s_None,\ SomeChar, SomeInteger, SomeUnicodeCodePoint, SomeUnicodeString + +# -------------- public API for string functions ----------------------- +def split(value, by, maxsplit=-1): + bylen = len(by) + if bylen == 0: + raise ValueError("empty separator") + + res = [] + start = 0 + while maxsplit != 0: + next = value.find(by, start) + if next < 0: + break + res.append(value[start:next]) + start = next + bylen + maxsplit -= 1 # NB. if it's already < 0, it stays < 0 + + res.append(value[start:len(value)]) + return res + +def rsplit(value, by, maxsplit=-1): + res = [] + end = len(value) + bylen = len(by) + if bylen == 0: + raise ValueError("empty separator") + + while maxsplit != 0: + next = value.rfind(by, 0, end) + if next < 0: + break + res.append(value[next+bylen:end]) + end = next + maxsplit -= 1 # NB. if it's already < 0, it stays < 0 + + res.append(value[:end]) + res.reverse() + return res + # -------------- public API --------------------------------- INIT_SIZE = 100 # XXX tweak Modified: pypy/branch/py12/pypy/rlib/rweakref.py ============================================================================== --- pypy/branch/py12/pypy/rlib/rweakref.py (original) +++ pypy/branch/py12/pypy/rlib/rweakref.py Sun May 2 18:05:10 2010 @@ -1,6 +1,7 @@ """ Weakref support in RPython. Supports ref() without callbacks, -and a limited version of WeakValueDictionary. LLType only for now! +a form of WeakKeyDictionary, and a limited version of WeakValueDictionary. +LLType only for now! """ import weakref @@ -27,6 +28,36 @@ self._dict[key] = value +class RWeakKeyDictionary(object): + """A dictionary containing weak keys. + Keys and values must be instances. + Prebuilt RWeakKeyDictionaries must be empty. + """ + + def __init__(self, keyclass, valueclass): + self._dict = weakref.WeakKeyDictionary() + self._keyclass = keyclass + self._valueclass = valueclass + + def get(self, key): + """Get the value associated to 'key', or None by default.""" + assert isinstance(key, self._keyclass) + return self._dict.get(key, None) + + def set(self, key, value): + """Set the key/value pair (or delete it if value is None).""" + assert isinstance(key, self._keyclass) + if value is None: + self._dict.pop(key, None) + else: + assert isinstance(value, self._valueclass) + self._dict[key] = value + + def length(self): + """Mostly for debugging. Slow, don't use in real code.""" + return len(self._dict) + + # ____________________________________________________________ from pypy.rpython import extregistry @@ -41,8 +72,8 @@ self.valueclassdef = valueclassdef def rtyper_makerepr(self, rtyper): - from pypy.rlib import rweakrefimpl - return rweakrefimpl.WeakValueDictRepr(rtyper) + from pypy.rlib import _rweakvaldict + return _rweakvaldict.WeakValueDictRepr(rtyper) def rtyper_makekey_ex(self, rtyper): return self.__class__, @@ -65,14 +96,11 @@ _about_ = RWeakValueDictionary def compute_result_annotation(self, s_valueclass): - assert isinstance(s_valueclass, annmodel.SomePBC) - assert s_valueclass.is_constant() - [desc] = s_valueclass.descriptions - return SomeWeakValueDict(desc.getuniqueclassdef()) + return SomeWeakValueDict(_getclassdef(s_valueclass)) def specialize_call(self, hop): - from pypy.rlib import rweakrefimpl - return rweakrefimpl.specialize_make_weakdict(hop) + from pypy.rlib import _rweakvaldict + return _rweakvaldict.specialize_make_weakdict(hop) class Entry(extregistry.ExtRegistryEntry): _type_ = RWeakValueDictionary @@ -81,3 +109,65 @@ bk = self.bookkeeper x = self.instance return SomeWeakValueDict(bk.getuniqueclassdef(x._valueclass)) + +def _getclassdef(s_instance): + assert isinstance(s_instance, annmodel.SomePBC) + assert s_instance.is_constant() + [desc] = s_instance.descriptions + return desc.getuniqueclassdef() + +# ____________________________________________________________ + +class SomeWeakKeyDict(annmodel.SomeObject): + knowntype = RWeakKeyDictionary + + def __init__(self, keyclassdef, valueclassdef): + self.keyclassdef = keyclassdef + self.valueclassdef = valueclassdef + + def rtyper_makerepr(self, rtyper): + from pypy.rlib import _rweakkeydict + return _rweakkeydict.WeakKeyDictRepr(rtyper) + + def rtyper_makekey_ex(self, rtyper): + return self.__class__, + + def method_get(self, s_key): + assert isinstance(s_key, annmodel.SomeInstance) + assert s_key.classdef.issubclass(self.keyclassdef) + return annmodel.SomeInstance(self.valueclassdef, can_be_None=True) + + def method_set(self, s_key, s_value): + s_oldvalue = self.method_get(s_key) + assert s_oldvalue.contains(s_value) + + def method_length(self): + return annmodel.SomeInteger(nonneg=True) + +class __extend__(pairtype(SomeWeakKeyDict, SomeWeakKeyDict)): + def union((s_wkd1, s_wkd2)): + if s_wkd1.keyclassdef is not s_wkd2.keyclassdef: + return SomeObject() # not the same key class! complain... + if s_wkd1.valueclassdef is not s_wkd2.valueclassdef: + return SomeObject() # not the same value class! complain... + return SomeWeakKeyDict(s_wkd1.keyclassdef, s_wkd1.valueclassdef) + +class Entry(extregistry.ExtRegistryEntry): + _about_ = RWeakKeyDictionary + + def compute_result_annotation(self, s_keyclass, s_valueclass): + return SomeWeakKeyDict(_getclassdef(s_keyclass), + _getclassdef(s_valueclass)) + + def specialize_call(self, hop): + from pypy.rlib import _rweakkeydict + return _rweakkeydict.specialize_make_weakdict(hop) + +class Entry(extregistry.ExtRegistryEntry): + _type_ = RWeakKeyDictionary + + def compute_annotation(self): + bk = self.bookkeeper + x = self.instance + return SomeWeakKeyDict(bk.getuniqueclassdef(x._keyclass), + bk.getuniqueclassdef(x._valueclass)) Modified: pypy/branch/py12/pypy/rlib/test/test_rarithmetic.py ============================================================================== --- pypy/branch/py12/pypy/rlib/test/test_rarithmetic.py (original) +++ pypy/branch/py12/pypy/rlib/test/test_rarithmetic.py Sun May 2 18:05:10 2010 @@ -365,3 +365,7 @@ def test_isnan(): assert isnan(NAN) + +def test_int_real_union(): + from pypy.rpython.lltypesystem.rffi import r_int_real + assert compute_restype(r_int_real, r_int_real) is r_int_real Modified: pypy/branch/py12/pypy/rlib/test/test_rbigint.py ============================================================================== --- pypy/branch/py12/pypy/rlib/test/test_rbigint.py (original) +++ pypy/branch/py12/pypy/rlib/test/test_rbigint.py Sun May 2 18:05:10 2010 @@ -474,10 +474,11 @@ def test_args_from_rarith_int(self): from pypy.rpython.tool.rfficache import platform from pypy.rlib.rarithmetic import r_int + from pypy.rpython.lltypesystem.rffi import r_int_real classlist = platform.numbertype_to_rclass.values() fnlist = [] for r in classlist: - if r is r_int: # and also r_longlong on 64-bit + if r in (r_int, r_int_real): # and also r_longlong on 64-bit continue if r is int: mask = sys.maxint*2+1 Modified: pypy/branch/py12/pypy/rlib/test/test_rstring.py ============================================================================== --- pypy/branch/py12/pypy/rlib/test/test_rstring.py (original) +++ pypy/branch/py12/pypy/rlib/test/test_rstring.py Sun May 2 18:05:10 2010 @@ -1,5 +1,26 @@ -from pypy.rlib.rstring import StringBuilder, UnicodeBuilder +from pypy.rlib.rstring import StringBuilder, UnicodeBuilder, split, rsplit + +def test_split(): + assert split("", 'x') == [''] + assert split("a", "a", 1) == ['', ''] + assert split(" ", " ", 1) == ['', ''] + assert split("aa", "a", 2) == ['', '', ''] + assert split('a|b|c|d', '|') == ['a', 'b', 'c', 'd'] + assert split('a|b|c|d', '|', 2) == ['a', 'b', 'c|d'] + assert split('a//b//c//d', '//') == ['a', 'b', 'c', 'd'] + assert split('endcase test', 'test') == ['endcase ', ''] + raises(ValueError, split, 'abc', '') + +def test_rsplit(): + assert rsplit("a", "a", 1) == ['', ''] + assert rsplit(" ", " ", 1) == ['', ''] + assert rsplit("aa", "a", 2) == ['', '', ''] + assert rsplit('a|b|c|d', '|') == ['a', 'b', 'c', 'd'] + assert rsplit('a|b|c|d', '|', 2) == ['a|b', 'c', 'd'] + assert rsplit('a//b//c//d', '//') == ['a', 'b', 'c', 'd'] + assert rsplit('endcase test', 'test') == ['endcase ', ''] + raises(ValueError, rsplit, "abc", '') def test_string_builder(): s = StringBuilder() Modified: pypy/branch/py12/pypy/rpython/annlowlevel.py ============================================================================== --- pypy/branch/py12/pypy/rpython/annlowlevel.py (original) +++ pypy/branch/py12/pypy/rpython/annlowlevel.py Sun May 2 18:05:10 2010 @@ -34,6 +34,7 @@ else: s = compact() return s + 'Const' + __repr__ = __str__ class LowLevelAnnotatorPolicy(AnnotatorPolicy): allow_someobjects = False Modified: pypy/branch/py12/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/py12/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/py12/pypy/rpython/lltypesystem/ll2ctypes.py Sun May 2 18:05:10 2010 @@ -675,7 +675,8 @@ if T is lltype.Void: return None if isinstance(T, lltype.Ptr): - if not cobj: # NULL pointer + if not cobj or not ctypes.cast(cobj, ctypes.c_void_p).value: # NULL pointer + # CFunctionType.__nonzero__ is broken before Python 2.6 return lltype.nullptr(T.TO) if isinstance(T.TO, lltype.Struct): REAL_TYPE = T.TO @@ -963,7 +964,7 @@ def invoke_via_ctypes(*argvalues): global _callback_exc_info cargs = [] - for i in range(len(FUNCTYPE.ARGS)): + for i in range(len(argvalues)): if i not in void_arguments: cvalue = lltype2ctypes(argvalues[i]) if i in container_arguments: @@ -1001,7 +1002,7 @@ elif isinstance(cvalue, (str, unicode)): cvalue = ord(cvalue) # character -> integer - if not isinstance(cvalue, (int, long)): + if not isinstance(cvalue, (int, long, float)): raise NotImplementedError("casting %r to %r" % (TYPE1, RESTYPE)) if isinstance(RESTYPE, lltype.Ptr): Modified: pypy/branch/py12/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/py12/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/py12/pypy/rpython/lltypesystem/lltype.py Sun May 2 18:05:10 2010 @@ -1,3 +1,8 @@ +import StringIO +import traceback +import sys + +import py from pypy.rlib.rarithmetic import (r_int, r_uint, intmask, r_singlefloat, r_ulonglong, r_longlong, base_int, normalizedinttype) @@ -12,6 +17,20 @@ TLS = tlsobject() +# Track allocations to detect memory leaks +# Don't track 'gc' and immortal mallocs +TRACK_ALLOCATIONS = False +ALLOCATED = identity_dict() + +def start_tracking_allocations(): + global TRACK_ALLOCATIONS + TRACK_ALLOCATIONS = True + ALLOCATED.clear() + +def stop_tracking_allocations(): + global TRACK_ALLOCATIONS + TRACK_ALLOCATIONS = False + class _uninitialized(object): def __init__(self, TYPE): self.TYPE = TYPE @@ -1318,21 +1337,41 @@ __slots__ = ('_TYPE', '_parent_type', '_parent_index', '_keepparent', '_wrparent', - '__weakref__', - '_storage') + '__weakref__', '_traceback', + '__storage') - def __init__(self, TYPE): + def __init__(self, TYPE, track_allocation=None): self._wrparent = None self._TYPE = TYPE self._storage = True # means "use default storage", as opposed to: # None - container was freed # - using ctypes # (see ll2ctypes.py) + if track_allocation is not False and TRACK_ALLOCATIONS: + self._traceback = self._get_traceback() + ALLOCATED[self] = None + else: + self._traceback = None + + def _get_traceback(self): + frame = sys._getframe().f_back.f_back.f_back.f_back + sio = StringIO.StringIO() + traceback.print_stack(frame, file=sio) + return sio.getvalue() def _free(self): self._check() # no double-frees self._storage = None + def _storage_get(self): + return self.__storage + + def _storage_set(self, value): + self.__storage = value + if value is not True and self in ALLOCATED: + del ALLOCATED[self] + _storage = property(_storage_get, _storage_set) + def _was_freed(self): if self._storage is None: return True @@ -1409,14 +1448,14 @@ class _struct(_parentable): _kind = "structure" - __slots__ = ('_hash_cache_',) + __slots__ = ('_hash_cache_', '_compilation_info') - def __new__(self, TYPE, n=None, initialization=None, parent=None, parentindex=None): + def __new__(self, TYPE, n=None, initialization=None, parent=None, parentindex=None, track_allocation=None): my_variety = _struct_variety(TYPE._names) return object.__new__(my_variety) - def __init__(self, TYPE, n=None, initialization=None, parent=None, parentindex=None): - _parentable.__init__(self, TYPE) + def __init__(self, TYPE, n=None, initialization=None, parent=None, parentindex=None, track_allocation=None): + _parentable.__init__(self, TYPE, track_allocation) if n is not None and TYPE._arrayfld is None: raise TypeError("%r is not variable-sized" % (TYPE,)) if n is None and TYPE._arrayfld is not None: @@ -1424,7 +1463,8 @@ first, FIRSTTYPE = TYPE._first_struct() for fld, typ in TYPE._flds.items(): if fld == TYPE._arrayfld: - value = _array(typ, n, initialization=initialization, parent=self, parentindex=fld) + value = _array(typ, n, initialization=initialization, parent=self, parentindex=fld, + track_allocation=track_allocation) else: value = typ._allocate(initialization=initialization, parent=self, parentindex=fld) setattr(self, fld, value) @@ -1485,12 +1525,12 @@ __slots__ = ('items',) - def __init__(self, TYPE, n, initialization=None, parent=None, parentindex=None): + def __init__(self, TYPE, n, initialization=None, parent=None, parentindex=None, track_allocation=None): if not isinstance(n, int): raise TypeError, "array length must be an int" if n < 0: raise ValueError, "negative array length" - _parentable.__init__(self, TYPE) + _parentable.__init__(self, TYPE, track_allocation) try: myrange = range(n) except OverflowError: @@ -1557,7 +1597,7 @@ _cache = weakref.WeakKeyDictionary() # parentarray -> {subarrays} def __init__(self, TYPE, parent, baseoffset_or_fieldname): - _parentable.__init__(self, TYPE) + _parentable.__init__(self, TYPE, track_allocation=False) self._setparentstructure(parent, baseoffset_or_fieldname) # Keep the parent array alive, we share the same allocation. # Don't do it if we are inside a GC object, though -- it's someone @@ -1787,9 +1827,9 @@ else: initialization = 'malloc' if isinstance(T, Struct): - o = _struct(T, n, initialization=initialization) + o = _struct(T, n, initialization=initialization, track_allocation=flavor == "raw" and not immortal) elif isinstance(T, Array): - o = _array(T, n, initialization=initialization) + o = _array(T, n, initialization=initialization, track_allocation=flavor == "raw" and not immortal) elif isinstance(T, OpaqueType): assert n is None o = _opaque(T, initialization=initialization) Modified: pypy/branch/py12/pypy/rpython/lltypesystem/rdict.py ============================================================================== --- pypy/branch/py12/pypy/rpython/lltypesystem/rdict.py (original) +++ pypy/branch/py12/pypy/rpython/lltypesystem/rdict.py Sun May 2 18:05:10 2010 @@ -508,12 +508,14 @@ def ll_dict_lookup(d, key, hash): entries = d.entries + ENTRIES = lltype.typeOf(entries).TO + direct_compare = not hasattr(ENTRIES, 'no_direct_compare') mask = len(entries) - 1 i = hash & mask # do the first try before any looping if entries.valid(i): checkingkey = entries[i].key - if checkingkey == key: + if direct_compare and checkingkey == key: return i # found the entry if d.keyeq is not None and entries.hash(i) == hash: # correct hash, maybe the key is e.g. a different pointer to @@ -548,7 +550,7 @@ return freeslot elif entries.valid(i): checkingkey = entries[i].key - if checkingkey == key: + if direct_compare and checkingkey == key: return i if d.keyeq is not None and entries.hash(i) == hash: # correct hash, maybe the key is e.g. a different pointer to Modified: pypy/branch/py12/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/py12/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/py12/pypy/rpython/lltypesystem/rffi.py Sun May 2 18:05:10 2010 @@ -48,6 +48,7 @@ result = isinstance(s_p, annmodel.SomePtr) return self.bookkeeper.immutablevalue(result) def specialize_call(self, hop): + hop.exception_cannot_occur() return hop.inputconst(lltype.Bool, hop.s_result.const) def llexternal(name, args, result, _callable=None, @@ -388,6 +389,7 @@ r_int_real = rarithmetic.build_int("r_int_real", r_int.SIGN, r_int.BITS, True) INT_real = lltype.build_number("INT", r_int_real) platform.numbertype_to_rclass[INT_real] = r_int_real +NUMBER_TYPES.append(INT_real) # ^^^ this creates at least the following names: # -------------------------------------------------------------------- Modified: pypy/branch/py12/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/branch/py12/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/branch/py12/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Sun May 2 18:05:10 2010 @@ -432,6 +432,29 @@ rffi.free_charp(p) assert not ALLOCATED # detects memory leaks in the test + def test_funcptr_cast(self): + eci = ExternalCompilationInfo( + separate_module_sources=[""" + long mul(long x, long y) { return x*y; } + long(*get_mul(long x)) () { return &mul; } + """], + export_symbols=['get_mul']) + get_mul = rffi.llexternal( + 'get_mul', [], + lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Signed)), + compilation_info=eci) + # This call returns a pointer to a function taking one argument + funcptr = get_mul() + # cast it to the "real" function type + FUNCTYPE2 = lltype.FuncType([lltype.Signed, lltype.Signed], + lltype.Signed) + cmul = rffi.cast(lltype.Ptr(FUNCTYPE2), funcptr) + # and it can be called with the expected number of arguments + res = cmul(41, 42) + assert res == 41 * 42 + raises(TypeError, cmul, 41) + raises(TypeError, cmul, 41, 42, 43) + def test_qsort(self): CMPFUNC = lltype.FuncType([rffi.VOIDP, rffi.VOIDP], rffi.INT) qsort = rffi.llexternal('qsort', [rffi.VOIDP, Modified: pypy/branch/py12/pypy/rpython/lltypesystem/test/test_lltype.py ============================================================================== --- pypy/branch/py12/pypy/rpython/lltypesystem/test/test_lltype.py (original) +++ pypy/branch/py12/pypy/rpython/lltypesystem/test/test_lltype.py Sun May 2 18:05:10 2010 @@ -1,5 +1,6 @@ import py from pypy.rpython.lltypesystem.lltype import * +from pypy.rpython.lltypesystem import lltype, rffi from pypy.lib.identity_dict import identity_dict def isweak(p, T): @@ -777,3 +778,42 @@ assert hash1 == identityhash(a) p = cast_opaque_ptr(llmemory.GCREF, a) assert hash1 == identityhash(p) + +class TestTrackAllocation: + def setup_method(self, func): + start_tracking_allocations() + + def teardown_method(self, func): + assert not lltype.ALLOCATED, "Memory was not correctly freed" + stop_tracking_allocations() + + def test_track_allocation(self): + """A malloc'd buffer fills the ALLOCATED dictionary""" + assert lltype.TRACK_ALLOCATIONS + assert not lltype.ALLOCATED + buf = malloc(Array(Signed), 1, flavor="raw") + assert len(lltype.ALLOCATED) == 1 + assert lltype.ALLOCATED.keys() == [buf._obj] + free(buf, flavor="raw") + assert not lltype.ALLOCATED + + def test_str_from_buffer(self): + """gc-managed memory does not need to be freed""" + size = 50 + raw_buf, gc_buf = rffi.alloc_buffer(size) + for i in range(size): raw_buf[i] = 'a' + rstr = rffi.str_from_buffer(raw_buf, gc_buf, size, size) + rffi.keep_buffer_alive_until_here(raw_buf, gc_buf) + assert not lltype.ALLOCATED + + def test_leak_traceback(self): + """Test info stored for allocated items""" + buf = malloc(Array(Signed), 1, flavor="raw") + traceback = lltype.ALLOCATED.keys()[0]._traceback + lines = traceback.splitlines() + assert 'malloc(' in lines[-1] and 'flavor="raw")' in lines[-1] + + # XXX The traceback should not be too long + print traceback + + free(buf, flavor="raw") Modified: pypy/branch/py12/pypy/rpython/test/test_llinterp.py ============================================================================== --- pypy/branch/py12/pypy/rpython/test/test_llinterp.py (original) +++ pypy/branch/py12/pypy/rpython/test/test_llinterp.py Sun May 2 18:05:10 2010 @@ -43,7 +43,7 @@ t = TranslationContext(config=config) t.config.set(**extraconfigopts) a = t.buildannotator(policy=policy) - timelog("annotating", a.build_types, func, argtypes) + timelog("annotating", a.build_types, func, argtypes, main_entry_point=True) if viewbefore == 'auto': viewbefore = getattr(conftest.option, 'view', False) if viewbefore: Modified: pypy/branch/py12/pypy/tool/fixeol ============================================================================== --- pypy/branch/py12/pypy/tool/fixeol (original) +++ pypy/branch/py12/pypy/tool/fixeol Sun May 2 18:05:10 2010 @@ -48,7 +48,7 @@ return True def checkeolfile(path): - return path.ext in ('.txt', '.py', '.asc') + return path.ext in ('.txt', '.py', '.asc', '.h', '.c') def fixdirectory(path): print "+ checking directory", path, Modified: pypy/branch/py12/pypy/translator/c/gcc/test/test_asmgcroot.py ============================================================================== --- pypy/branch/py12/pypy/translator/c/gcc/test/test_asmgcroot.py (original) +++ pypy/branch/py12/pypy/translator/c/gcc/test/test_asmgcroot.py Sun May 2 18:05:10 2010 @@ -1,10 +1,14 @@ import py -import sys, os +import sys, os, gc from pypy.translator.c.test import test_newgc from pypy.translator.translator import TranslationContext from pypy.translator.c.genc import CStandaloneBuilder from pypy.annotation.listdef import s_list_of_strings from pypy import conftest +from pypy.translator.tool.cbuild import ExternalCompilationInfo +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rlib.entrypoint import entrypoint, secondary_entrypoints +from pypy.rpython.lltypesystem.lloperation import llop class AbstractTestAsmGCRoot: # the asmgcroot gc transformer doesn't generate gc_reload_possibly_moved @@ -34,11 +38,15 @@ config = cls.make_config() t = TranslationContext(config=config) a = t.buildannotator() + sec_ep = getattr(cls, 'secondary_entrypoints', []) + for f, inputtypes in sec_ep: + a.build_types(f, inputtypes, False) a.build_types(main, [s_list_of_strings]) t.buildrtyper().specialize() t.checkgraphs() - cbuilder = CStandaloneBuilder(t, main, config=config) + cbuilder = CStandaloneBuilder(t, main, config=config, + secondary_entrypoints=sec_ep) c_source_filename = cbuilder.generate_source( defines = cbuilder.DEBUG_DEFINES) cls._patch_makefile(cbuilder.targetdir) @@ -53,7 +61,12 @@ redirect = ' 2> NUL' else: redirect = '' - g = os.popen('"%s" %s %d%s' % (exe_name, arg0, arg1, redirect), 'r') + if config.translation.shared and os.name == 'posix': + env = 'LD_LIBRARY_PATH="%s" ' % (exe_name.dirpath(),) + else: + env = '' + g = os.popen( + '%s"%s" %s %d%s' % (env, exe_name, arg0, arg1, redirect), 'r') for line in g: print >> sys.stderr, 'RUN:', line.rstrip() lines.append(line) @@ -101,6 +114,7 @@ test_newgc.TestSemiSpaceGC): # for the individual tests see # ====> ../../test/test_newgc.py + secondary_entrypoints = [] def define_large_function(cls): class A(object): @@ -123,11 +137,6 @@ assert res == 1000 def define_callback_simple(cls): - import gc - from pypy.rpython.lltypesystem import lltype, rffi - from pypy.rpython.annlowlevel import llhelper - from pypy.translator.tool.cbuild import ExternalCompilationInfo - c_source = py.code.Source(""" int mystuff(int(*cb)(int, int)) { @@ -153,11 +162,50 @@ return f - def test_callback_simple(self): res = self.run('callback_simple') assert res == 4900 + def define_secondary_entrypoint_callback(cls): + # XXX this is baaaad, cleanup global state + try: + del secondary_entrypoints["x42"] + except KeyError: + pass + + @entrypoint("x42", [lltype.Signed, lltype.Signed], c_name='callback') + def mycallback(a, b): + llop.gc_stack_bottom(lltype.Void) + rffi.stackcounter.stacks_counter += 1 + gc.collect() + rffi.stackcounter.stacks_counter -= 1 + return a + b + + c_source = py.code.Source(""" + int mystuff2() + { + return callback(40, 2) + callback(3, 4); + } + """) + + eci = ExternalCompilationInfo(separate_module_sources=[c_source]) + z = rffi.llexternal('mystuff2', [], lltype.Signed, + compilation_info=eci) + S = lltype.GcStruct('S', ('x', lltype.Signed)) + + cls.secondary_entrypoints = secondary_entrypoints["x42"] + + def f(): + p = lltype.malloc(S) + p.x = 100 + result = z() + return result * p.x + + return f + + def test_secondary_entrypoint_callback(self): + res = self.run('secondary_entrypoint_callback') + assert res == 4900 class TestAsmGCRootWithSemiSpaceGC_Mingw32(TestAsmGCRootWithSemiSpaceGC): # for the individual tests see @@ -186,6 +234,13 @@ def define_callback_with_collect(cls): return lambda: 0 +class TestAsmGCRootWithSemiSpaceGC_Shared(TestAsmGCRootWithSemiSpaceGC): + @classmethod + def make_config(cls): + config = TestAsmGCRootWithSemiSpaceGC.make_config() + config.translation.shared = True + return config + class TestAsmGCRootWithHybridTagged(AbstractTestAsmGCRoot, test_newgc.TestHybridTaggedPointers): pass Modified: pypy/branch/py12/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/py12/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/py12/pypy/translator/c/gcc/trackgcroot.py Sun May 2 18:05:10 2010 @@ -1640,6 +1640,7 @@ format = 'mingw32' else: format = 'elf' + entrypoint = 'main' while len(sys.argv) > 1: if sys.argv[1] == '-v': del sys.argv[1] @@ -1653,6 +1654,9 @@ elif sys.argv[1].startswith('-f'): format = sys.argv[1][2:] del sys.argv[1] + elif sys.argv[1].startswith('-m'): + entrypoint = sys.argv[1][2:] + del sys.argv[1] else: break tracker = GcRootTracker(verbose=verbose, shuffle=shuffle, format=format) @@ -1669,7 +1673,7 @@ lblfn = fn[:-2] + '.lbl.s' g = open(lblfn, 'w') try: - tracker.process(f, g, filename=fn) + tracker.process(f, g, entrypoint=entrypoint, filename=fn) except: g.close() os.unlink(lblfn) Modified: pypy/branch/py12/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/py12/pypy/translator/c/genc.py (original) +++ pypy/branch/py12/pypy/translator/c/genc.py Sun May 2 18:05:10 2010 @@ -13,6 +13,7 @@ from pypy.translator.c.support import log, c_string_constant from pypy.rpython.typesystem import getfunctionptr from pypy.translator.c import gc +from pypy.rlib import exports def import_module_from_directory(dir, modname): file, pathname, description = imp.find_module(modname, [str(dir)]) @@ -77,14 +78,22 @@ self.outputfilename = outputfilename self.profbased = profbased - def _build(self, eci=ExternalCompilationInfo()): + def _build(self, eci=ExternalCompilationInfo(), shared=False): + outputfilename = self.outputfilename + if shared: + if outputfilename: + basename = outputfilename + else: + basename = self.cfiles[0].purebasename + outputfilename = 'lib' + basename return self.platform.compile(self.cfiles, self.eci.merge(eci), - outputfilename=self.outputfilename) + outputfilename=outputfilename, + standalone=not shared) - def build(self): + def build(self, shared=False): if self.profbased: return self._do_profbased() - return self._build() + return self._build(shared=shared) def _do_profbased(self): ProfDriver, args = self.profbased @@ -103,7 +112,8 @@ modulename = None split = False - def __init__(self, translator, entrypoint, config, gcpolicy=None): + def __init__(self, translator, entrypoint, config, gcpolicy=None, + secondary_entrypoints=()): self.translator = translator self.entrypoint = entrypoint self.entrypoint_name = getattr(self.entrypoint, 'func_name', None) @@ -113,6 +123,7 @@ if gcpolicy is not None and gcpolicy.requires_stackless: config.translation.stackless = True self.eci = self.get_eci() + self.secondary_entrypoints = secondary_entrypoints def get_eci(self): pypy_include_dir = py.path.local(autopath.pypydir).join('translator', 'c') @@ -159,7 +170,16 @@ self.c_entrypoint_name = None else: pfname = db.get(pf) + + for func, _ in self.secondary_entrypoints: + bk = translator.annotator.bookkeeper + db.get(getfunctionptr(bk.getdesc(func).getuniquegraph())) + self.c_entrypoint_name = pfname + + for obj in exports.EXPORTS_obj2name.keys(): + db.getcontainernode(obj) + exports.clear() db.complete() self.collect_compilation_info(db) @@ -240,6 +260,10 @@ if CBuilder.have___thread: if not self.config.translation.no__thread: defines['USE___THREAD'] = 1 + if self.config.translation.shared: + defines['PYPY_MAIN_FUNCTION'] = "pypy_main_startup" + self.eci = self.eci.merge(ExternalCompilationInfo( + export_symbols=["pypy_main_startup"])) self.eci, cfile, extra = gen_source_standalone(db, modulename, targetdir, self.eci, @@ -404,12 +428,13 @@ if isinstance(self._module, isolate.Isolate): isolate.close_isolate(self._module) - def gen_makefile(self, targetdir): + def gen_makefile(self, targetdir, exe_name=None): pass class CStandaloneBuilder(CBuilder): standalone = True executable_name = None + shared_library_name = None def getprofbased(self): profbased = None @@ -450,9 +475,40 @@ return res.out, res.err return res.out - def compile(self): + def build_main_for_shared(self, shared_library_name, entrypoint, exe_name): + import time + time.sleep(1) + self.shared_library_name = shared_library_name + # build main program + eci = self.get_eci() + kw = {} + if self.translator.platform.so_ext == 'so': + kw['libraries'] = [self.shared_library_name.purebasename[3:]] + kw['library_dirs'] = [self.targetdir] + else: + kw['libraries'] = [self.shared_library_name.new(ext='')] + eci = eci.merge(ExternalCompilationInfo( + separate_module_sources=[''' + int %s(int argc, char* argv[]); + + int main(int argc, char* argv[]) + { return %s(argc, argv); } + ''' % (entrypoint, entrypoint) + ], + **kw + )) + eci = eci.convert_sources_to_files( + cache_dir=self.targetdir) + return self.translator.platform.compile( + [], eci, + outputfilename=exe_name) + + def compile(self, exe_name=None): assert self.c_source_filename assert not self._compiled + + shared = self.config.translation.shared + if (self.config.translation.gcrootfinder == "asmgcc" or self.config.translation.force_make): extra_opts = [] @@ -463,16 +519,23 @@ else: compiler = CCompilerDriver(self.translator.platform, [self.c_source_filename] + self.extrafiles, - self.eci, profbased=self.getprofbased()) - self.executable_name = compiler.build() + self.eci, profbased=self.getprofbased(), + outputfilename=exe_name) + self.executable_name = compiler.build(shared=shared) + if shared: + self.executable_name = self.build_main_for_shared( + self.executable_name, "pypy_main_startup", exe_name) assert self.executable_name self._compiled = True return self.executable_name - def gen_makefile(self, targetdir): + def gen_makefile(self, targetdir, exe_name=None): cfiles = [self.c_source_filename] + self.extrafiles - mk = self.translator.platform.gen_makefile(cfiles, self.eci, - path=targetdir) + mk = self.translator.platform.gen_makefile( + cfiles, self.eci, + path=targetdir, exe_name=exe_name, + shared=self.config.translation.shared) + if self.has_profopt(): profopt = self.config.translation.profopt mk.definition('ABS_TARGET', '$(shell python -c "import sys,os; print os.path.abspath(sys.argv[1])" $(TARGET))') @@ -516,6 +579,11 @@ mk.definition('GCMAPFILES', gcmapfiles) mk.definition('DEBUGFLAGS', '-O2 -fomit-frame-pointer -g') + if self.config.translation.shared: + mk.definition('PYPY_MAIN_FUNCTION', "pypy_main_startup") + else: + mk.definition('PYPY_MAIN_FUNCTION', "main") + if sys.platform == 'win32': python = sys.executable.replace('\\', '/') + ' ' else: @@ -541,7 +609,7 @@ 'cmd /c $(MASM) /nologo /Cx /Cp /Zm /coff /Fo$@ /c $< $(INCLUDEDIRS)') mk.rule('.c.gcmap', '', ['$(CC) /nologo $(ASM_CFLAGS) /c /FAs /Fa$*.s $< $(INCLUDEDIRS)', - 'cmd /c ' + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -fmsvc -t $*.s > $@'] + 'cmd /c ' + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -fmsvc -m$(PYPY_MAIN_FUNCTION) -t $*.s > $@'] ) mk.rule('gcmaptable.c', '$(GCMAPFILES)', 'cmd /c ' + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -fmsvc $(GCMAPFILES) > $@') @@ -550,7 +618,7 @@ mk.definition('OBJECTS', '$(ASMLBLFILES) gcmaptable.s') mk.rule('%.s', '%.c', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -frandom-seed=$< -o $@ -S $< $(INCLUDEDIRS)') mk.rule('%.lbl.s %.gcmap', '%.s', - python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -t $< > $*.gcmap') + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -m$(PYPY_MAIN_FUNCTION) -t $< > $*.gcmap') mk.rule('gcmaptable.s', '$(GCMAPFILES)', python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py $(GCMAPFILES) > $@') Modified: pypy/branch/py12/pypy/translator/c/node.py ============================================================================== --- pypy/branch/py12/pypy/translator/c/node.py (original) +++ pypy/branch/py12/pypy/translator/c/node.py Sun May 2 18:05:10 2010 @@ -10,6 +10,7 @@ from pypy.translator.c.support import cdecl, forward_cdecl, somelettersfrom from pypy.translator.c.support import c_char_array_constant, barebonearray from pypy.translator.c.primitive import PrimitiveType, name_signed +from pypy.rlib import exports from pypy.rlib.rarithmetic import isinf, isnan from pypy.rlib.rstackovf import _StackOverflow from pypy.translator.c import extfunc @@ -468,7 +469,7 @@ if USESLOTS: __slots__ = """db T obj typename implementationtypename - name ptrname + name ptrname compilation_info globalcontainer""".split() def __init__(self, db, T, obj): @@ -479,7 +480,10 @@ self.typename = db.gettype(T) #, who_asks=self) self.implementationtypename = db.gettype(T, varlength=self.getlength()) parent, parentindex = parentlink(obj) - if parent is None: + if obj in exports.EXPORTS_obj2name: + self.name = exports.EXPORTS_obj2name[obj] + self.globalcontainer = True + elif parent is None: self.name = db.namespace.uniquename('g_' + self.basename()) self.globalcontainer = True else: @@ -490,6 +494,7 @@ if self.typename != self.implementationtypename: if db.gettypedefnode(T).extra_union_for_varlength: self.name += '.b' + self.compilation_info = getattr(obj, '_compilation_info', None) self.ptrname = '(&%s)' % self.name def is_thread_local(self): @@ -922,6 +927,8 @@ else: assert fnobj.external == 'CPython' return [CExternalFunctionCodeGenerator(fnobj, db)] + elif hasattr(fnobj._callable, "c_name"): + return [] else: raise ValueError, "don't know how to generate code for %r" % (fnobj,) Modified: pypy/branch/py12/pypy/translator/c/src/commondefs.h ============================================================================== --- pypy/branch/py12/pypy/translator/c/src/commondefs.h (original) +++ pypy/branch/py12/pypy/translator/c/src/commondefs.h Sun May 2 18:05:10 2010 @@ -15,6 +15,13 @@ #include +#ifndef LLONG_MAX +#define LLONG_MAX __LONG_LONG_MAX__ +#endif +#ifndef LLONG_MIN +#define LLONG_MIN (-LLONG_MAX - 1LL) +#endif + #if INT_MAX != 2147483647 # error "unsupported value for INT_MAX" #endif @@ -62,9 +69,6 @@ /********************************************************/ -typedef long Py_intptr_t; -typedef unsigned long Py_uintptr_t; - #if ((-1) >> 1) > 0 # define Py_ARITHMETIC_RIGHT_SHIFT(TYPE, I, J) \ ((I) < 0 ? -1-((-1-(I)) >> (J)) : (I) >> (J)) Modified: pypy/branch/py12/pypy/translator/c/src/main.h ============================================================================== --- pypy/branch/py12/pypy/translator/c/src/main.h (original) +++ pypy/branch/py12/pypy/translator/c/src/main.h Sun May 2 18:05:10 2010 @@ -15,11 +15,15 @@ #ifndef PYPY_NOT_MAIN_FILE +#ifndef PYPY_MAIN_FUNCTION +#define PYPY_MAIN_FUNCTION main +#endif + #ifdef MS_WINDOWS #include "src/winstuff.c" #endif -int main(int argc, char *argv[]) +int PYPY_MAIN_FUNCTION(int argc, char *argv[]) { char *errmsg; int i, exitcode; Modified: pypy/branch/py12/pypy/translator/c/src/obmalloc.c ============================================================================== --- pypy/branch/py12/pypy/translator/c/src/obmalloc.c (original) +++ pypy/branch/py12/pypy/translator/c/src/obmalloc.c Sun May 2 18:05:10 2010 @@ -225,7 +225,7 @@ #define ulong unsigned long /* assuming >= 32 bits */ #undef uptr -#define uptr Py_uintptr_t +#define uptr unsigned long /* When you say memory, my mind reasons in terms of (pointers to) blocks */ typedef uchar block; @@ -439,7 +439,7 @@ arenabase = bp; nfreepools = ARENA_SIZE / POOL_SIZE; assert(POOL_SIZE * nfreepools == ARENA_SIZE); - excess = (uint) ((Py_uintptr_t)bp & POOL_SIZE_MASK); + excess = (uint) ((uint)bp & POOL_SIZE_MASK); if (excess != 0) { --nfreepools; arenabase += POOL_SIZE - excess; Modified: pypy/branch/py12/pypy/translator/c/test/test_genc.py ============================================================================== --- pypy/branch/py12/pypy/translator/c/test/test_genc.py (original) +++ pypy/branch/py12/pypy/translator/c/test/test_genc.py Sun May 2 18:05:10 2010 @@ -12,6 +12,7 @@ from pypy.translator.gensupp import uniquemodulename from pypy.translator.backendopt.all import backend_optimizations from pypy.translator.interactive import Translation +from pypy.rlib.entrypoint import entrypoint def compile(fn, argtypes, view=False, gcpolicy="ref", backendopt=True, annotatorpolicy=None): @@ -396,3 +397,85 @@ if py.test.config.option.view: t.view() assert 'pypy_xyz_f' in t.driver.cbuilder.c_source_filename.read() + +def test_entrypoints(): + def f(): + return 3 + + key = "test_entrypoints42" + @entrypoint(key, [int], "foobar") + def g(x): + return x + 42 + + t = Translation(f, [], backend="c", secondaryentrypoints="test_entrypoints42") + t.annotate() + compiled_fn = t.compile_c() + if py.test.config.option.view: + t.view() + assert 'foobar' in t.driver.cbuilder.c_source_filename.read() + +def test_exportstruct(): + from pypy.rlib.exports import export_struct + def f(): + return 42 + FOO = Struct("FOO", ("field1", Signed)) + foo = malloc(FOO, flavor="raw") + foo.field1 = 43 + export_struct("BarStruct", foo._obj) + t = Translation(f, [], backend="c") + t.annotate() + compiled_fn = t.compile_c() + if py.test.config.option.view: + t.view() + assert ' BarStruct ' in t.driver.cbuilder.c_source_filename.read() + +def test_recursive_llhelper(): + from pypy.rpython.annlowlevel import llhelper + from pypy.rpython.lltypesystem import lltype + from pypy.rlib.objectmodel import specialize + from pypy.rlib.nonconst import NonConstant + FT = lltype.ForwardReference() + FTPTR = lltype.Ptr(FT) + STRUCT = lltype.Struct("foo", ("bar", FTPTR)) + FT.become(lltype.FuncType([lltype.Ptr(STRUCT)], lltype.Signed)) + + class A: + def __init__(self, func, name): + self.func = func + self.name = name + def _freeze_(self): + return True + @specialize.memo() + def make_func(self): + f = getattr(self, "_f", None) + if f is not None: + return f + f = lambda *args: self.func(*args) + f.c_name = self.name + f.relax_sig_check = True + f.__name__ = "WRAP%s" % (self.name, ) + self._f = f + return f + def get_llhelper(self): + return llhelper(FTPTR, self.make_func()) + def f(s): + if s.bar == t.bar: + lltype.free(s, flavor="raw") + return 1 + lltype.free(s, flavor="raw") + return 0 + def g(x): + return 42 + def chooser(x): + s = lltype.malloc(STRUCT, flavor="raw") + if x: + s.bar = llhelper(FTPTR, a_f.make_func()) + else: + s.bar = llhelper(FTPTR, a_g.make_func()) + return f(s) + a_f = A(f, "f") + a_g = A(g, "g") + t = lltype.malloc(STRUCT, flavor="raw") + t.bar = llhelper(FTPTR, a_f.make_func()) + fn = compile(chooser, [bool]) + assert fn(True) Modified: pypy/branch/py12/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/py12/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/py12/pypy/translator/c/test/test_newgc.py Sun May 2 18:05:10 2010 @@ -66,6 +66,12 @@ for fullname in dir(cls): if not fullname.startswith('define'): continue + keyword = conftest.option.keyword + if keyword: + if keyword.startswith('test_'): + keyword = keyword[len('test_'):] + if keyword not in fullname: + continue prefix, name = fullname.split('_', 1) definefunc = getattr(cls, fullname) func = definefunc.im_func(cls) Modified: pypy/branch/py12/pypy/translator/c/test/test_standalone.py ============================================================================== --- pypy/branch/py12/pypy/translator/c/test/test_standalone.py (original) +++ pypy/branch/py12/pypy/translator/c/test/test_standalone.py Sun May 2 18:05:10 2010 @@ -16,11 +16,13 @@ class StandaloneTests(object): config = None - def compile(self, entry_point, debug=True): + def compile(self, entry_point, debug=True, shared=False): t = TranslationContext(self.config) t.buildannotator().build_types(entry_point, [s_list_of_strings]) t.buildrtyper().specialize() + t.config.translation.shared = shared + cbuilder = CStandaloneBuilder(t, entry_point, t.config) if debug: cbuilder.generate_source(defines=cbuilder.DEBUG_DEFINES) @@ -587,6 +589,19 @@ # The traceback stops at f() because it's the first function that # captures the AssertionError, which makes the program abort. + def test_shared(self, monkeypatch): + def f(argv): + print len(argv) + def entry_point(argv): + f(argv) + return 0 + t, cbuilder = self.compile(entry_point, shared=True) + assert cbuilder.shared_library_name is not None + assert cbuilder.shared_library_name != cbuilder.executable_name + monkeypatch.setenv('LD_LIBRARY_PATH', + cbuilder.shared_library_name.dirpath()) + out, err = cbuilder.cmdexec("a b") + assert out == "3" class TestMaemo(TestStandalone): def setup_class(cls): Modified: pypy/branch/py12/pypy/translator/driver.py ============================================================================== --- pypy/branch/py12/pypy/translator/driver.py (original) +++ pypy/branch/py12/pypy/translator/driver.py Sun May 2 18:05:10 2010 @@ -12,6 +12,7 @@ import optparse from pypy.tool.udir import udir from pypy.rlib.jit import DEBUG_OFF, DEBUG_DETAILED, DEBUG_PROFILE, DEBUG_STEPS +from pypy.rlib.entrypoint import secondary_entrypoints import py from pypy.tool.ansi_print import ansi_log @@ -93,7 +94,7 @@ if setopts is not None: self.config.set(**setopts) - + self.exe_name = exe_name self.extmod_name = extmod_name @@ -210,12 +211,25 @@ self.entry_point = entry_point self.translator = translator self.libdef = None + self.secondary_entrypoints = [] + + if self.config.translation.secondaryentrypoints: + for key in self.config.translation.secondaryentrypoints.split(","): + try: + points = secondary_entrypoints[key] + except KeyError: + raise KeyError( + "Entrypoints not found. I only know the keys %r." % + (", ".join(secondary_entrypoints.keys()), )) + self.secondary_entrypoints.extend(points) self.translator.driver_instrument_result = self.instrument_result def setup_library(self, libdef, policy=None, extra={}, empty_translator=None): + """ Used by carbon python only. """ self.setup(None, None, policy, extra, empty_translator) self.libdef = libdef + self.secondary_entrypoints = libdef.functions def instrument_result(self, args): backend, ts = self.get_backend_and_type_system() @@ -304,23 +318,26 @@ annmodel.DEBUG = self.config.translation.debug annotator = translator.buildannotator(policy=policy) + if self.secondary_entrypoints is not None: + for func, inputtypes in self.secondary_entrypoints: + if inputtypes == Ellipsis: + continue + rettype = annotator.build_types(func, inputtypes, False) + if self.entry_point: s = annotator.build_types(self.entry_point, self.inputtypes) - - self.sanity_check_annotation() - if self.standalone and s.knowntype != int: - raise Exception("stand-alone program entry point must return an " - "int (and not, e.g., None or always raise an " - "exception).") - annotator.simplify() - return s + translator.entry_point_graph = annotator.bookkeeper.getdesc(self.entry_point).getuniquegraph() else: - assert self.libdef is not None - for func, inputtypes in self.libdef.functions: - annotator.build_types(func, inputtypes) - func.c_name = func.func_name - self.sanity_check_annotation() - annotator.simplify() + s = None + + self.sanity_check_annotation() + if self.entry_point and self.standalone and s.knowntype != int: + raise Exception("stand-alone program entry point must return an " + "int (and not, e.g., None or always raise an " + "exception).") + annotator.simplify() + return s + # task_annotate = taskdef(task_annotate, [], "Annotating&simplifying") @@ -471,7 +488,8 @@ else: from pypy.translator.c.genc import CExtModuleBuilder as CBuilder cbuilder = CBuilder(self.translator, self.entry_point, - config=self.config) + config=self.config, + secondary_entrypoints=self.secondary_entrypoints) cbuilder.stackless = self.config.translation.stackless if not standalone: # xxx more messy cbuilder.modulename = self.extmod_name @@ -521,6 +539,10 @@ exename = mkexename(self.c_entryp) newexename = self.compute_exe_name() shutil.copy(str(exename), str(newexename)) + if self.cbuilder.shared_library_name is not None: + soname = self.cbuilder.shared_library_name + newsoname = newexename.new(basename=soname.basename) + shutil.copy(str(soname), str(newsoname)) self.c_entryp = newexename self.log.info("created: %s" % (self.c_entryp,)) @@ -529,7 +551,10 @@ translator/platform """ cbuilder = self.cbuilder - cbuilder.compile() + kwds = {} + if self.standalone and self.exe_name is not None: + kwds['exe_name'] = self.compute_exe_name().basename + cbuilder.compile(**kwds) if self.standalone: self.c_entryp = cbuilder.executable_name Modified: pypy/branch/py12/pypy/translator/goal/ann_override.py ============================================================================== --- pypy/branch/py12/pypy/translator/goal/ann_override.py (original) +++ pypy/branch/py12/pypy/translator/goal/ann_override.py Sun May 2 18:05:10 2010 @@ -81,7 +81,7 @@ def attach_lookup(pol, t, attr): cached = "cached_%s" % attr - if not t.is_heaptype(): + if not t.is_heaptype() and not t.is_cpytype(): pol._remember_immutable(t, cached) setattr(t, cached, t._lookup(attr)) return True @@ -89,7 +89,7 @@ def attach_lookup_in_type_where(pol, t, attr): cached = "cached_where_%s" % attr - if not t.is_heaptype(): + if not t.is_heaptype() and not t.is_cpytype(): pol._remember_immutable(t, cached) setattr(t, cached, t._lookup_where(attr)) return True @@ -177,14 +177,14 @@ CACHED_LOOKUP = """ def lookup_%(attr)s(space, w_obj, name): w_type = space.type(w_obj) - if not w_type.is_heaptype(): + if not w_type.is_heaptype() and not w_type.is_cpytype(): return w_type.cached_%(attr)s return w_type.lookup("%(attr)s") """ CACHED_LOOKUP_IN_TYPE_WHERE = """ def lookup_in_type_where_%(attr)s(space, w_type, name): - if not w_type.is_heaptype(): + if not w_type.is_heaptype() and not w_type.is_cpytype(): return w_type.cached_where_%(attr)s return w_type.lookup_where("%(attr)s") """ Modified: pypy/branch/py12/pypy/translator/platform/__init__.py ============================================================================== --- pypy/branch/py12/pypy/translator/platform/__init__.py (original) +++ pypy/branch/py12/pypy/translator/platform/__init__.py Sun May 2 18:05:10 2010 @@ -69,7 +69,8 @@ env) return ExecutionResult(returncode, stdout, stderr) - def gen_makefile(self, cfiles, eci, exe_name=None, path=None): + def gen_makefile(self, cfiles, eci, exe_name=None, path=None, + shared=False): raise NotImplementedError("Pure abstract baseclass") def __repr__(self): @@ -104,10 +105,8 @@ errorfile = outname.new(ext='errors') errorfile.write(stderr, 'wb') stderrlines = stderr.splitlines() - for line in stderrlines[:50]: + for line in stderrlines: log.ERROR(line) - if len(stderrlines) > 50: - log.ERROR('...') raise CompilationError(stdout, stderr) else: for line in stderr.splitlines(): Modified: pypy/branch/py12/pypy/translator/platform/darwin.py ============================================================================== --- pypy/branch/py12/pypy/translator/platform/darwin.py (original) +++ pypy/branch/py12/pypy/translator/platform/darwin.py Sun May 2 18:05:10 2010 @@ -18,7 +18,7 @@ self.cc = cc def _args_for_shared(self, args): - return (self.shared_only + ['-bundle', '-undefined', 'dynamic_lookup'] + return (self.shared_only + ['-dynamiclib', '-undefined', 'dynamic_lookup'] + args) def include_dirs_for_libffi(self): Modified: pypy/branch/py12/pypy/translator/platform/linux.py ============================================================================== --- pypy/branch/py12/pypy/translator/platform/linux.py (original) +++ pypy/branch/py12/pypy/translator/platform/linux.py Sun May 2 18:05:10 2010 @@ -7,7 +7,7 @@ name = "linux" link_flags = ['-pthread', '-lrt'] - cflags = ['-O3', '-pthread', '-fomit-frame-pointer', '-Wall'] + cflags = ['-O3', '-pthread', '-fomit-frame-pointer', '-Wall', '-Wno-unused'] standalone_only = [] shared_only = ['-fPIC'] so_ext = 'so' Modified: pypy/branch/py12/pypy/translator/platform/posix.py ============================================================================== --- pypy/branch/py12/pypy/translator/platform/posix.py (original) +++ pypy/branch/py12/pypy/translator/platform/posix.py Sun May 2 18:05:10 2010 @@ -36,6 +36,10 @@ cwd=str(cfile.dirpath())) return oname + def _link_args_from_eci(self, eci, standalone): + eci = eci.convert_exportsymbols_to_file() + return Platform._link_args_from_eci(self, eci, standalone) + def _link(self, cc, ofiles, link_args, standalone, exe_name): args = [str(ofile) for ofile in ofiles] + link_args args += ['-o', str(exe_name)] @@ -55,7 +59,9 @@ # strip compiler flags return [entry[2:] for entry in out.split()] - def gen_makefile(self, cfiles, eci, exe_name=None, path=None): + def gen_makefile(self, cfiles, eci, exe_name=None, path=None, + shared=False): + eci = eci.convert_exportsymbols_to_file() cfiles = [py.path.local(f) for f in cfiles] cfiles += [py.path.local(f) for f in eci.separate_module_files] @@ -67,6 +73,16 @@ if exe_name is None: exe_name = cfiles[0].new(ext=self.exe_ext) + linkflags = self.link_flags + if shared: + linkflags = self._args_for_shared(linkflags) + + if shared: + libname = exe_name.new(ext='').basename + target_name = 'lib' + exe_name.new(ext=self.so_ext).basename + else: + target_name = exe_name.basename + m = GnuMakefile(path) m.exe_name = exe_name m.eci = eci @@ -88,8 +104,8 @@ m.comment('automatically generated makefile') definitions = [ ('PYPYDIR', autopath.pypydir), - ('TARGET', exe_name.basename), - ('DEFAULT_TARGET', '$(TARGET)'), + ('TARGET', target_name), + ('DEFAULT_TARGET', exe_name.basename), ('SOURCES', rel_cfiles), ('OBJECTS', rel_ofiles), ('LIBS', self._libs(eci.libraries)), @@ -97,7 +113,7 @@ ('INCLUDEDIRS', self._includedirs(rel_includedirs)), ('CFLAGS', self.cflags), ('CFLAGSEXTRA', list(eci.compile_extra)), - ('LDFLAGS', self.link_flags), + ('LDFLAGS', linkflags), ('LDFLAGSEXTRA', list(eci.link_extra)), ('CC', self.cc), ('CC_LINK', eci.use_cpp_linker and 'g++' or '$(CC)'), @@ -115,6 +131,16 @@ for rule in rules: m.rule(*rule) + if shared: + m.definition('SHARED_IMPORT_LIB', libname), + m.rule('main.c', '', + 'echo "' + 'int $(PYPY_MAIN_FUNCTION)(int, char*[]); ' + 'int main(int argc, char* argv[]) ' + '{ return $(PYPY_MAIN_FUNCTION)(argc, argv); }" > $@') + m.rule('$(DEFAULT_TARGET)', ['$(TARGET)', 'main.o'], + '$(CC_LINK) main.o -L. -l$(SHARED_IMPORT_LIB) -o $@') + return m def execute_makefile(self, path_to_makefile, extra_opts=[]): Modified: pypy/branch/py12/pypy/translator/platform/windows.py ============================================================================== --- pypy/branch/py12/pypy/translator/platform/windows.py (original) +++ pypy/branch/py12/pypy/translator/platform/windows.py Sun May 2 18:05:10 2010 @@ -186,7 +186,8 @@ raise CompilationError(stdout, stderr) - def gen_makefile(self, cfiles, eci, exe_name=None, path=None): + def gen_makefile(self, cfiles, eci, exe_name=None, path=None, + shared=False): cfiles = [py.path.local(f) for f in cfiles] cfiles += [py.path.local(f) for f in eci.separate_module_files] @@ -202,6 +203,17 @@ m.exe_name = exe_name m.eci = eci + linkflags = self.link_flags + if shared: + linkflags = self._args_for_shared(linkflags) + [ + '/EXPORT:$(PYPY_MAIN_FUNCTION)'] + + if shared: + so_name = exe_name.new(ext=self.so_ext) + target_name = so_name.basename + else: + target_name = exe_name.basename + def pypyrel(fpath): rel = py.path.local(fpath).relto(pypypath) if rel: @@ -218,8 +230,8 @@ m.comment('automatically generated makefile') definitions = [ ('PYPYDIR', autopath.pypydir), - ('TARGET', exe_name.basename), - ('DEFAULT_TARGET', '$(TARGET)'), + ('TARGET', target_name), + ('DEFAULT_TARGET', exe_name.basename), ('SOURCES', rel_cfiles), ('OBJECTS', rel_ofiles), ('LIBS', self._libs(eci.libraries)), @@ -227,12 +239,13 @@ ('INCLUDEDIRS', self._includedirs(rel_includedirs)), ('CFLAGS', self.cflags), ('CFLAGSEXTRA', list(eci.compile_extra)), - ('LDFLAGS', self.link_flags), + ('LDFLAGS', linkflags), ('LDFLAGSEXTRA', list(eci.link_extra)), ('CC', self.cc), ('CC_LINK', self.link), ('MASM', self.masm), ] + for args in definitions: m.definition(*args) @@ -253,6 +266,16 @@ 'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1', ]) + if shared: + m.definition('SHARED_IMPORT_LIB', so_name.new(ext='lib').basename), + m.rule('main.c', '', + 'echo ' + 'int $(PYPY_MAIN_FUNCTION)(int, char*[]); ' + 'int main(int argc, char* argv[]) ' + '{ return $(PYPY_MAIN_FUNCTION)(argc, argv); } > $@') + m.rule('$(DEFAULT_TARGET)', ['$(TARGET)', 'main.obj'], + '$(CC_LINK) /nologo main.obj $(SHARED_IMPORT_LIB) /out:$@') + return m def execute_makefile(self, path_to_makefile, extra_opts=[]): Modified: pypy/branch/py12/pypy/translator/test/test_unsimplify.py ============================================================================== --- pypy/branch/py12/pypy/translator/test/test_unsimplify.py (original) +++ pypy/branch/py12/pypy/translator/test/test_unsimplify.py Sun May 2 18:05:10 2010 @@ -9,6 +9,7 @@ def translate(func, argtypes, type_system="lltype"): t = TranslationContext() t.buildannotator().build_types(func, argtypes) + t.entry_point_graph = graphof(t, func) t.buildrtyper(type_system=type_system).specialize() return graphof(t, func), t Modified: pypy/branch/py12/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/branch/py12/pypy/translator/tool/cbuild.py (original) +++ pypy/branch/py12/pypy/translator/tool/cbuild.py Sun May 2 18:05:10 2010 @@ -260,24 +260,49 @@ d['separate_module_files'] += tuple(files) return ExternalCompilationInfo(**d) + def convert_exportsymbols_to_file(self): + if not self.export_symbols: + return self + num = 0 + while 1: + file_name = udir.join('dynamic-symbols-%i' % num) + num += 1 + if not file_name.check(): + break + + f = file_name.open("w") + f.write("{\n") + for sym in self.export_symbols: + f.write("%s;\n" % (sym,)) + f.write("};") + f.close() + d = self._copy_attributes() + d['link_extra'] += ("-Wl,--dynamic-list=" + str(file_name), ) + d['export_symbols'] = () + return ExternalCompilationInfo(**d) + + def get_module_files(self): d = self._copy_attributes() files = d['separate_module_files'] d['separate_module_files'] = () return files, ExternalCompilationInfo(**d) - def compile_shared_lib(self): + def compile_shared_lib(self, outputfilename=None): self = self.convert_sources_to_files() if not self.separate_module_files: return self - # find more or less unique name there - basepath = py.path.local(self.separate_module_files[0]).dirpath() - pth = basepath.join('externmod').new(ext=host.so_ext) - num = 0 - while pth.check(): - pth = basepath.join('externmod_%d' % (num,)).new(ext=host.so_ext) - num += 1 - lib = str(host.compile([], self, outputfilename=pth.purebasename, + if outputfilename is None: + # find more or less unique name there + basepath = py.path.local(self.separate_module_files[0]).dirpath() + pth = basepath.join('externmod').new(ext=host.so_ext) + num = 0 + while pth.check(): + pth = basepath.join( + 'externmod_%d' % (num,)).new(ext=host.so_ext) + num += 1 + outputfilename=pth.purebasename + lib = str(host.compile([], self, outputfilename=outputfilename, standalone=False)) d = self._copy_attributes() d['libraries'] += (lib,) Modified: pypy/branch/py12/pypy/translator/tool/test/test_cbuild.py ============================================================================== --- pypy/branch/py12/pypy/translator/tool/test/test_cbuild.py (original) +++ pypy/branch/py12/pypy/translator/tool/test/test_cbuild.py Sun May 2 18:05:10 2010 @@ -71,6 +71,17 @@ e = ExternalCompilationInfo() assert e.convert_sources_to_files() is e + def test_convert_sources_to_c_files(self): + eci = ExternalCompilationInfo( + export_symbols=("foo", ) + ) + neweci = eci.convert_exportsymbols_to_file() + le = neweci.link_extra[-1] + assert "foo;" in file(le.rsplit("=", 1)[1]).read() + e = ExternalCompilationInfo() + assert e.convert_exportsymbols_to_file() is e + + def test_make_shared_lib(self): eci = ExternalCompilationInfo( separate_module_sources = [''' Modified: pypy/branch/py12/pypy/translator/unsimplify.py ============================================================================== --- pypy/branch/py12/pypy/translator/unsimplify.py (original) +++ pypy/branch/py12/pypy/translator/unsimplify.py Sun May 2 18:05:10 2010 @@ -168,7 +168,7 @@ if own_annhelper: annhelper.finish() - entry_point = translator.graphs[0] + entry_point = translator.entry_point_graph v = copyvar(translator.annotator, entry_point.getreturnvar()) extrablock = Block([v]) v_none = varoftype(lltype.Void) From fijal at codespeak.net Mon May 3 00:44:29 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 3 May 2010 00:44:29 +0200 (CEST) Subject: [pypy-svn] r74326 - pypy/extradoc/planning Message-ID: <20100502224429.150FD282BD4@codespeak.net> Author: fijal Date: Mon May 3 00:44:28 2010 New Revision: 74326 Modified: pypy/extradoc/planning/jit.txt Log: Things I was thinking about during hiking Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Mon May 3 00:44:28 2010 @@ -14,6 +14,15 @@ does not call can_enter_jit after an iteration in which there was a yield. obviously. +- think about handling hidden registers - mayb we can use couple of + first spots on the stack as registers + +- think again about perfect specialization. Check if we loose anything + if we turn it off. Another approach to specialization: specialize things + in locals even if they're not referenced (that will solve bridges using + new local variables, which I think is pretty common). Specialize things + that are never changed, but read from (an into which is a loop constant + will stay as a getfield). TASKS ----- @@ -31,6 +40,10 @@ _trace_and_drag_out_of_nursery. Requires thinking about card-marking GC, which is hard, postpone + Card marking GC is hard in our model. Think about solutions to chunked + list (a list that if big enough is a list of lists of stuff). Experiments + show that it helps a lot for some examples. + - improve tracing/blackholing speed Essential, especially blackholing in translate.py, as well as html5lib. From fijal at codespeak.net Mon May 3 02:03:04 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 3 May 2010 02:03:04 +0200 (CEST) Subject: [pypy-svn] r74327 - pypy/trunk/pypy/module/cpyext/test Message-ID: <20100503000304.C7A6D282BD4@codespeak.net> Author: fijal Date: Mon May 3 02:03:02 2010 New Revision: 74327 Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Log: A test (skipped) that tries to create a type. I think code in typeobject.py is roughly ok, not sure about the C part. Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Mon May 3 02:03:02 2010 @@ -87,6 +87,14 @@ raises(TypeError, "obj.char_member = 'spam'") raises(TypeError, "obj.char_member = 42") + def test_new(self): + module = self.import_module(name='foo') + obj = module.new() + # call __new__ + newobj = type(obj)() + assert type(newobj) is type(obj) + + XXX # stuff below has to be rethought a = module.fooType assert "cannot create" in raises(TypeError, "a()").value.message class bar(module.fooType): From fijal at codespeak.net Mon May 3 02:04:30 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 3 May 2010 02:04:30 +0200 (CEST) Subject: [pypy-svn] r74328 - pypy/trunk/pypy/module/cpyext/test Message-ID: <20100503000430.2E894282BD4@codespeak.net> Author: fijal Date: Mon May 3 02:04:28 2010 New Revision: 74328 Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Log: Skip the test Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Mon May 3 02:04:28 2010 @@ -88,6 +88,7 @@ raises(TypeError, "obj.char_member = 42") def test_new(self): + skip("skip") module = self.import_module(name='foo') obj = module.new() # call __new__ From fijal at codespeak.net Mon May 3 02:04:49 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 3 May 2010 02:04:49 +0200 (CEST) Subject: [pypy-svn] r74329 - pypy/trunk/pypy/module/cpyext/test Message-ID: <20100503000449.6ADA6282BD8@codespeak.net> Author: fijal Date: Mon May 3 02:04:47 2010 New Revision: 74329 Modified: pypy/trunk/pypy/module/cpyext/test/foo.c Log: This belongs to previous commit Modified: pypy/trunk/pypy/module/cpyext/test/foo.c ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/foo.c (original) +++ pypy/trunk/pypy/module/cpyext/test/foo.c Mon May 3 02:04:47 2010 @@ -175,6 +175,12 @@ {NULL} /* Sentinel */ }; +static PyObject * +Fuu_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds) +{ + return subtype->tp_alloc(subtype, 0); +} + PyTypeObject FuuType = { PyObject_HEAD_INIT(NULL) 0, @@ -224,7 +230,7 @@ 0, /*tp_init*/ 0, /*tp_alloc will be set to PyType_GenericAlloc in module init*/ - 0, /*tp_new*/ + Fuu_new, /*tp_new*/ 0, /*tp_free Low-level free-memory routine */ 0, /*tp_is_gc For PyObject_IS_GC */ 0, /*tp_bases*/ @@ -249,7 +255,6 @@ return (PyObject *)foop; } - /* List of functions exported by this module */ static PyMethodDef foo_functions[] = { From arigo at codespeak.net Mon May 3 10:22:49 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 3 May 2010 10:22:49 +0200 (CEST) Subject: [pypy-svn] r74331 - in pypy/branch/blackhole-improvement/pypy/jit: backend backend/llgraph codewriter codewriter/test metainterp Message-ID: <20100503082249.54446282BD8@codespeak.net> Author: arigo Date: Mon May 3 10:22:48 2010 New Revision: 74331 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py pypy/branch/blackhole-improvement/pypy/jit/backend/model.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/jitprof.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/typesystem.py Log: General progress. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py Mon May 3 10:22:48 2010 @@ -278,16 +278,6 @@ self.class_vtables_as_int[descr] = vtable_int return descr - def cast_adr_to_int(self, adr): - return llimpl.cast_adr_to_int(self.memo_cast, adr) - - def cast_int_to_adr(self, int): - return llimpl.cast_int_to_adr(self.memo_cast, int) - - def cast_gcref_to_int(self, gcref): - return self.cast_adr_to_int(llmemory.cast_ptr_to_adr(gcref)) - - class LLtypeCPU(BaseCPU): is_oo = False @@ -377,8 +367,6 @@ def bh_getfield_gc_i(self, struct, fielddescr): assert isinstance(fielddescr, Descr) return llimpl.do_getfield_gc_int(struct, fielddescr.ofs) - bh_getfield_gc_c = bh_getfield_gc_i - bh_getfield_gc_u = bh_getfield_gc_i def bh_getfield_gc_r(self, struct, fielddescr): assert isinstance(fielddescr, Descr) return llimpl.do_getfield_gc_ptr(struct, fielddescr.ofs) @@ -407,8 +395,6 @@ def bh_getfield_raw_i(self, struct, fielddescr): assert isinstance(fielddescr, Descr) return llimpl.do_getfield_raw_int(struct, fielddescr.ofs) - bh_getfield_raw_c = bh_getfield_raw_i - bh_getfield_raw_u = bh_getfield_raw_i def bh_getfield_raw_r(self, struct, fielddescr): assert isinstance(fielddescr, Descr) return llimpl.do_getfield_raw_ptr(struct, fielddescr.ofs) @@ -485,8 +471,6 @@ def bh_setfield_gc_i(self, struct, fielddescr, newvalue): assert isinstance(fielddescr, Descr) llimpl.do_setfield_gc_int(struct, fielddescr.ofs, newvalue) - bh_setfield_gc_c = bh_setfield_gc_i - bh_setfield_gc_u = bh_setfield_gc_i def bh_setfield_gc_r(self, struct, fielddescr, newvalue): assert isinstance(fielddescr, Descr) llimpl.do_setfield_gc_ptr(struct, fielddescr.ofs, newvalue) @@ -515,8 +499,6 @@ def bh_setfield_raw_i(self, struct, fielddescr, newvalue): assert isinstance(fielddescr, Descr) llimpl.do_setfield_raw_int(struct, fielddescr.ofs, newvalue) - bh_setfield_raw_c = bh_setfield_raw_i - bh_setfield_raw_u = bh_setfield_raw_i def bh_setfield_raw_r(self, struct, fielddescr, newvalue): assert isinstance(fielddescr, Descr) llimpl.do_setfield_raw_ptr(struct, fielddescr.ofs, newvalue) Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/model.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/model.py Mon May 3 10:22:48 2010 @@ -3,6 +3,7 @@ class AbstractCPU(object): supports_floats = False + _got_exception = None # assembler_helper_ptr - a pointer to helper to call after a direct # assembler call portal_calldescr = None @@ -157,10 +158,6 @@ def bh_getfield_gc_i(self, struct, fielddescr): raise NotImplementedError - def bh_getfield_gc_c(self, struct, fielddescr): - raise NotImplementedError - def bh_getfield_gc_u(self, struct, fielddescr): - raise NotImplementedError def bh_getfield_gc_r(self, struct, fielddescr): raise NotImplementedError def bh_getfield_gc_f(self, struct, fielddescr): @@ -168,10 +165,6 @@ def bh_getfield_raw_i(self, struct, fielddescr): raise NotImplementedError - def bh_getfield_raw_c(self, struct, fielddescr): - raise NotImplementedError - def bh_getfield_raw_u(self, struct, fielddescr): - raise NotImplementedError def bh_getfield_raw_r(self, struct, fielddescr): raise NotImplementedError def bh_getfield_raw_f(self, struct, fielddescr): @@ -196,10 +189,6 @@ def bh_setfield_gc_i(self, struct, fielddescr, newvalue): raise NotImplementedError - def bh_setfield_gc_c(self, struct, fielddescr, newvalue): - raise NotImplementedError - def bh_setfield_gc_u(self, struct, fielddescr, newvalue): - raise NotImplementedError def bh_setfield_gc_r(self, struct, fielddescr, newvalue): raise NotImplementedError def bh_setfield_gc_f(self, struct, fielddescr, newvalue): @@ -207,10 +196,6 @@ def bh_setfield_raw_i(self, struct, fielddescr, newvalue): raise NotImplementedError - def bh_setfield_raw_c(self, struct, fielddescr, newvalue): - raise NotImplementedError - def bh_setfield_raw_u(self, struct, fielddescr, newvalue): - raise NotImplementedError def bh_setfield_raw_r(self, struct, fielddescr, newvalue): raise NotImplementedError def bh_setfield_raw_f(self, struct, fielddescr, newvalue): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Mon May 3 10:22:48 2010 @@ -156,6 +156,8 @@ # An exception block. See test_exc_exitswitch in test_flatten.py # for an example of what kind of code this makes. lastopname = block.operations[-1].opname + if lastopname == '-live-': + lastopname = block.operations[-2].opname assert block.exits[0].exitcase is None # is this always True? self.emitline('catch_exception', TLabel(block.exits[0])) self.make_link(block.exits[0]) @@ -238,11 +240,13 @@ # A switch with several possible answers, though not too # many of them -- a chain of int_eq comparisons is fine assert kind == 'int' # XXX + color = self.getcolor(block.exitswitch) + self.emitline('int_guard_value', color, color) for switch in switches: # make the case described by 'switch' self.emitline('goto_if_not_int_eq', TLabel(switch), - self.getcolor(block.exitswitch), + color, Constant(switch.llexitcase, block.exitswitch.concretetype)) # emit code for the "taken" path Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Mon May 3 10:22:48 2010 @@ -190,7 +190,6 @@ def rewrite_op_hint(self, op): hints = op.args[1].value if hints.get('promote') and op.args[0].concretetype is not lltype.Void: - #self.minimize_variables() assert op.args[0].concretetype != lltype.Ptr(rstr.STR) kind = getkind(op.args[0].concretetype) return SpaceOperation('%s_guard_value' % kind, @@ -274,10 +273,7 @@ argname = getattr(v_inst.concretetype.TO, '_gckind', 'gc') descr = self.cpu.fielddescrof(v_inst.concretetype.TO, c_fieldname.value) - if isinstance(RESULT, lltype.Primitive): - kind = primitive_type_size[RESULT] - else: - kind = getkind(RESULT)[0] + kind = getkind(RESULT)[0] return SpaceOperation('getfield_%s_%s%s' % (argname, kind, pure), [v_inst, descr], op.result) @@ -302,10 +298,7 @@ argname = getattr(v_inst.concretetype.TO, '_gckind', 'gc') descr = self.cpu.fielddescrof(v_inst.concretetype.TO, c_fieldname.value) - if isinstance(RESULT, lltype.Primitive): - kind = primitive_type_size[RESULT] - else: - kind = getkind(RESULT)[0] + kind = getkind(RESULT)[0] return SpaceOperation('setfield_%s_%s' % (argname, kind), [v_inst, descr, v_value], None) @@ -406,12 +399,3 @@ return result _rewrite_ops = _with_prefix('rewrite_op_') - -primitive_type_size = { - lltype.Signed: 'i', - lltype.Unsigned: 'i', - lltype.Bool: 'c', - lltype.Char: 'c', - lltype.UniChar: 'u', - lltype.Float: 'f', - } Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py Mon May 3 10:22:48 2010 @@ -13,6 +13,7 @@ DEFAULT_OPNAMES_REQUIRING_LIVENESS = [ 'residual_call_', '(int|ref|float)_guard_value', + 'guard_class', ] # ____________________________________________________________ Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Mon May 3 10:22:48 2010 @@ -3,7 +3,6 @@ from pypy.jit.codewriter.flatten import flatten_graph, reorder_renaming_list from pypy.jit.codewriter.flatten import GraphFlattener, ListOfKind, Register from pypy.jit.codewriter.format import assert_format -from pypy.jit.codewriter.jitter import transform_graph from pypy.jit.metainterp.history import AbstractDescr from pypy.rpython.lltypesystem import lltype, rclass, rstr from pypy.objspace.flow.model import SpaceOperation, Variable, Constant @@ -72,10 +71,15 @@ self.rtyper = support.annotate(func, values, type_system=type_system) return self.rtyper.annotator.translator.graphs - def encoding_test(self, func, args, expected, transform=False): + def encoding_test(self, func, args, expected, + transform=False, liveness=False): graphs = self.make_graphs(func, args) if transform: + from pypy.jit.codewriter.jitter import transform_graph transform_graph(graphs[0], FakeCPU()) + if liveness: + from pypy.jit.codewriter.liveness import compute_liveness + compute_liveness(graphs[0]) ssarepr = flatten_graph(graphs[0], fake_regallocs()) assert_format(ssarepr, expected) @@ -187,6 +191,7 @@ elif n == 7: return 1212 else: return 42 self.encoding_test(f, [65], """ + int_guard_value %i0, %i0 goto_if_not_int_eq L1, %i0, $-5 int_return $12 L1: @@ -376,8 +381,9 @@ return 42 self.encoding_test(f, [7, 2], """ int_add_ovf %i0, %i1, %i2 + -live- %i2 catch_exception L1 int_return %i2 L1: int_return $42 - """, transform=True) + """, transform=True, liveness=True) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Mon May 3 10:22:48 2010 @@ -161,9 +161,9 @@ ('ps1', 'i'), ('ps2', 'r'), ('flt', 'f'), - ('boo', 'c'), - ('chr', 'c'), - ('unc', 'u')]: + ('boo', 'i'), + ('chr', 'i'), + ('unc', 'i')]: v_parent = varoftype(lltype.Ptr(S)) c_name = Constant(name, lltype.Void) v_result = varoftype(getattr(S, name)) @@ -199,9 +199,9 @@ ('ps1', 'i'), ('ps2', 'r'), ('flt', 'f'), - ('boo', 'c'), - ('chr', 'c'), - ('unc', 'u')]: + ('boo', 'i'), + ('chr', 'i'), + ('unc', 'i')]: v_parent = varoftype(lltype.Ptr(S)) c_name = Constant(name, lltype.Void) v_newvalue = varoftype(getattr(S, name)) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Mon May 3 10:22:48 2010 @@ -30,6 +30,25 @@ NULL = lltype.nullptr(llmemory.GCREF.TO) +def get_standard_error_llexception(rtyper, Class): + exdata = rtyper.getexceptiondata() + clsdef = rtyper.annotator.bookkeeper.getuniqueclassdef(Class) + evalue = exdata.get_standard_ll_exc_instance(rtyper, clsdef) + etype = rclass.ll_type(evalue) + return LLException(etype, evalue) + +def get_llexception(cpu, e): + if we_are_translated(): + return e + if isinstance(e, LLException): + return e # ok + if isinstance(e, OverflowError): + return get_standard_error_llexception(cpu.rtyper, + OverflowError) + raise # leave other exceptions be propagated + +# ____________________________________________________________ + class BlackholeInterpBuilder(object): verbose = True @@ -258,13 +277,7 @@ #except JitException: # ... except Exception, e: - if not we_are_translated(): - if isinstance(e, LLException): - pass # ok - elif isinstance(e, OverflowError): - e = self.get_standard_error_exception(OverflowError) - else: - raise # leave other exceptions be propagated + e = get_llexception(self.cpu, e) position = self.handle_exception_in_frame(e, code) def get_result_i(self): @@ -309,14 +322,6 @@ target = ord(code[position+1]) | (ord(code[position+2])<<8) return target - def get_standard_error_exception(self, Class): - rtyper = self.cpu.rtyper - exdata = rtyper.getexceptiondata() - clsdef = rtyper.annotator.bookkeeper.getuniqueclassdef(Class) - evalue = exdata.get_standard_ll_exc_instance(rtyper, clsdef) - etype = rclass.ll_type(evalue) - return LLException(etype, evalue) - # XXX must be specialized # XXX the real performance impact of the following loop is unclear def copy_constants(self, registers, constants): @@ -757,12 +762,6 @@ @arguments("cpu", "r", "d", returns="i") def bhimpl_getfield_gc_i(cpu, struct, fielddescr): return cpu.bh_getfield_gc_i(struct, fielddescr) - @arguments("cpu", "r", "d", returns="i") - def bhimpl_getfield_gc_c(cpu, struct, fielddescr): - return cpu.bh_getfield_gc_c(struct, fielddescr) - @arguments("cpu", "r", "d", returns="i") - def bhimpl_getfield_gc_u(cpu, struct, fielddescr): - return cpu.bh_getfield_gc_u(struct, fielddescr) @arguments("cpu", "r", "d", returns="r") def bhimpl_getfield_gc_r(cpu, struct, fielddescr): return cpu.bh_getfield_gc_r(struct, fielddescr) @@ -771,20 +770,12 @@ return cpu.bh_getfield_gc_f(struct, fielddescr) bhimpl_getfield_gc_i_pure = bhimpl_getfield_gc_i - bhimpl_getfield_gc_c_pure = bhimpl_getfield_gc_c - bhimpl_getfield_gc_u_pure = bhimpl_getfield_gc_u bhimpl_getfield_gc_r_pure = bhimpl_getfield_gc_r bhimpl_getfield_gc_f_pure = bhimpl_getfield_gc_f @arguments("cpu", "i", "d", returns="i") def bhimpl_getfield_raw_i(cpu, struct, fielddescr): return cpu.bh_getfield_raw_i(struct, fielddescr) - @arguments("cpu", "i", "d", returns="i") - def bhimpl_getfield_raw_c(cpu, struct, fielddescr): - return cpu.bh_getfield_raw_c(struct, fielddescr) - @arguments("cpu", "i", "d", returns="i") - def bhimpl_getfield_raw_u(cpu, struct, fielddescr): - return cpu.bh_getfield_raw_u(struct, fielddescr) @arguments("cpu", "i", "d", returns="r") def bhimpl_getfield_raw_r(cpu, struct, fielddescr): return cpu.bh_getfield_raw_r(struct, fielddescr) @@ -793,20 +784,12 @@ return cpu.bh_getfield_raw_f(struct, fielddescr) bhimpl_getfield_raw_i_pure = bhimpl_getfield_raw_i - bhimpl_getfield_raw_c_pure = bhimpl_getfield_raw_c - bhimpl_getfield_raw_u_pure = bhimpl_getfield_raw_u bhimpl_getfield_raw_r_pure = bhimpl_getfield_raw_r bhimpl_getfield_raw_f_pure = bhimpl_getfield_raw_f @arguments("cpu", "r", "d", "i") def bhimpl_setfield_gc_i(cpu, struct, fielddescr, newvalue): cpu.bh_setfield_gc_i(struct, fielddescr, newvalue) - @arguments("cpu", "r", "d", "i") - def bhimpl_setfield_gc_c(cpu, struct, fielddescr, newvalue): - cpu.bh_setfield_gc_c(struct, fielddescr, newvalue) - @arguments("cpu", "r", "d", "i") - def bhimpl_setfield_gc_u(cpu, struct, fielddescr, newvalue): - cpu.bh_setfield_gc_u(struct, fielddescr, newvalue) @arguments("cpu", "r", "d", "r") def bhimpl_setfield_gc_r(cpu, struct, fielddescr, newvalue): cpu.bh_setfield_gc_r(struct, fielddescr, newvalue) @@ -817,12 +800,6 @@ @arguments("cpu", "i", "d", "i") def bhimpl_setfield_raw_i(cpu, struct, fielddescr, newvalue): cpu.bh_setfield_raw_i(struct, fielddescr, newvalue) - @arguments("cpu", "i", "d", "i") - def bhimpl_setfield_raw_c(cpu, struct, fielddescr, newvalue): - cpu.bh_setfield_raw_c(struct, fielddescr, newvalue) - @arguments("cpu", "i", "d", "i") - def bhimpl_setfield_raw_u(cpu, struct, fielddescr, newvalue): - cpu.bh_setfield_raw_u(struct, fielddescr, newvalue) @arguments("cpu", "i", "d", "r") def bhimpl_setfield_raw_r(cpu, struct, fielddescr, newvalue): cpu.bh_setfield_raw_r(struct, fielddescr, newvalue) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py Mon May 3 10:22:48 2010 @@ -12,10 +12,11 @@ from pypy.jit.metainterp import resoperation from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.blackhole import BlackholeInterpreter, NULL +from pypy.jit.metainterp.blackhole import get_llexception # ____________________________________________________________ -def do_call(cpu, argboxes, descr): +def do_call(metainterp, argboxes, descr): # count the number of arguments of the different types count_i = count_r = count_f = 0 for i in range(1, len(argboxes)): @@ -45,23 +46,41 @@ count_f += 1 # get the function address as an integer func = argboxes[0].getint() + cpu = metainterp.cpu + metainterp.latest_exception_might_be = 'E' # any Exception # do the call using the correct function from the cpu rettype = descr.get_return_type() if rettype == INT: - result = cpu.bh_call_i(func, descr, args_i, args_r, args_f) + try: + result = cpu.bh_call_i(func, descr, args_i, args_r, args_f) + except Exception, e: + metainterp.got_exception = get_llexception(cpu, e) + result = 0 return BoxInt(result) if rettype == REF: - result = cpu.bh_call_r(func, descr, args_i, args_r, args_f) + try: + result = cpu.bh_call_r(func, descr, args_i, args_r, args_f) + except Exception, e: + metainterp.got_exception = get_llexception(cpu, e) + result = NULL return BoxPtr(result) if rettype == FLOAT: - result = cpu.bh_call_f(func, descr, args_i, args_r, args_f) + try: + result = cpu.bh_call_f(func, descr, args_i, args_r, args_f) + except Exception, e: + metainterp.got_exception = get_llexception(cpu, e) + result = 0.0 return BoxFloat(result) if rettype == 'v': # void - cpu.bh_call_v(func, descr, args_i, args_r, args_f) + try: + cpu.bh_call_v(func, descr, args_i, args_r, args_f) + except Exception, e: + metainterp.got_exception = get_llexception(cpu, e) return None raise AssertionError("bad rettype") -def do_setarrayitem_gc(cpu, arraybox, indexbox, itembox, arraydescr): +def do_setarrayitem_gc(metainterp, arraybox, indexbox, itembox, arraydescr): + cpu = metainterp.cpu array = arraybox.getref_base() index = indexbox.getint() if itembox.type == INT: @@ -76,6 +95,59 @@ else: raise AssertionError("bad itembox.type") +def do_getfield_gc(metainterp, structbox, fielddescr): + cpu = metainterp.cpu + struct = structbox.getref_base() + if fielddescr.is_pointer_field(): + return BoxPtr(cpu.bh_getfield_gc_p(struct, fielddescr)) + elif fielddescr.is_float_field(): + return BoxFloat(cpu.bh_getfield_gc_f(struct, fielddescr)) + else: + return BoxInt(cpu.bh_getfield_gc_i(struct, fielddescr)) + +def do_getfield_raw(metainterp, structbox, fielddescr): + cpu = metainterp.cpu + struct = structbox.getint() + if fielddescr.is_pointer_field(): + return BoxPtr(cpu.bh_getfield_raw_p(struct, fielddescr)) + elif fielddescr.is_float_field(): + return BoxFloat(cpu.bh_getfield_raw_f(struct, fielddescr)) + else: + return BoxInt(cpu.bh_getfield_raw_i(struct, fielddescr)) + +def do_int_add_ovf(metainterp, box1, box2): + metainterp.latest_exception_might_be = 'O' # only OverflowError + a = box1.getint() + b = box2.getint() + try: + z = ovfcheck(a + b) + except OverflowError, e: + metainterp.got_exception = get_llexception(metainterp.cpu, e) + z = 0 + return BoxInt(z) + +def do_int_sub_ovf(metainterp, box1, box2): + metainterp.latest_exception_might_be = 'O' # only OverflowError + a = box1.getint() + b = box2.getint() + try: + z = ovfcheck(a - b) + except OverflowError, e: + metainterp.got_exception = get_llexception(metainterp.cpu, e) + z = 0 + return BoxInt(z) + +def do_int_mul_ovf(metainterp, box1, box2): + metainterp.latest_exception_might_be = 'O' # only OverflowError + a = box1.getint() + b = box2.getint() + try: + z = ovfcheck(a * b) + except OverflowError, e: + metainterp.got_exception = get_llexception(metainterp.cpu, e) + z = 0 + return BoxInt(z) + # ____________________________________________________________ ##def do_force_token(cpu): @@ -165,11 +237,11 @@ argtypes = unrolling_iterable(func.argtypes) resulttype = func.resulttype # - def do(cpu, *argboxes): + def do(metainterp, *argboxes): newargs = () for argtype in argtypes: if argtype == 'cpu': - value = cpu + value = metainterp.cpu elif argtype == 'd': value = argboxes[-1] argboxes = argboxes[:-1] @@ -211,25 +283,25 @@ has_descr._annspecialcase_ = 'specialize:memo' -def execute(cpu, opnum, descr, *argboxes): +def execute(metainterp, opnum, descr, *argboxes): # only for opnums with a fixed arity if has_descr(opnum): check_descr(descr) argboxes = argboxes + (descr,) else: assert descr is None - func = get_execute_function(cpu, opnum, len(argboxes)) + func = get_execute_function(metainterp.cpu, opnum, len(argboxes)) assert func is not None - return func(cpu, *argboxes) # note that the 'argboxes' tuple - # optionally ends with the descr + return func(metainterp, *argboxes) # note that the 'argboxes' tuple + # optionally ends with the descr execute._annspecialcase_ = 'specialize:arg(1)' -def execute_varargs(cpu, opnum, argboxes, descr): +def execute_varargs(metainterp, opnum, argboxes, descr): # only for opnums with a variable arity (calls, typically) check_descr(descr) - func = get_execute_function(cpu, opnum, -1) + func = get_execute_function(metainterp.cpu, opnum, -1) assert func is not None - return func(cpu, argboxes, descr) + return func(metainterp, argboxes, descr) execute_varargs._annspecialcase_ = 'specialize:arg(1)' Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/jitprof.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/jitprof.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/jitprof.py Mon May 3 10:22:48 2010 @@ -13,7 +13,6 @@ BLACKHOLE OPS RECORDED_OPS -BLACKHOLED_OPS GUARDS OPT_OPS OPT_GUARDS Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Mon May 3 10:22:48 2010 @@ -11,7 +11,7 @@ from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp import executor from pypy.jit.metainterp.logger import Logger -from pypy.jit.metainterp.jitprof import BLACKHOLED_OPS, EmptyProfiler +from pypy.jit.metainterp.jitprof import EmptyProfiler from pypy.jit.metainterp.jitprof import GUARDS, RECORDED_OPS, ABORT_ESCAPE from pypy.jit.metainterp.jitprof import ABORT_TOO_LONG, ABORT_BRIDGE from pypy.rlib.rarithmetic import intmask @@ -19,6 +19,7 @@ from pypy.rlib.jit import DEBUG_OFF, DEBUG_PROFILE, DEBUG_STEPS, DEBUG_DETAILED from pypy.jit.metainterp.compile import GiveUp from pypy.jit.codewriter.assembler import JitCode +from pypy.jit.codewriter.flatten import SwitchDictDescr # ____________________________________________________________ @@ -42,8 +43,13 @@ class MIFrame(object): - exception_box = None - exc_value_box = None + + last_exception = None # the most recently raised-and-caught exception + # ^^^ not to be confused with MetaInterp.got_exception. The idea is that + # got_exception is only briefly != None; as soon as the exception + # is caught by opimpl_catch_exception, it is reset to None and + # transferred to last_exception here. + # for resume.py operation parent_resumedata_snapshot = None parent_resumedata_frame_info_list = None @@ -128,6 +134,7 @@ # ------------------------------ for _opimpl in ['int_add', 'int_sub', 'int_mul', 'int_floordiv', 'int_mod', + 'int_add_ovf', 'int_sub_ovf', 'int_mul_ovf', 'int_lt', 'int_le', 'int_eq', 'int_ne', 'int_gt', 'int_ge', 'int_and', 'int_or', 'int_xor', @@ -144,14 +151,6 @@ return self.execute(rop.%s, b1, b2) ''' % (_opimpl, _opimpl.upper())).compile() - for _opimpl in ['int_add_ovf', 'int_sub_ovf', 'int_mul_ovf']: - exec py.code.Source(''' - @XXX #arguments("box", "box") - def opimpl_%s(self, b1, b2): - self.execute(rop.%s, b1, b2) - return self.metainterp.handle_overflow_error() - ''' % (_opimpl, _opimpl.upper())).compile() - for _opimpl in ['int_is_true', 'int_neg', 'int_invert', 'bool_not', 'cast_ptr_to_int', 'cast_float_to_int', 'cast_int_to_float', 'float_neg', 'float_abs', @@ -175,11 +174,62 @@ def opimpl_void_return(self): self.metainterp.finishframe(None) - @arguments("label") - def opimpl_catch_exception(self, target): - pass # see comment in blackhole.py:opimpl_catch_exception. + @arguments("box") + def _opimpl_any_copy(self, box): + return box - @XXX #arguments("jumptarget") + opimpl_int_copy = _opimpl_any_copy + opimpl_ref_copy = _opimpl_any_copy + opimpl_float_copy = _opimpl_any_copy + + @arguments("box") + def _opimpl_any_push(self, box): + self.pushed_box = box + + opimpl_int_push = _opimpl_any_push + opimpl_ref_push = _opimpl_any_push + opimpl_float_push = _opimpl_any_push + + @arguments() + def _opimpl_any_pop(self): + box = self.pushed_box + self.pushed_box = None + return box + + opimpl_int_pop = _opimpl_any_pop + opimpl_ref_pop = _opimpl_any_pop + opimpl_float_pop = _opimpl_any_pop + + @arguments("orgpc", "catched_exception", "label") + def opimpl_catch_exception(self, pc, catched_exception, target): + if catched_exception is not None: + self.last_exception = catched_exception + self.pc = target + # + extraargs = [] + latest_exception_might_be = self.metainterp.latest_exception_might_be + if not we_are_translated(): + del self.metainterp.latest_exception_might_be + if latest_exception_might_be == 'O': + # only for int_xxx_ovf operations + if catched_exception is None: + opnum = rop.GUARD_NO_OVERFLOW + else: + opnum = rop.GUARD_OVERFLOW + else: + # for all residual calls + if catched_exception is None: + opnum = rop.GUARD_NO_EXCEPTION + else: + etype = catched_exception.typeptr # XXX use the typesystem + etype = llmemory.cast_ptr_to_adr(etype) + etype = llmemory.cast_adr_to_int(etype) + exception_box = self.metainterp.cpu.ts.get_exception_box(etype) + extraargs = [exception_box] + opnum = rop.GUARD_EXCEPTION + self.generate_guard(pc, opnum, extraargs=extraargs) + + @arguments("label") def opimpl_goto(self, target): self.pc = target @@ -236,22 +286,13 @@ self.load_int() # past the 'box' argument self.ignore_varargs() # past the 'livelist' argument - @XXX #arguments("orgpc", "box", "constargs", "jumptargets") - def opimpl_switch(self, pc, valuebox, constargs, jumptargets): - box = self.implement_guard_value(pc, valuebox) - for i in range(len(constargs)): - casebox = constargs[i] - if box.same_constant(casebox): - self.pc = jumptargets[i] - break - - @XXX #arguments("orgpc", "box", "constbox") - def opimpl_switch_dict(self, pc, valuebox, switchdict): + @arguments("orgpc", "box", "descr") + def opimpl_switch(self, pc, valuebox, switchdict): box = self.implement_guard_value(pc, valuebox) - search_value = box.getint() - assert isinstance(switchdict, codewriter.SwitchDict) + switchvalue = box.getint() + assert isinstance(switchdict, SwitchDictDescr) try: - self.pc = switchdict.dict[search_value] + self.pc = switchdict.dict[switchvalue] except KeyError: pass @@ -436,28 +477,47 @@ def opimpl_ptr_ne(self, box1, box2): self.execute(rop.OOISNOT, box1, box2) - opimpl_oois = opimpl_ptr_eq - opimpl_ooisnot = opimpl_ptr_ne - - @XXX #arguments("box", "descr") - def opimpl_getfield_gc(self, box, fielddesc): - self.execute_with_descr(rop.GETFIELD_GC, fielddesc, box) - @XXX #arguments("box", "descr") - def opimpl_getfield_gc_pure(self, box, fielddesc): - self.execute_with_descr(rop.GETFIELD_GC_PURE, fielddesc, box) - @XXX #arguments("box", "descr", "box") - def opimpl_setfield_gc(self, box, fielddesc, valuebox): - self.execute_with_descr(rop.SETFIELD_GC, fielddesc, box, valuebox) - - @XXX #arguments("box", "descr") - def opimpl_getfield_raw(self, box, fielddesc): - self.execute_with_descr(rop.GETFIELD_RAW, fielddesc, box) - @XXX #arguments("box", "descr") - def opimpl_getfield_raw_pure(self, box, fielddesc): - self.execute_with_descr(rop.GETFIELD_RAW_PURE, fielddesc, box) - @XXX #arguments("box", "descr", "box") - def opimpl_setfield_raw(self, box, fielddesc, valuebox): - self.execute_with_descr(rop.SETFIELD_RAW, fielddesc, box, valuebox) + @arguments("box", "descr") + def _opimpl_getfield_gc_any(self, box, fielddescr): + return self.execute_with_descr(rop.GETFIELD_GC, fielddescr, box) + opimpl_getfield_gc_i = _opimpl_getfield_gc_any + opimpl_getfield_gc_r = _opimpl_getfield_gc_any + opimpl_getfield_gc_f = _opimpl_getfield_gc_any + + @arguments("box", "descr") + def _opimpl_getfield_gc_pure_any(self, box, fielddescr): + return self.execute_with_descr(rop.GETFIELD_GC_PURE, fielddescr, box) + opimpl_getfield_gc_i_pure = _opimpl_getfield_gc_pure_any + opimpl_getfield_gc_r_pure = _opimpl_getfield_gc_pure_any + opimpl_getfield_gc_f_pure = _opimpl_getfield_gc_pure_any + + @arguments("box", "descr", "box") + def _opimpl_setfield_gc_any(self, box, fielddescr, valuebox): + self.execute_with_descr(rop.SETFIELD_GC, fielddescr, box, valuebox) + opimpl_setfield_gc_i = _opimpl_setfield_gc_any + opimpl_setfield_gc_r = _opimpl_setfield_gc_any + opimpl_setfield_gc_f = _opimpl_setfield_gc_any + + @arguments("box", "descr") + def _opimpl_getfield_raw_any(self, box, fielddescr): + return self.execute_with_descr(rop.GETFIELD_RAW, fielddescr, box) + opimpl_getfield_raw_i = _opimpl_getfield_raw_any + opimpl_getfield_raw_r = _opimpl_getfield_raw_any + opimpl_getfield_raw_f = _opimpl_getfield_raw_any + + @arguments("box", "descr") + def _opimpl_getfield_raw_pure_any(self, box, fielddescr): + return self.execute_with_descr(rop.GETFIELD_RAW_PURE, fielddescr, box) + opimpl_getfield_raw_i_pure = _opimpl_getfield_raw_pure_any + opimpl_getfield_raw_r_pure = _opimpl_getfield_raw_pure_any + opimpl_getfield_raw_f_pure = _opimpl_getfield_raw_pure_any + + @arguments("box", "descr", "box") + def _opimpl_setfield_raw_any(self, box, fielddescr, valuebox): + self.execute_with_descr(rop.SETFIELD_RAW, fielddescr, box, valuebox) + opimpl_setfield_raw_i = _opimpl_setfield_raw_any + opimpl_setfield_raw_r = _opimpl_setfield_raw_any + opimpl_setfield_raw_f = _opimpl_setfield_raw_any def _nonstandard_virtualizable(self, pc, box): # returns True if 'box' is actually not the "standard" virtualizable @@ -742,12 +802,11 @@ constbox = self.implement_guard_value(pc, box) self.env[boxindex] = constbox - @XXX #arguments("orgpc", "box") + @arguments("orgpc", "box") def opimpl_guard_class(self, pc, box): clsbox = self.cls_of_box(box) - if isinstance(box, Box): - self.generate_guard(pc, rop.GUARD_CLASS, box, [clsbox]) - self.make_result_box(clsbox) + self.generate_guard(pc, rop.GUARD_CLASS, box, [clsbox]) + return clsbox ## @XXX #arguments("orgpc", "box", "builtin") ## def opimpl_guard_builtin(self, pc, box, builtin): @@ -835,37 +894,26 @@ def opimpl_teardown_exception_block(self): self.exception_target = -1 - @XXX #arguments("constbox", "jumptarget", "orgpc") - def opimpl_goto_if_exception_mismatch(self, vtableref, next_exc_target, pc): - # XXX used to be: - # assert isinstance(self.exception_box, Const) # XXX - # seems this can happen that self.exception_box is not a Const, - # but I failed to write a test so far :-( - self.exception_box = self.implement_guard_value(pc, self.exception_box) - cpu = self.metainterp.cpu - ts = self.metainterp.cpu.ts - if not ts.subclassOf(cpu, self.exception_box, vtableref): + @arguments("box", "label") + def opimpl_goto_if_exception_mismatch(self, vtablebox, next_exc_target): + # XXX use typesystem.py + from pypy.rpython.lltypesystem import rclass + assert self.last_exception is not None + adr = vtablebox.getaddr(self.metainterp.cpu) + bounding_class = llmemory.cast_adr_to_ptr(adr, rclass.CLASSTYPE) + if not rclass.ll_isinstance(self.last_exception, bounding_class): self.pc = next_exc_target - @XXX #arguments("int") - def opimpl_put_last_exception(self, index): - assert index >= 0 - self.env.insert(index, self.exception_box) - - @XXX #arguments("int") - def opimpl_put_last_exc_value(self, index): - assert index >= 0 - self.env.insert(index, self.exc_value_box) - @XXX #arguments() def opimpl_raise(self): assert len(self.env) == 2 return self.metainterp.finishframe_exception(self.env[0], self.env[1]) - @XXX #arguments() + @arguments() def opimpl_reraise(self): - return self.metainterp.finishframe_exception(self.exception_box, - self.exc_value_box) + assert self.last_exception is not None + self.metainterp.got_exception = self.last_exception + self.metainterp.propagate_exception_out_of_frame() @XXX #arguments("box") def opimpl_virtual_ref(self, box): @@ -950,7 +998,7 @@ except ChangeFrame: pass - def generate_guard(self, pc, opnum, box, extraargs=[]): + def generate_guard(self, pc, opnum, box=None, extraargs=[]): if isinstance(box, Const): # no need for a guard return metainterp = self.metainterp @@ -1217,6 +1265,7 @@ class MetaInterp(object): in_recursion = 0 + got_exception = None _already_allocated_resume_virtuals = None def __init__(self, staticdata): @@ -1355,15 +1404,11 @@ @specialize.arg(1) def execute_and_record(self, opnum, descr, *argboxes): history.check_descr(descr) - assert (opnum != rop.CALL and opnum != rop.CALL_MAY_FORCE - and opnum != rop.OOSEND) + assert not (rop._CANRAISE_FIRST <= opnum <= rop._CANRAISE_LAST) # execute the operation profiler = self.staticdata.profiler profiler.count_ops(opnum) - resbox = executor.execute(self.cpu, opnum, descr, *argboxes) - if self.is_blackholing(): - profiler.count_ops(opnum, BLACKHOLED_OPS) - return resbox + resbox = executor.execute(self, opnum, descr, *argboxes) if rop._ALWAYS_PURE_FIRST <= opnum <= rop._ALWAYS_PURE_LAST: return self._record_helper_pure(opnum, resbox, descr, *argboxes) else: @@ -1376,7 +1421,7 @@ # execute the operation profiler = self.staticdata.profiler profiler.count_ops(opnum) - resbox = executor.execute_varargs(self.cpu, opnum, argboxes, descr) + resbox = executor.execute_varargs(self, opnum, argboxes, descr) if self.is_blackholing(): profiler.count_ops(opnum, BLACKHOLED_OPS) else: @@ -1848,6 +1893,7 @@ return False def handle_overflow_error(self): + xxx frame = self.framestack[-1] if self.cpu._overflow_flag: self.cpu._overflow_flag = False @@ -2096,10 +2142,19 @@ position = position3 + 1 + length3 elif argtype == "orgpc": value = orgpc + elif argtype == "catched_exception": + value = self.metainterp.got_exception + self.metainterp.got_exception = None else: raise AssertionError("bad argtype: %r" % (argtype,)) args += (value,) # + if self.metainterp.got_exception is not None: + # the previous operation raised and the current operation + # is not catching it. Propagate it. + self.metainterp.propagate_exception_out_of_frame() + return + # num_return_args = len(argcodes) - next_argcode assert num_return_args == 0 or num_return_args == 1 self.pc = position + num_return_args Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/typesystem.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/typesystem.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/typesystem.py Mon May 3 10:22:48 2010 @@ -74,7 +74,7 @@ def cls_of_box(self, cpu, box): obj = box.getref(lltype.Ptr(rclass.OBJECT)) cls = llmemory.cast_ptr_to_adr(obj.typeptr) - return history.ConstInt(cpu.cast_adr_to_int(cls)) + return history.ConstInt(llmemory.cast_adr_to_int(cls)) def subclassOf(self, cpu, clsbox1, clsbox2): adr = clsbox2.getaddr(cpu) @@ -120,7 +120,7 @@ def cast_vtable_to_hashable(self, cpu, ptr): adr = llmemory.cast_ptr_to_adr(ptr) - return cpu.cast_adr_to_int(adr) + return llmemory.cast_adr_to_int(adr) def cast_from_ref(self, TYPE, value): return lltype.cast_opaque_ptr(TYPE, value) From afa at codespeak.net Mon May 3 13:16:45 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 3 May 2010 13:16:45 +0200 (CEST) Subject: [pypy-svn] r74335 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100503111645.5B9AF282BD8@codespeak.net> Author: afa Date: Mon May 3 13:16:43 2010 New Revision: 74335 Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py pypy/trunk/pypy/module/cpyext/typeobject.py Log: Implement PyTypeObject::tp_mro, also add a basic test for tp_alloc. Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Mon May 3 13:16:43 2010 @@ -1,6 +1,8 @@ +from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref +from pypy.module.cpyext.typeobject import PyTypeObjectPtr import py import sys @@ -19,7 +21,7 @@ print "type of obj has type", type(type(obj)) print "type of type of obj has type", type(type(type(obj))) - def test_typeobject2(self): + def test_typeobject_method_descriptor(self): module = self.import_module(name='foo') obj = module.new() obj2 = obj.copy() @@ -37,7 +39,7 @@ assert obj.foo == 42 assert obj.int_member == obj.foo - def test_typeobject3(self): + def test_typeobject_data_member(self): module = self.import_module(name='foo') obj = module.new() obj.int_member = 23 @@ -53,7 +55,7 @@ assert module.fooType.broken_member.__doc__ is None assert module.fooType.object_member.__doc__ == "A Python object." - def test_typeobject4(self): + def test_typeobject_object_member(self): module = self.import_module(name='foo') obj = module.new() assert obj.object_member is None @@ -70,7 +72,7 @@ del obj.object_member_ex raises(AttributeError, "del obj.object_member_ex") - def test_typeobject5(self): + def test_typeobject_string_member(self): module = self.import_module(name='foo') obj = module.new() assert obj.string_member == "Hello from PyPy" @@ -128,6 +130,20 @@ re._cache_repl.clear() class TestTypes(BaseApiTest): + def test_type_attributes(self, space, api): + w_class = space.appexec([], """(): + class A(object): + pass + return A + """) + ref = make_ref(space, w_class) + + py_type = rffi.cast(PyTypeObjectPtr, ref) + assert py_type.c_tp_alloc + assert from_ref(space, py_type.c_tp_mro).wrappeditems is w_class.mro_w + + api.Py_DecRef(ref) + def test_multiple_inheritance(self, space, api): w_class = space.appexec([], """(): class A(object): Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Mon May 3 13:16:43 2010 @@ -428,6 +428,7 @@ type_pto = obj.c_ob_type base_pyo = rffi.cast(PyObject, obj_pto.c_tp_base) Py_DecRef(space, obj_pto.c_tp_bases) + Py_DecRef(space, obj_pto.c_tp_mro) Py_DecRef(space, obj_pto.c_tp_cache) # lets do it like cpython if obj_pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE: if obj_pto.c_tp_as_buffer: @@ -569,7 +570,8 @@ bases = space.newtuple([from_ref(space, base_pyo)]) pto.c_tp_bases = make_ref(space, bases) if w_obj is None: - PyPyType_Register(space, pto) + w_obj = PyPyType_Register(space, pto) + pto.c_tp_mro = make_ref(space, space.newtuple(w_obj.mro_w)) if base: inherit_special(space, pto, base) for w_base in space.fixedview(from_ref(space, pto.c_tp_bases)): @@ -596,7 +598,7 @@ state.py_objects_w2r[w_obj] = pyo w_obj.__init__(space, pto) w_obj.ready() - return 1 + return w_obj @cpython_api([PyTypeObjectPtr, PyTypeObjectPtr], rffi.INT_real, error=CANNOT_FAIL) def PyType_IsSubtype(space, a, b): From afa at codespeak.net Mon May 3 13:18:22 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 3 May 2010 13:18:22 +0200 (CEST) Subject: [pypy-svn] r74336 - pypy/trunk/pypy/module/cpyext/include Message-ID: <20100503111822.B7074282BD8@codespeak.net> Author: afa Date: Mon May 3 13:18:21 2010 New Revision: 74336 Modified: pypy/trunk/pypy/module/cpyext/include/patchlevel.h Log: Add PYPY_VERSION in Python.h, Some extension modules can use it to avoid CPython specific constructs. Modified: pypy/trunk/pypy/module/cpyext/include/patchlevel.h ============================================================================== --- pypy/trunk/pypy/module/cpyext/include/patchlevel.h (original) +++ pypy/trunk/pypy/module/cpyext/include/patchlevel.h Mon May 3 13:18:21 2010 @@ -28,6 +28,9 @@ /* Version as a string */ #define PY_VERSION "2.5.5" +/* PyPy version as a string */ +#define PYPY_VERSION "1.2.0" + /* Subversion Revision number of this file (not of the repository) */ #define PY_PATCHLEVEL_REVISION "$Revision: 77872 $" From arigo at codespeak.net Mon May 3 14:46:43 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 3 May 2010 14:46:43 +0200 (CEST) Subject: [pypy-svn] r74337 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20100503124643.BBFB8282BD8@codespeak.net> Author: arigo Date: Mon May 3 14:46:42 2010 New Revision: 74337 Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/test/test_exception.py Log: Test and fix. Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Mon May 3 14:46:42 2010 @@ -1396,13 +1396,13 @@ etype, evalue = self.cpu.get_overflow_error() return self.finishframe_exception( self.cpu.ts.get_exception_box(etype), - self.cpu.ts.get_exc_value_box(evalue)) + self.cpu.ts.get_exc_value_box(evalue).constbox()) def raise_zero_division_error(self): etype, evalue = self.cpu.get_zero_division_error() return self.finishframe_exception( self.cpu.ts.get_exception_box(etype), - self.cpu.ts.get_exc_value_box(evalue)) + self.cpu.ts.get_exc_value_box(evalue).constbox()) def create_empty_history(self): warmrunnerstate = self.staticdata.state Modified: pypy/trunk/pypy/jit/metainterp/test/test_exception.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_exception.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_exception.py Mon May 3 14:46:42 2010 @@ -556,6 +556,20 @@ optimizer=OPTIMIZER_SIMPLE) assert res == 8 + + def test_overflowerror_escapes(self): + def g(x): + return ovfcheck(x + 1) + def f(x): + try: + return g(x) + except Exception, e: + if isinstance(e, OverflowError): + return -42 + raise + res = self.interp_operations(f, [sys.maxint]) + assert res == -42 + class MyError(Exception): def __init__(self, n): self.n = n From arigo at codespeak.net Mon May 3 15:22:29 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 3 May 2010 15:22:29 +0200 (CEST) Subject: [pypy-svn] r74338 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100503132229.0DD44282BD8@codespeak.net> Author: arigo Date: Mon May 3 15:22:28 2010 New Revision: 74338 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/typesystem.py Log: Whack whack whack. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Mon May 3 15:22:28 2010 @@ -39,13 +39,13 @@ def get_llexception(cpu, e): if we_are_translated(): - return e + return XXX(e) if isinstance(e, LLException): return e # ok if isinstance(e, OverflowError): return get_standard_error_llexception(cpu.rtyper, OverflowError) - raise # leave other exceptions be propagated + raise # leave other exceptions to be propagated # ____________________________________________________________ @@ -274,8 +274,6 @@ self.dispatch_loop(self, code, position) except LeaveFrame: return - #except JitException: - # ... except Exception, e: e = get_llexception(self.cpu, e) position = self.handle_exception_in_frame(e, code) @@ -313,7 +311,8 @@ position = self.exception_pc # <-- just after the insn that raised opcode = ord(code[position]) if opcode != self.op_catch_exception: - raise e # no 'catch_exception' insn follows: just reraise + # no 'catch_exception' insn follows: just reraise + raise Exception, e else: # else store the exception on 'self', and jump to the handler if not we_are_translated(): # get the lltyped exception Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py Mon May 3 15:22:28 2010 @@ -12,7 +12,6 @@ from pypy.jit.metainterp import resoperation from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.blackhole import BlackholeInterpreter, NULL -from pypy.jit.metainterp.blackhole import get_llexception # ____________________________________________________________ @@ -47,35 +46,42 @@ # get the function address as an integer func = argboxes[0].getint() cpu = metainterp.cpu - metainterp.latest_exception_might_be = 'E' # any Exception # do the call using the correct function from the cpu rettype = descr.get_return_type() if rettype == INT: try: result = cpu.bh_call_i(func, descr, args_i, args_r, args_f) except Exception, e: - metainterp.got_exception = get_llexception(cpu, e) + metainterp.execute_raised(e) result = 0 + else: + metainterp.execute_did_not_raise() return BoxInt(result) if rettype == REF: try: result = cpu.bh_call_r(func, descr, args_i, args_r, args_f) except Exception, e: - metainterp.got_exception = get_llexception(cpu, e) + metainterp.execute_raised(e) result = NULL + else: + metainterp.execute_did_not_raise() return BoxPtr(result) if rettype == FLOAT: try: result = cpu.bh_call_f(func, descr, args_i, args_r, args_f) except Exception, e: - metainterp.got_exception = get_llexception(cpu, e) + metainterp.execute_raised(e) result = 0.0 + else: + metainterp.execute_did_not_raise() return BoxFloat(result) if rettype == 'v': # void try: cpu.bh_call_v(func, descr, args_i, args_r, args_f) except Exception, e: - metainterp.got_exception = get_llexception(cpu, e) + metainterp.execute_raised(e) + else: + metainterp.execute_did_not_raise() return None raise AssertionError("bad rettype") @@ -116,36 +122,39 @@ return BoxInt(cpu.bh_getfield_raw_i(struct, fielddescr)) def do_int_add_ovf(metainterp, box1, box2): - metainterp.latest_exception_might_be = 'O' # only OverflowError a = box1.getint() b = box2.getint() try: z = ovfcheck(a + b) - except OverflowError, e: - metainterp.got_exception = get_llexception(metainterp.cpu, e) + except OverflowError: + metainterp.execute_raised(OverflowError(), constant=True) z = 0 + else: + metainterp.execute_did_not_raise() return BoxInt(z) def do_int_sub_ovf(metainterp, box1, box2): - metainterp.latest_exception_might_be = 'O' # only OverflowError a = box1.getint() b = box2.getint() try: z = ovfcheck(a - b) - except OverflowError, e: - metainterp.got_exception = get_llexception(metainterp.cpu, e) + except OverflowError: + metainterp.execute_raised(OverflowError(), constant=True) z = 0 + else: + metainterp.execute_did_not_raise() return BoxInt(z) def do_int_mul_ovf(metainterp, box1, box2): - metainterp.latest_exception_might_be = 'O' # only OverflowError a = box1.getint() b = box2.getint() try: z = ovfcheck(a * b) - except OverflowError, e: - metainterp.got_exception = get_llexception(metainterp.cpu, e) + except OverflowError: + metainterp.execute_raised(OverflowError(), constant=True) z = 0 + else: + metainterp.execute_did_not_raise() return BoxInt(z) # ____________________________________________________________ Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Mon May 3 15:22:28 2010 @@ -1,6 +1,7 @@ import py, os from pypy.rpython.lltypesystem import llmemory from pypy.rpython.ootypesystem import ootype +from pypy.rpython.llinterp import LLException from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.debug import debug_start, debug_stop, debug_print @@ -14,6 +15,7 @@ from pypy.jit.metainterp.jitprof import EmptyProfiler from pypy.jit.metainterp.jitprof import GUARDS, RECORDED_OPS, ABORT_ESCAPE from pypy.jit.metainterp.jitprof import ABORT_TOO_LONG, ABORT_BRIDGE +from pypy.jit.metainterp.blackhole import get_llexception from pypy.rlib.rarithmetic import intmask from pypy.rlib.objectmodel import specialize from pypy.rlib.jit import DEBUG_OFF, DEBUG_PROFILE, DEBUG_STEPS, DEBUG_DETAILED @@ -44,12 +46,6 @@ class MIFrame(object): - last_exception = None # the most recently raised-and-caught exception - # ^^^ not to be confused with MetaInterp.got_exception. The idea is that - # got_exception is only briefly != None; as soon as the exception - # is caught by opimpl_catch_exception, it is reset to None and - # transferred to last_exception here. - # for resume.py operation parent_resumedata_snapshot = None parent_resumedata_frame_info_list = None @@ -134,7 +130,6 @@ # ------------------------------ for _opimpl in ['int_add', 'int_sub', 'int_mul', 'int_floordiv', 'int_mod', - 'int_add_ovf', 'int_sub_ovf', 'int_mul_ovf', 'int_lt', 'int_le', 'int_eq', 'int_ne', 'int_gt', 'int_ge', 'int_and', 'int_or', 'int_xor', @@ -151,6 +146,15 @@ return self.execute(rop.%s, b1, b2) ''' % (_opimpl, _opimpl.upper())).compile() + for _opimpl in ['int_add_ovf', 'int_sub_ovf', 'int_mul_ovf']: + exec py.code.Source(''' + @arguments("box", "box") + def opimpl_%s(self, b1, b2): + resbox = self.execute(rop.%s, b1, b2) + self.metainterp.handle_possible_overflow_error() + return resbox + ''' % (_opimpl, _opimpl.upper())).compile() + for _opimpl in ['int_is_true', 'int_neg', 'int_invert', 'bool_not', 'cast_ptr_to_int', 'cast_float_to_int', 'cast_int_to_float', 'float_neg', 'float_abs', @@ -200,34 +204,14 @@ opimpl_ref_pop = _opimpl_any_pop opimpl_float_pop = _opimpl_any_pop - @arguments("orgpc", "catched_exception", "label") - def opimpl_catch_exception(self, pc, catched_exception, target): - if catched_exception is not None: - self.last_exception = catched_exception - self.pc = target - # - extraargs = [] - latest_exception_might_be = self.metainterp.latest_exception_might_be - if not we_are_translated(): - del self.metainterp.latest_exception_might_be - if latest_exception_might_be == 'O': - # only for int_xxx_ovf operations - if catched_exception is None: - opnum = rop.GUARD_NO_OVERFLOW - else: - opnum = rop.GUARD_OVERFLOW - else: - # for all residual calls - if catched_exception is None: - opnum = rop.GUARD_NO_EXCEPTION - else: - etype = catched_exception.typeptr # XXX use the typesystem - etype = llmemory.cast_ptr_to_adr(etype) - etype = llmemory.cast_adr_to_int(etype) - exception_box = self.metainterp.cpu.ts.get_exception_box(etype) - extraargs = [exception_box] - opnum = rop.GUARD_EXCEPTION - self.generate_guard(pc, opnum, extraargs=extraargs) + @arguments("label") + def opimpl_catch_exception(self, target): + """This is a no-op when run normally. We can check that + last_exc_value_box is None; it should have been set to None + by the previous instruction. If the previous instruction + raised instead, finishframe_exception() should have been + called and we would not be there.""" + assert self.metainterp.last_exc_value_box is None @arguments("label") def opimpl_goto(self, target): @@ -618,26 +602,10 @@ self.make_result_box(ConstInt(result)) def perform_call(self, jitcode, varargs, greenkey=None): - if (self.metainterp.is_blackholing() and - jitcode.calldescr is not None): - # when producing only a BlackHole, we can implement this by - # calling the subfunction directly instead of interpreting it - if jitcode.cfnptr is not None: - # for non-oosends - varargs = [jitcode.cfnptr] + varargs - res = self.execute_varargs(rop.CALL, varargs, - descr=jitcode.calldescr, exc=True) - else: - # for oosends (ootype only): calldescr is a MethDescr - res = self.execute_varargs(rop.OOSEND, varargs, - descr=jitcode.calldescr, exc=True) - self.metainterp.load_fields_from_virtualizable() - return res - else: - # when tracing, this bytecode causes the subfunction to be entered - f = self.metainterp.newframe(jitcode, greenkey) - f.setup_call(varargs) - return True + # when tracing, this bytecode causes the subfunction to be entered + f = self.metainterp.newframe(jitcode, greenkey) + f.setup_call(varargs) + return True @XXX #arguments("bytecode", "varargs") def opimpl_call(self, callee, varargs): @@ -886,14 +854,6 @@ self.metainterp.history.record(rop.DEBUG_MERGE_POINT, [constloc], None) - @XXX #arguments("jumptarget") - def opimpl_setup_exception_block(self, exception_target): - self.exception_target = exception_target - - @XXX #arguments() - def opimpl_teardown_exception_block(self): - self.exception_target = -1 - @arguments("box", "label") def opimpl_goto_if_exception_mismatch(self, vtablebox, next_exc_target): # XXX use typesystem.py @@ -904,16 +864,17 @@ if not rclass.ll_isinstance(self.last_exception, bounding_class): self.pc = next_exc_target - @XXX #arguments() - def opimpl_raise(self): - assert len(self.env) == 2 - return self.metainterp.finishframe_exception(self.env[0], self.env[1]) + @arguments("box") + def opimpl_raise(self, exc_value_box): + self.metainterp.last_exc_value_box = exc_value_box + self.metainterp.popframe() + self.metainterp.finishframe_exception() @arguments() def opimpl_reraise(self): - assert self.last_exception is not None - self.metainterp.got_exception = self.last_exception - self.metainterp.propagate_exception_out_of_frame() + assert self.metainterp.last_exc_value_box is not None + self.metainterp.popframe() + self.metainterp.finishframe_exception() @XXX #arguments("box") def opimpl_virtual_ref(self, box): @@ -972,11 +933,11 @@ self.pc = 0 self.env = argboxes - def setup_resume_at_op(self, pc, exception_target, env): + def setup_resume_at_op(self, pc, env): if not we_are_translated(): check_args(*env) self.pc = pc - self.exception_target = exception_target + xxxxxxxxxxxxxxxxxxxxxxx self.env = env ## values = ' '.join([box.repr_rpython() for box in self.env]) ## log('setup_resume_at_op %s:%d [%s] %d' % (self.jitcode.name, @@ -1044,7 +1005,7 @@ return promoted_box def cls_of_box(self, box): - return self.metainterp.cpu.ts.cls_of_box(self.metainterp.cpu, box) + return self.metainterp.cpu.ts.cls_of_box(box) @specialize.arg(1) def execute(self, opnum, *argboxes): @@ -1056,14 +1017,13 @@ @specialize.arg(1) def execute_varargs(self, opnum, argboxes, descr, exc): - return self.metainterp.execute_and_record_varargs(opnum, argboxes, - descr=descr) -## if resbox is not None: -## self.make_result_box(resbox) -## if exc: -## return self.metainterp.handle_exception() -## else: -## return self.metainterp.assert_no_exception() + resbox = self.metainterp.execute_and_record_varargs(opnum, argboxes, + descr=descr) + if exc: + self.metainterp.handle_possible_exception() + else: + self.metainterp.assert_no_exception() + return resbox def do_residual_call(self, funcbox, descr, argboxes, exc): allboxes = [funcbox] + argboxes @@ -1112,9 +1072,6 @@ self.indirectcall_values = [] self.warmrunnerdesc = warmrunnerdesc - #self._op_goto_if_not = self.find_opcode('goto_if_not') - #self._op_ooisnull = self.find_opcode('ooisnull') - #self._op_oononnull = self.find_opcode('oononnull') backendmodule = self.cpu.__module__ backendmodule = backendmodule.split('.')[-2] @@ -1141,6 +1098,7 @@ name, argcodes = key.split('/') opimpl = _get_opimpl_method(name, argcodes) self.opcode_implementations[value] = opimpl + self.op_catch_exception = insns.get('catch_exception/L', -1) def setup_descrs(self, descrs): self.opcode_descrs = descrs @@ -1265,7 +1223,6 @@ class MetaInterp(object): in_recursion = 0 - got_exception = None _already_allocated_resume_virtuals = None def __init__(self, staticdata): @@ -1309,7 +1266,7 @@ if self.framestack: if resultbox is not None: self.framestack[-1].make_result_box(resultbox) - return True + raise ChangeFrame else: if not self.is_blackholing(): try: @@ -1329,7 +1286,7 @@ else: assert False - def finishframe_exception(self, exceptionbox, excvaluebox): + def finishframe_exception(self): # detect and propagate some exceptions early: # - AssertionError # - all subclasses of JitException @@ -1341,12 +1298,14 @@ # while self.framestack: frame = self.framestack[-1] - if frame.exception_target >= 0: - frame.pc = frame.exception_target - frame.exception_target = -1 - frame.exception_box = exceptionbox - frame.exc_value_box = excvaluebox - return True + code = frame.bytecode + position = frame.pc # <-- just after the insn that raised + opcode = ord(code[position]) + if opcode == self.staticdata.op_catch_exception: + # found a 'catch_exception' instruction; jump to the handler + target = ord(code[position+1]) | (ord(code[position+2])<<8) + frame.pc = target + raise ChangeFrame self.popframe() if not self.is_blackholing(): try: @@ -1373,18 +1332,6 @@ print jitcode.name raise Exception - def raise_overflow_error(self): - etype, evalue = self.cpu.get_overflow_error() - return self.finishframe_exception( - self.cpu.ts.get_exception_box(etype), - self.cpu.ts.get_exc_value_box(evalue)) - - def raise_zero_division_error(self): - etype, evalue = self.cpu.get_zero_division_error() - return self.finishframe_exception( - self.cpu.ts.get_exception_box(etype), - self.cpu.ts.get_exc_value_box(evalue)) - def create_empty_history(self): warmrunnerstate = self.staticdata.state self.history = history.History() @@ -1466,6 +1413,24 @@ op.pc = self.framestack[-1].pc op.name = self.framestack[-1].jitcode.name + def execute_raised(self, exception, constant=False): + # Exception handling: when execute.do_call() gets an exception it + # calls metainterp.execute_raised(), which puts it into + # 'self.last_exc_value_box'. This is used shortly afterwards + # to generate either GUARD_EXCEPTION or GUARD_NO_EXCEPTION, and also + # to handle the following opcodes 'goto_if_exception_mismatch'. + llexception = get_llexception(self.cpu, exception) + if not we_are_translated(): + llexception = llexception.args[1] + llexception = self.cpu.ts.cast_to_ref(llexception) + exc_value_box = self.cpu.ts.get_exc_value_box(llexception) + if constant: + exc_value_box = exc_value_box.constbox() + self.last_exc_value_box = exc_value_box + + def execute_did_not_raise(self): + self.last_exc_value_box = None + def switch_to_blackhole(self, reason): self.staticdata.profiler.count(reason) debug_print('~~~ ABORTING TRACING') @@ -1870,38 +1835,29 @@ # mark by replacing it with ConstPtr(NULL) self.virtualref_boxes[i+1] = self.cpu.ts.CONST_NULL - def handle_exception(self): - etype = self.cpu.get_exception() - evalue = self.cpu.get_exc_value() - assert bool(etype) == bool(evalue) - self.cpu.clear_exception() + def handle_possible_exception(self): frame = self.framestack[-1] - if etype: - exception_box = self.cpu.ts.get_exception_box(etype) - exc_value_box = self.cpu.ts.get_exc_value_box(evalue) + if self.last_exc_value_box: + exception_box = self.cpu.ts.cls_of_box(self.last_exc_value_box) op = frame.generate_guard(frame.pc, rop.GUARD_EXCEPTION, None, [exception_box]) if op: - op.result = exc_value_box - return self.finishframe_exception(exception_box, exc_value_box) + op.result = self.last_exc_value_box + self.finishframe_exception() else: frame.generate_guard(frame.pc, rop.GUARD_NO_EXCEPTION, None, []) - return False - def assert_no_exception(self): - assert not self.cpu.get_exception() - return False - - def handle_overflow_error(self): - xxx + def handle_possible_overflow_error(self): frame = self.framestack[-1] - if self.cpu._overflow_flag: - self.cpu._overflow_flag = False - frame.generate_guard(frame.pc, rop.GUARD_OVERFLOW, None, []) - return self.raise_overflow_error() + if self.last_exc_value_box: + frame.generate_guard(frame.pc, rop.GUARD_OVERFLOW, None) + assert isinstance(self.last_exc_value_box, Const) + self.finishframe_exception() else: - frame.generate_guard(frame.pc, rop.GUARD_NO_OVERFLOW, None, []) - return False + frame.generate_guard(frame.pc, rop.GUARD_NO_OVERFLOW, None) + + def assert_no_exception(self): + assert not self.last_exc_value_box def rebuild_state_after_failure(self, resumedescr, newboxes): vinfo = self.staticdata.virtualizable_info @@ -2069,7 +2025,8 @@ # ____________________________________________________________ class ChangeFrame(Exception): - pass + """Raised after we mutated metainterp.framestack, in order to force + it to reload the current top-of-stack frame that gets interpreted.""" def _get_opimpl_method(name, argcodes): from pypy.jit.metainterp.blackhole import signedord @@ -2142,19 +2099,10 @@ position = position3 + 1 + length3 elif argtype == "orgpc": value = orgpc - elif argtype == "catched_exception": - value = self.metainterp.got_exception - self.metainterp.got_exception = None else: raise AssertionError("bad argtype: %r" % (argtype,)) args += (value,) # - if self.metainterp.got_exception is not None: - # the previous operation raised and the current operation - # is not catching it. Propagate it. - self.metainterp.propagate_exception_out_of_frame() - return - # num_return_args = len(argcodes) - next_argcode assert num_return_args == 0 or num_return_args == 1 self.pc = position + num_return_args Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/typesystem.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/typesystem.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/typesystem.py Mon May 3 15:22:28 2010 @@ -71,7 +71,7 @@ def cast_fnptr_to_root(self, fnptr): return llmemory.cast_ptr_to_adr(fnptr) - def cls_of_box(self, cpu, box): + def cls_of_box(self, box): obj = box.getref(lltype.Ptr(rclass.OBJECT)) cls = llmemory.cast_ptr_to_adr(obj.typeptr) return history.ConstInt(llmemory.cast_adr_to_int(cls)) From fijal at codespeak.net Mon May 3 16:03:21 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 3 May 2010 16:03:21 +0200 (CEST) Subject: [pypy-svn] r74340 - pypy/trunk/pypy/module/cpyext/test Message-ID: <20100503140321.CAA2B282BD8@codespeak.net> Author: fijal Date: Mon May 3 16:03:20 2010 New Revision: 74340 Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Log: update test Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Mon May 3 16:03:20 2010 @@ -94,10 +94,9 @@ module = self.import_module(name='foo') obj = module.new() # call __new__ - newobj = type(obj)() - assert type(newobj) is type(obj) + newobj = module.FuuType(u"xyz") + assert newobj == u"xyz" - XXX # stuff below has to be rethought a = module.fooType assert "cannot create" in raises(TypeError, "a()").value.message class bar(module.fooType): From fijal at codespeak.net Mon May 3 16:03:58 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 3 May 2010 16:03:58 +0200 (CEST) Subject: [pypy-svn] r74341 - pypy/trunk/pypy/module/cpyext Message-ID: <20100503140358.ED18E282BD8@codespeak.net> Author: fijal Date: Mon May 3 16:03:57 2010 New Revision: 74341 Modified: pypy/trunk/pypy/module/cpyext/typeobject.py Log: Replace one piece of broken untested code with something that is slightly more correct (IMO). IN-PROGRESS Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Mon May 3 16:03:57 2010 @@ -158,11 +158,11 @@ add_tp_new_wrapper(space, dict_w, pto) @cpython_api([PyObject, PyObject, PyObject], PyObject, external=False) -def tp_new_wrapper(space, w_self, w_args, w_kwds): # XXX untested code - args_w = space.listview(w_args)[:] - args_w.insert(0, w_self) - w_args_new = space.newlist(args_w) - return space.call(space.lookup(space.w_type, "__new__"), w_args_new, w_kwds) +def tp_new_wrapper(space, w_self, w_args, w_kwds): + fn = rffi.cast(PyTypeObjectPtr, make_ref(space, w_self)).c_tp_new + pyo = make_ref(space, w_self) + pto = rffi.cast(PyTypeObjectPtr, pyo) + return generic_cpy_call(space, fn, pto, w_args, w_kwds) @specialize.memo() def get_new_method_def(space): From arigo at codespeak.net Mon May 3 16:25:17 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 3 May 2010 16:25:17 +0200 (CEST) Subject: [pypy-svn] r74344 - pypy/extradoc/talk/ep2010 Message-ID: <20100503142517.51730282BD8@codespeak.net> Author: arigo Date: Mon May 3 16:25:15 2010 New Revision: 74344 Added: pypy/extradoc/talk/ep2010/ pypy/extradoc/talk/ep2010/abstract.txt (contents, props changed) Log: Start the abstract for EuroPython 2010. Added: pypy/extradoc/talk/ep2010/abstract.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ep2010/abstract.txt Mon May 3 16:25:15 2010 @@ -0,0 +1,67 @@ +We will give an update on the state of PyPy, the Python +implementation in Python. The planned talk would fit ~45 +minutes to 1 hour. + + +Part 1 - introduction and status +=============================================================== + +PyPy is a slowly emerging Python implementation written in +Python. This part of the talk is going to give an introduction +about the project and a description of its current status. We +will focus on what it gives to Python programmers: + +* what works on PyPy: + + - mostly everything, as long as it works on Python 2.5 + - (NEW) most existing extension modules (written in C) work too + +* the differences with the standard CPython: + + - (NEW) the JIT is 100% complete and 99.9% ready to use; + we will see the kind of performance that can be expected + - as usual, we will warn against some "corner case" behavioral + differences in the language. + +* in summary, we will see why you should (or should not) consider + switching to PyPy, and for what kind of application. + + +Part 2 - the Just-in-Time compiler generator +=========================================================== + +PyPy now comes with an integrated JIT compiler. In this part of +the talk we will see: + +* more in-depth, how fast can Python code be: + + - with plots of various benchmarks + - in comparison to the speed of regular CPython, + CPython+Psyco, and of Unladen Swallow + - for various algorithmic or non-algorithmic kind + of code. + +* how it all works: + + - a reminder about tracing JITs in general + - the tracing JIT included in PyPy + - the differences between a "classical" tracing + JIT (like TraceMonkey for JavaScript) and + PyPy (a general tracing JIT for any language). + +* future or in-progress tasks: + + - the x86-64 backend (currently only x86-32 is supported) + - a quick tour of other missing improvements + - finally, how to use the JIT generator with your own + language's interpreter (we already have a JITing Prolog + interpreter :-) + + +Part 3 - cpyext, or how C extension modules can work in PyPy +============================================================= + +This part of the talk is ... + + +Authors: Antonio Cuni, Amaury Forgeot d'Arc, Armin Rigo From arigo at codespeak.net Mon May 3 16:33:54 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 3 May 2010 16:33:54 +0200 (CEST) Subject: [pypy-svn] r74347 - pypy/trunk/pypy/translator/tool Message-ID: <20100503143354.C1DF5282BD8@codespeak.net> Author: arigo Date: Mon May 3 16:33:53 2010 New Revision: 74347 Modified: pypy/trunk/pypy/translator/tool/cbuild.py Log: Mac OS/X support. Temporary quick fix. Modified: pypy/trunk/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/trunk/pypy/translator/tool/cbuild.py (original) +++ pypy/trunk/pypy/translator/tool/cbuild.py Mon May 3 16:33:53 2010 @@ -270,14 +270,20 @@ if not file_name.check(): break + # XXX this logic should be moved to translator/platform/*.py + d = self._copy_attributes() f = file_name.open("w") - f.write("{\n") - for sym in self.export_symbols: - f.write("%s;\n" % (sym,)) - f.write("};") + if host.name.startswith('darwin'): + for sym in self.export_symbols: + f.write("_%s\n" % (sym,)) + d['link_extra'] += ("-Wl,-exported_symbols_list,"+str(file_name), ) + else: + f.write("{\n") + for sym in self.export_symbols: + f.write("%s;\n" % (sym,)) + f.write("};") + d['link_extra'] += ("-Wl,--dynamic-list=" + str(file_name), ) f.close() - d = self._copy_attributes() - d['link_extra'] += ("-Wl,--dynamic-list=" + str(file_name), ) d['export_symbols'] = () return ExternalCompilationInfo(**d) From fijal at codespeak.net Mon May 3 17:10:35 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 3 May 2010 17:10:35 +0200 (CEST) Subject: [pypy-svn] r74348 - pypy/branch/chunked-list/pypy/translator/goal Message-ID: <20100503151035.30FA5282BD8@codespeak.net> Author: fijal Date: Mon May 3 17:10:33 2010 New Revision: 74348 Added: pypy/branch/chunked-list/pypy/translator/goal/targettest.py (contents, props changed) Log: A benchmark Added: pypy/branch/chunked-list/pypy/translator/goal/targettest.py ============================================================================== --- (empty file) +++ pypy/branch/chunked-list/pypy/translator/goal/targettest.py Mon May 3 17:10:33 2010 @@ -0,0 +1,33 @@ + +import os + +x = [chr(i) for i in range(256)] + +class ChunkedList(object): + def __init__(self, chunk_size=256): + self.l = [] + self.count = 0 + self.chunk_size = chunk_size + + def append(self, element): + if self.count % self.chunk_size == 0: + # end of chunk + self.l.append([None] * self.chunk_size) + self.l[-1][self.count % self.chunk_size] = element + self.count += 1 + +def main(no, chunk_size): + l = ChunkedList(chunk_size) + for i in range(no): + l.append("abc" + x[i % 256]) + return 3 + +def entry_point(argv): + if len(argv) != 3: + print "Provide 2 ints as args" + return 1 + main(int(argv[1]), int(argv[2])) + return 0 + +def target(*args): + return entry_point, None From arigo at codespeak.net Mon May 3 17:31:10 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 3 May 2010 17:31:10 +0200 (CEST) Subject: [pypy-svn] r74349 - pypy/trunk/pypy/module/posix/test Message-ID: <20100503153110.4B3C0282BD8@codespeak.net> Author: arigo Date: Mon May 3 17:31:08 2010 New Revision: 74349 Modified: pypy/trunk/pypy/module/posix/test/test_posix2.py Log: Windows fixes. Modified: pypy/trunk/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/trunk/pypy/module/posix/test/test_posix2.py (original) +++ pypy/trunk/pypy/module/posix/test/test_posix2.py Mon May 3 17:31:08 2010 @@ -180,16 +180,18 @@ assert 0 def test_filename_exception(self): - for fn in [self.posix.unlink, self.posix.remove, - self.posix.chdir, self.posix.mkdir, self.posix.rmdir, - self.posix.listdir, self.posix.readlink, - self.posix.chroot]: - try: - fn('qowieuqw/oeiu') - except OSError, e: - assert e.filename == 'qowieuqw/oeiu' - else: - assert 0 + for fname in ['unlink', 'remove', + 'chdir', 'mkdir', 'rmdir', + 'listdir', 'readlink', + 'chroot']: + if hasattr(self.posix, fname): + func = getattr(self.posix, fname) + try: + func('qowieuqw/oeiu') + except OSError, e: + assert e.filename == 'qowieuqw/oeiu' + else: + assert 0 def test_chmod_exception(self): try: @@ -200,12 +202,13 @@ assert 0 def test_chown_exception(self): - try: - self.posix.chown('qowieuqw/oeiu', 0, 0) - except OSError, e: - assert e.filename == 'qowieuqw/oeiu' - else: - assert 0 + if hasattr(self.posix, 'chown'): + try: + self.posix.chown('qowieuqw/oeiu', 0, 0) + except OSError, e: + assert e.filename == 'qowieuqw/oeiu' + else: + assert 0 def test_utime_exception(self): for arg in [None, (0, 0)]: From antocuni at codespeak.net Mon May 3 21:49:35 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 3 May 2010 21:49:35 +0200 (CEST) Subject: [pypy-svn] r74353 - pypy/extradoc/talk/ep2010 Message-ID: <20100503194935.C6C2F282BD8@codespeak.net> Author: antocuni Date: Mon May 3 21:49:32 2010 New Revision: 74353 Modified: pypy/extradoc/talk/ep2010/abstract.txt Log: complete this section. I don't really know much about cpyext, so I had to write it as general as possibile :-) Modified: pypy/extradoc/talk/ep2010/abstract.txt ============================================================================== --- pypy/extradoc/talk/ep2010/abstract.txt (original) +++ pypy/extradoc/talk/ep2010/abstract.txt Mon May 3 21:49:32 2010 @@ -61,7 +61,14 @@ Part 3 - cpyext, or how C extension modules can work in PyPy ============================================================= -This part of the talk is ... +In this part, we will talk about the new cpyext module, that allow PyPy to use +the extension modules of CPython written in C. In particular: + +* How it works + +* What are the drawbacks, and the current status + +* What are the incompatibilities, and how to solve them Authors: Antonio Cuni, Amaury Forgeot d'Arc, Armin Rigo From afa at codespeak.net Tue May 4 00:27:17 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 4 May 2010 00:27:17 +0200 (CEST) Subject: [pypy-svn] r74354 - pypy/extradoc/talk/ep2010 Message-ID: <20100503222717.02126282B9D@codespeak.net> Author: afa Date: Tue May 4 00:27:16 2010 New Revision: 74354 Modified: pypy/extradoc/talk/ep2010/abstract.txt Log: Add some items. I don't know if all this belongs to the abstract, but they should be part of the talk. Modified: pypy/extradoc/talk/ep2010/abstract.txt ============================================================================== --- pypy/extradoc/talk/ep2010/abstract.txt (original) +++ pypy/extradoc/talk/ep2010/abstract.txt Tue May 4 00:27:16 2010 @@ -66,9 +66,29 @@ * How it works + - "normal" integration with distutils + - PyObjects are "shadows" of living interpreter objects + - reference counted + - Most of the API is written in RPython + - Some C functions are copied verbatim + - PyArg_ParseTuple, PyString_Format, the buffer object... + - Testable from interpreted PyPy + * What are the drawbacks, and the current status + - built on top of another Python VM + - Concrete Layer built on top of the Abtract Layer + ex: PyDict_Size(x) calls len(x) + - Constant round-turns between PyObjects and the interpreter objects + - no threads yet + - extensions only, cannot embed PyPy yet + - PyPy still has a GIL... + * What are the incompatibilities, and how to solve them + - No direct field access + - except for types + - Borrowed references are evil + Authors: Antonio Cuni, Amaury Forgeot d'Arc, Armin Rigo From afa at codespeak.net Tue May 4 00:37:13 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 4 May 2010 00:37:13 +0200 (CEST) Subject: [pypy-svn] r74355 - pypy/trunk/lib-python/modified-2.5.2/distutils Message-ID: <20100503223713.F37D9282B9D@codespeak.net> Author: afa Date: Tue May 4 00:37:12 2010 New Revision: 74355 Modified: pypy/trunk/lib-python/modified-2.5.2/distutils/msvccompiler.py Log: Try to fix the windows buildbot, by returning the version of the compiler installed there. Obviously this function should return the version used to compile the current executable, but pypy does not gives this information yet (in sys.version). Modified: pypy/trunk/lib-python/modified-2.5.2/distutils/msvccompiler.py ============================================================================== --- pypy/trunk/lib-python/modified-2.5.2/distutils/msvccompiler.py (original) +++ pypy/trunk/lib-python/modified-2.5.2/distutils/msvccompiler.py Tue May 4 00:37:12 2010 @@ -165,7 +165,7 @@ For Python 2.3 and up, the version number is included in sys.version. For earlier versions, assume the compiler is MSVC 6. """ - return 8.0 + return 9.0 def normalize_and_reduce_paths(paths): """Return a list of normalized paths with duplicates removed. From afa at codespeak.net Tue May 4 14:04:20 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 4 May 2010 14:04:20 +0200 (CEST) Subject: [pypy-svn] r74356 - pypy/trunk/pypy/module/cpyext Message-ID: <20100504120420.8446F282B9D@codespeak.net> Author: afa Date: Tue May 4 14:04:18 2010 New Revision: 74356 Modified: pypy/trunk/pypy/module/cpyext/presetup.py Log: - on Windows, distutils uses the _winreg module - no need to call build_bridge() twice Modified: pypy/trunk/pypy/module/cpyext/presetup.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/presetup.py (original) +++ pypy/trunk/pypy/module/cpyext/presetup.py Tue May 4 14:04:18 2010 @@ -4,7 +4,7 @@ Run python setup.py build in your project directory -You can import resulting .so with py.py --allworingmodules +You can import resulting .so with py.py --allworkingmodules """ import sys, os @@ -21,8 +21,10 @@ from pypy.conftest import gettestobjspace from pypy.module.cpyext.api import build_bridge -space = gettestobjspace(usemodules=['cpyext', 'thread']) -build_bridge(space) +usemodules = ['cpyext', 'thread'] +if sys.platform == 'win32': + usemodules.append('_winreg') # necessary in distutils +space = gettestobjspace(usemodules=usemodules) inc_paths = str(udir) From afa at codespeak.net Tue May 4 14:07:41 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 4 May 2010 14:07:41 +0200 (CEST) Subject: [pypy-svn] r74357 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100504120741.01F63282B9E@codespeak.net> Author: afa Date: Tue May 4 14:07:40 2010 New Revision: 74357 Modified: pypy/trunk/pypy/module/cpyext/test/test_weakref.py pypy/trunk/pypy/module/cpyext/weakrefobject.py Log: Add the new function PyWeakref_LockObject, similar to PyWeakref_GetObject except that it returns a *new reference*. PyWeakref_GetObject does not work on cpyext and should be removed. Modified: pypy/trunk/pypy/module/cpyext/test/test_weakref.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_weakref.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_weakref.py Tue May 4 14:07:40 2010 @@ -7,6 +7,7 @@ w_ref = api.PyWeakref_NewRef(w_obj, space.w_None) assert w_ref is not None assert space.is_w(api.PyWeakref_GetObject(w_ref), w_obj) + assert space.is_w(api.PyWeakref_LockObject(w_ref), w_obj) w_obj = space.newtuple([]) assert api.PyWeakref_NewRef(w_obj, space.w_None) is None Modified: pypy/trunk/pypy/module/cpyext/weakrefobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/weakrefobject.py (original) +++ pypy/trunk/pypy/module/cpyext/weakrefobject.py Tue May 4 14:07:40 2010 @@ -18,9 +18,16 @@ @cpython_api([PyObject], PyObject, borrowed=True) def PyWeakref_GetObject(space, w_ref): - """Return the referenced object from a weak reference, ref. If the referent is - no longer live, returns None. + """Return the referenced object from a weak reference. If the referent is + no longer live, returns None. This function returns a borrowed reference. """ register_container(space, w_ref) return space.call_function(w_ref) + at cpython_api([PyObject], PyObject) +def PyWeakref_LockObject(space, w_ref): + """Return the referenced object from a weak reference. If the referent is + no longer live, returns None. This function returns a new reference. + """ + return space.call_function(w_ref) + From agaynor at codespeak.net Tue May 4 14:36:14 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Tue, 4 May 2010 14:36:14 +0200 (CEST) Subject: [pypy-svn] r74358 - pypy/trunk/pypy/module/cpyext/test Message-ID: <20100504123614.0A295282B9D@codespeak.net> Author: agaynor Date: Tue May 4 14:36:13 2010 New Revision: 74358 Modified: pypy/trunk/pypy/module/cpyext/test/foo.c Log: Normalized whitespace (removed tabs). Modified: pypy/trunk/pypy/module/cpyext/test/foo.c ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/foo.c (original) +++ pypy/trunk/pypy/module/cpyext/test/foo.c Tue May 4 14:36:13 2010 @@ -2,8 +2,8 @@ #include "structmember.h" typedef struct { - PyObject_HEAD - int foo; /* the context holder */ + PyObject_HEAD + int foo; /* the context holder */ PyObject *foo_object; char *foo_string; char foo_string_inplace[5]; @@ -14,17 +14,17 @@ static fooobject * newfooobject(void) { - fooobject *foop; + fooobject *foop; - foop = PyObject_New(fooobject, &footype); - if (foop == NULL) - return NULL; - - foop->foo = 42; - foop->foo_object = NULL; - foop->foo_string = "Hello from PyPy"; - strncpy(foop->foo_string_inplace, "spam", 5); - return foop; + foop = PyObject_New(fooobject, &footype); + if (foop == NULL) + return NULL; + + foop->foo = 42; + foop->foo_object = NULL; + foop->foo_string = "Hello from PyPy"; + strncpy(foop->foo_string_inplace, "spam", 5); + return foop; } @@ -33,7 +33,7 @@ static void foo_dealloc(fooobject *foop) { - PyObject_Del(foop); + PyObject_Del(foop); } @@ -42,14 +42,14 @@ static PyObject * foo_copy(fooobject *self) { - fooobject *foop; + fooobject *foop; - if ((foop = newfooobject()) == NULL) - return NULL; + if ((foop = newfooobject()) == NULL) + return NULL; - foop->foo = self->foo; + foop->foo = self->foo; - return (PyObject *)foop; + return (PyObject *)foop; } static PyObject * @@ -61,9 +61,9 @@ static PyMethodDef foo_methods[] = { - {"copy", (PyCFunction)foo_copy, METH_NOARGS, NULL}, - {"unset_string_member", (PyCFunction)foo_unset, METH_NOARGS, NULL}, - {NULL, NULL} /* sentinel */ + {"copy", (PyCFunction)foo_copy, METH_NOARGS, NULL}, + {"unset_string_member", (PyCFunction)foo_unset, METH_NOARGS, NULL}, + {NULL, NULL} /* sentinel */ }; static PyObject * @@ -126,37 +126,37 @@ }; static PyTypeObject footype = { - PyVarObject_HEAD_INIT(NULL, 0) - "foo.foo", /*tp_name*/ - sizeof(fooobject), /*tp_size*/ - 0, /*tp_itemsize*/ - /* methods */ - (destructor)foo_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare*/ - foo_repr, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - foo_call, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT, /*tp_flags*/ - 0, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - foo_methods, /*tp_methods*/ - foo_members, /*tp_members*/ - foo_getseters, /*tp_getset*/ + PyVarObject_HEAD_INIT(NULL, 0) + "foo.foo", /*tp_name*/ + sizeof(fooobject), /*tp_size*/ + 0, /*tp_itemsize*/ + /* methods */ + (destructor)foo_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + foo_repr, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + foo_call, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + foo_methods, /*tp_methods*/ + foo_members, /*tp_members*/ + foo_getseters, /*tp_getset*/ }; typedef struct { @@ -246,20 +246,20 @@ static PyObject * foo_new(PyObject *self, PyObject *args) { - fooobject *foop; + fooobject *foop; - if ((foop = newfooobject()) == NULL) { - return NULL; - } - - return (PyObject *)foop; + if ((foop = newfooobject()) == NULL) { + return NULL; + } + + return (PyObject *)foop; } /* List of functions exported by this module */ static PyMethodDef foo_functions[] = { - {"new", (PyCFunction)foo_new, METH_NOARGS, NULL}, - {NULL, NULL} /* Sentinel */ + {"new", (PyCFunction)foo_new, METH_NOARGS, NULL}, + {NULL, NULL} /* Sentinel */ }; @@ -267,7 +267,7 @@ void initfoo(void) { - PyObject *m, *d; + PyObject *m, *d; footype.ob_type = &PyType_Type; @@ -279,14 +279,14 @@ return; if (PyType_Ready(&FuuType) < 0) return; - m = Py_InitModule("foo", foo_functions); - if (m == NULL) - return; - d = PyModule_GetDict(m); - if (d) { - if (PyDict_SetItemString(d, "fooType", (PyObject *)&footype) < 0) + m = Py_InitModule("foo", foo_functions); + if (m == NULL) + return; + d = PyModule_GetDict(m); + if (d) { + if (PyDict_SetItemString(d, "fooType", (PyObject *)&footype) < 0) return; PyDict_SetItemString(d, "FuuType", (PyObject *) &FuuType); } - /* No need to check the error here, the caller will do that */ + /* No need to check the error here, the caller will do that */ } From afa at codespeak.net Tue May 4 14:39:35 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 4 May 2010 14:39:35 +0200 (CEST) Subject: [pypy-svn] r74359 - pypy/trunk/pypy/module/cpyext/test Message-ID: <20100504123935.793D4282B9D@codespeak.net> Author: afa Date: Tue May 4 14:39:34 2010 New Revision: 74359 Modified: pypy/trunk/pypy/module/cpyext/test/test_weakref.py Log: test weakref on a collected object. Modified: pypy/trunk/pypy/module/cpyext/test/test_weakref.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_weakref.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_weakref.py Tue May 4 14:39:34 2010 @@ -13,3 +13,17 @@ assert api.PyWeakref_NewRef(w_obj, space.w_None) is None assert api.PyErr_Occurred() is space.w_TypeError api.PyErr_Clear() + + def test_weakref_lockobject(self, space, api): + # some new weakrefable object + w_obj = space.call_function(space.w_type, space.wrap("newtype"), + space.newtuple([]), space.newdict()) + assert w_obj is not None + + w_ref = api.PyWeakref_NewRef(w_obj, space.w_None) + assert w_obj is not None + + assert space.is_w(api.PyWeakref_LockObject(w_ref), w_obj) + del w_obj + import gc; gc.collect() + assert space.is_w(api.PyWeakref_LockObject(w_ref), space.w_None) From afa at codespeak.net Tue May 4 16:34:36 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 4 May 2010 16:34:36 +0200 (CEST) Subject: [pypy-svn] r74360 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100504143436.1145C282BAD@codespeak.net> Author: afa Date: Tue May 4 16:34:34 2010 New Revision: 74360 Modified: pypy/trunk/pypy/module/cpyext/api.py pypy/trunk/pypy/module/cpyext/pyerrors.py pypy/trunk/pypy/module/cpyext/state.py pypy/trunk/pypy/module/cpyext/test/test_api.py pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py Log: Store an OperationError instead of the (exc_type, exc_value) attributes pair. This is a little faster when errors are passed from C to pypy and vice-versa, and it will be easier to add the traceback. Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Tue May 4 16:34:34 2010 @@ -210,8 +210,7 @@ if not hasattr(api_function, "error_value"): raise state = space.fromcache(State) - e.normalize_exception(space) - state.set_exception(e.w_type, e.get_w_value(space)) + state.set_exception(e) if restype is PyObject: return None else: @@ -426,11 +425,11 @@ print >>sys.stderr, " DONE" except OperationError, e: failed = True - e.normalize_exception(space) - state.set_exception(e.w_type, e.get_w_value(space)) + state.set_exception(e) except BaseException, e: failed = True - state.set_exception(space.w_SystemError, space.wrap(str(e))) + state.set_exception(OperationError(space.w_SystemError, + space.wrap(str(e)))) if not we_are_translated(): import traceback traceback.print_exc() Modified: pypy/trunk/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyerrors.py (original) +++ pypy/trunk/pypy/module/cpyext/pyerrors.py Tue May 4 16:34:34 2010 @@ -14,7 +14,7 @@ """This function is similar to PyErr_SetString() but lets you specify an arbitrary Python object for the "value" of the exception.""" state = space.fromcache(State) - state.set_exception(w_type, w_value) + state.set_exception(OperationError(w_type, w_value)) @cpython_api([PyObject, CONST_STRING], lltype.Void) def PyErr_SetString(space, w_type, message_ptr): @@ -29,8 +29,10 @@ @cpython_api([], PyObject, borrowed=True) def PyErr_Occurred(space): state = space.fromcache(State) + if state.operror is None: + return None register_container(space, lltype.nullptr(PyObject.TO)) - return state.exc_type + return state.operror.w_type @cpython_api([], lltype.Void) def PyErr_Clear(space): @@ -47,10 +49,13 @@ This function is normally only used by code that needs to handle exceptions or by code that needs to save and restore the error indicator temporarily.""" state = space.fromcache(State) - ptype[0] = make_ref(space, state.exc_type, steal=True) - pvalue[0] = make_ref(space, state.exc_type, steal=True) - state.exc_type = None - state.exc_value = None + operror = state.clear_exception() + if operror: + ptype[0] = make_ref(space, operror.w_type) + pvalue[0] = make_ref(space, operror.get_w_value(space)) + else: + ptype[0] = lltype.nullptr(PyObject.TO) + pvalue[0] = lltype.nullptr(PyObject.TO) ptraceback[0] = lltype.nullptr(PyObject.TO) @cpython_api([PyObject, PyObject, PyObject], lltype.Void) @@ -69,8 +74,7 @@ error indicator temporarily; use PyErr_Fetch() to save the current exception state.""" state = space.fromcache(State) - state.exc_type = w_type - state.exc_value = w_value + state.set_exception(OperationError(w_type, w_value)) Py_DecRef(space, w_type) Py_DecRef(space, w_value) Modified: pypy/trunk/pypy/module/cpyext/state.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/state.py (original) +++ pypy/trunk/pypy/module/cpyext/state.py Tue May 4 16:34:34 2010 @@ -18,8 +18,7 @@ self.borrowed_objects = {} # { addr of containee -> None } self.non_heaptypes = [] # list of wrapped objects self.last_container = 0 # addr of last container - self.exc_type = None - self.exc_value = None + self.operror = None self.new_method_def = lltype.nullptr(PyMethodDef) # When importing a package, use this to keep track of its name. This is @@ -43,31 +42,32 @@ ptr = rffi.cast(ADDR, obj) self.py_objects_r2w[ptr] = w_obj - def set_exception(self, w_type, w_value): + def set_exception(self, operror): self.clear_exception() - self.exc_type = w_type - self.exc_value = w_value + self.operror = operror def clear_exception(self): + """Clear the current exception state, and return the operror. + Also frees the borrowed reference returned by PyErr_Occurred() + """ from pypy.module.cpyext.pyobject import Py_DecRef, make_ref from pypy.module.cpyext.api import ADDR # handling of borrowed objects, remove when we have # a weakkeydict - exc_type = make_ref(self.space, self.exc_type, borrowed=True) - if exc_type: + operror = self.operror + if operror is not None: + exc_type = make_ref(self.space, operror.w_type, borrowed=True) Py_DecRef(self.space, exc_type) containee_ptr = rffi.cast(ADDR, exc_type) del self.borrowed_objects[containee_ptr] - self.exc_type = None - self.exc_value = None + self.operror = None + return operror def check_and_raise_exception(self, always=False): - exc_value = self.exc_value - exc_type = self.exc_type - if exc_type is not None or exc_value is not None: + operror = self.operror + if operror: self.clear_exception() - op_err = OperationError(exc_type, exc_value) - raise op_err + raise operror if always: raise OperationError(self.space.w_SystemError, self.space.wrap( "Function returned an error result without setting an exception")) Modified: pypy/trunk/pypy/module/cpyext/test/test_api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_api.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_api.py Tue May 4 16:34:34 2010 @@ -30,10 +30,10 @@ raise Exception("%s is not callable" % (f,)) f(*args) state = space.fromcache(State) - tp = state.exc_type - if not tp: + operror = state.operror + if not operror: raise Exception("DID NOT RAISE") - if getattr(space, 'w_' + expected_exc.__name__) is not tp: + if getattr(space, 'w_' + expected_exc.__name__) is not operror.w_type: raise Exception("Wrong exception") state.clear_exception() Modified: pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py Tue May 4 16:34:34 2010 @@ -50,25 +50,26 @@ api.PyErr_SetObject(space.w_ValueError, space.wrap("a value")) assert api.PyErr_Occurred() is space.w_ValueError state = space.fromcache(State) - assert space.eq_w(state.exc_value, space.wrap("a value")) + assert space.eq_w(state.operror.get_w_value(space), + space.wrap("a value")) api.PyErr_Clear() def test_SetNone(self, space, api): api.PyErr_SetNone(space.w_KeyError) state = space.fromcache(State) - assert space.eq_w(state.exc_type, space.w_KeyError) - assert space.eq_w(state.exc_value, space.w_None) + assert space.eq_w(state.operror.w_type, space.w_KeyError) + assert space.eq_w(state.operror.get_w_value(space), space.w_None) api.PyErr_Clear() api.PyErr_NoMemory() - assert space.eq_w(state.exc_type, space.w_MemoryError) + assert space.eq_w(state.operror.w_type, space.w_MemoryError) api.PyErr_Clear() def test_BadArgument(self, space, api): api.PyErr_BadArgument() state = space.fromcache(State) - assert space.eq_w(state.exc_type, space.w_TypeError) + assert space.eq_w(state.operror.w_type, space.w_TypeError) api.PyErr_Clear() class AppTestFetch(AppTestCpythonExtensionBase): @@ -92,7 +93,7 @@ ''' ), ]) - module.check_error() + assert module.check_error() def test_fetch_and_restore(self): module = self.import_extension('foo', [ @@ -106,8 +107,6 @@ return NULL; if (type != PyExc_TypeError) Py_RETURN_FALSE; - if (val->ob_type != type) - Py_RETURN_FALSE; PyErr_Restore(type, val, tb); if (!PyErr_Occurred()) Py_RETURN_FALSE; @@ -116,7 +115,7 @@ ''' ), ]) - module.check_error() + assert module.check_error() def test_SetFromErrno(self): skip("The test does not set the errno in a way which " From afa at codespeak.net Tue May 4 18:22:59 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 4 May 2010 18:22:59 +0200 (CEST) Subject: [pypy-svn] r74361 - pypy/trunk/pypy/module/cpyext Message-ID: <20100504162259.7D67B282B9E@codespeak.net> Author: afa Date: Tue May 4 18:22:57 2010 New Revision: 74361 Modified: pypy/trunk/pypy/module/cpyext/pyobject.py pypy/trunk/pypy/module/cpyext/typeobject.py Log: suppress many translation warnings Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Tue May 4 18:22:57 2010 @@ -14,7 +14,18 @@ # type description class BaseCpyTypedescr(object): - pass + basestruct = PyObject.TO + + def get_dealloc(self, space): + raise NotImplementedError + def allocate(self, space, w_type, itemcount=0): + raise NotImplementedError + def make_ref(self, space, w_type, w_obj, itemcount=0): + raise NotImplementedError + def attach(self, space, pyobj, w_obj): + raise NotImplementedError + def realize(self, space, ref): + raise NotImplementedError typedescr_cache = {} Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Tue May 4 18:22:57 2010 @@ -219,12 +219,14 @@ class GettersAndSetters: def getter(self, space, w_self): + assert isinstance(self, W_GetSetPropertyEx) check_descr(space, w_self, self.w_type) return generic_cpy_call( space, self.getset.c_get, w_self, self.getset.c_closure) def setter(self, space, w_self, w_value): + assert isinstance(self, W_GetSetPropertyEx) check_descr(space, w_self, self.w_type) res = generic_cpy_call( space, self.getset.c_set, w_self, w_value, @@ -234,14 +236,17 @@ state.check_and_raise_exception() def member_getter(self, space, w_self): + assert isinstance(self, W_MemberDescr) check_descr(space, w_self, self.w_type) return PyMember_GetOne(space, w_self, self.member) def member_delete(self, space, w_self): + assert isinstance(self, W_MemberDescr) check_descr(space, w_self, self.w_type) PyMember_SetOne(space, w_self, self.member, None) def member_setter(self, space, w_self, w_value): + assert isinstance(self, W_MemberDescr) check_descr(space, w_self, self.w_type) PyMember_SetOne(space, w_self, self.member, w_value) From afa at codespeak.net Tue May 4 19:23:53 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 4 May 2010 19:23:53 +0200 (CEST) Subject: [pypy-svn] r74362 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100504172353.6F5CE282B9D@codespeak.net> Author: afa Date: Tue May 4 19:23:51 2010 New Revision: 74362 Modified: pypy/trunk/pypy/module/cpyext/api.py pypy/trunk/pypy/module/cpyext/pyerrors.py pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py Log: It's not so difficult to set errno from C code... unskip test then fix PyErr_SetFromErrno which was completely broken. Also add a few assertions (len(args) == len(argtypes)) in the generated wrappers Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Tue May 4 19:23:51 2010 @@ -178,6 +178,7 @@ from pypy.module.cpyext.pyobject import make_ref, from_ref newargs = () to_decref = [] + assert len(args) == len(api_function.argtypes) for i, (ARG, is_wrapped) in types_names_enum_ui: input_arg = args[i] if is_PyObject(ARG) and not is_wrapped: @@ -408,6 +409,7 @@ try: if not we_are_translated() and DEBUG_WRAPPER: print >>sys.stderr, callable, + assert len(args) == len(callable.api_func.argtypes) for i, (typ, is_wrapped) in argtypes_enum_ui: arg = args[i] if typ is PyObject and is_wrapped: @@ -817,6 +819,7 @@ def generic_cpy_call(space, func, *args): boxed_args = () to_decref = [] + assert len(args) == len(FT.ARGS) for i, ARG in unrolling_arg_types: arg = args[i] if ARG is PyObject: Modified: pypy/trunk/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyerrors.py (original) +++ pypy/trunk/pypy/module/cpyext/pyerrors.py Tue May 4 19:23:51 2010 @@ -1,13 +1,14 @@ import os from pypy.rpython.lltypesystem import rffi, lltype -from pypy.interpreter.error import OperationError +from pypy.interpreter.error import OperationError, wrap_oserror from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING from pypy.module.exceptions.interp_exceptions import W_RuntimeWarning from pypy.module.cpyext.pyobject import ( PyObject, PyObjectP, make_ref, Py_DecRef, register_container) from pypy.module.cpyext.state import State from pypy.rlib.rposix import get_errno +from pypy.rlib.rposix import get_errno @cpython_api([PyObject, PyObject], lltype.Void) def PyErr_SetObject(space, w_type, w_value): @@ -77,6 +78,7 @@ state.set_exception(OperationError(w_type, w_value)) Py_DecRef(space, w_type) Py_DecRef(space, w_value) + Py_DecRef(space, w_traceback) @cpython_api([], lltype.Void) def PyErr_BadArgument(space): @@ -114,9 +116,7 @@ Return value: always NULL.""" # XXX Doesn't actually do anything with PyErr_CheckSignals. errno = get_errno() - errno_w = space.wrap(errno) - message_w = space.wrap(os.strerror(errno)) - PyErr_SetObject(space, w_type, errno_w, message_w) + raise wrap_oserror(space, OSError(errno, "PyErr_SetFromErrno")) @cpython_api([], rffi.INT_real, error=-1) def PyErr_CheckSignals(space): Modified: pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py Tue May 4 19:23:51 2010 @@ -77,11 +77,6 @@ AppTestCpythonExtensionBase.setup_class.im_func(cls) space = cls.space - def set_errno(num): - ll2ctypes.TLS.errno = num - - cls.w_set_errno = space.wrap(interp2app(set_errno, unwrap_spec=[int])) - def test_occurred(self): module = self.import_extension('foo', [ ("check_error", "METH_NOARGS", @@ -118,23 +113,21 @@ assert module.check_error() def test_SetFromErrno(self): - skip("The test does not set the errno in a way which " - "untranslated pypy can actually notice") - - import errno + import errno, os module = self.import_extension('foo', [ ("set_from_errno", "METH_NOARGS", ''' + errno = EBADF; PyErr_SetFromErrno(PyExc_OSError); return NULL; '''), - ]) + ], + prologue="#include ") try: - self.set_errno(errno.EBADF) module.set_from_errno() except OSError, e: assert e.errno == errno.EBADF - assert e.message == os.strerror(errno.EBADF) + assert e.strerror == os.strerror(errno.EBADF) From afa at codespeak.net Tue May 4 19:39:20 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 4 May 2010 19:39:20 +0200 (CEST) Subject: [pypy-svn] r74363 - pypy/trunk/pypy/module/cpyext Message-ID: <20100504173920.7D5A836C228@codespeak.net> Author: afa Date: Tue May 4 19:39:18 2010 New Revision: 74363 Modified: pypy/trunk/pypy/module/cpyext/pyerrors.py Log: remove duplicate line Modified: pypy/trunk/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyerrors.py (original) +++ pypy/trunk/pypy/module/cpyext/pyerrors.py Tue May 4 19:39:18 2010 @@ -8,7 +8,6 @@ PyObject, PyObjectP, make_ref, Py_DecRef, register_container) from pypy.module.cpyext.state import State from pypy.rlib.rposix import get_errno -from pypy.rlib.rposix import get_errno @cpython_api([PyObject, PyObject], lltype.Void) def PyErr_SetObject(space, w_type, w_value): From hpk at codespeak.net Tue May 4 21:32:06 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 4 May 2010 21:32:06 +0200 (CEST) Subject: [pypy-svn] r74364 - pypy/branch/py12/py/_plugin Message-ID: <20100504193206.48F7C282B9E@codespeak.net> Author: hpk Date: Tue May 4 21:32:03 2010 New Revision: 74364 Modified: pypy/branch/py12/py/_plugin/pytest__pytest.py pypy/branch/py12/py/_plugin/pytest_skipping.py pypy/branch/py12/py/_plugin/pytest_terminal.py pypy/branch/py12/py/_plugin/pytest_terminal.py.orig Log: update from py-trunk to have new xfail(run=False) marker and improved reporting Modified: pypy/branch/py12/py/_plugin/pytest__pytest.py ============================================================================== --- pypy/branch/py12/py/_plugin/pytest__pytest.py (original) +++ pypy/branch/py12/py/_plugin/pytest__pytest.py Tue May 4 21:32:03 2010 @@ -46,7 +46,8 @@ recorder = RecordCalls() self._recorders[hookspec] = recorder self._registry.register(recorder) - self.hook = HookRelay(hookspecs, registry=self._registry) + self.hook = HookRelay(hookspecs, registry=self._registry, + prefix="pytest_") def finish_recording(self): for recorder in self._recorders.values(): Modified: pypy/branch/py12/py/_plugin/pytest_skipping.py ============================================================================== --- pypy/branch/py12/py/_plugin/pytest_skipping.py (original) +++ pypy/branch/py12/py/_plugin/pytest_skipping.py Tue May 4 21:32:03 2010 @@ -83,10 +83,17 @@ depending on platform:: @py.test.mark.xfail("sys.version_info >= (3,0)") - def test_function(): ... +To not run a test and still regard it as "xfailed":: + + @py.test.mark.xfail(..., run=False) + +To specify an explicit reason to be shown with xfailure detail:: + + @py.test.mark.xfail(..., reason="my reason") + skipping on a missing import dependency -------------------------------------------------- @@ -116,34 +123,94 @@ py.test.skip("unsuppored configuration") """ -# XXX py.test.skip, .importorskip and the Skipped class -# should also be defined in this plugin, requires thought/changes import py +def pytest_addoption(parser): + group = parser.getgroup("general") + group.addoption('--runxfail', + action="store_true", dest="runxfail", default=False, + help="run tests even if they are marked xfail") + +class MarkEvaluator: + def __init__(self, item, name): + self.item = item + self.name = name + self.holder = getattr(item.obj, name, None) + + def __bool__(self): + return bool(self.holder) + __nonzero__ = __bool__ + + def istrue(self): + if self.holder: + d = {'os': py.std.os, 'sys': py.std.sys, 'config': self.item.config} + self.result = True + for expr in self.holder.args: + self.expr = expr + if isinstance(expr, str): + result = cached_eval(self.item.config, expr, d) + else: + result = expr + if not result: + self.result = False + self.expr = expr + break + return getattr(self, 'result', False) + + def get(self, attr, default=None): + return self.holder.kwargs.get(attr, default) + + def getexplanation(self): + expl = self.get('reason', None) + if not expl: + if not hasattr(self, 'expr'): + return "condition: True" + else: + return "condition: " + self.expr + return expl + def pytest_runtest_setup(item): - expr, result = evalexpression(item, 'skipif') - if result: - py.test.skip(expr) + if not isinstance(item, py.test.collect.Function): + return + evalskip = MarkEvaluator(item, 'skipif') + if evalskip.istrue(): + py.test.skip(evalskip.getexplanation()) + item._evalxfail = MarkEvaluator(item, 'xfail') + if not item.config.getvalue("runxfail"): + if item._evalxfail.istrue(): + if not item._evalxfail.get('run', True): + py.test.skip("xfail") def pytest_runtest_makereport(__multicall__, item, call): - if call.when != "call": + if not isinstance(item, py.test.collect.Function): + return + evalxfail = getattr(item, '_evalxfail', None) + if not evalxfail: return - expr, result = evalexpression(item, 'xfail') - rep = __multicall__.execute() - if result: - if call.excinfo: - rep.skipped = True - rep.failed = rep.passed = False + if call.when == "setup": + rep = __multicall__.execute() + if rep.skipped and evalxfail.istrue(): + expl = evalxfail.getexplanation() + if not evalxfail.get("run", True): + expl = "[NOTRUN] " + expl + rep.keywords['xfail'] = expl + return rep + elif call.when == "call": + rep = __multicall__.execute() + if not item.config.getvalue("runxfail") and evalxfail.istrue(): + if call.excinfo: + rep.skipped = True + rep.failed = rep.passed = False + else: + rep.skipped = rep.passed = False + rep.failed = True + rep.keywords['xfail'] = evalxfail.getexplanation() else: - rep.skipped = rep.passed = False - rep.failed = True - rep.keywords['xfail'] = expr - else: - if 'xfail' in rep.keywords: - del rep.keywords['xfail'] - return rep + if 'xfail' in rep.keywords: + del rep.keywords['xfail'] + return rep # called by terminalreporter progress reporting def pytest_report_teststatus(report): @@ -151,7 +218,7 @@ if report.skipped: return "xfailed", "x", "xfail" elif report.failed: - return "xpassed", "P", "xpass" + return "xpassed", "P", "XPASS" # called by the terminalreporter instance/plugin def pytest_terminal_summary(terminalreporter): @@ -169,40 +236,17 @@ return tr.write_sep("_", "expected failures") for rep in xfailed: - entry = rep.longrepr.reprcrash - modpath = rep.item.getmodpath(includemodule=True) - pos = "%s %s:%d: " %(modpath, entry.path, entry.lineno) - reason = rep.longrepr.reprcrash.message - i = reason.find("\n") - if i != -1: - reason = reason[:i] + pos = terminalreporter.gettestid(rep.item) + reason = rep.keywords['xfail'] tr._tw.line("%s %s" %(pos, reason)) xpassed = terminalreporter.stats.get("xpassed") if xpassed: tr.write_sep("_", "UNEXPECTEDLY PASSING TESTS") for rep in xpassed: - fspath, lineno, modpath = rep.item.reportinfo() - pos = "%s %s:%d: unexpectedly passing" %(modpath, fspath, lineno) - tr._tw.line(pos) - - -def evalexpression(item, keyword): - if isinstance(item, py.test.collect.Function): - markholder = getattr(item.obj, keyword, None) - result = False - if markholder: - d = {'os': py.std.os, 'sys': py.std.sys, 'config': item.config} - expr, result = None, True - for expr in markholder.args: - if isinstance(expr, str): - result = cached_eval(item.config, expr, d) - else: - result = expr - if not result: - break - return expr, result - return None, False + pos = terminalreporter.gettestid(rep.item) + reason = rep.keywords['xfail'] + tr._tw.line("%s %s" %(pos, reason)) def cached_eval(config, expr, d): if not hasattr(config, '_evalcache'): Modified: pypy/branch/py12/py/_plugin/pytest_terminal.py ============================================================================== --- pypy/branch/py12/py/_plugin/pytest_terminal.py (original) +++ pypy/branch/py12/py/_plugin/pytest_terminal.py Tue May 4 21:32:03 2010 @@ -128,6 +128,20 @@ else: return "???", dict(red=True) + def gettestid(self, item, relative=True): + fspath = item.fspath + chain = [x for x in item.listchain() if x.fspath == fspath] + chain = chain[1:] + names = [x.name for x in chain if x.name != "()"] + path = item.fspath + if relative: + relpath = path.relto(self.curdir) + if relpath: + path = relpath + names.insert(0, str(path)) + return "::".join(names) + + def pytest_internalerror(self, excrepr): for line in str(excrepr).split("\n"): self.write_line("INTERNALERROR> " + line) Modified: pypy/branch/py12/py/_plugin/pytest_terminal.py.orig ============================================================================== --- pypy/branch/py12/py/_plugin/pytest_terminal.py.orig (original) +++ pypy/branch/py12/py/_plugin/pytest_terminal.py.orig Tue May 4 21:32:03 2010 @@ -257,7 +257,7 @@ self._sessionstarttime = py.std.time.time() verinfo = ".".join(map(str, sys.version_info[:3])) - msg = "python: platform %s -- Python %s" % (sys.platform, verinfo) + msg = "platform %s -- Python %s" % (sys.platform, verinfo) msg += " -- pytest-%s" % (py.__version__) if self.config.option.verbose or self.config.option.debug or getattr(self.config.option, 'pastebin', None): msg += " -- " + str(sys.executable) @@ -420,8 +420,6 @@ self._failed = [] def outindent(self, line): - s = self.indent + str(line) - print ("printing: %s" % s) self.out.line(self.indent + str(line)) def pytest_internalerror(self, excrepr): From hpk at codespeak.net Tue May 4 21:34:05 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 4 May 2010 21:34:05 +0200 (CEST) Subject: [pypy-svn] r74365 - in pypy/branch/py12/pypy: . interpreter/test lib/distributed/test module/_sre/test module/gc/test module/imp/test tool/pytest/test tool/test Message-ID: <20100504193405.13744282B9E@codespeak.net> Author: hpk Date: Tue May 4 21:34:03 2010 New Revision: 74365 Added: pypy/branch/py12/pypy/tool/pytest/test/test_pytestsupport.py - copied, changed from r74323, pypy/branch/py12/pypy/tool/test/test_pytestsupport.py Removed: pypy/branch/py12/pypy/tool/test/test_pytestsupport.py Modified: pypy/branch/py12/pypy/conftest.py pypy/branch/py12/pypy/interpreter/test/test_zzpickle_and_slow.py pypy/branch/py12/pypy/lib/distributed/test/test_distributed.py pypy/branch/py12/pypy/module/_sre/test/test_app_sre.py pypy/branch/py12/pypy/module/gc/test/test_gc.py pypy/branch/py12/pypy/module/imp/test/test_import.py pypy/branch/py12/pypy/tool/pytest/test/test_appsupport.py Log: Various bits to make buildbot applevel tests run * extended conftest.py mechanism for space configuration: defining "spaceconfig = dict-instance" on a test class will instantiate (or skip) an according space object. * fixes related to the fact that py.test since 1.2 adheres to the convention of running teardown_* functions even if the corresponding setup_* failed. * more tests for pytestsupport itself Modified: pypy/branch/py12/pypy/conftest.py ============================================================================== --- pypy/branch/py12/pypy/conftest.py (original) +++ pypy/branch/py12/pypy/conftest.py Tue May 4 21:34:03 2010 @@ -43,7 +43,8 @@ help="set up tests to use specified platform as compile/run target") def pytest_funcarg__space(request): - return gettestobjspace() + spaceconfig = getattr(request.cls, 'spaceconfig', {}) + return gettestobjspace(**spaceconfig) _SPACECACHE={} def gettestobjspace(name=None, **kwds): @@ -93,6 +94,14 @@ space.eq_w = appsupport.eq_w.__get__(space) return space +def pytest_runtest_setup(item): + if isinstance(item, PyPyTestFunction): + appclass = item.getparent(PyPyClassCollector) + if appclass is not None: + spaceconfig = getattr(appclass.obj, 'spaceconfig', None) + if spaceconfig: + appclass.obj.space = gettestobjspace(**spaceconfig) + class TinyObjSpace(object): def __init__(self, **kwds): import sys @@ -408,11 +417,17 @@ class PyPyClassCollector(py.test.collect.Class): def setup(self): cls = self.obj - cls.space = LazyObjSpaceGetter() + if not hasattr(cls, 'spaceconfig'): + cls.space = LazyObjSpaceGetter() + else: + assert hasattr(cls, 'space') # set by pytest_runtest_setup super(PyPyClassCollector, self).setup() + +class IntInstanceCollector(py.test.collect.Instance): + Function = IntTestFunction class IntClassCollector(PyPyClassCollector): - Function = IntTestFunction + Instance = IntInstanceCollector def _haskeyword(self, keyword): return keyword == 'interplevel' or \ Modified: pypy/branch/py12/pypy/interpreter/test/test_zzpickle_and_slow.py ============================================================================== --- pypy/branch/py12/pypy/interpreter/test/test_zzpickle_and_slow.py (original) +++ pypy/branch/py12/pypy/interpreter/test/test_zzpickle_and_slow.py Tue May 4 21:34:03 2010 @@ -73,10 +73,8 @@ space.wrap('restore_top_frame')) class AppTestInterpObjectPickling: - + pytestmark = py.test.mark.skipif("config.option.runappdirect") def setup_class(cls): - if conftest.option.runappdirect: - py.test.skip("not for py.test -A") _attach_helpers(cls.space) def teardown_class(cls): Modified: pypy/branch/py12/pypy/lib/distributed/test/test_distributed.py ============================================================================== --- pypy/branch/py12/pypy/lib/distributed/test/test_distributed.py (original) +++ pypy/branch/py12/pypy/lib/distributed/test/test_distributed.py Tue May 4 21:34:03 2010 @@ -94,9 +94,11 @@ assert len(item) == 11 class AppTestDistributedTasklets(object): + spaceconfig = {"objspace.std.withtproxy": True, + "objspace.usemodules._stackless": True} def setup_class(cls): - cls.space = gettestobjspace(**{"objspace.std.withtproxy": True, - "usemodules":("_stackless",)}) + #cls.space = gettestobjspace(**{"objspace.std.withtproxy": True, + # "usemodules":("_stackless",)}) cls.w_test_env = cls.space.appexec([], """(): from distributed import test_env return test_env Modified: pypy/branch/py12/pypy/module/_sre/test/test_app_sre.py ============================================================================== --- pypy/branch/py12/pypy/module/_sre/test/test_app_sre.py (original) +++ pypy/branch/py12/pypy/module/_sre/test/test_app_sre.py Tue May 4 21:34:03 2010 @@ -1,5 +1,6 @@ """Regular expression tests specific to _sre.py and accumulated during TDD.""" import autopath +import py from py.test import raises, skip from pypy.interpreter.gateway import app2interp_temp from pypy.conftest import gettestobjspace, option Modified: pypy/branch/py12/pypy/module/gc/test/test_gc.py ============================================================================== --- pypy/branch/py12/pypy/module/gc/test/test_gc.py (original) +++ pypy/branch/py12/pypy/module/gc/test/test_gc.py Tue May 4 21:34:03 2010 @@ -1,4 +1,5 @@ from pypy.conftest import gettestobjspace +import py class AppTestGC(object): def test_collect(self): @@ -76,9 +77,10 @@ assert gc.isenabled() class AppTestGcDumpHeap(object): + pytestmark = py.test.mark.xfail(run=False) + def setup_class(cls): import py - py.test.skip("Disabled") from pypy.tool.udir import udir from pypy.rlib import rgc class X(object): Modified: pypy/branch/py12/pypy/module/imp/test/test_import.py ============================================================================== --- pypy/branch/py12/pypy/module/imp/test/test_import.py (original) +++ pypy/branch/py12/pypy/module/imp/test/test_import.py Tue May 4 21:34:03 2010 @@ -920,16 +920,15 @@ sys.path_hooks.pop() class AppTestNoPycFile(object): - usepycfiles = False - lonepycfiles = False - + spaceconfig = { + "objspace.usepycfiles": False, + "objspace.lonepycfiles": False + } def setup_class(cls): - cls.space = gettestobjspace(**{ - "objspace.usepycfiles": cls.usepycfiles, - "objspace.lonepycfiles": cls.lonepycfiles, - }) - cls.w_usepycfiles = cls.space.wrap(cls.usepycfiles) - cls.w_lonepycfiles = cls.space.wrap(cls.lonepycfiles) + usepycfiles = cls.spaceconfig['objspace.usepycfiles'] + lonepycfiles = cls.spaceconfig['objspace.lonepycfiles'] + cls.w_usepycfiles = cls.space.wrap(usepycfiles) + cls.w_lonepycfiles = cls.space.wrap(lonepycfiles) cls.saved_modules = _setup(cls.space) def teardown_class(cls): @@ -950,9 +949,13 @@ assert lone.__file__.endswith('lone.pyc') class AppTestNoLonePycFile(AppTestNoPycFile): - usepycfiles = True - lonepycfiles = False + spaceconfig = { + "objspace.usepycfiles": True, + "objspace.lonepycfiles": False + } class AppTestLonePycFile(AppTestNoPycFile): - usepycfiles = True - lonepycfiles = True + spaceconfig = { + "objspace.usepycfiles": True, + "objspace.lonepycfiles": True + } Modified: pypy/branch/py12/pypy/tool/pytest/test/test_appsupport.py ============================================================================== --- pypy/branch/py12/pypy/tool/pytest/test/test_appsupport.py (original) +++ pypy/branch/py12/pypy/tool/pytest/test/test_appsupport.py Tue May 4 21:34:03 2010 @@ -1,4 +1,74 @@ +import sys +import py +import pypy +pytest_plugins = "pytester" + +def setpypyconftest(testdir): + path = str(py.path.local(pypy.__file__).dirpath().dirpath()) + testdir.makeconftest(""" + import sys + sys.path.insert(0, %r) + from pypy.conftest import * + """ % path) + +def test_pypy_collection(testdir): + testdir.makepyfile(""" + def test_func(): + pass + class TestClassInt: + def test_method(self, space): + pass + class AppTestClass: + def test_method(self): + pass + """) + setpypyconftest(testdir) + result = testdir.runpytest("--collectonly") + assert result.ret == 0 + result.stdout.fnmatch_lines([ + "*IntTestFunction*test_func*", + "*IntClassCollector*TestClassInt*", + "*IntTestFunction*test_method*", + "*AppClassCollector*AppTestClass*", + "*AppTestMethod*", + ]) + +class TestSpaceConfig: + def test_applevel_skipped_on_cpython_and_spaceconfig(self, testdir): + setpypyconftest(testdir) + testdir.makepyfile(""" + class AppTestClass: + spaceconfig = {"objspace.usemodules._stackless": True} + def setup_class(cls): + assert 0 + def test_applevel(self): + pass + """) + result = testdir.runpytest("-A") + assert result.ret == 0 + if hasattr(sys, 'pypy_translation_info') and \ + sys.pypy_translation_info.get('objspace.usemodules._stackless'): + result.stdout.fnmatch_lines(["*1 error*"]) + else: + # setup_class didn't get called, otherwise it would error + result.stdout.fnmatch_lines(["*1 skipped*"]) + + def test_interp_spaceconfig(self, testdir): + setpypyconftest(testdir) + p = testdir.makepyfile(""" + class TestClass: + spaceconfig = {"objspace.usemodules._stackless": False} + def setup_class(cls): + assert not cls.space.config.objspace.usemodules._stackless + def test_interp(self, space): + assert self.space is space + def test_interp2(self, space): + assert self.space is space + """) + result = testdir.runpytest(p) + assert result.ret == 0 + result.stdout.fnmatch_lines(["*2 passed*"]) def app_test_raises(): info = raises(TypeError, id) Copied: pypy/branch/py12/pypy/tool/pytest/test/test_pytestsupport.py (from r74323, pypy/branch/py12/pypy/tool/test/test_pytestsupport.py) ============================================================================== --- pypy/branch/py12/pypy/tool/test/test_pytestsupport.py (original) +++ pypy/branch/py12/pypy/tool/pytest/test/test_pytestsupport.py Tue May 4 21:34:03 2010 @@ -1,4 +1,3 @@ -import autopath from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import app2interp_temp from pypy.interpreter.argument import Arguments From benjamin at codespeak.net Tue May 4 22:55:10 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 4 May 2010 22:55:10 +0200 (CEST) Subject: [pypy-svn] r74366 - pypy/trunk/pypy/module/cpyext/test Message-ID: <20100504205510.B0356282BAD@codespeak.net> Author: benjamin Date: Tue May 4 22:55:09 2010 New Revision: 74366 Modified: pypy/trunk/pypy/module/cpyext/test/test_thread.py Log: don't need to pass space Modified: pypy/trunk/pypy/module/cpyext/test/test_thread.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_thread.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_thread.py Tue May 4 22:55:09 2010 @@ -11,7 +11,7 @@ def test_get_thread_ident(self, space, api): results = [] def some_thread(): - res = api.PyThread_get_thread_ident(space) + res = api.PyThread_get_thread_ident() results.append((res, thread.get_ident())) some_thread() From benjamin at codespeak.net Tue May 4 22:57:18 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 4 May 2010 22:57:18 +0200 (CEST) Subject: [pypy-svn] r74367 - pypy/trunk/pypy/module/cpyext/test Message-ID: <20100504205718.5EA7A282BAD@codespeak.net> Author: benjamin Date: Tue May 4 22:57:16 2010 New Revision: 74367 Modified: pypy/trunk/pypy/module/cpyext/test/test_mapping.py Log: getitem doesn't take a value :) Modified: pypy/trunk/pypy/module/cpyext/test/test_mapping.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_mapping.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_mapping.py Tue May 4 22:57:16 2010 @@ -26,7 +26,7 @@ key = rffi.str2charp("key") api.PyMapping_SetItemString(w_d, key, space.wrap(42)) assert 42 == space.unwrap( - api.PyMapping_GetItemString(w_d, key, space.wrap(42))) + api.PyMapping_GetItemString(w_d, key)) rffi.free_charp(key) def test_haskey(self, space, api): From hpk at codespeak.net Wed May 5 10:11:54 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 5 May 2010 10:11:54 +0200 (CEST) Subject: [pypy-svn] r74373 - pypy/branch/py12/pypy/lib/app_test/ctypes_tests Message-ID: <20100505081154.04CDB282BDD@codespeak.net> Author: hpk Date: Wed May 5 10:11:53 2010 New Revision: 74373 Modified: pypy/branch/py12/pypy/lib/app_test/ctypes_tests/support.py Log: simplifying and fixing applevel skipping Modified: pypy/branch/py12/pypy/lib/app_test/ctypes_tests/support.py ============================================================================== --- pypy/branch/py12/pypy/lib/app_test/ctypes_tests/support.py (original) +++ pypy/branch/py12/pypy/lib/app_test/ctypes_tests/support.py Wed May 5 10:11:53 2010 @@ -1,36 +1,29 @@ import py import ctypes -if ctypes.__version__ < "1.0.2": - py.test.skip("we expect a ctypes implementation with ver >= 1.0.2") +py.test.importorskip("ctypes", "1.0.2") + +try: + import _rawffi +except ImportError: + _rawffi = None class WhiteBoxTests: def setup_class(cls): - try: - import _rawffi - except ImportError: - py.test.skip("these tests are white-box tests for pypy _rawffi based ctypes impl") + if _rawffi: + py.test.skip("white-box tests for pypy _rawffi based ctypes impl") class BaseCTypesTestChecker: def setup_class(cls): - try: - import _rawffi - except ImportError: - pass - else: + if _rawffi: import gc for _ in range(4): gc.collect() cls.old_num = _rawffi._num_of_allocated_objects() def teardown_class(cls): - #return - try: - import _rawffi - except ImportError: - pass - else: + if hasattr(cls, 'old_num'): import gc for _ in range(4): gc.collect() From afa at codespeak.net Wed May 5 11:08:40 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 5 May 2010 11:08:40 +0200 (CEST) Subject: [pypy-svn] r74374 - pypy/trunk/pypy/module/cpyext Message-ID: <20100505090840.C5C18282BDD@codespeak.net> Author: afa Date: Wed May 5 11:08:38 2010 New Revision: 74374 Modified: pypy/trunk/pypy/module/cpyext/import_.py Log: Use space.call_function instead of space.call Modified: pypy/trunk/pypy/module/cpyext/import_.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/import_.py (original) +++ pypy/trunk/pypy/module/cpyext/import_.py Wed May 5 11:08:38 2010 @@ -30,11 +30,11 @@ else: w_import = space.getattr(w_builtin, space.wrap("__import__")) - # Call the __import__ function with the proper argument list - # Always use absolute import here. - return space.call(w_import, space.newtuple( - [w_name, w_globals, w_globals, - space.newlist([space.wrap("__doc__")])])) + # Call the __import__ function with the proper argument list + # Always use absolute import here. + return space.call_function(w_import, + w_name, w_globals, w_globals, + space.newlist([space.wrap("__doc__")])) @cpython_api([CONST_STRING], PyObject) def PyImport_ImportModule(space, name): From afa at codespeak.net Wed May 5 11:10:00 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 5 May 2010 11:10:00 +0200 (CEST) Subject: [pypy-svn] r74375 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100505091000.43ABF282BDD@codespeak.net> Author: afa Date: Wed May 5 11:09:58 2010 New Revision: 74375 Modified: pypy/trunk/pypy/module/cpyext/pyerrors.py pypy/trunk/pypy/module/cpyext/test/test_api.py pypy/trunk/pypy/module/cpyext/test/test_import.py pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py Log: PyErr_WarnEx now uses the real warnings module Modified: pypy/trunk/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyerrors.py (original) +++ pypy/trunk/pypy/module/cpyext/pyerrors.py Wed May 5 11:09:58 2010 @@ -7,6 +7,7 @@ from pypy.module.cpyext.pyobject import ( PyObject, PyObjectP, make_ref, Py_DecRef, register_container) from pypy.module.cpyext.state import State +from pypy.module.cpyext.import_ import PyImport_Import from pypy.rlib.rposix import get_errno @cpython_api([PyObject, PyObject], lltype.Void) @@ -188,10 +189,14 @@ For information about warning control, see the documentation for the warnings module and the -W option in the command line documentation. There is no C API for warning control.""" - message = rffi.charp2str(message_ptr) if w_category is None: - w_category = space.gettypeobject(W_RuntimeWarning.typedef) - os.write(2, "WARNING: " + message + "\n") + w_category = space.w_None + w_message = space.wrap(rffi.charp2str(message_ptr)) + w_stacklevel = space.wrap(stacklevel) + + w_module = PyImport_Import(space, space.wrap("warnings")) + w_warn = space.getattr(w_module, space.wrap("warn")) + space.call_function(w_warn, w_message, w_category, w_stacklevel) return 0 @cpython_api([PyObject, CONST_STRING], rffi.INT_real, error=-1) Modified: pypy/trunk/pypy/module/cpyext/test/test_api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_api.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_api.py Wed May 5 11:09:58 2010 @@ -7,6 +7,7 @@ PyObject = api.PyObject from pypy.interpreter.error import OperationError from pypy.module.cpyext.state import State +import os @api.cpython_api([PyObject], lltype.Void) def PyPy_GetWrapped(space, w_arg): @@ -17,8 +18,17 @@ class BaseApiTest(LeakCheckingTest): def setup_class(cls): - cls.space = gettestobjspace(usemodules=['cpyext', 'thread']) - cls.space.getbuiltinmodule("cpyext") + cls.space = space = gettestobjspace(usemodules=['cpyext', 'thread']) + + # warm up reference counts: + # - the posix module allocates a HCRYPTPROV on Windows + # - writing to stderr allocates a file lock + space.getbuiltinmodule("cpyext") + space.getbuiltinmodule(os.name) + space.call_function(space.getattr(space.sys.get("stderr"), + space.wrap("write")), + space.wrap("")) + class CAPI: def __getattr__(self, name): return getattr(cls.space, name) Modified: pypy/trunk/pypy/module/cpyext/test/test_import.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_import.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_import.py Wed May 5 11:09:58 2010 @@ -2,11 +2,6 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase class TestImport(BaseApiTest): - def setup_method(self, func): - from pypy.module.imp.importing import importhook - importhook(self.space, "os") # warm up reference counts - BaseApiTest.setup_method(self, func) - def test_import(self, space, api): pdb = api.PyImport_Import(space.wrap("pdb")) assert pdb Modified: pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py Wed May 5 11:09:58 2010 @@ -72,6 +72,13 @@ assert space.eq_w(state.operror.w_type, space.w_TypeError) api.PyErr_Clear() + def test_Warning(self, space, api, capfd): + message = rffi.str2charp("this is a warning") + api.PyErr_WarnEx(None, message, 1) + out, err = capfd.readouterr() + assert ": UserWarning: this is a warning" in err + rffi.free_charp(message) + class AppTestFetch(AppTestCpythonExtensionBase): def setup_class(cls): AppTestCpythonExtensionBase.setup_class.im_func(cls) @@ -129,5 +136,3 @@ except OSError, e: assert e.errno == errno.EBADF assert e.strerror == os.strerror(errno.EBADF) - - From arigo at codespeak.net Wed May 5 11:20:29 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 May 2010 11:20:29 +0200 (CEST) Subject: [pypy-svn] r74376 - in pypy/branch/blackhole-improvement/pypy/jit: backend/llgraph codewriter metainterp Message-ID: <20100505092029.EBB8A282BDD@codespeak.net> Author: arigo Date: Wed May 5 11:20:28 2010 New Revision: 74376 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/logger.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/typesystem.py Log: Pass the correct location of exceptions. Complete some more executor.py. Kill ConstAddr entirely, now that we have AddressAsInt. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py Wed May 5 11:20:28 2010 @@ -7,7 +7,7 @@ import sys from pypy.objspace.flow.model import Variable, Constant from pypy.annotation import model as annmodel -from pypy.jit.metainterp.history import (ConstInt, ConstPtr, ConstAddr, +from pypy.jit.metainterp.history import (ConstInt, ConstPtr, BoxInt, BoxPtr, BoxObj, BoxFloat, REF, INT, FLOAT) from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py Wed May 5 11:20:28 2010 @@ -14,6 +14,7 @@ 'residual_call_', '(int|ref|float)_guard_value', 'guard_class', + 'int_(add|sub|mul)_ovf', ] # ____________________________________________________________ Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py Wed May 5 11:20:28 2010 @@ -85,27 +85,34 @@ return None raise AssertionError("bad rettype") +def do_getarrayitem_gc(metainterp, arraybox, indexbox, arraydescr): + cpu = metainterp.cpu + array = arraybox.getref_base() + index = indexbox.getint() + if arraydescr.is_array_of_pointers(): + return BoxPtr(cpu.bh_setarrayitem_gc_r(arraydescr, array, index)) + elif arraydescr.is_array_of_floats(): + return BoxFloat(cpu.bh_setarrayitem_gc_f(arraydescr, array, index)) + else: + return BoxInt(cpu.bh_setarrayitem_gc_i(arraydescr, array, index)) + def do_setarrayitem_gc(metainterp, arraybox, indexbox, itembox, arraydescr): cpu = metainterp.cpu array = arraybox.getref_base() index = indexbox.getint() - if itembox.type == INT: - item = itembox.getint() - cpu.bh_setarrayitem_gc_i(arraydescr, array, index, item) - elif itembox.type == REF: - item = itembox.getref_base() - cpu.bh_setarrayitem_gc_r(arraydescr, array, index, item) - elif itembox.type == FLOAT: - item = itembox.getfloat() - cpu.bh_setarrayitem_gc_f(arraydescr, array, index, item) + if arraydescr.is_array_of_pointers(): + cpu.bh_setarrayitem_gc_r(arraydescr, array, index, + itembox.getref_base()) + elif arraydescr.is_array_of_floats(): + cpu.bh_setarrayitem_gc_f(arraydescr, array, index, itembox.getfloat()) else: - raise AssertionError("bad itembox.type") + cpu.bh_setarrayitem_gc_i(arraydescr, array, index, itembox.getint()) def do_getfield_gc(metainterp, structbox, fielddescr): cpu = metainterp.cpu struct = structbox.getref_base() if fielddescr.is_pointer_field(): - return BoxPtr(cpu.bh_getfield_gc_p(struct, fielddescr)) + return BoxPtr(cpu.bh_getfield_gc_r(struct, fielddescr)) elif fielddescr.is_float_field(): return BoxFloat(cpu.bh_getfield_gc_f(struct, fielddescr)) else: @@ -115,12 +122,32 @@ cpu = metainterp.cpu struct = structbox.getint() if fielddescr.is_pointer_field(): - return BoxPtr(cpu.bh_getfield_raw_p(struct, fielddescr)) + return BoxPtr(cpu.bh_getfield_raw_r(struct, fielddescr)) elif fielddescr.is_float_field(): return BoxFloat(cpu.bh_getfield_raw_f(struct, fielddescr)) else: return BoxInt(cpu.bh_getfield_raw_i(struct, fielddescr)) +def do_setfield_gc(metainterp, structbox, itembox, fielddescr): + cpu = metainterp.cpu + struct = structbox.getref_base() + if fielddescr.is_pointer_field(): + cpu.bh_setfield_gc_r(struct, fielddescr, itembox.getref_base()) + elif fielddescr.is_float_field(): + cpu.bh_setfield_gc_f(struct, fielddescr, itembox.getfloat()) + else: + cpu.bh_setfield_gc_i(struct, fielddescr, itembox.getint()) + +def do_setfield_raw(metainterp, structbox, itembox, fielddescr): + cpu = metainterp.cpu + struct = structbox.getint() + if fielddescr.is_pointer_field(): + cpu.bh_setfield_raw_r(struct, fielddescr, itembox.getref_base()) + elif fielddescr.is_float_field(): + cpu.bh_setfield_raw_f(struct, fielddescr, itembox.getfloat()) + else: + cpu.bh_setfield_raw_i(struct, fielddescr, itembox.getint()) + def do_int_add_ovf(metainterp, box1, box2): a = box1.getint() b = box2.getint() Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py Wed May 5 11:20:28 2010 @@ -98,7 +98,7 @@ def nonconstbox(self): raise NotImplementedError - def getaddr(self, cpu): + def getaddr(self): raise NotImplementedError def sort_key(self): @@ -182,18 +182,13 @@ __slots__ = () @staticmethod - def _new(x, cpu): + def _new(x): "NOT_RPYTHON" T = lltype.typeOf(x) kind = getkind(T) if kind == "int": if isinstance(T, lltype.Ptr): - if not we_are_translated(): - # cannot store integers representing casted addresses - # inside ConstInt() instances that are going through - # translation; must use the special ConstAddr instead. - return ConstAddr(x, cpu) - intval = cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x)) + intval = llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(x)) else: intval = lltype.cast_primitive(lltype.Signed, x) return ConstInt(intval) @@ -256,8 +251,8 @@ def getint(self): return self.value - def getaddr(self, cpu): - return cpu.cast_int_to_adr(self.value) + def getaddr(self): + return llmemory.cast_int_to_adr(self.value) def _get_hash_(self): return self.value @@ -281,50 +276,6 @@ CONST_FALSE = ConstInt(0) CONST_TRUE = ConstInt(1) -class ConstAddr(Const): # only for constants built before translation - type = INT - _attrs_ = ('value', 'cpu') - - def __init__(self, adrvalue, cpu): - "NOT_RPYTHON" - assert not we_are_translated() - if isinstance(lltype.typeOf(adrvalue), lltype.Ptr): - adrvalue = llmemory.cast_ptr_to_adr(adrvalue) # convenience - else: - assert lltype.typeOf(adrvalue) == llmemory.Address - self.value = adrvalue - self.cpu = cpu - - def clonebox(self): - return BoxInt(self.cpu.cast_adr_to_int(self.value)) - - nonconstbox = clonebox - - def getint(self): - return self.cpu.cast_adr_to_int(self.value) - - def getaddr(self, cpu): - return self.value - - def _get_hash_(self): - return llmemory.cast_adr_to_int(self.value) - - def set_future_value(self, cpu, j): - cpu.set_future_value_int(j, self.getint()) - - def same_constant(self, other): - assert isinstance(other, Const) - return self.value == other.getaddr(self.cpu) - - def nonnull(self): - return bool(self.value) - - def _getrepr_(self): - return self.value - - def repr_rpython(self): - return repr_rpython(self, 'ca') - class ConstFloat(Const): type = FLOAT value = 0.0 @@ -388,7 +339,7 @@ else: return 0 - def getaddr(self, cpu): + def getaddr(self): return llmemory.cast_ptr_to_adr(self.value) def set_future_value(self, cpu, j): @@ -441,7 +392,7 @@ def set_future_value(self, cpu, j): cpu.set_future_value_ref(j, self.value) -## def getaddr(self, cpu): +## def getaddr(self): ## # so far this is used only when calling ## # CodeWriter.IndirectCallset.bytecode_for_address. We don't need a ## # real addr, but just a key for the dictionary @@ -470,7 +421,7 @@ is_box = True # hint that we want to make links in graphviz from this @staticmethod - def _new(x, cpu): + def _new(x): "NOT_RPYTHON" kind = getkind(lltype.typeOf(x)) if kind == "int": @@ -533,8 +484,8 @@ def getint(self): return self.value - def getaddr(self, cpu): - return cpu.cast_int_to_adr(self.value) + def getaddr(self): + return llmemory.cast_int_to_adr(self.value) def _get_hash_(self): return self.value @@ -604,7 +555,7 @@ return lltype.cast_opaque_ptr(PTR, self.getref_base()) getref._annspecialcase_ = 'specialize:arg(1)' - def getaddr(self, cpu): + def getaddr(self): return llmemory.cast_ptr_to_adr(self.value) def _get_hash_(self): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/logger.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/logger.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/logger.py Wed May 5 11:20:28 2010 @@ -1,9 +1,11 @@ import os from pypy.rlib.debug import have_debug_prints from pypy.rlib.debug import debug_start, debug_stop, debug_print +from pypy.rlib.objectmodel import we_are_translated +from pypy.rpython.lltypesystem import llmemory from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.history import Const, ConstInt, Box, \ - BoxInt, ConstAddr, ConstFloat, BoxFloat, AbstractFailDescr + BoxInt, ConstFloat, BoxFloat, AbstractFailDescr class Logger(object): @@ -46,10 +48,16 @@ mv = len(memo) memo[arg] = mv if isinstance(arg, ConstInt): + if int_could_be_an_address(arg.value): + addr = arg.getaddr() + name = self.metainterp_sd.get_name_from_address(addr) + if name: + return 'ConstAddr(' + name + ')' return str(arg.value) elif isinstance(arg, BoxInt): return 'i' + str(mv) elif isinstance(arg, self.ts.ConstRef): + # XXX for ootype, this should also go through get_name_from_address return 'ConstPtr(ptr' + str(mv) + ')' elif isinstance(arg, self.ts.BoxRef): return 'p' + str(mv) @@ -57,12 +65,6 @@ return str(arg.value) elif isinstance(arg, BoxFloat): return 'f' + str(mv) - elif isinstance(arg, self.ts.ConstAddr): - addr = arg.getaddr(self.metainterp_sd.cpu) - name = self.metainterp_sd.get_name_from_address(addr) - if not name: - name = 'cls' + str(mv) - return 'ConstClass(' + name + ')' elif arg is None: return 'None' else: @@ -102,3 +104,10 @@ fail_args = '' debug_print(res + op.getopname() + '(' + args + ')' + fail_args) + + +def int_could_be_an_address(x): + if we_are_translated(): + return not (-32768 <= x <= 32767) + else: + return isinstance(x, llmemory.AddressAsInt) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Wed May 5 11:20:28 2010 @@ -148,10 +148,10 @@ for _opimpl in ['int_add_ovf', 'int_sub_ovf', 'int_mul_ovf']: exec py.code.Source(''' - @arguments("box", "box") - def opimpl_%s(self, b1, b2): + @arguments("orgpc", "box", "box") + def opimpl_%s(self, pc, b1, b2): resbox = self.execute(rop.%s, b1, b2) - self.metainterp.handle_possible_overflow_error() + self.metainterp.handle_possible_overflow_error(pc) return resbox ''' % (_opimpl, _opimpl.upper())).compile() @@ -280,13 +280,13 @@ except KeyError: pass - @XXX #arguments("descr") - def opimpl_new(self, size): - self.execute_with_descr(rop.NEW, descr=size) - - @XXX #arguments("constbox") - def opimpl_new_with_vtable(self, vtablebox): - self.execute(rop.NEW_WITH_VTABLE, vtablebox) + @arguments("descr") + def opimpl_new(self, sizedescr): + return self.execute_with_descr(rop.NEW, sizedescr) + + @arguments("descr") + def opimpl_new_with_vtable(self, sizevtabledescr): + return self.execute_with_descr(rop.NEW_WITH_VTABLE, sizevtabledescr) @XXX #arguments("box") def opimpl_runtimenew(self, classbox): @@ -611,15 +611,15 @@ def opimpl_call(self, callee, varargs): return self.perform_call(callee, varargs) - @arguments("box", "descr", "boxes") - def _opimpl_residual_call1(self, funcbox, calldescr, argboxes): - return self.do_residual_call(funcbox, calldescr, argboxes, exc=True) - @arguments("box", "descr", "boxes2") - def _opimpl_residual_call2(self, funcbox, calldescr, argboxes): - return self.do_residual_call(funcbox, calldescr, argboxes, exc=True) - @arguments("box", "descr", "boxes3") - def _opimpl_residual_call3(self, funcbox, calldescr, argboxes): - return self.do_residual_call(funcbox, calldescr, argboxes, exc=True) + @arguments("orgpc", "box", "descr", "boxes") + def _opimpl_residual_call1(self, pc, funcbox, calldescr, argboxes): + return self.do_residual_call(funcbox, calldescr, argboxes, exc_at=pc) + @arguments("orgpc", "box", "descr", "boxes2") + def _opimpl_residual_call2(self, pc, funcbox, calldescr, argboxes): + return self.do_residual_call(funcbox, calldescr, argboxes, exc_at=pc) + @arguments("orgpc", "box", "descr", "boxes3") + def _opimpl_residual_call3(self, pc, funcbox, calldescr, argboxes): + return self.do_residual_call(funcbox, calldescr, argboxes, exc_at=pc) opimpl_residual_call_r_i = _opimpl_residual_call1 opimpl_residual_call_r_r = _opimpl_residual_call1 @@ -856,12 +856,10 @@ @arguments("box", "label") def opimpl_goto_if_exception_mismatch(self, vtablebox, next_exc_target): - # XXX use typesystem.py - from pypy.rpython.lltypesystem import rclass - assert self.last_exception is not None - adr = vtablebox.getaddr(self.metainterp.cpu) - bounding_class = llmemory.cast_adr_to_ptr(adr, rclass.CLASSTYPE) - if not rclass.ll_isinstance(self.last_exception, bounding_class): + metainterp = self.metainterp + last_exc_value_box = metainterp.last_exc_value_box + assert last_exc_value_box is not None + if not metainterp.cpu.ts.instanceOf(last_exc_value_box, vtablebox): self.pc = next_exc_target @arguments("box") @@ -1016,16 +1014,16 @@ return self.metainterp.execute_and_record(opnum, descr, *argboxes) @specialize.arg(1) - def execute_varargs(self, opnum, argboxes, descr, exc): + def execute_varargs(self, opnum, argboxes, descr, exc_at): resbox = self.metainterp.execute_and_record_varargs(opnum, argboxes, descr=descr) - if exc: - self.metainterp.handle_possible_exception() + if exc_at >= 0: + self.metainterp.handle_possible_exception(exc_at) else: self.metainterp.assert_no_exception() return resbox - def do_residual_call(self, funcbox, descr, argboxes, exc): + def do_residual_call(self, funcbox, descr, argboxes, exc_at): allboxes = [funcbox] + argboxes effectinfo = descr.get_extra_info() if 0:# XXX effectinfo is None or effectinfo.forces_virtual_or_virtualizable: @@ -1043,7 +1041,7 @@ else: return self.metainterp.assert_no_exception() else: - return self.execute_varargs(rop.CALL, allboxes, descr, exc) + return self.execute_varargs(rop.CALL, allboxes, descr, exc_at) # ____________________________________________________________ @@ -1835,26 +1833,26 @@ # mark by replacing it with ConstPtr(NULL) self.virtualref_boxes[i+1] = self.cpu.ts.CONST_NULL - def handle_possible_exception(self): + def handle_possible_exception(self, pc): frame = self.framestack[-1] if self.last_exc_value_box: exception_box = self.cpu.ts.cls_of_box(self.last_exc_value_box) - op = frame.generate_guard(frame.pc, rop.GUARD_EXCEPTION, + op = frame.generate_guard(pc, rop.GUARD_EXCEPTION, None, [exception_box]) if op: op.result = self.last_exc_value_box self.finishframe_exception() else: - frame.generate_guard(frame.pc, rop.GUARD_NO_EXCEPTION, None, []) + frame.generate_guard(pc, rop.GUARD_NO_EXCEPTION, None, []) - def handle_possible_overflow_error(self): + def handle_possible_overflow_error(self, pc): frame = self.framestack[-1] if self.last_exc_value_box: - frame.generate_guard(frame.pc, rop.GUARD_OVERFLOW, None) + frame.generate_guard(pc, rop.GUARD_OVERFLOW, None) assert isinstance(self.last_exc_value_box, Const) self.finishframe_exception() else: - frame.generate_guard(frame.pc, rop.GUARD_NO_OVERFLOW, None) + frame.generate_guard(pc, rop.GUARD_NO_OVERFLOW, None) def assert_no_exception(self): assert not self.last_exc_value_box Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py Wed May 5 11:20:28 2010 @@ -206,7 +206,7 @@ 'GETFIELD_GC/1d', 'GETFIELD_RAW/1d', 'NEW/0d', - 'NEW_WITH_VTABLE/1', + 'NEW_WITH_VTABLE/0d', 'NEW_ARRAY/1d', 'FORCE_TOKEN/0', 'VIRTUAL_REF/2', Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/typesystem.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/typesystem.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/typesystem.py Wed May 5 11:20:28 2010 @@ -44,7 +44,6 @@ BASETYPE = llmemory.GCREF BoxRef = history.BoxPtr ConstRef = history.ConstPtr - ConstAddr = history.ConstAddr loops_done_with_this_frame_ref = None # patched by compile.py CONST_NULL = history.ConstPtr(history.ConstPtr.value) CVAL_NULLREF = None # patched by optimizeopt.py @@ -76,12 +75,11 @@ cls = llmemory.cast_ptr_to_adr(obj.typeptr) return history.ConstInt(llmemory.cast_adr_to_int(cls)) - def subclassOf(self, cpu, clsbox1, clsbox2): - adr = clsbox2.getaddr(cpu) + def instanceOf(self, instbox, clsbox): + adr = clsbox.getaddr() bounding_class = llmemory.cast_adr_to_ptr(adr, rclass.CLASSTYPE) - adr = clsbox1.getaddr(cpu) - real_class = llmemory.cast_adr_to_ptr(adr, rclass.CLASSTYPE) - return rclass.ll_issubclass(real_class, bounding_class) + real_instance = instbox.getref(rclass.OBJECTPTR) + return rclass.ll_isinstance(real_instance, bounding_class) def get_exception_box(self, etype): return history.ConstInt(etype) @@ -151,7 +149,6 @@ BASETYPE = ootype.Object BoxRef = history.BoxObj ConstRef = history.ConstObj - ConstAddr = history.ConstObj loops_done_with_this_frame_ref = None # patched by compile.py CONST_NULL = history.ConstObj(history.ConstObj.value) CVAL_NULLREF = None # patched by optimizeopt.py From arigo at codespeak.net Wed May 5 11:23:25 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 May 2010 11:23:25 +0200 (CEST) Subject: [pypy-svn] r74377 - in pypy/branch/blackhole-improvement/pypy/jit: backend/llgraph backend/test backend/x86 codewriter codewriter/test metainterp metainterp/test Message-ID: <20100505092325.2B8E7282BDD@codespeak.net> Author: arigo Date: Wed May 5 11:23:22 2010 New Revision: 74377 Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py (contents, props changed) pypy/branch/blackhole-improvement/pypy/jit/codewriter/policy.py - copied, changed from r74271, pypy/branch/blackhole-improvement/pypy/jit/metainterp/policy.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_call.py (contents, props changed) Removed: pypy/branch/blackhole-improvement/pypy/jit/metainterp/policy.py Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py pypy/branch/blackhole-improvement/pypy/jit/backend/x86/assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_codewriter.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py Log: In-progress. Start to write or copy code in codewriter/* to handle more than one JitCode. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py Wed May 5 11:23:22 2010 @@ -103,7 +103,6 @@ 'float_ge' : (('float', 'float'), 'bool'), 'float_neg' : (('float',), 'float'), 'float_abs' : (('float',), 'float'), - 'float_is_true' : (('float',), 'bool'), 'cast_float_to_int':(('float',), 'int'), 'cast_int_to_float':(('int',), 'float'), 'same_as' : (('int',), 'int'), # could also be ptr=>ptr Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py Wed May 5 11:23:22 2010 @@ -1085,7 +1085,6 @@ operator.truediv, no_zero_divison yield nan_and_infinity, rop.FLOAT_NEG, operator.neg, all_cases_unary yield nan_and_infinity, rop.FLOAT_ABS, abs, all_cases_unary - yield nan_and_infinity, rop.FLOAT_IS_TRUE, bool, all_cases_unary yield nan_and_infinity, rop.FLOAT_LT, operator.lt, all_cases_binary yield nan_and_infinity, rop.FLOAT_LE, operator.le, all_cases_binary yield nan_and_infinity, rop.FLOAT_EQ, operator.eq, all_cases_binary Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/x86/assembler.py Wed May 5 11:23:22 2010 @@ -695,32 +695,6 @@ # Following what gcc does: res = x & 0x7FFFFFFFFFFFFFFF self.mc.ANDPD(arglocs[0], self.loc_float_const_abs) - def genop_guard_float_is_true(self, op, guard_op, addr, arglocs, resloc): - guard_opnum = guard_op.opnum - loc0, loc1 = arglocs - self.mc.XORPD(loc0, loc0) - self.mc.UCOMISD(loc0, loc1) - mc = self.mc._mc - if guard_opnum == rop.GUARD_TRUE: - mc.JP(rel8(6)) - mc.JZ(rel32(addr)) - return mc.tell() - 4 - else: - mc.JP(rel8(2)) - mc.JZ(rel8(5)) - return self.implement_guard(addr, mc.JMP) - - def genop_float_is_true(self, op, arglocs, resloc): - loc0, loc1 = arglocs - self.mc.XORPD(loc0, loc0) - self.mc.UCOMISD(loc0, loc1) - rl = resloc.lowest8bits() - rh = resloc.higher8bits() - self.mc.SETNE(rl) - self.mc.SETP(rh) - self.mc.OR(rl, rh) - self.mc.MOVZX(resloc, rl) - def genop_cast_float_to_int(self, op, arglocs, resloc): self.mc.CVTTSD2SI(resloc, arglocs[0]) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Wed May 5 11:23:22 2010 @@ -42,31 +42,39 @@ def num_regs_f(self): return self.num_regs_encoded & 0x1FF + def has_liveness_info(self, pc): + return pc in self.liveness + def enumerate_live_vars(self, pc, callback, arg, registers_i, registers_r, registers_f): # 'pc' gives a position in this bytecode. This invokes # 'callback' for each variable that is live across the # instruction which starts at 'pc'. (It excludes the arguments # of that instruction which are no longer used afterwards, and - # also the return value of that instruction.) More precisely, - # this invokes 'callback(arg, box)' where 'box' comes from one - # of the three lists of registers. If the callback returns a - # box, then it is stored back. + # excludes the return value of that instruction.) More precisely, + # this invokes 'callback(arg, box, index)' where 'box' comes from one + # of the three lists of registers and 'index' is 0, 1, 2... + # If the callback returns a box, then it is stored back. if not we_are_translated() and pc not in self.liveness: self._missing_liveness(pc) live_i, live_r, live_f = self.liveness[pc] # XXX compactify!! + index = 0 for c in live_i: - newbox = callback(arg, registers_i[ord(c)]) + newbox = callback(arg, registers_i[ord(c)], index) + index += 1 if newbox is not None: registers_i[ord(c)] = newbox for c in live_r: - newbox = callback(arg, registers_r[ord(c)]) + newbox = callback(arg, registers_r[ord(c)], index) + index += 1 if newbox is not None: registers_r[ord(c)] = newbox for c in live_f: - newbox = callback(arg, registers_f[ord(c)]) + newbox = callback(arg, registers_f[ord(c)], index) + index += 1 if newbox is not None: registers_f[ord(c)] = newbox + return index enumerate_live_vars._annspecialcase_ = 'specialize:arg(2)' def _live_vars(self, pc): @@ -76,7 +84,7 @@ self.kind = kind def __getitem__(self, index): return '%%%s%d' % (self.kind, index) - def callback(lst, reg): + def callback(lst, reg, index): lst.append(reg) lst = [] self.enumerate_live_vars(pc, callback, lst, @@ -221,7 +229,9 @@ else: raise NotImplementedError(x) # - key = insn[0] + '/' + ''.join(argcodes) + opname = insn[0] + if opname.startswith('G_'): opname = opname[2:] + key = opname + '/' + ''.join(argcodes) num = self.insns.setdefault(key, len(self.insns)) self.code[startposition] = chr(num) Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py ============================================================================== --- (empty file) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py Wed May 5 11:23:22 2010 @@ -0,0 +1,113 @@ +# +# Contains the logic to decide, based on the policy, which graphs +# to transform to JitCodes or not. +# + +from pypy.jit.codewriter import support +from pypy.translator.simplify import get_funcobj + + +class CallControl(object): + + def __init__(self, rtyper=None, portal_runner_obj=None): + self.rtyper = rtyper + self.portal_runner_obj = portal_runner_obj + + def find_all_graphs(self, portal_graph, policy): + def is_candidate(graph): + return policy.look_inside_graph(graph) + + todo = [portal_graph] + candidate_graphs = set(todo) + + def callers(): + graph = top_graph + print graph + while graph in coming_from: + graph = coming_from[graph] + print '<-', graph + coming_from = {} + + while todo: + top_graph = todo.pop() + for _, op in top_graph.iterblockops(): + if op.opname not in ("direct_call", "indirect_call", "oosend"): + continue + kind = self.guess_call_kind(op, is_candidate) + # use callers() to view the calling chain in pdb + if kind != "regular": + continue + for graph in self.graphs_from(op, is_candidate): + if graph in candidate_graphs: + continue + assert is_candidate(graph) + todo.append(graph) + candidate_graphs.add(graph) + coming_from[graph] = top_graph + self.candidate_graphs = candidate_graphs + return candidate_graphs + + def graphs_from(self, op, is_candidate=None): + if is_candidate is None: + is_candidate = self.is_candidate + if op.opname == 'direct_call': + funcobj = get_funcobj(op.args[0].value) + graph = funcobj.graph + if is_candidate(graph): + return [graph] # common case: look inside this graph + else: + assert op.opname in ('indirect_call', 'oosend') + if op.opname == 'indirect_call': + graphs = op.args[-1].value + else: + v_obj = op.args[1].concretetype + graphs = v_obj._lookup_graphs(op.args[0].value) + if graphs is not None: + result = [] + for graph in graphs: + if is_candidate(graph): + result.append(graph) + if result: + return result # common case: look inside these graphs, + # and ignore the others if there are any + else: + # special case: handle the indirect call that goes to + # the 'instantiate' methods. This check is a bit imprecise + # but it's not too bad if we mistake a random indirect call + # for the one to 'instantiate'. + from pypy.rpython.lltypesystem import rclass + CALLTYPE = op.args[0].concretetype + if (op.opname == 'indirect_call' and len(op.args) == 2 and + CALLTYPE == rclass.OBJECT_VTABLE.instantiate): + return list(self._graphs_of_all_instantiate()) + # residual call case: we don't need to look into any graph + return None + + def _graphs_of_all_instantiate(self): + for vtable in self.rtyper.lltype2vtable.values(): + if vtable.instantiate: + yield vtable.instantiate._obj.graph + + def guess_call_kind(self, op, is_candidate=None): + if op.opname == 'direct_call': + funcptr = op.args[0].value + funcobj = get_funcobj(funcptr) + if getattr(funcobj, 'graph', None) is None: + return 'residual' + if funcobj is self.portal_runner_obj: + return 'recursive' + targetgraph = funcobj.graph + if (hasattr(targetgraph, 'func') and + hasattr(targetgraph.func, 'oopspec')): + return 'builtin' + elif op.opname == 'oosend': + SELFTYPE, methname, opargs = support.decompose_oosend(op) + if SELFTYPE.oopspec_name is not None: + return 'builtin' + if self.graphs_from(op, is_candidate) is None: + return 'residual' + return 'regular' + + def is_candidate(self, graph): + # used only after find_all_graphs() + return graph in self.candidate_graphs Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Wed May 5 11:23:22 2010 @@ -165,7 +165,7 @@ for link in block.exits[1:]: if (link.exitcase is Exception or (link.exitcase is OverflowError and - lastopname.startswith('int_') and + lastopname.startswith('G_int_') and lastopname.endswith('_ovf'))): # this link captures all exceptions self.make_exception_link(link) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py Wed May 5 11:23:22 2010 @@ -74,7 +74,7 @@ print "Got: " + asm print "Expected: " + exp lgt = 0 - for i in range(len(asm)): + for i in range(min(len(asm), len(exp))): if exp[i] == asm[i]: lgt += 1 else: @@ -82,3 +82,45 @@ print " " + " " * lgt + "^^^^" raise AssertionError assert len(asmlines) == len(explines) + +def unformat_assembler(text, registers=None): + # XXX limited to simple assembler right now + # + def unformat_arg(s): + if s[0] == '%': + try: + return registers[s] + except KeyError: + num = int(s[2:]) + if s[1] == 'i': reg = Register('int', num) + elif s[1] == 'r': reg = Register('ref', num) + elif s[1] == 'f': reg = Register('float', num) + else: raise AssertionError("bad register type") + registers[s] = reg + return reg + elif s[0] == '$': + intvalue = int(s[1:]) + return Constant(intvalue, lltype.Signed) + elif s[0] == 'L': + return TLabel(s) + else: + raise AssertionError("unsupported argument: %r" % (s,)) + # + if registers is None: + registers = {} + ssarepr = SSARepr('test') + for line in text.splitlines(): + line = line.strip() + if not line: + continue + if line.startswith('L') and line.endswith(':'): + ssarepr.insns.append((Label(line[:-1]),)) + else: + try: + opname, line = line.split(None, 1) + except ValueError: + opname, line = line, '' + line = [s.strip() for s in line.split(',')] + insn = [opname] + [unformat_arg(s) for s in line if s] + ssarepr.insns.append(tuple(insn)) + return ssarepr Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Wed May 5 11:23:22 2010 @@ -121,15 +121,46 @@ return rewrite(self, op) def rewrite_op_same_as(self, op): raise NoOp - def rewrite_op_cast_int_to_char(self, op): raise NoOp - def rewrite_op_cast_int_to_unichar(self, op): raise NoOp + def rewrite_op_cast_pointer(self, op): raise NoOp + def rewrite_op_cast_primitive(self, op): raise NoOp + def rewrite_op_cast_bool_to_int(self, op): raise NoOp + def rewrite_op_cast_bool_to_uint(self, op): raise NoOp def rewrite_op_cast_char_to_int(self, op): raise NoOp def rewrite_op_cast_unichar_to_int(self, op): raise NoOp - def rewrite_op_cast_bool_to_int(self, op): raise NoOp - def rewrite_op_cast_pointer(self, op): raise NoOp + def rewrite_op_cast_int_to_char(self, op): raise NoOp + def rewrite_op_cast_int_to_unichar(self, op): raise NoOp def rewrite_op_cast_int_to_uint(self, op): raise NoOp def rewrite_op_cast_uint_to_int(self, op): raise NoOp + def _rewrite_symmetric(self, op): + """Rewrite 'c1+v2' into 'v2+c1' in an attempt to avoid generating + too many variants of the bytecode.""" + if (isinstance(op.args[0], Constant) and + isinstance(op.args[1], Variable)): + reversename = {'int_lt': 'int_gt', + 'int_le': 'int_ge', + 'int_gt': 'int_lt', + 'int_ge': 'int_le', + 'float_lt': 'float_gt', + 'float_le': 'float_ge', + 'float_gt': 'float_lt', + 'float_ge': 'float_le', + }.get(op.opname, op.opname) + return SpaceOperation(reversename, + [op.args[1], op.args[0]] + op.args[2:], + op.result) + else: + return op + + rewrite_op_int_add = _rewrite_symmetric + rewrite_op_int_mul = _rewrite_symmetric + rewrite_op_int_and = _rewrite_symmetric + rewrite_op_int_or = _rewrite_symmetric + rewrite_op_int_xor = _rewrite_symmetric + + rewrite_op_G_int_add_ovf = _rewrite_symmetric + rewrite_op_G_int_mul_ovf = _rewrite_symmetric + def rewrite_op_direct_call(self, op): """Turn 'i0 = direct_call(fn, i1, i2, ref1, ref2)' into e.g. 'i0 = residual_call_ir_i(fn, [i1, i2], [ref1, ref2])'. @@ -150,7 +181,7 @@ FUNC = op.args[0].concretetype.TO NONVOIDARGS = tuple([ARG for ARG in FUNC.ARGS if ARG != lltype.Void]) calldescr = self.cpu.calldescrof(FUNC, NONVOIDARGS, FUNC.RESULT) - return SpaceOperation('residual_call_%s_%s' % (kinds, reskind), + return SpaceOperation('G_residual_call_%s_%s' % (kinds, reskind), [op.args[0], calldescr] + sublists, op.result) @@ -184,7 +215,7 @@ rewrite_op_int_mod_ovf_zer = _do_builtin_call rewrite_op_int_mod_ovf = _do_builtin_call rewrite_op_int_mod_zer = _do_builtin_call - + rewrite_op_int_lshift_ovf = _do_builtin_call rewrite_op_gc_identityhash = _do_builtin_call def rewrite_op_hint(self, op): @@ -192,7 +223,9 @@ if hints.get('promote') and op.args[0].concretetype is not lltype.Void: assert op.args[0].concretetype != lltype.Ptr(rstr.STR) kind = getkind(op.args[0].concretetype) - return SpaceOperation('%s_guard_value' % kind, + # note: the 'G_' prefix tells that the operation might generate + # a guard in pyjitpl (see liveness.py) + return SpaceOperation('G_%s_guard_value' % kind, [op.args[0]], op.result) else: log.WARNING('ignoring hint %r at %r' % (hints, self.graph)) @@ -308,7 +341,9 @@ op.args[0].concretetype.TO._hints.get('typeptr')) def handle_getfield_typeptr(self, op): - return SpaceOperation('guard_class', [op.args[0]], op.result) + # note: the 'G_' prefix tells that the operation might generate + # a guard in pyjitpl (see liveness.py) + return SpaceOperation('G_guard_class', [op.args[0]], op.result) def rewrite_op_malloc(self, op): assert op.args[1].value == {'flavor': 'gc'} @@ -343,7 +378,7 @@ elif isinstance(arg1, Constant) and not arg1.value: return SpaceOperation(opname, [arg0], op.result) else: - return op + return self._rewrite_symmetric(op) def _is_gc(self, v): return v.concretetype.TO._gckind == 'gc' @@ -381,14 +416,69 @@ else: raise NoOp + # ---------- + # Renames, from the _old opname to the _new one. + # The new operation is optionally further processed by rewrite_operation(). for _old, _new in [('bool_not', 'int_is_zero'), ('cast_bool_to_float', 'cast_int_to_float'), + ('cast_uint_to_float', 'cast_int_to_float'), + ('cast_float_to_uint', 'cast_float_to_int'), + + ('int_add_ovf', 'G_int_add_ovf'), + ('int_add_nonneg_ovf', 'G_int_add_ovf'), + ('int_sub_ovf', 'G_int_sub_ovf'), + ('int_mul_ovf', 'G_int_mul_ovf'), + + ('char_lt', 'int_lt'), + ('char_le', 'int_le'), + ('char_eq', 'int_eq'), + ('char_ne', 'int_ne'), + ('char_gt', 'int_gt'), + ('char_ge', 'int_ge'), + ('unichar_eq', 'int_eq'), + ('unichar_ne', 'int_ne'), + + ('uint_is_true', 'int_is_true'), + ('uint_invert', 'int_invert'), + ('uint_add', 'int_add'), + ('uint_sub', 'int_sub'), + ('uint_mul', 'int_mul'), + ('uint_floordiv', 'int_floordiv'), + ('uint_floordiv_zer', 'int_floordiv_zer'), + ('uint_mod', 'int_mod'), + ('uint_mod_zer', 'int_mod_zer'), + ('uint_lt', 'int_lt'), + ('uint_le', 'int_le'), + ('uint_eq', 'int_eq'), + ('uint_ne', 'int_ne'), + ('uint_gt', 'int_gt'), + ('uint_ge', 'int_ge'), + ('uint_and', 'int_and'), + ('uint_or', 'int_or'), + ('uint_lshift', 'int_lshift'), + ('uint_rshift', 'int_rshift'), + ('uint_xor', 'int_xor'), + + ]: exec py.code.Source(''' def rewrite_op_%s(self, op): - return SpaceOperation(%r, op.args, op.result) + op1 = SpaceOperation(%r, op.args, op.result) + return self.rewrite_operation(op1) ''' % (_old, _new)).compile() + def rewrite_op_int_neg_ovf(self, op): + op1 = SpaceOperation('int_sub_ovf', + [Constant(0, lltype.Signed), op.args[0]], + op.result) + return self.rewrite_operation(op1) + + def rewrite_op_float_is_true(self, op): + op1 = SpaceOperation('float_ne', + [op.args[0], Constant(0.0, lltype.Float)], + op.result) + return self.rewrite_operation(op1) + # ____________________________________________________________ def _with_prefix(prefix): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py Wed May 5 11:23:22 2010 @@ -1,50 +1,37 @@ -import re -from pypy.objspace.flow.model import Variable, SpaceOperation +from pypy.objspace.flow.model import Variable, SpaceOperation, c_last_exception -def compile_re(opnames): - return re.compile('|'.join(opnames)) - -# List of instruction prefixes that might need the liveness information -# (they are the ones that can end up in generate_guard() in pyjitpl.py). -# Note that the goto_if_not_* operations are not present in the control -# flow graph format; their liveness information is attached by setting -# 'switches_require_liveness' to True. -DEFAULT_OPNAMES_REQUIRING_LIVENESS = [ - 'residual_call_', - '(int|ref|float)_guard_value', - 'guard_class', - 'int_(add|sub|mul)_ovf', - ] +# Some instruction require liveness information (the ones that can end up +# in generate_guard() in pyjitpl.py); jitter.py prefixes these opnames +# with a 'G_'. Additionally, boolean and general switches in the flow graph +# will turn in 'goto_if_not_*' operations, which also require liveness info. # ____________________________________________________________ -def compute_liveness(graph, - switches_require_liveness = True, - opnames_requiring_liveness = - compile_re(DEFAULT_OPNAMES_REQUIRING_LIVENESS)): - if isinstance(opnames_requiring_liveness, list): - opnames_requiring_liveness = compile_re(opnames_requiring_liveness) +def compute_liveness(graph, switches_require_liveness=True): for block in graph.iterblocks(): num_operations = len(block.operations) alive = set() for link in block.exits: for v in link.args: - alive.add(v) - if switches_require_liveness and len(block.exits) > 1: - block.operations.append(_livespaceop(alive)) + if (v is not link.last_exception and + v is not link.last_exc_value): + alive.add(v) + if switches_require_liveness: + if len(block.exits) > 1 and block.exitswitch != c_last_exception: + block.operations.append(_livespaceop(alive)) if isinstance(block.exitswitch, tuple): for v in block.exitswitch[1:]: alive.add(v) else: - alive.add(v) + alive.add(block.exitswitch) for i in range(num_operations-1, -1, -1): op = block.operations[i] try: alive.remove(op.result) except KeyError: pass - if opnames_requiring_liveness.match(op.opname): + if op.opname.startswith('G_'): block.operations.insert(i, _livespaceop(alive)) for v in op.args: alive.add(v) Copied: pypy/branch/blackhole-improvement/pypy/jit/codewriter/policy.py (from r74271, pypy/branch/blackhole-improvement/pypy/jit/metainterp/policy.py) ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/policy.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/policy.py Wed May 5 11:23:22 2010 @@ -3,6 +3,12 @@ from pypy.rpython.lltypesystem import lltype, rclass from pypy.tool.udir import udir +import py, sys +from pypy.tool.ansi_print import ansi_log +log = py.log.Producer('jitcodewriter') +py.log.setconsumer('jitcodewriter', ansi_log) + + class JitPolicy(object): def __init__(self): self.unsafe_loopy_graphs = set() @@ -40,9 +46,6 @@ # string builder interface if mod == 'pypy.rpython.lltypesystem.rbuilder': return True - # rweakvaluedict implementation - if mod == 'pypy.rlib.rweakrefimpl': - return True return False @@ -76,7 +79,6 @@ getkind(v.concretetype, supports_floats) getkind(op.result.concretetype, supports_floats) except NotImplementedError, e: - from pypy.jit.metainterp.codewriter import log log.WARNING('%s, ignoring graph' % (e,)) log.WARNING(' %s' % (graph,)) return True Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py Wed May 5 11:23:22 2010 @@ -157,6 +157,9 @@ def _ll_2_int_mod_zer(x, y): return llop.int_mod_zer(lltype.Signed, x, y) +def _ll_2_int_lshift_ovf(x, y): + return llop.int_lshift_ovf(lltype.Signed, x, y) + class LLtypeHelpers: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py Wed May 5 11:23:22 2010 @@ -161,15 +161,15 @@ i3, i4, i5 = Register('int', 3), Register('int', 4), Register('int', 5) ssarepr.insns = [ ('-live-', i0), - ('int_add', i0, Constant(10, lltype.Signed), i1), + ('G_int_add', i0, Constant(10, lltype.Signed), i1), ('-live-', i0, i1), - ('int_add', i0, Constant(3, lltype.Signed), i2), + ('G_int_add', i0, Constant(3, lltype.Signed), i2), ('-live-', i0), - ('int_mul', i1, i2, i3), + ('G_int_mul', i1, i2, i3), ('-live-', i3), - ('int_add', i0, Constant(6, lltype.Signed), i4), + ('G_int_add', i0, Constant(6, lltype.Signed), i4), ('-live-',), - ('int_mul', i3, i4, i5), + ('G_int_mul', i3, i4, i5), ('int_return', i5), ] assembler = Assembler() Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_call.py ============================================================================== --- (empty file) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_call.py Wed May 5 11:23:22 2010 @@ -0,0 +1,208 @@ +from pypy.objspace.flow.model import SpaceOperation, Constant, Variable +from pypy.rpython.lltypesystem import lltype +from pypy.translator.unsimplify import varoftype +from pypy.rlib import jit +from pypy.jit.codewriter.call import CallControl +from pypy.jit.codewriter import support +from pypy.jit.codewriter.policy import JitPolicy + + +def test_graphs_from_direct_call(): + cc = CallControl() + F = lltype.FuncType([], lltype.Signed) + f = lltype.functionptr(F, 'f', graph='fgraph') + v = varoftype(lltype.Signed) + op = SpaceOperation('direct_call', [Constant(f, lltype.Ptr(F))], v) + # + lst = cc.graphs_from(op, {}.__contains__) + assert lst is None # residual call + # + lst = cc.graphs_from(op, {'fgraph': True}.__contains__) + assert lst == ['fgraph'] # normal call + +def test_graphs_from_indirect_call(): + cc = CallControl() + F = lltype.FuncType([], lltype.Signed) + v = varoftype(lltype.Signed) + graphlst = ['f1graph', 'f2graph'] + op = SpaceOperation('indirect_call', [varoftype(lltype.Ptr(F)), + Constant(graphlst, lltype.Void)], v) + # + lst = cc.graphs_from(op, {'f1graph': True, 'f2graph': True}.__contains__) + assert lst == ['f1graph', 'f2graph'] # normal indirect call + # + lst = cc.graphs_from(op, {'f1graph': True}.__contains__) + assert lst == ['f1graph'] # indirect call, look only inside some graphs + # + lst = cc.graphs_from(op, {}.__contains__) + assert lst is None # indirect call, don't look inside any graph + +def test_graphs_from_no_target(): + cc = CallControl() + F = lltype.FuncType([], lltype.Signed) + v = varoftype(lltype.Signed) + op = SpaceOperation('indirect_call', [varoftype(lltype.Ptr(F)), + Constant(None, lltype.Void)], v) + lst = cc.graphs_from(op, {}.__contains__) + assert lst is None + +# ____________________________________________________________ + +def test_find_all_graphs(): + def f(x): + if x < 0: + return f(-x) + return x + 1 + @jit.purefunction + def g(x): + return x + 2 + @jit.dont_look_inside + def h(x): + return x + 3 + def i(x): + return f(x) * g(x) * h(x) + + rtyper = support.annotate(i, [7]) + cc = CallControl() + jitpolicy = JitPolicy() + res = cc.find_all_graphs(rtyper.annotator.translator.graphs[0], + jitpolicy) + translator = rtyper.annotator.translator + + funcs = set([graph.func for graph in res]) + assert funcs == set([i, f]) + +def test_find_all_graphs_without_floats(): + def g(x): + return int(x * 12.5) + def f(x): + return g(x) + 1 + rtyper = support.annotate(f, [7]) + cc = CallControl() + jitpolicy = JitPolicy() + jitpolicy.set_supports_floats(True) + translator = rtyper.annotator.translator + res = cc.find_all_graphs(translator.graphs[0], jitpolicy) + funcs = set([graph.func for graph in res]) + assert funcs == set([f, g]) + + cc = CallControl() + jitpolicy.set_supports_floats(False) + res = cc.find_all_graphs(translator.graphs[0], jitpolicy) + funcs = [graph.func for graph in res] + assert funcs == [f] + +def test_find_all_graphs_loops(): + def g(x): + i = 0 + while i < x: + i += 1 + return i + @jit.unroll_safe + def h(x): + i = 0 + while i < x: + i += 1 + return i + + def f(x): + i = 0 + while i < x*x: + i += g(x) + h(x) + return i + + rtyper = support.annotate(f, [7]) + cc = CallControl() + jitpolicy = JitPolicy() + translator = rtyper.annotator.translator + res = cc.find_all_graphs(translator.graphs[0], jitpolicy) + funcs = set([graph.func for graph in res]) + assert funcs == set([f, h]) + +def test_unroll_safe_and_inline(): + @jit.unroll_safe + def h(x): + i = 0 + while i < x: + i += 1 + return i + h._always_inline_ = True + + def g(x): + return h(x) + + rtyper = support.annotate(g, [7]) + cc = CallControl() + jitpolicy = JitPolicy() + translator = rtyper.annotator.translator + res = cc.find_all_graphs(translator.graphs[0], jitpolicy) + funcs = set([graph.func for graph in res]) + assert funcs == set([g, h]) + +def test_find_all_graphs_str_join(): + def i(x, y): + return "hello".join([str(x), str(y), "bye"]) + + rtyper = support.annotate(i, [7, 100]) + cc = CallControl() + jitpolicy = JitPolicy() + translator = rtyper.annotator.translator + # does not explode + cc.find_all_graphs(translator.graphs[0], jitpolicy) + +# ____________________________________________________________ + +def test_guess_call_kind_and_calls_from_graphs(): + class portal_runner_obj: + graph = object() + g = object() + g1 = object() + cc = CallControl(portal_runner_obj=portal_runner_obj) + cc.candidate_graphs = [g, g1] + + op = SpaceOperation('direct_call', [Constant(portal_runner_obj)], + Variable()) + assert cc.guess_call_kind(op) == 'recursive' + + op = SpaceOperation('direct_call', [Constant(object())], + Variable()) + assert cc.guess_call_kind(op) == 'residual' + + class funcptr: + class graph: + class func: + oopspec = "spec" + op = SpaceOperation('direct_call', [Constant(funcptr)], + Variable()) + assert cc.guess_call_kind(op) == 'builtin' + + class funcptr: + graph = g + op = SpaceOperation('direct_call', [Constant(funcptr)], + Variable()) + res = cc.graphs_from(op) + assert res == [g] + assert cc.guess_call_kind(op) == 'regular' + + class funcptr: + graph = object() + op = SpaceOperation('direct_call', [Constant(funcptr)], + Variable()) + res = cc.graphs_from(op) + assert res is None + assert cc.guess_call_kind(op) == 'residual' + + h = object() + op = SpaceOperation('indirect_call', [Variable(), + Constant([g, g1, h])], + Variable()) + res = cc.graphs_from(op) + assert res == [g, g1] + assert cc.guess_call_kind(op) == 'regular' + + op = SpaceOperation('indirect_call', [Variable(), + Constant([h])], + Variable()) + res = cc.graphs_from(op) + assert res is None + assert cc.guess_call_kind(op) == 'residual' Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Wed May 5 11:23:22 2010 @@ -8,6 +8,7 @@ from pypy.objspace.flow.model import SpaceOperation, Variable, Constant from pypy.translator.unsimplify import varoftype from pypy.rlib.rarithmetic import ovfcheck +from pypy.rlib.jit import dont_look_inside class FakeRegAlloc: @@ -261,6 +262,7 @@ def test_exc_exitswitch_2(self): class FooError(Exception): pass + @dont_look_inside def g(i): FooError().num = 1 FooError().num = 2 @@ -275,7 +277,7 @@ return 4 self.encoding_test(f, [65], """ - residual_call_ir_v $<* fn g>, , I[%i0], R[] + G_residual_call_ir_v $<* fn g>, , I[%i0], R[] catch_exception L1 int_return $4 L1: @@ -340,7 +342,7 @@ except ZeroDivisionError: return -42 self.encoding_test(f, [7, 2], """ - residual_call_ir_i $<* fn int_floordiv_ovf_zer>, , I[%i0, %i1], R[], %i2 + G_residual_call_ir_i $<* fn int_floordiv_ovf_zer>, , I[%i0, %i1], R[], %i2 catch_exception L1 int_return %i2 L1: @@ -363,7 +365,7 @@ return 42 # XXX so far, this really produces a int_mod_ovf_zer... self.encoding_test(f, [7, 2], """ - residual_call_ir_i $<* fn int_mod_ovf_zer>, , I[%i0, %i1], R[], %i2 + G_residual_call_ir_i $<* fn int_mod_ovf_zer>, , I[%i0, %i1], R[], %i2 catch_exception L1 int_return %i2 L1: @@ -380,10 +382,44 @@ except OverflowError: return 42 self.encoding_test(f, [7, 2], """ - int_add_ovf %i0, %i1, %i2 - -live- %i2 + -live- + G_int_add_ovf %i0, %i1, %i2 catch_exception L1 int_return %i2 L1: int_return $42 """, transform=True, liveness=True) + + def test_residual_call_raising(self): + @dont_look_inside + def g(i, j): + return ovfcheck(i + j) + def f(i, j): + try: + return g(i, j) + except Exception: + return 42 + j + self.encoding_test(f, [7, 2], """ + -live- %i1 + G_residual_call_ir_i $<* fn g>, , I[%i0, %i1], R[], %i2 + catch_exception L1 + int_return %i2 + L1: + int_copy %i1, %i3 + int_add %i3, $42, %i4 + int_return %i4 + """, transform=True, liveness=True) + + def test_residual_call_nonraising(self): + @dont_look_inside + def g(i, j): + return i + j + def f(i, j): + try: + return g(i, j) + except Exception: + return 42 + j + self.encoding_test(f, [7, 2], """ + residual_call_ir_i $<* fn g>, , I[%i0, %i1], R[], %i2 + int_return %i2 + """, transform=True, liveness=True) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py Wed May 5 11:23:22 2010 @@ -1,6 +1,6 @@ import py from pypy.objspace.flow.model import Constant -from pypy.jit.codewriter.format import format_assembler +from pypy.jit.codewriter.format import format_assembler, unformat_assembler from pypy.jit.codewriter.flatten import Label, TLabel, SSARepr, Register from pypy.jit.codewriter.flatten import ListOfKind from pypy.jit.metainterp.history import AbstractDescr @@ -96,3 +96,41 @@ foobar hi_there! """ assert asm == str(py.code.Source(expected)).strip() + '\n' + +def test_unformat_assembler_simple(): + input = """ + int_add %i0, %i1, %i2 + int_return %i2 + """ + regs = {} + ssarepr = unformat_assembler(input, regs) + assert regs['%i2'].kind == 'int' + assert regs['%i2'].index == 2 + assert ssarepr.insns == [ + ('int_add', regs['%i0'], regs['%i1'], regs['%i2']), + ('int_return', regs['%i2']), + ] + +def test_unformat_assembler_consts(): + input = """ + foo $123 + """ + ssarepr = unformat_assembler(input) + assert ssarepr.insns == [ + ('foo', Constant(123, lltype.Signed)), + ] + +def test_unformat_assembler_label(): + input = """ + L1: + foo L2 + L2: + bar L1 + """ + ssarepr = unformat_assembler(input) + assert ssarepr.insns == [ + (Label('L1'),), + ('foo', TLabel('L2')), + (Label('L2'),), + ('bar', TLabel('L1')), + ] Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Wed May 5 11:23:22 2010 @@ -98,6 +98,23 @@ assert block.exitswitch == (opname, v1) assert block.exits == exits +def test_symmetric(): + ops = {'int_add': 'int_add', + 'int_gt': 'int_lt', + 'int_le': 'int_ge'} + v3 = varoftype(lltype.Signed) + for v1 in [varoftype(lltype.Signed), Constant(42, lltype.Signed)]: + for v2 in [varoftype(lltype.Signed), Constant(43, lltype.Signed)]: + for name1, name2 in ops.items(): + op = SpaceOperation(name1, [v1, v2], v3) + op1 = Transformer(FakeCPU()).rewrite_operation(op) + if isinstance(v1, Constant) and isinstance(v2, Variable): + assert op1.opname == name2 + assert op1.args == [v2, v1] + assert op1.result == v3 + else: + assert op1 is op + def test_residual_call(): for RESTYPE in [lltype.Signed, rclass.OBJECTPTR, lltype.Float, lltype.Void]: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py Wed May 5 11:23:22 2010 @@ -3,6 +3,7 @@ from pypy.jit.codewriter.test.test_flatten import fake_regallocs from pypy.jit.codewriter.flatten import flatten_graph from pypy.jit.codewriter.format import assert_format +from pypy.objspace.flow.model import SpaceOperation class TestFlatten: @@ -11,12 +12,22 @@ self.rtyper = support.annotate(func, values, type_system=type_system) return self.rtyper.annotator.translator.graphs + def add_G_prefix(self, graph): + """Add a 'G_' prefix to the opnames 'int_add' and 'int_mul'.""" + def with_prefix(op): + if op.opname in ('int_add', 'int_mul'): + return SpaceOperation('G_' + op.opname, op.args, op.result) + return op + # + for block in graph.iterblocks(): + if block.operations: + block.operations = map(with_prefix, block.operations) + def encoding_test(self, func, args, expected, switches_require_liveness=False): graphs = self.make_graphs(func, args) - compute_liveness(graphs[0], - switches_require_liveness=switches_require_liveness, - opnames_requiring_liveness=['int_add', 'int_mul']) + self.add_G_prefix(graphs[0]) + compute_liveness(graphs[0], switches_require_liveness) ssarepr = flatten_graph(graphs[0], fake_regallocs()) assert_format(ssarepr, expected) @@ -25,7 +36,7 @@ return n + 10 self.encoding_test(f, [5], """ -live- - int_add %i0, $10, %i1 + G_int_add %i0, $10, %i1 int_return %i1 """) @@ -34,15 +45,15 @@ return (n + 10) * (n + 3) * (n + 6) self.encoding_test(f, [5], """ -live- %i0 - int_add %i0, $10, %i1 + G_int_add %i0, $10, %i1 -live- %i0, %i1 - int_add %i0, $3, %i2 + G_int_add %i0, $3, %i2 -live- %i0 - int_mul %i1, %i2, %i3 + G_int_mul %i1, %i2, %i3 -live- %i3 - int_add %i0, $6, %i4 + G_int_add %i0, $6, %i4 -live- - int_mul %i3, %i4, %i5 + G_int_mul %i3, %i4, %i5 int_return %i5 """) @@ -53,17 +64,17 @@ return y+2 self.encoding_test(f, [5, 6], """ -live- %i0, %i1 - int_add %i0, $5, %i2 + G_int_add %i0, $5, %i2 int_is_true %i2, %i3 goto_if_not L1, %i3 int_copy %i0, %i4 -live- - int_add %i4, $1, %i5 + G_int_add %i4, $1, %i5 int_return %i5 L1: int_copy %i1, %i6 -live- - int_add %i6, $2, %i7 + G_int_add %i6, $2, %i7 int_return %i7 """) @@ -74,18 +85,18 @@ return x+2 self.encoding_test(f, [5, 6], """ -live- %i0, %i1 - int_add %i0, $5, %i2 + G_int_add %i0, $5, %i2 int_is_true %i2, %i3 goto_if_not L1, %i3 int_copy %i0, %i4 int_copy %i1, %i5 -live- - int_add %i4, %i5, %i6 + G_int_add %i4, %i5, %i6 int_return %i6 L1: int_copy %i0, %i7 -live- - int_add %i7, $2, %i8 + G_int_add %i7, $2, %i8 int_return %i8 """) @@ -96,17 +107,17 @@ return x+2 self.encoding_test(f, [5, 6], """ -live- %i0 - int_add %i0, %i1, %i2 + G_int_add %i0, %i1, %i2 int_is_true %i2, %i3 goto_if_not L1, %i3 int_copy %i0, %i4 -live- - int_add %i4, $5, %i5 + G_int_add %i4, $5, %i5 int_return %i5 L1: int_copy %i0, %i6 -live- - int_add %i6, $2, %i7 + G_int_add %i6, $2, %i7 int_return %i7 """) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Wed May 5 11:23:22 2010 @@ -5,6 +5,7 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.debug import debug_start, debug_stop, debug_print +from pypy.rlib.debug import make_sure_not_resized from pypy.jit.metainterp import history, compile, resume from pypy.jit.metainterp.history import Const, ConstInt, ConstPtr, ConstFloat @@ -97,17 +98,23 @@ prepare_list_of_boxes._annspecialcase_ = 'specialize:arg(4)' def get_list_of_active_boxes(self): - # XXX find a way to avoid needing the temporary 'env' as a - # variable-sized list - env = [] + count = self.jitcode.enumerate_live_vars( + self.pc, MIFrame._count_boxes, None, + self.registers_i, self.registers_r, self.registers_f) + env = [None] * count self.jitcode.enumerate_live_vars( self.pc, MIFrame._store_in_env, env, self.registers_i, self.registers_r, self.registers_f) - return env[:] + make_sure_not_resized(env) + return env + + @staticmethod + def _count_boxes(_, box, index): + pass # just used to count how many boxes there are @staticmethod - def _store_in_env(env, box): - env.append(box) + def _store_in_env(env, box, index): + env[index] = box def replace_active_box_in_frame(self, oldbox, newbox): if isinstance(oldbox, history.BoxInt): @@ -158,7 +165,6 @@ for _opimpl in ['int_is_true', 'int_neg', 'int_invert', 'bool_not', 'cast_ptr_to_int', 'cast_float_to_int', 'cast_int_to_float', 'float_neg', 'float_abs', - 'float_is_true', ]: exec py.code.Source(''' @arguments("box") Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py Wed May 5 11:23:22 2010 @@ -156,7 +156,6 @@ 'FLOAT_TRUEDIV/2', 'FLOAT_NEG/1', 'FLOAT_ABS/1', - 'FLOAT_IS_TRUE/1b', 'CAST_FLOAT_TO_INT/1', 'CAST_INT_TO_FLOAT/1', # Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Wed May 5 11:23:22 2010 @@ -118,7 +118,6 @@ assert get_stats().aborted_count >= count def meta_interp(self, *args, **kwds): - py.test.skip("XXX") kwds['CPUClass'] = self.CPUClass kwds['type_system'] = self.type_system if "backendopt" not in kwds: @@ -142,7 +141,6 @@ def check_operations_history(self, expected=None, **isns): # this can be used after interp_operations - py.test.skip("XXX") self.metainterp.staticdata.stats.check_history(expected, **isns) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_codewriter.py Wed May 5 11:23:22 2010 @@ -9,109 +9,6 @@ from pypy.translator.translator import graphof from pypy.rpython.lltypesystem.rbuiltin import ll_instantiate -def test_find_all_graphs(): - def f(x): - if x < 0: - return f(-x) - return x + 1 - @jit.purefunction - def g(x): - return x + 2 - @jit.dont_look_inside - def h(x): - return x + 3 - def i(x): - return f(x) * g(x) * h(x) - - rtyper = support.annotate(i, [7]) - cw = CodeWriter(rtyper) - jitpolicy = JitPolicy() - res = cw.find_all_graphs(rtyper.annotator.translator.graphs[0], - jitpolicy, True) - translator = rtyper.annotator.translator - - funcs = set([graph.func for graph in res]) - assert funcs == set([i, f]) - -def test_find_all_graphs_without_floats(): - def g(x): - return int(x * 12.5) - def f(x): - return g(x) + 1 - rtyper = support.annotate(f, [7]) - cw = CodeWriter(rtyper) - jitpolicy = JitPolicy() - translator = rtyper.annotator.translator - res = cw.find_all_graphs(translator.graphs[0], jitpolicy, - supports_floats=True) - funcs = set([graph.func for graph in res]) - assert funcs == set([f, g]) - - cw = CodeWriter(rtyper) - res = cw.find_all_graphs(translator.graphs[0], jitpolicy, - supports_floats=False) - funcs = [graph.func for graph in res] - assert funcs == [f] - -def test_find_all_graphs_loops(): - def g(x): - i = 0 - while i < x: - i += 1 - return i - @jit.unroll_safe - def h(x): - i = 0 - while i < x: - i += 1 - return i - - def f(x): - i = 0 - while i < x*x: - i += g(x) + h(x) - return i - - rtyper = support.annotate(f, [7]) - cw = CodeWriter(rtyper) - jitpolicy = JitPolicy() - translator = rtyper.annotator.translator - res = cw.find_all_graphs(translator.graphs[0], jitpolicy, - supports_floats=True) - funcs = set([graph.func for graph in res]) - assert funcs == set([f, h]) - -def test_unroll_safe_and_inline(): - @jit.unroll_safe - def h(x): - i = 0 - while i < x: - i += 1 - return i - h._always_inline_ = True - - def g(x): - return h(x) - - rtyper = support.annotate(g, [7]) - cw = CodeWriter(rtyper) - jitpolicy = JitPolicy() - translator = rtyper.annotator.translator - res = cw.find_all_graphs(translator.graphs[0], jitpolicy, - supports_floats=True) - funcs = set([graph.func for graph in res]) - assert funcs == set([g, h]) - -def test_find_all_graphs_str_join(): - def i(x, y): - return "hello".join([str(x), str(y), "bye"]) - - rtyper = support.annotate(i, [7, 100]) - cw = CodeWriter(rtyper) - jitpolicy = JitPolicy() - translator = rtyper.annotator.translator - # does not explode - cw.find_all_graphs(translator.graphs[0], jitpolicy, True) class SomeLabel(object): def __eq__(self, other): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py Wed May 5 11:23:22 2010 @@ -152,9 +152,8 @@ self.set_translator(translator) self.find_portal() self.codewriter = codewriter.CodeWriter(self.rtyper) - graphs = self.codewriter.find_all_graphs(self.portal_graph, - policy, - CPUClass.supports_floats) + policy.set_supports_floats(CPUClass.supports_floats) + graphs = self.codewriter.find_all_graphs(self.portal_graph, policy) policy.dump_unsafe_loops() self.check_access_directly_sanity(graphs) if backendopt: From afa at codespeak.net Wed May 5 14:09:30 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 5 May 2010 14:09:30 +0200 (CEST) Subject: [pypy-svn] r74378 - pypy/trunk/pypy/module/cpyext Message-ID: <20100505120930.766D5282BDD@codespeak.net> Author: afa Date: Wed May 5 14:09:28 2010 New Revision: 74378 Modified: pypy/trunk/pypy/module/cpyext/stringobject.py pypy/trunk/pypy/module/cpyext/unicodeobject.py Log: Document, factorize, rename some variables and reorganize imports. Modified: pypy/trunk/pypy/module/cpyext/stringobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/stringobject.py (original) +++ pypy/trunk/pypy/module/cpyext/stringobject.py Wed May 5 14:09:28 2010 @@ -1,11 +1,53 @@ from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import ( - cpython_api, bootstrap_function, PyVarObjectFields, Py_ssize_t, - cpython_struct, PyObjectFields, ADDR, CONST_STRING, CANNOT_FAIL, - build_type_checkers, PyObjectP, PyTypeObjectPtr, generic_cpy_call) -from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref, Py_DecRef, make_typedescr -from pypy.module.cpyext.state import State + cpython_api, cpython_struct, bootstrap_function, build_type_checkers, + PyObjectFields, Py_ssize_t, CONST_STRING) +from pypy.module.cpyext.pyobject import ( + PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference, + make_typedescr, get_typedescr) + +## +## Implementation of PyStringObject +## ================================ +## +## The problem +## ----------- +## +## PyString_AsString() must returns a (non-movable) pointer to the underlying +## buffer, whereas pypy strings are movable. C code may temporarily store +## this address and use it, as long as it owns a reference to the PyObject. +## There is no "release" function to specify that the pointer is not needed +## any more. +## +## Also, the pointer may be used to fill the initial value of string. This is +## valid only when the string was just allocated, and is not used elsewhere. +## +## Solution +## -------- +## +## PyStringObject contains two additional members: the size and a pointer to a +## char buffer; it may be NULL. +## +## - A string allocated by pypy will be converted into a PyStringObject with a +## NULL buffer. The first time PyString_AsString() is called, memory is +## allocated (with flavor='raw') and content is copied. +## +## - A string allocated with PyString_FromStringAndSize(NULL, size) will +## allocate a PyStringObject structure, and a buffer with the specified +## size, but the reference won't be stored in the global map; there is no +## corresponding object in pypy. When from_ref() or Py_INCREF() is called, +## the pypy string is created, and added to the global map of tracked +## objects. The buffer is then supposed to be immutable. +## +## - _PyString_Resize() works only on not-yet-pypy'd strings, and returns a +## similar object. +## +## - PyString_Size() doesn't need to force the object. +## +## - There could be an (expensive!) check in from_ref() that the buffer still +## corresponds to the pypy gc-managed string. +## PyStringObjectStruct = lltype.ForwardReference() PyStringObject = lltype.Ptr(PyStringObjectStruct) @@ -15,6 +57,7 @@ @bootstrap_function def init_stringobject(space): + "Type description of PyStringObject" make_typedescr(space.w_str.instancetypedef, basestruct=PyStringObject.TO, attach=string_attach, @@ -24,40 +67,53 @@ PyString_Check, PyString_CheckExact = build_type_checkers("String", "w_str") def new_empty_str(space, length): - py_str = lltype.malloc(PyStringObject.TO, flavor='raw') - py_str.c_ob_refcnt = 1 - + """ + Allocatse a PyStringObject and its buffer, but without a corresponding + interpreter object. The buffer may be mutated, until string_realize() is + called. + """ + typedescr = get_typedescr(space.w_str.instancetypedef) + py_obj = typedescr.allocate(space, space.w_str) + py_str = rffi.cast(PyStringObject, py_obj) + buflen = length + 1 + py_str.c_size = length py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, buflen, flavor='raw', zero=True) - py_str.c_size = length - py_str.c_ob_type = rffi.cast(PyTypeObjectPtr, make_ref(space, space.w_str)) return py_str def string_attach(space, py_obj, w_obj): + """ + Fills a newly allocated PyStringObject with the given string object. The + buffer must not be modified. + """ py_str = rffi.cast(PyStringObject, py_obj) py_str.c_size = len(space.str_w(w_obj)) py_str.c_buffer = lltype.nullptr(rffi.CCHARP.TO) -def string_realize(space, ref): - state = space.fromcache(State) - ref = rffi.cast(PyStringObject, ref) - s = rffi.charpsize2str(ref.c_buffer, ref.c_size) - ref = rffi.cast(PyObject, ref) - w_str = space.wrap(s) - state.py_objects_w2r[w_str] = ref - ptr = rffi.cast(ADDR, ref) - state.py_objects_r2w[ptr] = w_str - return w_str +def string_realize(space, py_obj): + """ + Creates the string in the interpreter. The PyStringObject buffer must not + be modified after this call. + """ + py_str = rffi.cast(PyStringObject, py_obj) + s = rffi.charpsize2str(py_str.c_buffer, py_str.c_size) + w_obj = space.wrap(s) + track_reference(space, py_obj, w_obj) + return w_obj @cpython_api([PyObject], lltype.Void, external=False) def string_dealloc(space, py_obj): + """Frees allocated PyStringObject resources. + """ py_str = rffi.cast(PyStringObject, py_obj) if py_str.c_buffer: lltype.free(py_str.c_buffer, flavor="raw") from pypy.module.cpyext.object import PyObject_dealloc PyObject_dealloc(space, py_obj) +#_______________________________________________________________________ + @cpython_api([CONST_STRING, Py_ssize_t], PyObject) def PyString_FromStringAndSize(space, char_p, length): if char_p: Modified: pypy/trunk/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/unicodeobject.py Wed May 5 14:09:28 2010 @@ -3,15 +3,17 @@ from pypy.module.unicodedata import unicodedb_4_1_0 as unicodedb from pypy.module.cpyext.api import ( CANNOT_FAIL, Py_ssize_t, build_type_checkers, cpython_api, - bootstrap_function, generic_cpy_call, PyObjectFields, - cpython_struct, CONST_STRING, CONST_WSTRING) + bootstrap_function, PyObjectFields, cpython_struct, CONST_STRING, + CONST_WSTRING) from pypy.module.cpyext.pyerrors import PyErr_BadArgument -from pypy.module.cpyext.pyobject import PyObject, from_ref, make_ref, Py_DecRef, make_typedescr -from pypy.module.cpyext.stringobject import PyString_Check +from pypy.module.cpyext.pyobject import PyObject, from_ref, make_typedescr from pypy.module.sys.interp_encoding import setdefaultencoding from pypy.objspace.std import unicodeobject, unicodetype import sys +## See comment in stringobject.py. PyUnicode_FromUnicode(NULL, size) is not +## yet supported. + PyUnicodeObjectStruct = lltype.ForwardReference() PyUnicodeObject = lltype.Ptr(PyUnicodeObjectStruct) PyUnicodeObjectFields = (PyObjectFields + @@ -35,6 +37,7 @@ Py_UNICODE = lltype.UniChar def unicode_attach(space, py_obj, w_obj): + "Fills a newly allocated PyUnicodeObject with a unicode string" py_unicode = rffi.cast(PyUnicodeObject, py_obj) py_unicode.c_size = len(space.unicode_w(w_obj)) py_unicode.c_buffer = lltype.nullptr(rffi.CWCHARP.TO) @@ -214,8 +217,7 @@ if not wchar_p: raise NotImplementedError s = rffi.wcharpsize2unicode(wchar_p, length) - ptr = make_ref(space, space.wrap(s)) - return ptr + return space.wrap(s) @cpython_api([CONST_WSTRING, Py_ssize_t], PyObject) def PyUnicode_FromWideChar(space, wchar_p, length): From afa at codespeak.net Wed May 5 16:02:20 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 5 May 2010 16:02:20 +0200 (CEST) Subject: [pypy-svn] r74379 - pypy/trunk/pypy/module/cpyext Message-ID: <20100505140220.11A82282BDD@codespeak.net> Author: afa Date: Wed May 5 16:02:17 2010 New Revision: 74379 Modified: pypy/trunk/pypy/module/cpyext/pyerrors.py Log: Translation fix Modified: pypy/trunk/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyerrors.py (original) +++ pypy/trunk/pypy/module/cpyext/pyerrors.py Wed May 5 16:02:17 2010 @@ -192,7 +192,7 @@ if w_category is None: w_category = space.w_None w_message = space.wrap(rffi.charp2str(message_ptr)) - w_stacklevel = space.wrap(stacklevel) + w_stacklevel = space.wrap(rffi.cast(lltype.Signed, stacklevel)) w_module = PyImport_Import(space, space.wrap("warnings")) w_warn = space.getattr(w_module, space.wrap("warn")) From afa at codespeak.net Wed May 5 16:56:29 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 5 May 2010 16:56:29 +0200 (CEST) Subject: [pypy-svn] r74380 - pypy/trunk/pypy/module/cpyext Message-ID: <20100505145629.460F3282BDD@codespeak.net> Author: afa Date: Wed May 5 16:56:27 2010 New Revision: 74380 Modified: pypy/trunk/pypy/module/cpyext/pyobject.py pypy/trunk/pypy/module/cpyext/typeobject.py Log: More docstrings, some light refactoring. Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Wed May 5 16:56:27 2010 @@ -164,22 +164,37 @@ print >>sys.stderr def create_ref(space, w_obj, items=0): + """ + Allocates a PyObject, and fills its fields with info from the given + intepreter object. + """ w_type = space.type(w_obj) metatypedescr = get_typedescr(w_type.typedef) return metatypedescr.make_ref(space, w_type, w_obj, itemcount=items) -def track_reference(space, py_obj, w_obj, borrowed=False): +def track_reference(space, py_obj, w_obj, borrowed=False, replace=False): + """ + Ties together a PyObject and an interpreter object. + """ # XXX looks like a PyObject_GC_TRACK ptr = rffi.cast(ADDR, py_obj) + state = space.fromcache(State) if DEBUG_REFCOUNT: debug_refcount("MAKREF", py_obj, w_obj) - state = space.fromcache(State) + if not replace: + assert w_obj not in state.py_objects_w2r + assert ptr not in state.py_objects_r2w + assert ptr not in state.borrowed_objects state.py_objects_w2r[w_obj] = py_obj state.py_objects_r2w[ptr] = w_obj - if borrowed and ptr not in state.borrowed_objects: + if borrowed: state.borrowed_objects[ptr] = None def make_ref(space, w_obj, borrowed=False, steal=False, items=0): + """ + Returns a reference to an intepreter object. + if borrowed=False, the caller owns the reference. + """ if w_obj is None: return lltype.nullptr(PyObject.TO) assert isinstance(w_obj, W_Root) @@ -204,6 +219,10 @@ def from_ref(space, ref, recurse=False): + """ + Finds the interpreter object corresponding to the given reference. If the + object is not yet realized (see stringobject.py), creates it. + """ assert lltype.typeOf(ref) == PyObject if not ref: return None Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Wed May 5 16:56:27 2010 @@ -6,18 +6,18 @@ from pypy.rlib.rweakref import RWeakKeyDictionary from pypy.interpreter.gateway import ObjSpace, W_Root, Arguments from pypy.interpreter.gateway import interp2app, unwrap_spec -from pypy.interpreter.baseobjspace import Wrappable, DescrMismatch +from pypy.interpreter.baseobjspace import DescrMismatch from pypy.objspace.std.typeobject import W_TypeObject, _CPYTYPE, call__Type from pypy.objspace.std.typetype import _precheck_for_new -from pypy.objspace.std.objectobject import W_ObjectObject from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.module.cpyext.api import cpython_api, cpython_struct, bootstrap_function, \ - PyVarObjectFields, Py_ssize_t, Py_TPFLAGS_READYING, generic_cpy_call, \ - Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE, ADDR, \ - Py_TPFLAGS_HAVE_CLASS, METH_VARARGS, METH_KEYWORDS, \ - CANNOT_FAIL, PyBufferProcs, build_type_checkers -from pypy.module.cpyext.pyobject import PyObject, make_ref, create_ref, from_ref -from pypy.module.cpyext.pyobject import get_typedescr, make_typedescr, track_reference +from pypy.module.cpyext.api import ( + cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, + generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING, + Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL, + PyBufferProcs, build_type_checkers) +from pypy.module.cpyext.pyobject import ( + PyObject, make_ref, create_ref, from_ref, get_typedescr, make_typedescr, + track_reference) from pypy.interpreter.module import Module from pypy.interpreter.function import FunctionWithFixedCode, StaticMethod from pypy.module.cpyext import structmemberdefs @@ -367,9 +367,9 @@ pto_tuple.c_tp_bases.c_ob_type = pto_tuple # Restore the mapping - track_reference(space, py_type, space.w_type) - track_reference(space, py_object, space.w_object) - track_reference(space, py_tuple, space.w_tuple) + track_reference(space, py_type, space.w_type, replace=True) + track_reference(space, py_object, space.w_object, replace=True) + track_reference(space, py_tuple, space.w_tuple, replace=True) @cpython_api([PyObject], lltype.Void, external=False) @@ -447,7 +447,9 @@ def type_attach(space, py_obj, w_type): - """ Allocates a PyTypeObject from a w_type which must be a PyPy type. """ + """ + Fills a newly allocated PyTypeObject from an existing type. + """ from pypy.module.cpyext.object import PyObject_Del assert isinstance(w_type, W_TypeObject) @@ -547,15 +549,17 @@ finally: Py_DecRef(space, base_pyo) -def type_realize(space, ref): - PyPyType_Ready(space, rffi.cast(PyTypeObjectPtr, ref), None) - return from_ref(space, ref, True) +def type_realize(space, py_obj): + """ + Creates an interpreter type from a PyTypeObject structure. + """ + return PyPyType_Ready(space, rffi.cast(PyTypeObjectPtr, py_obj), None) def PyPyType_Ready(space, pto, w_obj): try: pto.c_tp_dict = lltype.nullptr(PyObject.TO) # not supported if pto.c_tp_flags & Py_TPFLAGS_READY: - return 0 + return w_obj assert pto.c_tp_flags & Py_TPFLAGS_READYING == 0 pto.c_tp_flags |= Py_TPFLAGS_READYING base = pto.c_tp_base @@ -588,22 +592,21 @@ # tp_mro, tp_subclasses finally: pto.c_tp_flags &= ~Py_TPFLAGS_READYING - pto.c_tp_flags = (pto.c_tp_flags & ~Py_TPFLAGS_READYING) | Py_TPFLAGS_READY - return 0 + pto.c_tp_flags |= Py_TPFLAGS_READY + return w_obj def PyPyType_Register(space, pto): + w_obj = space.allocate_instance( + W_PyCTypeObject, + space.gettypeobject(W_PyCTypeObject.typedef)) state = space.fromcache(State) - ptr = rffi.cast(ADDR, pto) - if ptr not in state.py_objects_r2w: - w_obj = space.allocate_instance(W_PyCTypeObject, - space.gettypeobject(W_PyCTypeObject.typedef)) - state.non_heaptypes.append(w_obj) - pyo = rffi.cast(PyObject, pto) - state.py_objects_r2w[ptr] = w_obj - state.py_objects_w2r[w_obj] = pyo - w_obj.__init__(space, pto) - w_obj.ready() - return w_obj + state.non_heaptypes.append(w_obj) + py_obj = rffi.cast(PyObject, pto) + + track_reference(space, py_obj, w_obj) + w_obj.__init__(space, pto) + w_obj.ready() + return w_obj @cpython_api([PyTypeObjectPtr, PyTypeObjectPtr], rffi.INT_real, error=CANNOT_FAIL) def PyType_IsSubtype(space, a, b): From antocuni at codespeak.net Wed May 5 17:00:05 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 5 May 2010 17:00:05 +0200 (CEST) Subject: [pypy-svn] r74381 - in pypy/extradoc/talk/pycon-italy-2010/pypy_demo: . manderlbrot Message-ID: <20100505150005.93BAC282BDD@codespeak.net> Author: antocuni Date: Wed May 5 17:00:04 2010 New Revision: 74381 Added: pypy/extradoc/talk/pycon-italy-2010/pypy_demo/ pypy/extradoc/talk/pycon-italy-2010/pypy_demo/__init__.py (contents, props changed) pypy/extradoc/talk/pycon-italy-2010/pypy_demo/manage.py (contents, props changed) pypy/extradoc/talk/pycon-italy-2010/pypy_demo/manderlbrot/ pypy/extradoc/talk/pycon-italy-2010/pypy_demo/manderlbrot/__init__.py (contents, props changed) pypy/extradoc/talk/pycon-italy-2010/pypy_demo/manderlbrot/models.py (contents, props changed) pypy/extradoc/talk/pycon-italy-2010/pypy_demo/manderlbrot/py_kohn_bmp.py (contents, props changed) pypy/extradoc/talk/pycon-italy-2010/pypy_demo/manderlbrot/py_mandel.py (contents, props changed) pypy/extradoc/talk/pycon-italy-2010/pypy_demo/manderlbrot/tests.py (contents, props changed) pypy/extradoc/talk/pycon-italy-2010/pypy_demo/manderlbrot/views.py (contents, props changed) pypy/extradoc/talk/pycon-italy-2010/pypy_demo/settings.py (contents, props changed) pypy/extradoc/talk/pycon-italy-2010/pypy_demo/urls.py (contents, props changed) Log: a simple django demo that is faster on pypy than cpython. Moreover, it also shows how to run the application on cpython, but delegate the computational intensive tasks to pypy, through execnet Added: pypy/extradoc/talk/pycon-italy-2010/pypy_demo/__init__.py ============================================================================== Added: pypy/extradoc/talk/pycon-italy-2010/pypy_demo/manage.py ============================================================================== --- (empty file) +++ pypy/extradoc/talk/pycon-italy-2010/pypy_demo/manage.py Wed May 5 17:00:04 2010 @@ -0,0 +1,11 @@ +#!/usr/bin/python +from django.core.management import execute_manager +try: + import settings # Assumed to be in the same directory. +except ImportError: + import sys + sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__) + sys.exit(1) + +if __name__ == "__main__": + execute_manager(settings) Added: pypy/extradoc/talk/pycon-italy-2010/pypy_demo/manderlbrot/__init__.py ============================================================================== Added: pypy/extradoc/talk/pycon-italy-2010/pypy_demo/manderlbrot/models.py ============================================================================== --- (empty file) +++ pypy/extradoc/talk/pycon-italy-2010/pypy_demo/manderlbrot/models.py Wed May 5 17:00:04 2010 @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. Added: pypy/extradoc/talk/pycon-italy-2010/pypy_demo/manderlbrot/py_kohn_bmp.py ============================================================================== --- (empty file) +++ pypy/extradoc/talk/pycon-italy-2010/pypy_demo/manderlbrot/py_kohn_bmp.py Wed May 5 17:00:04 2010 @@ -0,0 +1,111 @@ + +# py_kohn_bmp - Copyright 2007 by Michael Kohn +# http://www.mikekohn.net/ +# mike at mikekohn.net +# +# Feel free to use this class in your own programs (commerical or not) +# as long as you don't remove my name, email address, webpage, or copyright. +# +# Example how to use: +# +# my_bmp.kohn_bmp("out.bmp",image_width,image_height,3) <-- depth of 3 is color +# my_bmp.write_pixel(red,green,blue) <-- do this width*height times +# my_bmp.close() +# +# if depth is set to 1 (black and white image) call write_pixel_bw(y) where +# y is between 0 and 255 + +# adapted by Antonio Cuni - 2010 + +from cStringIO import StringIO + +class kohn_bmp: + out=0 + width=0 + height=0 + depth=0 + bytes=0 + xpos=0 + width_bytes=0 + + def write_int(self,n): + str_out='%c%c%c%c' % ((n&255),(n>>8)&255,(n>>16)&255,(n>>24)&255) + self.out.write(str_out) + + def write_word(self,n): + str_out='%c%c' % ((n&255),(n>>8)&255) + self.out.write(str_out) + + def __init__(self,width,height,depth): + self.width=width + self.height=height + self.depth=depth + + self.width_bytes=width*depth + if (self.width_bytes%4)!=0: + self.width_bytes=self.width_bytes+(4-(self.width_bytes%4)) + + self.out=StringIO() + self.out.write("BM") # magic number + if depth==1: + self.write_int((self.width_bytes*height)+54+1024) + else: + self.write_int((self.width_bytes*height)+54) + self.write_word(0) + self.write_word(0) + if depth==1: + self.write_int(54+1024) + else: + self.write_int(54) + + self.write_int(40) # header_size + self.write_int(width) # width + self.write_int(height) # height + self.write_word(1) # planes + self.write_word(depth*8) # bits per pixel + self.write_int(0) # compression + self.write_int(self.width_bytes*height*depth) # image_size + self.write_int(0) # biXPelsperMetre + self.write_int(0) # biYPelsperMetre + + if depth==1: + self.write_int(256) # colors used + self.write_int(256) # colors important + + for c in range(256): + self.out.write('%c' % c) + self.out.write('%c' % c) + self.out.write('%c' % c) + self.out.write('%c' % 0) + + else: + self.write_int(0) # colors used - 0 since 24 bit + self.write_int(0) # colors important - 0 since 24 bit + + + def write_pixel_bw(self,y): + self.out.write(str("%c" % y)) + self.xpos=self.xpos+1 + if self.xpos==self.width: + while self.xpos0: + z=(z**2)+c + if abs(z)>2: break + count=count-1 + + c=int(count/10) + my_bmp.write_pixel(colors[c][0],colors[c][1],colors[c][2]) + + return my_bmp.dump() + Added: pypy/extradoc/talk/pycon-italy-2010/pypy_demo/manderlbrot/tests.py ============================================================================== --- (empty file) +++ pypy/extradoc/talk/pycon-italy-2010/pypy_demo/manderlbrot/tests.py Wed May 5 17:00:04 2010 @@ -0,0 +1,23 @@ +""" +This file demonstrates two different styles of tests (one doctest and one +unittest). These will both pass when you run "manage.py test". + +Replace these with more appropriate tests for your application. +""" + +from django.test import TestCase + +class SimpleTest(TestCase): + def test_basic_addition(self): + """ + Tests that 1 + 1 always equals 2. + """ + self.failUnlessEqual(1 + 1, 2) + +__test__ = {"doctest": """ +Another way to test that 1 + 1 is equal to 2. + +>>> 1 + 1 == 2 +True +"""} + Added: pypy/extradoc/talk/pycon-italy-2010/pypy_demo/manderlbrot/views.py ============================================================================== --- (empty file) +++ pypy/extradoc/talk/pycon-italy-2010/pypy_demo/manderlbrot/views.py Wed May 5 17:00:04 2010 @@ -0,0 +1,52 @@ +# Create your views here. + +from django.http import HttpResponse + +# only for benchmarking purposes +def empty(request): + return HttpResponse('') + +# render a manderlbrot image +def render(request): + w = int(request.GET.get('w', 320)) + h = int(request.GET.get('h', 240)) + + from py_mandel import manderlbrot + img = manderlbrot(w, h) + return HttpResponse(img, content_type="image/bmp") + + +# render a manderlbrot image through the execnet pypy child, which is set up +# below +def pypy_render(request): + w = int(request.GET.get('w', 320)) + h = int(request.GET.get('h', 240)) + + channel = pypy.remote_exec(""" + from py_mandel import manderlbrot + w, h = channel.receive() + img = manderlbrot(w, h) + channel.send(img) + """) + channel.send((w, h)) + img = channel.receive() + return HttpResponse(img, content_type="image/bmp") + + +# setup execnet and pypy child +# +# The setup is done here so that a single pypy-c is started and reused for all +# requests. sys.path and cwd are set up so that py_mandel can be imported +# from remote_exec + +import execnet +mygroup = execnet.Group() +pypy = mygroup.makegateway("popen//python=pypy-c") +pypy.remote_exec(""" + import sys + import os + os.chdir("manderlbrot") + sys.path.insert(0, '') +""") + + Added: pypy/extradoc/talk/pycon-italy-2010/pypy_demo/settings.py ============================================================================== --- (empty file) +++ pypy/extradoc/talk/pycon-italy-2010/pypy_demo/settings.py Wed May 5 17:00:04 2010 @@ -0,0 +1,80 @@ +# Django settings for pypy_demo project. + +DEBUG = True +TEMPLATE_DEBUG = DEBUG + +ADMINS = ( + # ('Your Name', 'your_email at domain.com'), +) + +MANAGERS = ADMINS + +DATABASE_ENGINE = '' # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. +DATABASE_NAME = '' # Or path to database file if using sqlite3. +DATABASE_USER = '' # Not used with sqlite3. +DATABASE_PASSWORD = '' # Not used with sqlite3. +DATABASE_HOST = '' # Set to empty string for localhost. Not used with sqlite3. +DATABASE_PORT = '' # Set to empty string for default. Not used with sqlite3. + +# Local time zone for this installation. Choices can be found here: +# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name +# although not all choices may be available on all operating systems. +# If running in a Windows environment this must be set to the same as your +# system time zone. +TIME_ZONE = 'America/Chicago' + +# Language code for this installation. All choices can be found here: +# http://www.i18nguy.com/unicode/language-identifiers.html +LANGUAGE_CODE = 'en-us' + +SITE_ID = 1 + +# If you set this to False, Django will make some optimizations so as not +# to load the internationalization machinery. +USE_I18N = True + +# Absolute path to the directory that holds media. +# Example: "/home/media/media.lawrence.com/" +MEDIA_ROOT = '' + +# URL that handles the media served from MEDIA_ROOT. Make sure to use a +# trailing slash if there is a path component (optional in other cases). +# Examples: "http://media.lawrence.com", "http://example.com/media/" +MEDIA_URL = '' + +# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a +# trailing slash. +# Examples: "http://foo.com/media/", "/media/". +ADMIN_MEDIA_PREFIX = '/media/' + +# Make this unique, and don't share it with anybody. +SECRET_KEY = '_+n^blk6hik*s1kfylp1pm8fpu9n0#=z_9cngayme5)-**$(#9' + +# List of callables that know how to import templates from various sources. +TEMPLATE_LOADERS = ( + 'django.template.loaders.filesystem.load_template_source', + 'django.template.loaders.app_directories.load_template_source', +# 'django.template.loaders.eggs.load_template_source', +) + +MIDDLEWARE_CLASSES = ( + 'django.middleware.common.CommonMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', +) + +ROOT_URLCONF = 'pypy_demo.urls' + +TEMPLATE_DIRS = ( + # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". + # Always use forward slashes, even on Windows. + # Don't forget to use absolute paths, not relative paths. +) + +INSTALLED_APPS = ( + ## 'django.contrib.auth', + ## 'django.contrib.contenttypes', + ## 'django.contrib.sessions', + ## 'django.contrib.sites', + 'pypy_demo.manderlbrot', +) Added: pypy/extradoc/talk/pycon-italy-2010/pypy_demo/urls.py ============================================================================== --- (empty file) +++ pypy/extradoc/talk/pycon-italy-2010/pypy_demo/urls.py Wed May 5 17:00:04 2010 @@ -0,0 +1,19 @@ +from django.conf.urls.defaults import * + +# Uncomment the next two lines to enable the admin: +# from django.contrib import admin +# admin.autodiscover() + +urlpatterns = patterns('', + # Example: + (r'^render$', 'pypy_demo.manderlbrot.views.render'), + (r'^pypy_render$', 'pypy_demo.manderlbrot.views.pypy_render'), + (r'^empty$', 'pypy_demo.manderlbrot.views.empty'), + + # Uncomment the admin/doc line below and add 'django.contrib.admindocs' + # to INSTALLED_APPS to enable admin documentation: + # (r'^admin/doc/', include('django.contrib.admindocs.urls')), + + # Uncomment the next line to enable the admin: + # (r'^admin/', include(admin.site.urls)), +) From afa at codespeak.net Wed May 5 18:29:22 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 5 May 2010 18:29:22 +0200 (CEST) Subject: [pypy-svn] r74382 - pypy/trunk/pypy/module/cpyext Message-ID: <20100505162922.16ABD282BDD@codespeak.net> Author: afa Date: Wed May 5 18:29:20 2010 New Revision: 74382 Modified: pypy/trunk/pypy/module/cpyext/typeobject.py Log: Reorganize type initialization code: finish_type_1() uses info already available in the structure itself, to compute things necessary to create the type. finish_type_2() completes the structure after the type is created in the interpreter (like slot inheritance) Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Wed May 5 18:29:20 2010 @@ -479,9 +479,8 @@ w_base = best_base(space, w_type.bases_w) pto.c_tp_base = rffi.cast(PyTypeObjectPtr, make_ref(space, w_base)) - pto.c_tp_bases = lltype.nullptr(PyObject.TO) - PyPyType_Ready(space, pto, w_type) - + finish_type_1(space, pto) + finish_type_2(space, pto, w_type) pto.c_tp_basicsize = rffi.sizeof(typedescr.basestruct) if pto.c_tp_base: @@ -493,11 +492,21 @@ if space.is_w(w_type, space.w_object): pto.c_tp_new = rffi.cast(newfunc, 1) update_all_slots(space, w_type, pto) + pto.c_tp_flags |= Py_TPFLAGS_READY return pto @cpython_api([PyTypeObjectPtr], rffi.INT_real, error=-1) def PyType_Ready(space, pto): - return PyPyType_Ready(space, pto, None) + if pto.c_tp_flags & Py_TPFLAGS_READY: + return + assert pto.c_tp_flags & Py_TPFLAGS_READYING == 0 + pto.c_tp_flags |= Py_TPFLAGS_READYING + try: + type_realize(space, rffi.cast(PyObject, pto)) + pto.c_tp_flags |= Py_TPFLAGS_READY + finally: + pto.c_tp_flags &= ~Py_TPFLAGS_READYING + return 0 def solid_base(space, w_type): typedef = w_type.instancetypedef @@ -553,61 +562,63 @@ """ Creates an interpreter type from a PyTypeObject structure. """ - return PyPyType_Ready(space, rffi.cast(PyTypeObjectPtr, py_obj), None) + # missing: + # setting __doc__ if not defined and tp_doc defined + # inheriting tp_as_* slots + # unsupported: + # tp_mro, tp_subclasses + py_type = rffi.cast(PyTypeObjectPtr, py_obj) + + if not py_type.c_tp_base: + base = make_ref(space, space.w_object, steal=True) + py_type.c_tp_base = rffi.cast(PyTypeObjectPtr, base) -def PyPyType_Ready(space, pto, w_obj): - try: - pto.c_tp_dict = lltype.nullptr(PyObject.TO) # not supported - if pto.c_tp_flags & Py_TPFLAGS_READY: - return w_obj - assert pto.c_tp_flags & Py_TPFLAGS_READYING == 0 - pto.c_tp_flags |= Py_TPFLAGS_READYING - base = pto.c_tp_base - if not base: - base_pyo = make_ref(space, space.w_object, steal=True) - base = pto.c_tp_base = rffi.cast(PyTypeObjectPtr, base_pyo) - else: - base_pyo = rffi.cast(PyObject, base) - if base and not base.c_tp_flags & Py_TPFLAGS_READY: - PyPyType_Ready(space, base, None) - if base and not pto.c_ob_type: # will be filled later - pto.c_ob_type = base.c_ob_type - if not pto.c_tp_bases: - if not base: - bases = space.newtuple([]) - else: - bases = space.newtuple([from_ref(space, base_pyo)]) - pto.c_tp_bases = make_ref(space, bases) - if w_obj is None: - w_obj = PyPyType_Register(space, pto) - pto.c_tp_mro = make_ref(space, space.newtuple(w_obj.mro_w)) - if base: - inherit_special(space, pto, base) - for w_base in space.fixedview(from_ref(space, pto.c_tp_bases)): - inherit_slots(space, pto, w_base) - # missing: - # setting __doc__ if not defined and tp_doc defined - # inheriting tp_as_* slots - # unsupported: - # tp_mro, tp_subclasses - finally: - pto.c_tp_flags &= ~Py_TPFLAGS_READYING - pto.c_tp_flags |= Py_TPFLAGS_READY - return w_obj + finish_type_1(space, py_type) -def PyPyType_Register(space, pto): w_obj = space.allocate_instance( W_PyCTypeObject, space.gettypeobject(W_PyCTypeObject.typedef)) + track_reference(space, py_obj, w_obj) + w_obj.__init__(space, py_type) + w_obj.ready() + + finish_type_2(space, py_type, w_obj) + state = space.fromcache(State) state.non_heaptypes.append(w_obj) - py_obj = rffi.cast(PyObject, pto) - track_reference(space, py_obj, w_obj) - w_obj.__init__(space, pto) - w_obj.ready() return w_obj +def finish_type_1(space, pto): + """ + Sets up tp_bases, necessary before creating the interpreter type. + """ + pto.c_tp_dict = lltype.nullptr(PyObject.TO) # not supported + + base = pto.c_tp_base + base_pyo = rffi.cast(PyObject, pto.c_tp_base) + if base and not base.c_tp_flags & Py_TPFLAGS_READY: + type_realize(space, rffi.cast(PyObject, base_pyo)) + if base and not pto.c_ob_type: # will be filled later + pto.c_ob_type = base.c_ob_type + if not pto.c_tp_bases: + if not base: + bases = space.newtuple([]) + else: + bases = space.newtuple([from_ref(space, base_pyo)]) + pto.c_tp_bases = make_ref(space, bases) + +def finish_type_2(space, pto, w_obj): + """ + Sets up other attributes, when the interpreter type has been created. + """ + pto.c_tp_mro = make_ref(space, space.newtuple(w_obj.mro_w)) + base = pto.c_tp_base + if base: + inherit_special(space, pto, base) + for w_base in space.fixedview(from_ref(space, pto.c_tp_bases)): + inherit_slots(space, pto, w_base) + @cpython_api([PyTypeObjectPtr, PyTypeObjectPtr], rffi.INT_real, error=CANNOT_FAIL) def PyType_IsSubtype(space, a, b): """Return true if a is a subtype of b. From afa at codespeak.net Wed May 5 19:43:25 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 5 May 2010 19:43:25 +0200 (CEST) Subject: [pypy-svn] r74383 - pypy/trunk/pypy/module/cpyext Message-ID: <20100505174325.ABEA7282BDD@codespeak.net> Author: afa Date: Wed May 5 19:43:23 2010 New Revision: 74383 Modified: pypy/trunk/pypy/module/cpyext/methodobject.py pypy/trunk/pypy/module/cpyext/object.py pypy/trunk/pypy/module/cpyext/pyobject.py pypy/trunk/pypy/module/cpyext/typeobject.py Log: Remove references to "state" where it is not necessary. Now only pyobject.py and state.py reference the r2w & w2r dictionaries Modified: pypy/trunk/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/methodobject.py (original) +++ pypy/trunk/pypy/module/cpyext/methodobject.py Wed May 5 19:43:23 2010 @@ -13,7 +13,6 @@ generic_cpy_call, cpython_api, PyObject, cpython_struct, METH_KEYWORDS, METH_O, CONST_STRING, METH_CLASS, METH_STATIC, METH_COEXIST, METH_NOARGS, METH_VARARGS, build_type_checkers, PyObjectFields, bootstrap_function) -from pypy.module.cpyext.state import State from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.rlib.objectmodel import we_are_translated from pypy.objspace.std.tupleobject import W_TupleObject Modified: pypy/trunk/pypy/module/cpyext/object.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/object.py (original) +++ pypy/trunk/pypy/module/cpyext/object.py Wed May 5 19:43:23 2010 @@ -4,7 +4,6 @@ Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT, Py_GE, CONST_STRING from pypy.module.cpyext.pyobject import PyObject, PyObjectP, make_ref, from_ref from pypy.module.cpyext.pyobject import Py_IncRef, Py_DecRef -from pypy.module.cpyext.state import State from pypy.module.cpyext.typeobject import PyTypeObjectPtr, W_PyCTypeObject from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall from pypy.objspace.std.objectobject import W_ObjectObject Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Wed May 5 19:43:23 2010 @@ -186,7 +186,8 @@ assert ptr not in state.py_objects_r2w assert ptr not in state.borrowed_objects state.py_objects_w2r[w_obj] = py_obj - state.py_objects_r2w[ptr] = w_obj + if ptr: # init_typeobject() bootstraps with NULL references + state.py_objects_r2w[ptr] = w_obj if borrowed: state.borrowed_objects[ptr] = None Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Wed May 5 19:43:23 2010 @@ -106,7 +106,6 @@ def update_all_slots(space, w_obj, pto): # XXX fill slots in pto - state = space.fromcache(State) for method_name, slot_name, slot_func, _, _, _ in slotdefs: w_descr = space.lookup(w_obj, method_name) if w_descr is None: @@ -135,7 +134,6 @@ def add_operators(space, dict_w, pto): # XXX support PyObject_HashNotImplemented - state = space.fromcache(State) for method_name, slot_name, _, wrapper_func, wrapper_func_kwds, doc in slotdefs: if method_name in dict_w: continue @@ -341,10 +339,9 @@ # - tuple.tp_bases is a tuple # insert null placeholders to please make_ref() - state = space.fromcache(State) - state.py_objects_w2r[space.w_type] = lltype.nullptr(PyObject.TO) - state.py_objects_w2r[space.w_object] = lltype.nullptr(PyObject.TO) - state.py_objects_w2r[space.w_tuple] = lltype.nullptr(PyObject.TO) + track_reference(space, lltype.nullptr(PyObject.TO), space.w_type) + track_reference(space, lltype.nullptr(PyObject.TO), space.w_object) + track_reference(space, lltype.nullptr(PyObject.TO), space.w_tuple) # create the objects py_type = create_ref(space, space.w_type) @@ -428,7 +425,6 @@ @cpython_api([PyObject], lltype.Void, external=False) def type_dealloc(space, obj): - state = space.fromcache(State) obj_pto = rffi.cast(PyTypeObjectPtr, obj) type_pto = obj.c_ob_type base_pyo = rffi.cast(PyObject, obj_pto.c_tp_base) From hpk at codespeak.net Wed May 5 20:20:23 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 5 May 2010 20:20:23 +0200 (CEST) Subject: [pypy-svn] r74384 - pypy/branch/py12/py/_plugin Message-ID: <20100505182023.3DBB9282BDD@codespeak.net> Author: hpk Date: Wed May 5 20:20:21 2010 New Revision: 74384 Modified: pypy/branch/py12/py/_plugin/pytest_skipping.py pypy/branch/py12/py/_plugin/pytest_terminal.py Log: pull over newest py.test-1.3.0 release candidate changes Modified: pypy/branch/py12/py/_plugin/pytest_skipping.py ============================================================================== --- pypy/branch/py12/py/_plugin/pytest_skipping.py (original) +++ pypy/branch/py12/py/_plugin/pytest_skipping.py Wed May 5 20:20:21 2010 @@ -10,9 +10,8 @@ If a test fails under all conditions then it's probably better to mark your test as 'xfail'. -By passing ``--report=xfailed,skipped`` to the terminal reporter -you will see summary information on skips and xfail-run tests -at the end of a test run. +By passing ``-rxs`` to the terminal reporter you will see extra +summary information on skips and xfail-run tests at the end of a test run. .. _skipif: @@ -165,7 +164,7 @@ expl = self.get('reason', None) if not expl: if not hasattr(self, 'expr'): - return "condition: True" + return "" else: return "condition: " + self.expr return expl @@ -222,31 +221,53 @@ # called by the terminalreporter instance/plugin def pytest_terminal_summary(terminalreporter): - show_xfailed(terminalreporter) - show_skipped(terminalreporter) - -def show_xfailed(terminalreporter): tr = terminalreporter - xfailed = tr.stats.get("xfailed") + if not tr.reportchars: + #for name in "xfailed skipped failed xpassed": + # if not tr.stats.get(name, 0): + # tr.write_line("HINT: use '-r' option to see extra " + # "summary info about tests") + # break + return + + lines = [] + for char in tr.reportchars: + if char == "x": + show_xfailed(terminalreporter, lines) + elif char == "X": + show_xpassed(terminalreporter, lines) + elif char == "f": + show_failed(terminalreporter, lines) + elif char == "s": + show_skipped(terminalreporter, lines) + if lines: + tr._tw.sep("=", "short test summary info") + for line in lines: + tr._tw.line(line) + +def show_failed(terminalreporter, lines): + tw = terminalreporter._tw + failed = terminalreporter.stats.get("failed") + if failed: + for rep in failed: + pos = terminalreporter.gettestid(rep.item) + lines.append("FAIL %s" %(pos, )) + +def show_xfailed(terminalreporter, lines): + xfailed = terminalreporter.stats.get("xfailed") if xfailed: - if not tr.hasopt('xfailed'): - tr.write_line( - "%d expected failures, use --report=xfailed for more info" % - len(xfailed)) - return - tr.write_sep("_", "expected failures") for rep in xfailed: pos = terminalreporter.gettestid(rep.item) reason = rep.keywords['xfail'] - tr._tw.line("%s %s" %(pos, reason)) + lines.append("XFAIL %s %s" %(pos, reason)) +def show_xpassed(terminalreporter, lines): xpassed = terminalreporter.stats.get("xpassed") if xpassed: - tr.write_sep("_", "UNEXPECTEDLY PASSING TESTS") for rep in xpassed: pos = terminalreporter.gettestid(rep.item) reason = rep.keywords['xfail'] - tr._tw.line("%s %s" %(pos, reason)) + lines.append("XPASS %s %s" %(pos, reason)) def cached_eval(config, expr, d): if not hasattr(config, '_evalcache'): @@ -271,17 +292,20 @@ l.append((len(events),) + key) return l -def show_skipped(terminalreporter): +def show_skipped(terminalreporter, lines): tr = terminalreporter skipped = tr.stats.get('skipped', []) if skipped: - if not tr.hasopt('skipped'): - tr.write_line( - "%d skipped tests, use --report=skipped for more info" % - len(skipped)) - return + #if not tr.hasopt('skipped'): + # tr.write_line( + # "%d skipped tests, specify -rs for more info" % + # len(skipped)) + # return fskips = folded_skips(skipped) if fskips: - tr.write_sep("_", "skipped test summary") + #tr.write_sep("_", "skipped test summary") for num, fspath, lineno, reason in fskips: - tr._tw.line("%s:%d: [%d] %s" %(fspath, lineno, num, reason)) + if reason.startswith("Skipped: "): + reason = reason[9:] + lines.append("SKIP [%d] %s:%d: %s" % + (num, fspath, lineno, reason)) Modified: pypy/branch/py12/py/_plugin/pytest_terminal.py ============================================================================== --- pypy/branch/py12/py/_plugin/pytest_terminal.py (original) +++ pypy/branch/py12/py/_plugin/pytest_terminal.py Wed May 5 20:20:21 2010 @@ -12,12 +12,16 @@ group = parser.getgroup("terminal reporting", "reporting", after="general") group._addoption('-v', '--verbose', action="count", dest="verbose", default=0, help="increase verbosity."), + group._addoption('-r', + action="store", dest="reportchars", default=None, metavar="chars", + help="show extra test summary info as specified by chars (f)ailed, " + "(s)skipped, (x)failed, (X)passed.") group._addoption('-l', '--showlocals', - action="store_true", dest="showlocals", default=False, - help="show locals in tracebacks (disabled by default).") - group.addoption('--report', - action="store", dest="report", default=None, metavar="opts", - help="show more info, valid: skipped,xfailed") + action="store_true", dest="showlocals", default=False, + help="show locals in tracebacks (disabled by default).") + group._addoption('--report', + action="store", dest="report", default=None, metavar="opts", + help="(deprecated, use -r)") group._addoption('--tb', metavar="style", action="store", dest="tbstyle", default='long', type="choice", choices=['long', 'short', 'no', 'line'], @@ -47,17 +51,25 @@ setattr(reporter._tw, name, getattr(config, attr)) config.pluginmanager.register(reporter, 'terminalreporter') -def getreportopt(optvalue): - d = {} +def getreportopt(config): + reportopts = "" + optvalue = config.getvalue("report") if optvalue: - for setting in optvalue.split(","): - setting = setting.strip() - val = True - if setting.startswith("no"): - val = False - setting = setting[2:] - d[setting] = val - return d + py.builtin.print_("DEPRECATED: use -r instead of --report option.", + file=py.std.sys.stderr) + if optvalue: + for setting in optvalue.split(","): + setting = setting.strip() + if setting == "skipped": + reportopts += "s" + elif setting == "xfailed": + reportopts += "x" + reportchars = config.getvalue("reportchars") + if reportchars: + for char in reportchars: + if char not in reportopts: + reportopts += char + return reportopts class TerminalReporter: def __init__(self, config, file=None): @@ -69,10 +81,11 @@ self._tw = py.io.TerminalWriter(file) self.currentfspath = None self.gateway2info = {} - self._reportopt = getreportopt(config.getvalue('report')) + self.reportchars = getreportopt(config) - def hasopt(self, name): - return self._reportopt.get(name, False) + def hasopt(self, char): + char = {'xfailed': 'x', 'skipped': 's'}.get(char,char) + return char in self.reportchars def write_fspath_result(self, fspath, res): fspath = self.curdir.bestrelpath(fspath) From afa at codespeak.net Wed May 5 20:41:10 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 5 May 2010 20:41:10 +0200 (CEST) Subject: [pypy-svn] r74385 - pypy/trunk/pypy/module/cpyext Message-ID: <20100505184110.8D221282BEC@codespeak.net> Author: afa Date: Wed May 5 20:41:08 2010 New Revision: 74385 Modified: pypy/trunk/pypy/module/cpyext/typeobject.py Log: Fix translation Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Wed May 5 20:41:08 2010 @@ -494,7 +494,7 @@ @cpython_api([PyTypeObjectPtr], rffi.INT_real, error=-1) def PyType_Ready(space, pto): if pto.c_tp_flags & Py_TPFLAGS_READY: - return + return 0 assert pto.c_tp_flags & Py_TPFLAGS_READYING == 0 pto.c_tp_flags |= Py_TPFLAGS_READYING try: From fijal at codespeak.net Wed May 5 21:32:43 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 5 May 2010 21:32:43 +0200 (CEST) Subject: [pypy-svn] r74386 - pypy/branch/chunked-list/pypy/translator Message-ID: <20100505193243.AB279282BEF@codespeak.net> Author: fijal Date: Wed May 5 21:32:41 2010 New Revision: 74386 Modified: pypy/branch/chunked-list/pypy/translator/simplify.py Log: Always check boundaries. A test of performance mostly Modified: pypy/branch/chunked-list/pypy/translator/simplify.py ============================================================================== --- pypy/branch/chunked-list/pypy/translator/simplify.py (original) +++ pypy/branch/chunked-list/pypy/translator/simplify.py Wed May 5 21:32:41 2010 @@ -229,8 +229,8 @@ if last_op.opname == 'getitem': postfx = [] for exit in block.exits: - if exit.exitcase is IndexError: - postfx.append('idx') + #if exit.exitcase is IndexError: + postfx.append('idx') elif exit.exitcase is KeyError: postfx.append('key') if postfx: From fijal at codespeak.net Wed May 5 21:39:21 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 5 May 2010 21:39:21 +0200 (CEST) Subject: [pypy-svn] r74387 - pypy/branch/chunked-list/pypy/translator Message-ID: <20100505193921.40603282BEF@codespeak.net> Author: fijal Date: Wed May 5 21:39:19 2010 New Revision: 74387 Modified: pypy/branch/chunked-list/pypy/translator/simplify.py Log: oops, sytax error Modified: pypy/branch/chunked-list/pypy/translator/simplify.py ============================================================================== --- pypy/branch/chunked-list/pypy/translator/simplify.py (original) +++ pypy/branch/chunked-list/pypy/translator/simplify.py Wed May 5 21:39:19 2010 @@ -231,7 +231,7 @@ for exit in block.exits: #if exit.exitcase is IndexError: postfx.append('idx') - elif exit.exitcase is KeyError: + if exit.exitcase is KeyError: postfx.append('key') if postfx: last_op.opname = last_op.opname + '_' + '_'.join(postfx) From fijal at codespeak.net Wed May 5 21:44:09 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 5 May 2010 21:44:09 +0200 (CEST) Subject: [pypy-svn] r74388 - pypy/branch/chunked-list/pypy/translator Message-ID: <20100505194409.83AA8282BEF@codespeak.net> Author: fijal Date: Wed May 5 21:44:07 2010 New Revision: 74388 Modified: pypy/branch/chunked-list/pypy/translator/simplify.py Log: Don't append _idx_idx_idx Modified: pypy/branch/chunked-list/pypy/translator/simplify.py ============================================================================== --- pypy/branch/chunked-list/pypy/translator/simplify.py (original) +++ pypy/branch/chunked-list/pypy/translator/simplify.py Wed May 5 21:44:07 2010 @@ -227,10 +227,9 @@ if block.operations and block.exitswitch == c_last_exception: last_op = block.operations[-1] if last_op.opname == 'getitem': - postfx = [] + postfx = ['idx'] for exit in block.exits: #if exit.exitcase is IndexError: - postfx.append('idx') if exit.exitcase is KeyError: postfx.append('key') if postfx: From fijal at codespeak.net Wed May 5 22:59:55 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 5 May 2010 22:59:55 +0200 (CEST) Subject: [pypy-svn] r74389 - pypy/benchmarks Message-ID: <20100505205955.95A6E282BEF@codespeak.net> Author: fijal Date: Wed May 5 22:59:54 2010 New Revision: 74389 Modified: pypy/benchmarks/runner.py Log: Add a possibility to customize baseline interpreter as well Modified: pypy/benchmarks/runner.py ============================================================================== --- pypy/benchmarks/runner.py (original) +++ pypy/benchmarks/runner.py Wed May 5 22:59:54 2010 @@ -11,7 +11,7 @@ def run_and_store(benchmark_set, result_filename, pypy_c_path, revision=0, options='', branch='trunk', args='', upload=False, - force_host=None, fast=False): + force_host=None, fast=False, baseline=sys.executable): funcs = perf.BENCH_FUNCS.copy() funcs.update(perf._FindAllBenchmarks(benchmarks.__dict__)) opts = ['-b', ','.join(benchmark_set), '--inherit_env=PATH', @@ -20,7 +20,7 @@ opts += ['--fast'] if args: opts += ['--args', args] - opts += [sys.executable, pypy_c_path] + opts += [baseline, pypy_c_path] results = perf.main(opts, funcs) f = open(str(result_filename), "w") res = [(name, result.__class__.__name__, result.__dict__) @@ -79,6 +79,8 @@ help='a string describing picked options, no spaces') parser.add_option('--branch', default='trunk', action='store', help="pypy's branch") + parser.add_option('--baseline', default=sys.executable, action='store', + help='baseline interpreter, defaults to host one') parser.add_option("-a", "--args", default="", help=("Pass extra arguments to the python binaries." " If there is a comma in this option's value, the" @@ -100,7 +102,8 @@ raise WrongBenchmark(benchmark) run_and_store(benchmarks, options.output_filename, options.pypy_c, options.revision, args=options.args, upload=options.upload, - force_host=options.force_host, fast=options.fast) + force_host=options.force_host, fast=options.fast, + baseline=options.baseline) if __name__ == '__main__': main(sys.argv[1:]) From fijal at codespeak.net Wed May 5 23:10:21 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 5 May 2010 23:10:21 +0200 (CEST) Subject: [pypy-svn] r74390 - pypy/branch/chunked-list/pypy/translator Message-ID: <20100505211021.7F710282BEF@codespeak.net> Author: fijal Date: Wed May 5 23:10:20 2010 New Revision: 74390 Modified: pypy/branch/chunked-list/pypy/translator/simplify.py Log: Revert last couple of changes, will hack differently Modified: pypy/branch/chunked-list/pypy/translator/simplify.py ============================================================================== --- pypy/branch/chunked-list/pypy/translator/simplify.py (original) +++ pypy/branch/chunked-list/pypy/translator/simplify.py Wed May 5 23:10:20 2010 @@ -227,10 +227,11 @@ if block.operations and block.exitswitch == c_last_exception: last_op = block.operations[-1] if last_op.opname == 'getitem': - postfx = ['idx'] + postfx = [] for exit in block.exits: - #if exit.exitcase is IndexError: - if exit.exitcase is KeyError: + if exit.exitcase is IndexError: + postfx.append('idx') + elif exit.exitcase is KeyError: postfx.append('key') if postfx: last_op.opname = last_op.opname + '_' + '_'.join(postfx) From afa at codespeak.net Wed May 5 23:46:18 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 5 May 2010 23:46:18 +0200 (CEST) Subject: [pypy-svn] r74391 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100505214618.A3102282BEF@codespeak.net> Author: afa Date: Wed May 5 23:46:17 2010 New Revision: 74391 Modified: pypy/trunk/pypy/module/cpyext/pyobject.py pypy/trunk/pypy/module/cpyext/test/test_cpyext.py pypy/trunk/pypy/module/cpyext/test/test_getargs.py Log: - Fix a test that returned a borrowed reference - Fix cleanup of the kept alive references: remove them from borrowed_objects before DecRef This allows a simplification in Py_DecRef The "borrowing" system starts to make sense to me Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Wed May 5 23:46:17 2010 @@ -258,10 +258,6 @@ if obj.c_ob_refcnt == 0: state = space.fromcache(State) ptr = rffi.cast(ADDR, obj) - try: - del state.borrowed_objects[ptr] - except KeyError: - pass if ptr not in state.py_objects_r2w: w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type)) if space.is_w(w_type, space.w_str) or space.is_w(w_type, space.w_unicode): @@ -277,11 +273,12 @@ _Py_Dealloc(space, obj) del state.py_objects_w2r[w_obj] if ptr in state.borrow_mapping: # move to lifeline __del__ + # The object is a borrow container, clear borrowed references for containee in state.borrow_mapping[ptr]: w_containee = state.py_objects_r2w.get(containee, None) if w_containee is not None: containee = state.py_objects_w2r[w_containee] - Py_DecRef(space, w_containee) + Py_DecRef(space, containee) containee_ptr = rffi.cast(ADDR, containee) try: del state.borrowed_objects[containee_ptr] @@ -342,10 +339,8 @@ raise NullPointerException if container_ptr == -1: return - borrowees = state.borrow_mapping.get(container_ptr, None) - if borrowees is None: - state.borrow_mapping[container_ptr] = borrowees = {} obj_ptr = rffi.cast(ADDR, obj) + borrowees = state.borrow_mapping.setdefault(container_ptr, {}) borrowees[obj_ptr] = None #___________________________________________________________ Modified: pypy/trunk/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Wed May 5 23:46:17 2010 @@ -263,10 +263,9 @@ Py_DecRef(self.space, w_obj) state.non_heaptypes[:] = [] while state.borrowed_objects: - addr = state.borrowed_objects.keys()[0] + addr, _ = state.borrowed_objects.popitem() w_obj = state.py_objects_r2w[addr] Py_DecRef(self.space, w_obj) - state.borrowed_objects = {} state.borrow_mapping = {} if self.check_and_print_leaks(): assert False, "Test leaks or loses object(s)." Modified: pypy/trunk/pypy/module/cpyext/test/test_getargs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_getargs.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_getargs.py Wed May 5 23:46:17 2010 @@ -47,9 +47,11 @@ if (!PyArg_ParseTuple(args, "O|O", &a, &b)) { return NULL; } - if (!b) { + if (b) + Py_INCREF(b); + else b = PyInt_FromLong(42); - } + /* return an owned reference */ return b; ''')]) assert mod.oneargint(1) == 1 From afa at codespeak.net Thu May 6 01:09:24 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 6 May 2010 01:09:24 +0200 (CEST) Subject: [pypy-svn] r74392 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100505230924.1307C282BEF@codespeak.net> Author: afa Date: Thu May 6 01:09:22 2010 New Revision: 74392 Modified: pypy/trunk/pypy/module/cpyext/api.py pypy/trunk/pypy/module/cpyext/dictobject.py pypy/trunk/pypy/module/cpyext/funcobject.py pypy/trunk/pypy/module/cpyext/listobject.py pypy/trunk/pypy/module/cpyext/modsupport.py pypy/trunk/pypy/module/cpyext/object.py pypy/trunk/pypy/module/cpyext/pyerrors.py pypy/trunk/pypy/module/cpyext/pyobject.py pypy/trunk/pypy/module/cpyext/sequence.py pypy/trunk/pypy/module/cpyext/state.py pypy/trunk/pypy/module/cpyext/sysmodule.py pypy/trunk/pypy/module/cpyext/test/test_borrow.py pypy/trunk/pypy/module/cpyext/tupleobject.py pypy/trunk/pypy/module/cpyext/weakrefobject.py Log: Start to refactor the "borrowed references" system: now functions may return a BorrowedPair that ties the container with the returned object the idiom is now: "return borrow_from(w_container, w_obj)" 'borrowed=True' disappears from the api signatures, state.last_container is gone. Let's see if this translates... Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Thu May 6 01:09:22 2010 @@ -104,13 +104,11 @@ # class ApiFunction: - def __init__(self, argtypes, restype, callable, - borrowed=False, error=_NOT_SPECIFIED): + def __init__(self, argtypes, restype, callable, error=_NOT_SPECIFIED): self.argtypes = argtypes self.restype = restype self.functype = lltype.Ptr(lltype.FuncType(argtypes, restype)) self.callable = callable - self.borrowed = borrowed if error is not _NOT_SPECIFIED: self.error_value = error @@ -141,7 +139,7 @@ wrapper.relax_sig_check = True return wrapper -def cpython_api(argtypes, restype, borrowed=False, error=_NOT_SPECIFIED, +def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, external=True, name=None): if error is _NOT_SPECIFIED: if restype is PyObject: @@ -157,7 +155,7 @@ else: func_name = name func = func_with_new_name(func, name) - api_function = ApiFunction(argtypes, restype, func, borrowed, error) + api_function = ApiFunction(argtypes, restype, func, error) func.api_func = api_function assert func_name not in FUNCTIONS @@ -176,6 +174,7 @@ def unwrapper(space, *args): from pypy.module.cpyext.pyobject import Py_DecRef from pypy.module.cpyext.pyobject import make_ref, from_ref + from pypy.module.cpyext.pyobject import BorrowedPair newargs = () to_decref = [] assert len(args) == len(api_function.argtypes) @@ -204,7 +203,7 @@ newargs += (arg, ) try: try: - return func(space, *newargs) + res = func(space, *newargs) except OperationError, e: if not catch_exception: raise @@ -216,10 +215,11 @@ return None else: return api_function.error_value + if isinstance(res, BorrowedPair): + return res.w_borrowed + else: + return res finally: - if api_function.borrowed: - state = space.fromcache(State) - state.last_container = 0 for arg in to_decref: Py_DecRef(space, arg) unwrapper.func = func @@ -398,7 +398,7 @@ @specialize.ll() def wrapper(*args): from pypy.module.cpyext.pyobject import make_ref, from_ref - from pypy.module.cpyext.pyobject import add_borrowed_object + from pypy.module.cpyext.pyobject import BorrowedPair from pypy.module.cpyext.pyobject import NullPointerException # we hope that malloc removal removes the newtuple() that is # inserted exactly here by the varargs specializer @@ -446,13 +446,12 @@ retval = error_value elif callable.api_func.restype is PyObject: - borrowed = callable.api_func.borrowed - if not rffi._isllptr(result): - retval = make_ref(space, result, borrowed=borrowed) + if isinstance(result, BorrowedPair): + retval = result.get_ref(space) + elif not rffi._isllptr(result): + retval = make_ref(space, result) else: retval = result - if borrowed: - add_borrowed_object(space, retval) elif callable.api_func.restype is not lltype.Void: retval = rffi.cast(callable.api_func.restype, result) except NullPointerException: Modified: pypy/trunk/pypy/module/cpyext/dictobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/dictobject.py (original) +++ pypy/trunk/pypy/module/cpyext/dictobject.py Thu May 6 01:09:22 2010 @@ -1,7 +1,7 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, build_type_checkers,\ Py_ssize_t, PyObjectP, CONST_STRING -from pypy.module.cpyext.pyobject import PyObject, register_container +from pypy.module.cpyext.pyobject import PyObject, borrow_from from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.interpreter.error import OperationError @@ -11,14 +11,13 @@ PyDict_Check, PyDict_CheckExact = build_type_checkers("Dict") - at cpython_api([PyObject, PyObject], PyObject, borrowed=True, error=CANNOT_FAIL) + at cpython_api([PyObject, PyObject], PyObject, error=CANNOT_FAIL) def PyDict_GetItem(space, w_dict, w_key): try: w_res = space.getitem(w_dict, w_key) except: return None - register_container(space, w_dict) - return w_res + return borrow_from(w_dict, w_res) @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) def PyDict_SetItem(space, w_dict, w_key, w_obj): @@ -47,7 +46,7 @@ else: PyErr_BadInternalCall(space) - at cpython_api([PyObject, CONST_STRING], PyObject, borrowed=True, error=CANNOT_FAIL) + at cpython_api([PyObject, CONST_STRING], PyObject, error=CANNOT_FAIL) def PyDict_GetItemString(space, w_dict, key): """This is the same as PyDict_GetItem(), but key is specified as a char*, rather than a PyObject*.""" @@ -57,8 +56,7 @@ w_res = None if w_res is None: return None - register_container(space, w_dict) - return w_res + return borrow_from(w_dict, w_res) @cpython_api([PyObject], Py_ssize_t, error=-1) def PyDict_Size(space, w_obj): Modified: pypy/trunk/pypy/module/cpyext/funcobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/funcobject.py (original) +++ pypy/trunk/pypy/module/cpyext/funcobject.py Thu May 6 01:09:22 2010 @@ -3,7 +3,7 @@ PyObjectFields, generic_cpy_call, cpython_api, bootstrap_function, cpython_struct, build_type_checkers) from pypy.module.cpyext.pyobject import ( - PyObject, make_ref, from_ref, Py_DecRef, make_typedescr, register_container) + PyObject, make_ref, from_ref, Py_DecRef, make_typedescr, borrow_from) from pypy.interpreter.function import Function, Method PyFunctionObjectStruct = lltype.ForwardReference() @@ -43,18 +43,16 @@ class which provides the unbound method.""" return Method(space, w_func, w_self, w_cls) - at cpython_api([PyObject], PyObject, borrowed=True) + at cpython_api([PyObject], PyObject) def PyMethod_Function(space, w_method): """Return the function object associated with the method meth.""" assert isinstance(w_method, Method) - register_container(space, w_method) - return w_method.w_function + return borrow_from(w_method, w_method.w_function) - at cpython_api([PyObject], PyObject, borrowed=True) + at cpython_api([PyObject], PyObject) def PyMethod_Class(space, w_method): """Return the class object from which the method meth was created; if this was created from an instance, it will be the class of the instance.""" assert isinstance(w_method, Method) - register_container(space, w_method) - return w_method.w_class + return borrow_from(w_method, w_method.w_class) Modified: pypy/trunk/pypy/module/cpyext/listobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/listobject.py (original) +++ pypy/trunk/pypy/module/cpyext/listobject.py Thu May 6 01:09:22 2010 @@ -3,7 +3,7 @@ from pypy.module.cpyext.api import (cpython_api, CANNOT_FAIL, Py_ssize_t, build_type_checkers) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall -from pypy.module.cpyext.pyobject import Py_DecRef, PyObject, register_container +from pypy.module.cpyext.pyobject import Py_DecRef, PyObject, borrow_from from pypy.objspace.std.listobject import W_ListObject from pypy.interpreter.error import OperationError @@ -39,20 +39,19 @@ wrappeditems[index] = w_item return 0 - at cpython_api([PyObject, Py_ssize_t], PyObject, borrowed=True) + at cpython_api([PyObject, Py_ssize_t], PyObject) def PyList_GetItem(space, w_list, index): """Return the object at position pos in the list pointed to by p. The position must be positive, indexing from the end of the list is not supported. If pos is out of bounds, return NULL and set an IndexError exception.""" - register_container(space, w_list) if not isinstance(w_list, W_ListObject): PyErr_BadInternalCall(space) wrappeditems = w_list.wrappeditems if index < 0 or index >= len(wrappeditems): raise OperationError(space.w_IndexError, space.wrap( "list index out of range")) - return wrappeditems[index] + return borrow_from(w_list, wrappeditems[index]) @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) Modified: pypy/trunk/pypy/module/cpyext/modsupport.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/modsupport.py (original) +++ pypy/trunk/pypy/module/cpyext/modsupport.py Thu May 6 01:09:22 2010 @@ -1,14 +1,14 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import cpython_api, cpython_struct, \ METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, CONST_STRING -from pypy.module.cpyext.pyobject import PyObject, register_container +from pypy.module.cpyext.pyobject import PyObject, borrow_from from pypy.interpreter.module import Module from pypy.module.cpyext.methodobject import W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod, PyMethodDef, PyCFunction from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.module.cpyext.state import State from pypy.interpreter.error import OperationError -#@cpython_api([rffi.CCHARP], PyObject, borrowed=True) +#@cpython_api([rffi.CCHARP], PyObject) def PyImport_AddModule(space, name): """Return the module object corresponding to a module name. The name argument may be of the form package.module. First check the modules dictionary if @@ -30,7 +30,7 @@ return w_mod @cpython_api([CONST_STRING, lltype.Ptr(PyMethodDef), CONST_STRING, - PyObject, rffi.INT_real], PyObject, borrowed=False) # we cannot borrow here + PyObject, rffi.INT_real], PyObject) def Py_InitModule4(space, name, methods, doc, w_self, apiver): """ Create a new module object based on a name and table of functions, returning @@ -58,6 +58,7 @@ if doc: space.setattr(w_mod, space.wrap("__doc__"), space.wrap(rffi.charp2str(doc))) + # we cannot borrow here return w_mod @@ -108,13 +109,12 @@ return int(space.is_w(w_type, w_obj_type) or space.is_true(space.issubtype(w_obj_type, w_type))) - at cpython_api([PyObject], PyObject, borrowed=True) + at cpython_api([PyObject], PyObject) def PyModule_GetDict(space, w_mod): if PyModule_Check(space, w_mod): assert isinstance(w_mod, Module) w_dict = w_mod.getdict() - register_container(space, w_mod) - return w_dict + return borrow_from(w_mod, w_dict) else: PyErr_BadInternalCall(space) Modified: pypy/trunk/pypy/module/cpyext/object.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/object.py (original) +++ pypy/trunk/pypy/module/cpyext/object.py Thu May 6 01:09:22 2010 @@ -144,7 +144,7 @@ This is the equivalent of the Python expression o[key].""" return space.getitem(w_obj, w_key) - at cpython_api([PyObject, PyTypeObjectPtr], PyObject, borrowed=True) + at cpython_api([PyObject, PyTypeObjectPtr], PyObject) def PyObject_Init(space, op, type): """Initialize a newly-allocated object op with its type and initial reference. Returns the initialized object. If type indicates that the @@ -157,7 +157,7 @@ op.c_ob_refcnt = 1 return from_ref(space, op) # XXX will give an exception - at cpython_api([PyVarObject, PyTypeObjectPtr, Py_ssize_t], PyObject, borrowed=True) + at cpython_api([PyVarObject, PyTypeObjectPtr, Py_ssize_t], PyObject) def PyObject_InitVar(space, op, type, size): """This does everything PyObject_Init() does, and also initializes the length information for a variable-size object.""" Modified: pypy/trunk/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyerrors.py (original) +++ pypy/trunk/pypy/module/cpyext/pyerrors.py Thu May 6 01:09:22 2010 @@ -5,7 +5,7 @@ from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING from pypy.module.exceptions.interp_exceptions import W_RuntimeWarning from pypy.module.cpyext.pyobject import ( - PyObject, PyObjectP, make_ref, Py_DecRef, register_container) + PyObject, PyObjectP, make_ref, Py_DecRef, borrow_from) from pypy.module.cpyext.state import State from pypy.module.cpyext.import_ import PyImport_Import from pypy.rlib.rposix import get_errno @@ -27,13 +27,12 @@ """This is a shorthand for PyErr_SetObject(type, Py_None).""" PyErr_SetObject(space, w_type, space.w_None) - at cpython_api([], PyObject, borrowed=True) + at cpython_api([], PyObject) def PyErr_Occurred(space): state = space.fromcache(State) if state.operror is None: return None - register_container(space, lltype.nullptr(PyObject.TO)) - return state.operror.w_type + return borrow_from(None, state.operror.w_type) @cpython_api([], lltype.Void) def PyErr_Clear(space): Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Thu May 6 01:09:22 2010 @@ -319,29 +319,28 @@ #___________________________________________________________ # Support for borrowed references - at cpython_api([PyObject], lltype.Void, external=False) -def register_container(space, container): - state = space.fromcache(State) - if not container: # self-managed - container_ptr = -1 - else: +class BorrowedPair: + def __init__(self, w_container, w_borrowed): + self.w_borrowed = w_borrowed + self.w_container = w_container + + def get_ref(self, space): + if self.w_borrowed is None: + return lltype.nullptr(PyObject.TO) + state = space.fromcache(State) + container = make_ref(space, self.w_container) + Py_DecRef(space, container) container_ptr = rffi.cast(ADDR, container) - assert not state.last_container, "Last container was not fetched" - state.last_container = container_ptr + ref = make_ref(space, self.w_borrowed, borrowed=True) + if self.w_container is None: # self-managed + return ref + obj_ptr = rffi.cast(ADDR, ref) + borrowees = state.borrow_mapping.setdefault(container_ptr, {}) + borrowees[obj_ptr] = None + return ref -def add_borrowed_object(space, obj): - state = space.fromcache(State) - container_ptr = state.last_container - state.last_container = 0 - if not obj: - return - if not container_ptr: - raise NullPointerException - if container_ptr == -1: - return - obj_ptr = rffi.cast(ADDR, obj) - borrowees = state.borrow_mapping.setdefault(container_ptr, {}) - borrowees[obj_ptr] = None +def borrow_from(container, borrowed): + return BorrowedPair(container, borrowed) #___________________________________________________________ Modified: pypy/trunk/pypy/module/cpyext/sequence.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/sequence.py (original) +++ pypy/trunk/pypy/module/cpyext/sequence.py Thu May 6 01:09:22 2010 @@ -2,7 +2,7 @@ from pypy.interpreter.error import OperationError from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, CONST_STRING, Py_ssize_t) -from pypy.module.cpyext.pyobject import PyObject, register_container +from pypy.module.cpyext.pyobject import PyObject, borrow_from from pypy.rpython.lltypesystem import rffi, lltype from pypy.objspace.std import listobject, tupleobject @@ -40,7 +40,7 @@ except OperationError: raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m))) - at cpython_api([PyObject, Py_ssize_t], PyObject, borrowed=True) + at cpython_api([PyObject, Py_ssize_t], PyObject) def PySequence_Fast_GET_ITEM(space, w_obj, index): """Return the ith element of o, assuming that o was returned by PySequence_Fast(), o is not NULL, and that i is within bounds. @@ -50,8 +50,7 @@ else: assert isinstance(w_obj, tupleobject.W_TupleObject) w_res = w_obj.wrappeditems[index] - register_container(space, w_obj) - return w_res + return borrow_from(w_obj, w_res) @cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL) def PySequence_Fast_GET_SIZE(space, w_obj): Modified: pypy/trunk/pypy/module/cpyext/state.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/state.py (original) +++ pypy/trunk/pypy/module/cpyext/state.py Thu May 6 01:09:22 2010 @@ -17,7 +17,6 @@ self.borrow_mapping = {} # { addr of container -> { addr of containee -> None } } self.borrowed_objects = {} # { addr of containee -> None } self.non_heaptypes = [] # list of wrapped objects - self.last_container = 0 # addr of last container self.operror = None self.new_method_def = lltype.nullptr(PyMethodDef) Modified: pypy/trunk/pypy/module/cpyext/sysmodule.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/sysmodule.py (original) +++ pypy/trunk/pypy/module/cpyext/sysmodule.py Thu May 6 01:09:22 2010 @@ -1,17 +1,16 @@ from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import CANNOT_FAIL, cpython_api, CONST_STRING -from pypy.module.cpyext.pyobject import PyObject, register_container +from pypy.module.cpyext.pyobject import PyObject, borrow_from - at cpython_api([CONST_STRING], PyObject, borrowed=True, error=CANNOT_FAIL) + at cpython_api([CONST_STRING], PyObject, error=CANNOT_FAIL) def PySys_GetObject(space, name): """Return the object name from the sys module or NULL if it does not exist, without setting an exception.""" name = rffi.charp2str(name) w_dict = space.sys.getdict() w_obj = space.finditem_str(w_dict, name) - register_container(space, w_dict) - return w_obj + return borrow_from(w_dict, w_obj) @cpython_api([CONST_STRING, PyObject], rffi.INT_real, error=-1) def PySys_SetObject(space, name, w_obj): Modified: pypy/trunk/pypy/module/cpyext/test/test_borrow.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_borrow.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_borrow.py Thu May 6 01:09:22 2010 @@ -2,7 +2,7 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.state import State -from pypy.module.cpyext.pyobject import make_ref, add_borrowed_object +from pypy.module.cpyext.pyobject import make_ref, borrow_from class TestBorrowing(BaseApiTest): @@ -11,9 +11,7 @@ w_int = space.wrap(1) w_tuple = space.newtuple([w_int]) api.Py_IncRef(w_tuple) - api.register_container(w_tuple) - one_pyo = make_ref(space, w_int, borrowed=True) - add_borrowed_object(space, one_pyo) + one_pyo = borrow_from(w_tuple, w_int).get_ref(space) print state.borrowed_objects api.Py_DecRef(w_tuple) state.print_refcounts() Modified: pypy/trunk/pypy/module/cpyext/tupleobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/tupleobject.py (original) +++ pypy/trunk/pypy/module/cpyext/tupleobject.py Thu May 6 01:09:22 2010 @@ -2,7 +2,7 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL, build_type_checkers) -from pypy.module.cpyext.pyobject import PyObject, Py_DecRef, register_container +from pypy.module.cpyext.pyobject import PyObject, Py_DecRef, borrow_from from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.objspace.std.tupleobject import W_TupleObject @@ -23,14 +23,13 @@ Py_DecRef(space, w_obj) # SetItem steals a reference! return 0 - at cpython_api([PyObject, Py_ssize_t], PyObject, borrowed=True) + at cpython_api([PyObject, Py_ssize_t], PyObject) def PyTuple_GetItem(space, w_t, pos): if not PyTuple_Check(space, w_t): PyErr_BadInternalCall(space) assert isinstance(w_t, W_TupleObject) w_obj = w_t.wrappeditems[pos] - register_container(space, w_t) - return w_obj + return borrow_from(w_t, w_obj) @cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL) def PyTuple_GET_SIZE(space, w_t): Modified: pypy/trunk/pypy/module/cpyext/weakrefobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/weakrefobject.py (original) +++ pypy/trunk/pypy/module/cpyext/weakrefobject.py Thu May 6 01:09:22 2010 @@ -1,5 +1,5 @@ from pypy.module.cpyext.api import cpython_api -from pypy.module.cpyext.pyobject import PyObject, register_container +from pypy.module.cpyext.pyobject import PyObject, borrow_from from pypy.module._weakref.interp__weakref import W_Weakref @cpython_api([PyObject, PyObject], PyObject) @@ -16,13 +16,12 @@ w_weakref = space.gettypeobject(W_Weakref.typedef) return space.call_function(w_weakref, w_obj, w_callback) - at cpython_api([PyObject], PyObject, borrowed=True) + at cpython_api([PyObject], PyObject) def PyWeakref_GetObject(space, w_ref): """Return the referenced object from a weak reference. If the referent is no longer live, returns None. This function returns a borrowed reference. """ - register_container(space, w_ref) - return space.call_function(w_ref) + return borrow_from(w_ref, space.call_function(w_ref)) @cpython_api([PyObject], PyObject) def PyWeakref_LockObject(space, w_ref): From afa at codespeak.net Thu May 6 01:15:42 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 6 May 2010 01:15:42 +0200 (CEST) Subject: [pypy-svn] r74393 - pypy/trunk/pypy/module/cpyext Message-ID: <20100505231542.D963D282BEF@codespeak.net> Author: afa Date: Thu May 6 01:15:40 2010 New Revision: 74393 Modified: pypy/trunk/pypy/module/cpyext/stubgen.py pypy/trunk/pypy/module/cpyext/stubs.py Log: Remove "borrowed=True" from the stubs Modified: pypy/trunk/pypy/module/cpyext/stubgen.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/stubgen.py (original) +++ pypy/trunk/pypy/module/cpyext/stubgen.py Thu May 6 01:15:40 2010 @@ -7,9 +7,10 @@ TEMPLATE = """ - at cpython_api([%(paramtypes)s], %(rettype)s%(borrows)s) + at cpython_api([%(paramtypes)s], %(rettype)s) def %(functionname)s(%(params)s): %(docstring)s raise NotImplementedError + %(borrows)s """ C_TYPE_TO_PYPY_TYPE = { @@ -77,7 +78,7 @@ if entry.result_refs is None: docstring += "\nReturn value: always NULL." else: - borrows = (", borrowed=True", "")[entry.result_refs] + borrows = ("borrow_from()", "")[entry.result_refs] docstring = "\n ".join(docstring.splitlines()) if docstring: docstring = ' """%s"""\n' % (docstring,) Modified: pypy/trunk/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/stubs.py (original) +++ pypy/trunk/pypy/module/cpyext/stubs.py Thu May 6 01:15:40 2010 @@ -32,16 +32,17 @@ def _PyObject_Del(space, op): raise NotImplementedError - at cpython_api([rffi.CCHARP, PyMethodDef], PyObject, borrowed=True) + at cpython_api([rffi.CCHARP, PyMethodDef], PyObject) def Py_InitModule(space, name, methods): """Create a new module object based on a name and table of functions, returning the new module object. Older versions of Python did not support NULL as the value for the methods argument.""" + borrow_from() raise NotImplementedError - at cpython_api([rffi.CCHARP, PyMethodDef, rffi.CCHARP], PyObject, borrowed=True) + at cpython_api([rffi.CCHARP, PyMethodDef, rffi.CCHARP], PyObject) def Py_InitModule3(space, name, methods, doc): """Create a new module object based on a name and table of functions, returning the new module object. If doc is non-NULL, it will be used @@ -49,6 +50,7 @@ Older versions of Python did not support NULL as the value for the methods argument.""" + borrow_from() raise NotImplementedError @cpython_api([PyObject, rffi.CCHARP, va_list], rffi.INT_real, error=0) @@ -337,10 +339,11 @@ """Return the contents of the cell cell.""" raise NotImplementedError - at cpython_api([PyObject], PyObject, borrowed=True) + at cpython_api([PyObject], PyObject) def PyCell_GET(space, cell): """Return the contents of the cell cell, but without checking that cell is non-NULL and a cell object.""" + borrow_from() raise NotImplementedError @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) @@ -1059,9 +1062,10 @@ raised if the end of the file is reached immediately.""" raise NotImplementedError - at cpython_api([PyObject], PyObject, borrowed=True) + at cpython_api([PyObject], PyObject) def PyFile_Name(space, p): """Return the name of the file specified by p as a string object.""" + borrow_from() raise NotImplementedError @cpython_api([PyFileObject, rffi.INT_real], lltype.Void) @@ -1184,27 +1188,31 @@ object, the argument defaults and closure are set to NULL.""" raise NotImplementedError - at cpython_api([PyObject], PyObject, borrowed=True) + at cpython_api([PyObject], PyObject) def PyFunction_GetCode(space, op): """Return the code object associated with the function object op.""" + borrow_from() raise NotImplementedError - at cpython_api([PyObject], PyObject, borrowed=True) + at cpython_api([PyObject], PyObject) def PyFunction_GetGlobals(space, op): """Return the globals dictionary associated with the function object op.""" + borrow_from() raise NotImplementedError - at cpython_api([PyObject], PyObject, borrowed=True) + at cpython_api([PyObject], PyObject) def PyFunction_GetModule(space, op): """Return the __module__ attribute of the function object op. This is normally a string containing the module name, but can be set to any other object by Python code.""" + borrow_from() raise NotImplementedError - at cpython_api([PyObject], PyObject, borrowed=True) + at cpython_api([PyObject], PyObject) def PyFunction_GetDefaults(space, op): """Return the argument default values of the function object op. This can be a tuple of arguments or NULL.""" + borrow_from() raise NotImplementedError @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) @@ -1215,10 +1223,11 @@ Raises SystemError and returns -1 on failure.""" raise NotImplementedError - at cpython_api([PyObject], PyObject, borrowed=True) + at cpython_api([PyObject], PyObject) def PyFunction_GetClosure(space, op): """Return the closure associated with the function object op. This can be NULL or a tuple of cell objects.""" + borrow_from() raise NotImplementedError @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) @@ -1352,7 +1361,7 @@ with an exception set on failure (the module still exists in this case).""" raise NotImplementedError - at cpython_api([rffi.CCHARP], PyObject, borrowed=True) + at cpython_api([rffi.CCHARP], PyObject) def PyImport_AddModule(space, name): """Return the module object corresponding to a module name. The name argument may be of the form package.module. First check the modules dictionary if @@ -1363,6 +1372,7 @@ loaded, you will get an empty module object. Use PyImport_ImportModule() or one of its variants to import a module. Package structures implied by a dotted name for name are not created if not already present.""" + borrow_from() raise NotImplementedError @cpython_api([rffi.CCHARP, PyObject], PyObject) @@ -1398,10 +1408,11 @@ of the bytecode file, in little-endian byte order.""" raise NotImplementedError - at cpython_api([], PyObject, borrowed=True) + at cpython_api([], PyObject) def PyImport_GetModuleDict(space, ): """Return the dictionary used for the module administration (a.k.a. sys.modules). Note that this is a per-interpreter variable.""" + borrow_from() raise NotImplementedError @cpython_api([PyObject], PyObject) @@ -1870,7 +1881,7 @@ the caller needn't check for NULL).""" raise NotImplementedError - at cpython_api([], PyObject, borrowed=True) + at cpython_api([], PyObject) def PyThreadState_GetDict(space, ): """Return a dictionary in which extensions can store thread-specific state information. Each extension should use a unique key to use to store state in @@ -1880,6 +1891,7 @@ Previously this could only be called when a current thread is active, and NULL meant that an exception was raised.""" + borrow_from() raise NotImplementedError @cpython_api([lltype.Signed, PyObject], rffi.INT_real, error=CANNOT_FAIL) @@ -2103,12 +2115,13 @@ """ raise NotImplementedError - at cpython_api([PyObject, Py_ssize_t], PyObject, borrowed=True) + at cpython_api([PyObject, Py_ssize_t], PyObject) def PyList_GET_ITEM(space, list, i): """Macro form of PyList_GetItem() without error checking. This macro used an int for i. This might require changes in your code for properly supporting 64-bit systems.""" + borrow_from() raise NotImplementedError @cpython_api([PyObject, Py_ssize_t, PyObject], lltype.Void) @@ -2293,15 +2306,17 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject], PyObject, borrowed=True) + at cpython_api([PyObject], PyObject) def PyMethod_Self(space, meth): """Return the instance associated with the method meth if it is bound, otherwise return NULL.""" + borrow_from() raise NotImplementedError - at cpython_api([PyObject], PyObject, borrowed=True) + at cpython_api([PyObject], PyObject) def PyMethod_GET_SELF(space, meth): """Macro version of PyMethod_Self() which avoids error checking.""" + borrow_from() raise NotImplementedError @cpython_api([], rffi.INT_real, error=CANNOT_FAIL) @@ -2442,28 +2457,32 @@ is active then NULL is returned but PyErr_Occurred() will return false.""" raise NotImplementedError - at cpython_api([], PyObject, borrowed=True) + at cpython_api([], PyObject) def PyEval_GetBuiltins(space, ): """Return a dictionary of the builtins in the current execution frame, or the interpreter of the thread state if no frame is currently executing.""" + borrow_from() raise NotImplementedError - at cpython_api([], PyObject, borrowed=True) + at cpython_api([], PyObject) def PyEval_GetLocals(space, ): """Return a dictionary of the local variables in the current execution frame, or NULL if no frame is currently executing.""" + borrow_from() raise NotImplementedError - at cpython_api([], PyObject, borrowed=True) + at cpython_api([], PyObject) def PyEval_GetGlobals(space, ): """Return a dictionary of the global variables in the current execution frame, or NULL if no frame is currently executing.""" + borrow_from() raise NotImplementedError - at cpython_api([], PyFrameObject, borrowed=True) + at cpython_api([], PyFrameObject) def PyEval_GetFrame(space, ): """Return the current thread state's frame, which is NULL if no frame is currently executing.""" + borrow_from() raise NotImplementedError @cpython_api([PyFrameObject], rffi.INT_real, error=CANNOT_FAIL) @@ -2958,12 +2977,13 @@ the cleanup function, no Python APIs should be called by func.""" raise NotImplementedError - at cpython_api([PyObject, Py_ssize_t], PyObject, borrowed=True) + at cpython_api([PyObject, Py_ssize_t], PyObject) def PyTuple_GET_ITEM(space, p, pos): """Like PyTuple_GetItem(), but does no checking of its arguments. This function used an int type for pos. This might require changes in your code for properly supporting 64-bit systems.""" + borrow_from() raise NotImplementedError @cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject) @@ -3856,9 +3876,10 @@ """ raise NotImplementedError - at cpython_api([PyObject], PyObject, borrowed=True) + at cpython_api([PyObject], PyObject) def PyWeakref_GET_OBJECT(space, ref): """Similar to PyWeakref_GetObject(), but implemented as a macro that does no error checking. """ + borrow_from() raise NotImplementedError From afa at codespeak.net Thu May 6 01:26:17 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 6 May 2010 01:26:17 +0200 (CEST) Subject: [pypy-svn] r74394 - pypy/trunk/pypy/module/cpyext Message-ID: <20100505232617.87EBF282BEF@codespeak.net> Author: afa Date: Thu May 6 01:26:16 2010 New Revision: 74394 Modified: pypy/trunk/pypy/module/cpyext/api.py Log: Attempt to fix translation Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Thu May 6 01:26:16 2010 @@ -215,7 +215,9 @@ return None else: return api_function.error_value - if isinstance(res, BorrowedPair): + if res is None: + return res + elif isinstance(res, BorrowedPair): return res.w_borrowed else: return res From afa at codespeak.net Thu May 6 01:34:55 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 6 May 2010 01:34:55 +0200 (CEST) Subject: [pypy-svn] r74395 - pypy/trunk/pypy/module/cpyext Message-ID: <20100505233455.6179F282BEF@codespeak.net> Author: afa Date: Thu May 6 01:34:53 2010 New Revision: 74395 Modified: pypy/trunk/pypy/module/cpyext/api.py Log: Another translation fix maybe Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Thu May 6 01:34:53 2010 @@ -448,6 +448,8 @@ retval = error_value elif callable.api_func.restype is PyObject: + if result is None: + retval = lltype.nullptr(PyObject.TO) if isinstance(result, BorrowedPair): retval = result.get_ref(space) elif not rffi._isllptr(result): From afa at codespeak.net Thu May 6 01:47:39 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 6 May 2010 01:47:39 +0200 (CEST) Subject: [pypy-svn] r74396 - pypy/trunk/pypy/module/cpyext Message-ID: <20100505234739.1437B282BEF@codespeak.net> Author: afa Date: Thu May 6 01:47:37 2010 New Revision: 74396 Modified: pypy/trunk/pypy/module/cpyext/api.py Log: Better fix I think Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Thu May 6 01:47:37 2010 @@ -450,7 +450,7 @@ elif callable.api_func.restype is PyObject: if result is None: retval = lltype.nullptr(PyObject.TO) - if isinstance(result, BorrowedPair): + elif isinstance(result, BorrowedPair): retval = result.get_ref(space) elif not rffi._isllptr(result): retval = make_ref(space, result) From afa at codespeak.net Thu May 6 01:57:22 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 6 May 2010 01:57:22 +0200 (CEST) Subject: [pypy-svn] r74397 - pypy/trunk/pypy/module/cpyext Message-ID: <20100505235722.16F0E282BEF@codespeak.net> Author: afa Date: Thu May 6 01:57:21 2010 New Revision: 74397 Modified: pypy/trunk/pypy/module/cpyext/api.py Log: Another attempt to fix translation. Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Thu May 6 01:57:21 2010 @@ -453,6 +453,7 @@ elif isinstance(result, BorrowedPair): retval = result.get_ref(space) elif not rffi._isllptr(result): + assert isinstance(retval, W_Root) retval = make_ref(space, result) else: retval = result From afa at codespeak.net Thu May 6 02:07:03 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 6 May 2010 02:07:03 +0200 (CEST) Subject: [pypy-svn] r74398 - pypy/trunk/pypy/module/cpyext Message-ID: <20100506000703.A7AEE282BEF@codespeak.net> Author: afa Date: Thu May 6 02:07:02 2010 New Revision: 74398 Modified: pypy/trunk/pypy/module/cpyext/api.py Log: Grr Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Thu May 6 02:07:02 2010 @@ -453,7 +453,7 @@ elif isinstance(result, BorrowedPair): retval = result.get_ref(space) elif not rffi._isllptr(result): - assert isinstance(retval, W_Root) + assert isinstance(result, W_Root) retval = make_ref(space, result) else: retval = result From afa at codespeak.net Thu May 6 02:34:05 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 6 May 2010 02:34:05 +0200 (CEST) Subject: [pypy-svn] r74399 - pypy/trunk/pypy/module/cpyext Message-ID: <20100506003405.18469282BEF@codespeak.net> Author: afa Date: Thu May 6 02:34:03 2010 New Revision: 74399 Modified: pypy/trunk/pypy/module/cpyext/api.py Log: Another attempt to fix translation. rtyper insist on trying to pass a BorrowedPair to make_ref() Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Thu May 6 02:34:03 2010 @@ -448,12 +448,9 @@ retval = error_value elif callable.api_func.restype is PyObject: - if result is None: - retval = lltype.nullptr(PyObject.TO) - elif isinstance(result, BorrowedPair): + if isinstance(result, BorrowedPair): retval = result.get_ref(space) elif not rffi._isllptr(result): - assert isinstance(result, W_Root) retval = make_ref(space, result) else: retval = result From afa at codespeak.net Thu May 6 02:38:06 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 6 May 2010 02:38:06 +0200 (CEST) Subject: [pypy-svn] r74400 - pypy/trunk/pypy/module/cpyext Message-ID: <20100506003806.797C6282BEF@codespeak.net> Author: afa Date: Thu May 6 02:38:04 2010 New Revision: 74400 Modified: pypy/trunk/pypy/module/cpyext/api.py Log: Last attempt Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Thu May 6 02:38:04 2010 @@ -451,6 +451,7 @@ if isinstance(result, BorrowedPair): retval = result.get_ref(space) elif not rffi._isllptr(result): + assert not isinstance(result, BorrowedPair) retval = make_ref(space, result) else: retval = result From afa at codespeak.net Thu May 6 02:46:04 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 6 May 2010 02:46:04 +0200 (CEST) Subject: [pypy-svn] r74401 - pypy/trunk/pypy/module/cpyext Message-ID: <20100506004604.37EFD282BEF@codespeak.net> Author: afa Date: Thu May 6 02:46:02 2010 New Revision: 74401 Modified: pypy/trunk/pypy/module/cpyext/pyobject.py pypy/trunk/pypy/module/cpyext/state.py Log: Finally manage to remove all "borrowed=True" from the code. Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Thu May 6 02:46:02 2010 @@ -172,7 +172,7 @@ metatypedescr = get_typedescr(w_type.typedef) return metatypedescr.make_ref(space, w_type, w_obj, itemcount=items) -def track_reference(space, py_obj, w_obj, borrowed=False, replace=False): +def track_reference(space, py_obj, w_obj, replace=False): """ Ties together a PyObject and an interpreter object. """ @@ -188,13 +188,10 @@ state.py_objects_w2r[w_obj] = py_obj if ptr: # init_typeobject() bootstraps with NULL references state.py_objects_r2w[ptr] = w_obj - if borrowed: - state.borrowed_objects[ptr] = None -def make_ref(space, w_obj, borrowed=False, steal=False, items=0): +def make_ref(space, w_obj, steal=False, items=0): """ Returns a reference to an intepreter object. - if borrowed=False, the caller owns the reference. """ if w_obj is None: return lltype.nullptr(PyObject.TO) @@ -205,17 +202,11 @@ except KeyError: assert not steal py_obj = create_ref(space, w_obj, items) - track_reference(space, py_obj, w_obj, borrowed=borrowed) + track_reference(space, py_obj, w_obj) return py_obj if not steal: - if borrowed: - py_obj_addr = rffi.cast(ADDR, py_obj) - if py_obj_addr not in state.borrowed_objects: - Py_IncRef(space, py_obj) - state.borrowed_objects[py_obj_addr] = None - else: - Py_IncRef(space, py_obj) + Py_IncRef(space, py_obj) return py_obj @@ -272,23 +263,8 @@ if not space.is_w(w_typetype, space.gettypeobject(W_PyCTypeObject.typedef)): _Py_Dealloc(space, obj) del state.py_objects_w2r[w_obj] - if ptr in state.borrow_mapping: # move to lifeline __del__ - # The object is a borrow container, clear borrowed references - for containee in state.borrow_mapping[ptr]: - w_containee = state.py_objects_r2w.get(containee, None) - if w_containee is not None: - containee = state.py_objects_w2r[w_containee] - Py_DecRef(space, containee) - containee_ptr = rffi.cast(ADDR, containee) - try: - del state.borrowed_objects[containee_ptr] - except KeyError: - pass - else: - if DEBUG_REFCOUNT: - print >>sys.stderr, "Borrowed object is already gone:", \ - hex(containee) - del state.borrow_mapping[ptr] + # if the object was a container for borrowed references + delete_borrower(space, obj) else: if not we_are_translated() and obj.c_ob_refcnt < 0: message = "Negative refcount for obj %s with type %s" % ( @@ -325,16 +301,28 @@ self.w_container = w_container def get_ref(self, space): + """ + Create a borrowed reference, which will live as long as the container + has a living reference (as a PyObject!) + """ if self.w_borrowed is None: return lltype.nullptr(PyObject.TO) + + ref = make_ref(space, self.w_borrowed) + state = space.fromcache(State) + obj_ptr = rffi.cast(ADDR, ref) + if obj_ptr not in state.borrowed_objects: + state.borrowed_objects[obj_ptr] = None + else: + Py_DecRef(space, ref) + + if self.w_container is None: # self-managed + return ref + container = make_ref(space, self.w_container) Py_DecRef(space, container) container_ptr = rffi.cast(ADDR, container) - ref = make_ref(space, self.w_borrowed, borrowed=True) - if self.w_container is None: # self-managed - return ref - obj_ptr = rffi.cast(ADDR, ref) borrowees = state.borrow_mapping.setdefault(container_ptr, {}) borrowees[obj_ptr] = None return ref @@ -342,6 +330,43 @@ def borrow_from(container, borrowed): return BorrowedPair(container, borrowed) +def forget_borrowee(space, w_obj): + "De-register an object from the list of borrowed references" + state = space.fromcache(State) + ref = state.py_objects_w2r.get(w_obj) + if not ref: + if DEBUG_REFCOUNT: + print >>sys.stderr, "Borrowed object is already gone:", \ + hex(containee) + return + + containee_ptr = rffi.cast(ADDR, ref) + try: + del state.borrowed_objects[containee_ptr] + except KeyError: + pass + else: + Py_DecRef(space, ref) + +def delete_borrower(space, py_obj): + """ + Called when a potential container for borrowed references has lost its + last reference. Removes the borrowed references it contains. + """ + ptr = rffi.cast(ADDR, py_obj) + state = space.fromcache(State) + if ptr in state.borrow_mapping: # move to lifeline __del__ + for containee in state.borrow_mapping[ptr]: + w_containee = state.py_objects_r2w.get(containee, None) + if w_containee is not None: + forget_borrowee(space, w_containee) + else: + if DEBUG_REFCOUNT: + print >>sys.stderr, "Borrowed object is already gone:", \ + hex(containee) + del state.borrow_mapping[ptr] + + #___________________________________________________________ @cpython_api([rffi.VOIDP_real], lltype.Signed, error=CANNOT_FAIL) Modified: pypy/trunk/pypy/module/cpyext/state.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/state.py (original) +++ pypy/trunk/pypy/module/cpyext/state.py Thu May 6 02:46:02 2010 @@ -49,16 +49,14 @@ """Clear the current exception state, and return the operror. Also frees the borrowed reference returned by PyErr_Occurred() """ - from pypy.module.cpyext.pyobject import Py_DecRef, make_ref from pypy.module.cpyext.api import ADDR # handling of borrowed objects, remove when we have # a weakkeydict operror = self.operror if operror is not None: - exc_type = make_ref(self.space, operror.w_type, borrowed=True) - Py_DecRef(self.space, exc_type) - containee_ptr = rffi.cast(ADDR, exc_type) - del self.borrowed_objects[containee_ptr] + # we have no explicit container + from pypy.module.cpyext.pyobject import forget_borrowee + forget_borrowee(self.space, operror.w_type) self.operror = None return operror From afa at codespeak.net Thu May 6 02:55:45 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 6 May 2010 02:55:45 +0200 (CEST) Subject: [pypy-svn] r74402 - pypy/trunk/pypy/module/cpyext Message-ID: <20100506005545.AF348282BEF@codespeak.net> Author: afa Date: Thu May 6 02:55:44 2010 New Revision: 74402 Modified: pypy/trunk/pypy/module/cpyext/pyobject.py Log: rdict.get() seems to require the default value Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Thu May 6 02:55:44 2010 @@ -333,7 +333,7 @@ def forget_borrowee(space, w_obj): "De-register an object from the list of borrowed references" state = space.fromcache(State) - ref = state.py_objects_w2r.get(w_obj) + ref = state.py_objects_w2r.get(w_obj, lltype.nullptr(PyObject.TO)) if not ref: if DEBUG_REFCOUNT: print >>sys.stderr, "Borrowed object is already gone:", \ From antocuni at codespeak.net Thu May 6 11:45:49 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 6 May 2010 11:45:49 +0200 (CEST) Subject: [pypy-svn] r74403 - pypy/extradoc/talk/pycon-italy-2010/talk Message-ID: <20100506094549.CBBA8282BDD@codespeak.net> Author: antocuni Date: Thu May 6 11:45:48 2010 New Revision: 74403 Added: pypy/extradoc/talk/pycon-italy-2010/talk/ - copied from r74348, pypy/extradoc/talk/rst2beamer-template/ Modified: pypy/extradoc/talk/pycon-italy-2010/talk/author.latex pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt Log: begginning of our talk Modified: pypy/extradoc/talk/pycon-italy-2010/talk/author.latex ============================================================================== --- pypy/extradoc/talk/rst2beamer-template/author.latex (original) +++ pypy/extradoc/talk/pycon-italy-2010/talk/author.latex Thu May 6 11:45:48 2010 @@ -1,8 +1,8 @@ \definecolor{rrblitbackground}{rgb}{0.0, 0.0, 0.0} -\title[PyPy: becoming fast]{PyPy: becoming fast} -\author[antocuni, cfbolz, pedronis] -{Antonio Cuni \\ Carl Friedrich Bolz\\ Samuele Pedroni} +\title[PyPy 1.2]{PyPy 1.2: snakes never crawled so fast} +\author[antocuni, arigato] +{Antonio Cuni \\ Armin Rigo} -\institute{EuroPython 2009} -\date{June 30 2009} +\institute{Pycon Italia Qu4ttro} +\date{May 8 2010} Modified: pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt ============================================================================== --- pypy/extradoc/talk/rst2beamer-template/talk.txt (original) +++ pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt Thu May 6 11:45:48 2010 @@ -1,7 +1,16 @@ .. include:: beamerdefs.txt -================================ -Title -================================ +======================================= +PyPy 1.2: snakes never crawled so fast +======================================= + + +Outline +------- + +- PyPy 1.2: what's new and status update + +- Overview of the JIT + +- Demo: how to use PyPy right now -XXX From antocuni at codespeak.net Thu May 6 12:25:06 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 6 May 2010 12:25:06 +0200 (CEST) Subject: [pypy-svn] r74404 - pypy/extradoc/talk/pycon-italy-2010/talk Message-ID: <20100506102506.6F3A5282BDD@codespeak.net> Author: antocuni Date: Thu May 6 12:25:04 2010 New Revision: 74404 Modified: pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt Log: (antocuni, arigo) outline Modified: pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt ============================================================================== --- pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt (original) +++ pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt Thu May 6 12:25:04 2010 @@ -14,3 +14,71 @@ - Demo: how to use PyPy right now + +Part 1 +------- + +- What's new and status update + +- Mandelbrot demo + + +What's new in PyPy 1.2 +----------------------- + +- Main theme: speed + +- JIT compiler + +- Memory savings + +- speed.pypy.org + +- ubuntu packages + +Speed +----- + +XXX show the performance graphs +(also with psyco) + + +What works on PyPy +------------------- + +XXX + + +What does not work on PyPy +-------------------------- + +XXX :-) + +cpyext +------ + +XXX + +PyPy 1.2.1 +---------- + +- Coming soon + +- Many bugfixes + +- beta version of cpyext + + +bPart 2 +------- + +JIT + + + + +Part 3 +------ + +Demo explained + From afa at codespeak.net Thu May 6 12:55:53 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 6 May 2010 12:55:53 +0200 (CEST) Subject: [pypy-svn] r74405 - pypy/trunk/pypy/module/cpyext Message-ID: <20100506105553.EA2ED282BDD@codespeak.net> Author: afa Date: Thu May 6 12:55:52 2010 New Revision: 74405 Modified: pypy/trunk/pypy/module/cpyext/api.py pypy/trunk/pypy/module/cpyext/pyobject.py Log: Fix translation, at last. Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Thu May 6 12:55:52 2010 @@ -216,7 +216,7 @@ else: return api_function.error_value if res is None: - return res + return None elif isinstance(res, BorrowedPair): return res.w_borrowed else: @@ -448,10 +448,11 @@ retval = error_value elif callable.api_func.restype is PyObject: - if isinstance(result, BorrowedPair): + if result is None: + retval = make_ref(space, None) + elif isinstance(result, BorrowedPair): retval = result.get_ref(space) elif not rffi._isllptr(result): - assert not isinstance(result, BorrowedPair) retval = make_ref(space, result) else: retval = result Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Thu May 6 12:55:52 2010 @@ -305,17 +305,17 @@ Create a borrowed reference, which will live as long as the container has a living reference (as a PyObject!) """ - if self.w_borrowed is None: - return lltype.nullptr(PyObject.TO) - ref = make_ref(space, self.w_borrowed) + if not ref: + return ref + # state.borrowed_objects owns the reference state = space.fromcache(State) obj_ptr = rffi.cast(ADDR, ref) if obj_ptr not in state.borrowed_objects: state.borrowed_objects[obj_ptr] = None else: - Py_DecRef(space, ref) + Py_DecRef(space, ref) # already in borrowed list if self.w_container is None: # self-managed return ref From afa at codespeak.net Thu May 6 13:23:51 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 6 May 2010 13:23:51 +0200 (CEST) Subject: [pypy-svn] r74406 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100506112351.A7037282BDD@codespeak.net> Author: afa Date: Thu May 6 13:23:50 2010 New Revision: 74406 Modified: pypy/trunk/pypy/module/cpyext/__init__.py pypy/trunk/pypy/module/cpyext/api.py pypy/trunk/pypy/module/cpyext/pyobject.py pypy/trunk/pypy/module/cpyext/state.py pypy/trunk/pypy/module/cpyext/test/test_borrow.py pypy/trunk/pypy/module/cpyext/test/test_cpyext.py pypy/trunk/pypy/module/cpyext/typeobject.py Log: Put management of reference counted objects in its own State structure. Modified: pypy/trunk/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/__init__.py (original) +++ pypy/trunk/pypy/module/cpyext/__init__.py Thu May 6 13:23:50 2010 @@ -34,8 +34,6 @@ for func in api.INIT_FUNCTIONS: func(space) state.check_and_raise_exception() - if not we_are_translated(): - state.non_heaptypes[:] = [] # import these modules to register api functions by side-effect import pypy.module.cpyext.thread Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Thu May 6 13:23:50 2010 @@ -174,7 +174,7 @@ def unwrapper(space, *args): from pypy.module.cpyext.pyobject import Py_DecRef from pypy.module.cpyext.pyobject import make_ref, from_ref - from pypy.module.cpyext.pyobject import BorrowedPair + from pypy.module.cpyext.pyobject import BorrowPair newargs = () to_decref = [] assert len(args) == len(api_function.argtypes) @@ -217,7 +217,7 @@ return api_function.error_value if res is None: return None - elif isinstance(res, BorrowedPair): + elif isinstance(res, BorrowPair): return res.w_borrowed else: return res @@ -400,7 +400,7 @@ @specialize.ll() def wrapper(*args): from pypy.module.cpyext.pyobject import make_ref, from_ref - from pypy.module.cpyext.pyobject import BorrowedPair + from pypy.module.cpyext.pyobject import BorrowPair from pypy.module.cpyext.pyobject import NullPointerException # we hope that malloc removal removes the newtuple() that is # inserted exactly here by the varargs specializer @@ -450,7 +450,7 @@ elif callable.api_func.restype is PyObject: if result is None: retval = make_ref(space, None) - elif isinstance(result, BorrowedPair): + elif isinstance(result, BorrowPair): retval = result.get_ref(space) elif not rffi._isllptr(result): retval = make_ref(space, result) Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Thu May 6 13:23:50 2010 @@ -146,6 +146,30 @@ #________________________________________________________ # refcounted object support +class RefcountState: + def __init__(self, space): + self.py_objects_w2r = {} # { w_obj -> raw PyObject } + self.py_objects_r2w = {} # { addr of raw PyObject -> w_obj } + self.borrow_mapping = {} # { addr of container -> { addr of containee -> None } } + self.borrowed_objects = {} # { addr of containee -> None } + self.non_heaptypes = [] # list of wrapped objects + + def _freeze_(self): + assert not self.borrowed_objects and not self.borrow_mapping + self.py_objects_r2w.clear() # is not valid anymore after translation + return False + + def init_r2w_from_w2r(self): + from pypy.module.cpyext.api import ADDR + for w_obj, obj in self.py_objects_w2r.items(): + ptr = rffi.cast(ADDR, obj) + self.py_objects_r2w[ptr] = w_obj + + def print_refcounts(self): + print "REFCOUNTS" + for w_obj, obj in self.py_objects_w2r.items(): + print "%r: %i" % (w_obj, obj.c_ob_refcnt) + class NullPointerException(Exception): pass @@ -178,7 +202,7 @@ """ # XXX looks like a PyObject_GC_TRACK ptr = rffi.cast(ADDR, py_obj) - state = space.fromcache(State) + state = space.fromcache(RefcountState) if DEBUG_REFCOUNT: debug_refcount("MAKREF", py_obj, w_obj) if not replace: @@ -196,7 +220,7 @@ if w_obj is None: return lltype.nullptr(PyObject.TO) assert isinstance(w_obj, W_Root) - state = space.fromcache(State) + state = space.fromcache(RefcountState) try: py_obj = state.py_objects_w2r[w_obj] except KeyError: @@ -218,7 +242,7 @@ assert lltype.typeOf(ref) == PyObject if not ref: return None - state = space.fromcache(State) + state = space.fromcache(RefcountState) ptr = rffi.cast(ADDR, ref) try: @@ -247,7 +271,7 @@ if DEBUG_REFCOUNT: debug_refcount("DECREF", obj, obj.c_ob_refcnt, frame_stackdepth=3) if obj.c_ob_refcnt == 0: - state = space.fromcache(State) + state = space.fromcache(RefcountState) ptr = rffi.cast(ADDR, obj) if ptr not in state.py_objects_r2w: w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type)) @@ -295,44 +319,49 @@ #___________________________________________________________ # Support for borrowed references -class BorrowedPair: +def make_borrowed_ref(space, w_container, w_borrowed): + """ + Create a borrowed reference, which will live as long as the container + has a living reference (as a PyObject!) + """ + ref = make_ref(space, w_borrowed) + if not ref: + return ref + + # state.borrowed_objects owns the reference + state = space.fromcache(RefcountState) + obj_ptr = rffi.cast(ADDR, ref) + if obj_ptr not in state.borrowed_objects: + state.borrowed_objects[obj_ptr] = None + else: + Py_DecRef(space, ref) # already in borrowed list + + container = make_ref(space, w_container) + if not container: # self-managed + return ref + Py_DecRef(space, container) + container_ptr = rffi.cast(ADDR, container) + borrowees = state.borrow_mapping.setdefault(container_ptr, {}) + borrowees[obj_ptr] = None + return ref + +class BorrowPair: + """ + Delays the creation of a borrowed reference. + """ def __init__(self, w_container, w_borrowed): - self.w_borrowed = w_borrowed self.w_container = w_container + self.w_borrowed = w_borrowed def get_ref(self, space): - """ - Create a borrowed reference, which will live as long as the container - has a living reference (as a PyObject!) - """ - ref = make_ref(space, self.w_borrowed) - if not ref: - return ref - - # state.borrowed_objects owns the reference - state = space.fromcache(State) - obj_ptr = rffi.cast(ADDR, ref) - if obj_ptr not in state.borrowed_objects: - state.borrowed_objects[obj_ptr] = None - else: - Py_DecRef(space, ref) # already in borrowed list - - if self.w_container is None: # self-managed - return ref - - container = make_ref(space, self.w_container) - Py_DecRef(space, container) - container_ptr = rffi.cast(ADDR, container) - borrowees = state.borrow_mapping.setdefault(container_ptr, {}) - borrowees[obj_ptr] = None - return ref + return make_borrowed_ref(space, self.w_container, self.w_borrowed) def borrow_from(container, borrowed): - return BorrowedPair(container, borrowed) + return BorrowPair(container, borrowed) def forget_borrowee(space, w_obj): "De-register an object from the list of borrowed references" - state = space.fromcache(State) + state = space.fromcache(RefcountState) ref = state.py_objects_w2r.get(w_obj, lltype.nullptr(PyObject.TO)) if not ref: if DEBUG_REFCOUNT: @@ -354,7 +383,7 @@ last reference. Removes the borrowed references it contains. """ ptr = rffi.cast(ADDR, py_obj) - state = space.fromcache(State) + state = space.fromcache(RefcountState) if ptr in state.borrow_mapping: # move to lifeline __del__ for containee in state.borrow_mapping[ptr]: w_containee = state.py_objects_r2w.get(containee, None) Modified: pypy/trunk/pypy/module/cpyext/state.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/state.py (original) +++ pypy/trunk/pypy/module/cpyext/state.py Thu May 6 13:23:50 2010 @@ -12,11 +12,6 @@ def reset(self): from pypy.module.cpyext.modsupport import PyMethodDef - self.py_objects_w2r = {} # { w_obj -> raw PyObject } - self.py_objects_r2w = {} # { addr of raw PyObject -> w_obj } - self.borrow_mapping = {} # { addr of container -> { addr of containee -> None } } - self.borrowed_objects = {} # { addr of containee -> None } - self.non_heaptypes = [] # list of wrapped objects self.operror = None self.new_method_def = lltype.nullptr(PyMethodDef) @@ -29,18 +24,6 @@ # already read it by that time. self.package_context = None - - def _freeze_(self): - assert not self.borrowed_objects and not self.borrow_mapping - self.py_objects_r2w.clear() # is not valid anymore after translation - return False - - def init_r2w_from_w2r(self): - from pypy.module.cpyext.api import ADDR - for w_obj, obj in self.py_objects_w2r.items(): - ptr = rffi.cast(ADDR, obj) - self.py_objects_r2w[ptr] = w_obj - def set_exception(self, operror): self.clear_exception() self.operror = operror @@ -68,8 +51,3 @@ if always: raise OperationError(self.space.w_SystemError, self.space.wrap( "Function returned an error result without setting an exception")) - - def print_refcounts(self): - print "REFCOUNTS" - for w_obj, obj in self.py_objects_w2r.items(): - print "%r: %i" % (w_obj, obj.c_ob_refcnt) Modified: pypy/trunk/pypy/module/cpyext/test/test_borrow.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_borrow.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_borrow.py Thu May 6 13:23:50 2010 @@ -1,19 +1,17 @@ import py from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.test.test_api import BaseApiTest -from pypy.module.cpyext.state import State -from pypy.module.cpyext.pyobject import make_ref, borrow_from +from pypy.module.cpyext.pyobject import make_ref, borrow_from, RefcountState class TestBorrowing(BaseApiTest): def test_borrowing(self, space, api): - state = space.fromcache(State) w_int = space.wrap(1) w_tuple = space.newtuple([w_int]) api.Py_IncRef(w_tuple) one_pyo = borrow_from(w_tuple, w_int).get_ref(space) - print state.borrowed_objects api.Py_DecRef(w_tuple) + state = space.fromcache(RefcountState) state.print_refcounts() py.test.raises(AssertionError, api.Py_DecRef, one_pyo) Modified: pypy/trunk/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Thu May 6 13:23:50 2010 @@ -12,6 +12,7 @@ from pypy.tool.udir import udir from pypy.module.cpyext import api, typeobject from pypy.module.cpyext.state import State +from pypy.module.cpyext.pyobject import RefcountState from pypy.module.cpyext.pyobject import Py_DecRef, InvalidPointerException from pypy.translator.goal import autopath from pypy.lib.identity_dict import identity_dict @@ -71,7 +72,7 @@ return str(pydname) def freeze_refcnts(self): - state = self.space.fromcache(State) + state = self.space.fromcache(RefcountState) self.frozen_refcounts = {} for w_obj, obj in state.py_objects_w2r.iteritems(): self.frozen_refcounts[w_obj] = obj.c_ob_refcnt @@ -85,7 +86,7 @@ import gc leaking = False - state = self.space.fromcache(State) + state = self.space.fromcache(RefcountState) gc.collect() lost_objects_w = identity_dict() lost_objects_w.update((key, None) for key in self.frozen_refcounts.keys()) @@ -136,6 +137,8 @@ cls.space.getbuiltinmodule("cpyext") from pypy.module.imp.importing import importhook importhook(cls.space, "os") # warm up reference counts + state = cls.space.fromcache(RefcountState) + state.non_heaptypes[:] = [] def compile_module(self, name, **kwds): """ @@ -258,7 +261,7 @@ def teardown_method(self, func): for name in self.imported_module_names: self.unimport_module(name) - state = self.space.fromcache(State) + state = self.space.fromcache(RefcountState) for w_obj in state.non_heaptypes: Py_DecRef(self.space, w_obj) state.non_heaptypes[:] = [] Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Thu May 6 13:23:50 2010 @@ -17,7 +17,7 @@ PyBufferProcs, build_type_checkers) from pypy.module.cpyext.pyobject import ( PyObject, make_ref, create_ref, from_ref, get_typedescr, make_typedescr, - track_reference) + track_reference, RefcountState) from pypy.interpreter.module import Module from pypy.interpreter.function import FunctionWithFixedCode, StaticMethod from pypy.module.cpyext import structmemberdefs @@ -580,7 +580,7 @@ finish_type_2(space, py_type, w_obj) - state = space.fromcache(State) + state = space.fromcache(RefcountState) state.non_heaptypes.append(w_obj) return w_obj From afa at codespeak.net Thu May 6 13:27:21 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 6 May 2010 13:27:21 +0200 (CEST) Subject: [pypy-svn] r74407 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100506112721.D8A9A282BDD@codespeak.net> Author: afa Date: Thu May 6 13:27:20 2010 New Revision: 74407 Modified: pypy/trunk/pypy/module/cpyext/pyobject.py pypy/trunk/pypy/module/cpyext/test/test_cpyext.py pypy/trunk/pypy/module/cpyext/typeobject.py Log: Rename member to follow convention Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Thu May 6 13:27:20 2010 @@ -152,7 +152,7 @@ self.py_objects_r2w = {} # { addr of raw PyObject -> w_obj } self.borrow_mapping = {} # { addr of container -> { addr of containee -> None } } self.borrowed_objects = {} # { addr of containee -> None } - self.non_heaptypes = [] # list of wrapped objects + self.non_heaptypes_w = [] def _freeze_(self): assert not self.borrowed_objects and not self.borrow_mapping Modified: pypy/trunk/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Thu May 6 13:27:20 2010 @@ -138,7 +138,7 @@ from pypy.module.imp.importing import importhook importhook(cls.space, "os") # warm up reference counts state = cls.space.fromcache(RefcountState) - state.non_heaptypes[:] = [] + state.non_heaptypes_w[:] = [] def compile_module(self, name, **kwds): """ @@ -262,9 +262,9 @@ for name in self.imported_module_names: self.unimport_module(name) state = self.space.fromcache(RefcountState) - for w_obj in state.non_heaptypes: + for w_obj in state.non_heaptypes_w: Py_DecRef(self.space, w_obj) - state.non_heaptypes[:] = [] + state.non_heaptypes_w[:] = [] while state.borrowed_objects: addr, _ = state.borrowed_objects.popitem() w_obj = state.py_objects_r2w[addr] Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Thu May 6 13:27:20 2010 @@ -581,7 +581,7 @@ finish_type_2(space, py_type, w_obj) state = space.fromcache(RefcountState) - state.non_heaptypes.append(w_obj) + state.non_heaptypes_w.append(w_obj) return w_obj From afa at codespeak.net Thu May 6 13:47:29 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 6 May 2010 13:47:29 +0200 (CEST) Subject: [pypy-svn] r74408 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100506114729.79870282BDD@codespeak.net> Author: afa Date: Thu May 6 13:47:27 2010 New Revision: 74408 Modified: pypy/trunk/pypy/module/cpyext/pyobject.py pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Log: Turns borrow_mapping into a dict(addr of container -> set of w_containees) Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Thu May 6 13:47:27 2010 @@ -148,9 +148,10 @@ class RefcountState: def __init__(self, space): + self.space = space self.py_objects_w2r = {} # { w_obj -> raw PyObject } self.py_objects_r2w = {} # { addr of raw PyObject -> w_obj } - self.borrow_mapping = {} # { addr of container -> { addr of containee -> None } } + self.borrow_mapping = {} # { addr of container -> { w_containee -> None } } self.borrowed_objects = {} # { addr of containee -> None } self.non_heaptypes_w = [] @@ -170,6 +171,13 @@ for w_obj, obj in self.py_objects_w2r.items(): print "%r: %i" % (w_obj, obj.c_ob_refcnt) + def reset_borrowed_references(self): + while self.borrowed_objects: + addr, _ = self.borrowed_objects.popitem() + w_obj = self.py_objects_r2w[addr] + Py_DecRef(self.space, w_obj) + self.borrow_mapping = {} + class NullPointerException(Exception): pass @@ -342,7 +350,7 @@ Py_DecRef(space, container) container_ptr = rffi.cast(ADDR, container) borrowees = state.borrow_mapping.setdefault(container_ptr, {}) - borrowees[obj_ptr] = None + borrowees[w_borrowed] = None return ref class BorrowPair: @@ -385,14 +393,8 @@ ptr = rffi.cast(ADDR, py_obj) state = space.fromcache(RefcountState) if ptr in state.borrow_mapping: # move to lifeline __del__ - for containee in state.borrow_mapping[ptr]: - w_containee = state.py_objects_r2w.get(containee, None) - if w_containee is not None: - forget_borrowee(space, w_containee) - else: - if DEBUG_REFCOUNT: - print >>sys.stderr, "Borrowed object is already gone:", \ - hex(containee) + for w_containee in state.borrow_mapping[ptr]: + forget_borrowee(space, w_containee) del state.borrow_mapping[ptr] Modified: pypy/trunk/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Thu May 6 13:47:27 2010 @@ -265,11 +265,7 @@ for w_obj in state.non_heaptypes_w: Py_DecRef(self.space, w_obj) state.non_heaptypes_w[:] = [] - while state.borrowed_objects: - addr, _ = state.borrowed_objects.popitem() - w_obj = state.py_objects_r2w[addr] - Py_DecRef(self.space, w_obj) - state.borrow_mapping = {} + state.reset_borrowed_references() if self.check_and_print_leaks(): assert False, "Test leaks or loses object(s)." From afa at codespeak.net Thu May 6 13:51:14 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 6 May 2010 13:51:14 +0200 (CEST) Subject: [pypy-svn] r74409 - pypy/trunk/pypy/module/cpyext Message-ID: <20100506115114.6A5F5282BDD@codespeak.net> Author: afa Date: Thu May 6 13:51:10 2010 New Revision: 74409 Modified: pypy/trunk/pypy/module/cpyext/pyobject.py Log: Actually a dict(w_container: set of w_borrowees) makes things easier Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Thu May 6 13:51:10 2010 @@ -151,7 +151,7 @@ self.space = space self.py_objects_w2r = {} # { w_obj -> raw PyObject } self.py_objects_r2w = {} # { addr of raw PyObject -> w_obj } - self.borrow_mapping = {} # { addr of container -> { w_containee -> None } } + self.borrow_mapping = {} # { w_container -> { w_containee -> None } } self.borrowed_objects = {} # { addr of containee -> None } self.non_heaptypes_w = [] @@ -295,8 +295,8 @@ if not space.is_w(w_typetype, space.gettypeobject(W_PyCTypeObject.typedef)): _Py_Dealloc(space, obj) del state.py_objects_w2r[w_obj] - # if the object was a container for borrowed references - delete_borrower(space, obj) + # if the object was a container for borrowed references + delete_borrower(space, w_obj) else: if not we_are_translated() and obj.c_ob_refcnt < 0: message = "Negative refcount for obj %s with type %s" % ( @@ -344,12 +344,9 @@ else: Py_DecRef(space, ref) # already in borrowed list - container = make_ref(space, w_container) - if not container: # self-managed + if w_container is None: # self-managed return ref - Py_DecRef(space, container) - container_ptr = rffi.cast(ADDR, container) - borrowees = state.borrow_mapping.setdefault(container_ptr, {}) + borrowees = state.borrow_mapping.setdefault(w_container, {}) borrowees[w_borrowed] = None return ref @@ -385,17 +382,16 @@ else: Py_DecRef(space, ref) -def delete_borrower(space, py_obj): +def delete_borrower(space, w_obj): """ Called when a potential container for borrowed references has lost its last reference. Removes the borrowed references it contains. """ - ptr = rffi.cast(ADDR, py_obj) state = space.fromcache(RefcountState) - if ptr in state.borrow_mapping: # move to lifeline __del__ - for w_containee in state.borrow_mapping[ptr]: + if w_obj in state.borrow_mapping: # move to lifeline __del__ + for w_containee in state.borrow_mapping[w_obj]: forget_borrowee(space, w_containee) - del state.borrow_mapping[ptr] + del state.borrow_mapping[w_obj] #___________________________________________________________ From afa at codespeak.net Thu May 6 13:56:14 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 6 May 2010 13:56:14 +0200 (CEST) Subject: [pypy-svn] r74410 - pypy/trunk/pypy/module/cpyext Message-ID: <20100506115614.A0E35282BDD@codespeak.net> Author: afa Date: Thu May 6 13:56:13 2010 New Revision: 74410 Modified: pypy/trunk/pypy/module/cpyext/pyobject.py Log: from_ref(): Detect recursion differently, and remove parameter Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Thu May 6 13:56:13 2010 @@ -242,7 +242,7 @@ return py_obj -def from_ref(space, ref, recurse=False): +def from_ref(space, ref): """ Finds the interpreter object corresponding to the given reference. If the object is not yet realized (see stringobject.py), creates it. @@ -256,13 +256,14 @@ try: return state.py_objects_r2w[ptr] except KeyError: - if recurse: - raise InvalidPointerException(str(ref)) + pass # This reference is not yet a real interpreter object. # Realize it. ref_type = rffi.cast(PyObject, ref.c_ob_type) - w_type = from_ref(space, ref_type, True) + if ref_type == ref: + raise InvalidPointerException(str(ref)) + w_type = from_ref(space, ref_type) assert isinstance(w_type, W_TypeObject) return get_typedescr(w_type.instancetypedef).realize(space, ref) From afa at codespeak.net Thu May 6 14:17:08 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 6 May 2010 14:17:08 +0200 (CEST) Subject: [pypy-svn] r74411 - pypy/trunk/pypy/module/cpyext Message-ID: <20100506121708.9B989282BEC@codespeak.net> Author: afa Date: Thu May 6 14:17:06 2010 New Revision: 74411 Modified: pypy/trunk/pypy/module/cpyext/object.py pypy/trunk/pypy/module/cpyext/pyobject.py pypy/trunk/pypy/module/cpyext/typeobject.py Log: make_ref() does not really need the "nbitems" argument Modified: pypy/trunk/pypy/module/cpyext/object.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/object.py (original) +++ pypy/trunk/pypy/module/cpyext/object.py Thu May 6 14:17:06 2010 @@ -2,8 +2,9 @@ from pypy.module.cpyext.api import cpython_api, generic_cpy_call, CANNOT_FAIL,\ Py_ssize_t, PyVarObject, Py_TPFLAGS_HEAPTYPE,\ Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT, Py_GE, CONST_STRING -from pypy.module.cpyext.pyobject import PyObject, PyObjectP, make_ref, from_ref -from pypy.module.cpyext.pyobject import Py_IncRef, Py_DecRef +from pypy.module.cpyext.pyobject import ( + PyObject, PyObjectP, create_ref, from_ref, Py_IncRef, Py_DecRef, + track_reference) from pypy.module.cpyext.typeobject import PyTypeObjectPtr, W_PyCTypeObject from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall from pypy.objspace.std.objectobject import W_ObjectObject @@ -21,7 +22,9 @@ w_type = from_ref(space, rffi.cast(PyObject, type)) if isinstance(w_type, W_PyCTypeObject): w_obj = space.allocate_instance(W_ObjectObject, w_type) - return make_ref(space, w_obj, items=size) + py_obj = create_ref(space, w_obj, items=size) + track_reference(space, py_obj, w_obj) + return py_obj assert False, "Please add more cases in _PyObject_New" @cpython_api([rffi.VOIDP_real], lltype.Void) Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Thu May 6 14:17:06 2010 @@ -221,7 +221,7 @@ if ptr: # init_typeobject() bootstraps with NULL references state.py_objects_r2w[ptr] = w_obj -def make_ref(space, w_obj, steal=False, items=0): +def make_ref(space, w_obj, steal=False): """ Returns a reference to an intepreter object. """ @@ -233,7 +233,7 @@ py_obj = state.py_objects_w2r[w_obj] except KeyError: assert not steal - py_obj = create_ref(space, w_obj, items) + py_obj = create_ref(space, w_obj) track_reference(space, py_obj, w_obj) return py_obj Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Thu May 6 14:17:06 2010 @@ -338,7 +338,7 @@ # - object.tp_bases is a tuple # - tuple.tp_bases is a tuple - # insert null placeholders to please make_ref() + # insert null placeholders to please create_ref() track_reference(space, lltype.nullptr(PyObject.TO), space.w_type) track_reference(space, lltype.nullptr(PyObject.TO), space.w_object) track_reference(space, lltype.nullptr(PyObject.TO), space.w_tuple) From afa at codespeak.net Thu May 6 14:54:15 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 6 May 2010 14:54:15 +0200 (CEST) Subject: [pypy-svn] r74412 - pypy/trunk/pypy/module/cpyext Message-ID: <20100506125415.5E19D282BF2@codespeak.net> Author: afa Date: Thu May 6 14:54:13 2010 New Revision: 74412 Modified: pypy/trunk/pypy/module/cpyext/pyobject.py pypy/trunk/pypy/module/cpyext/typeobject.py Log: Remove the "steal" parameter of make_ref. Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Thu May 6 14:54:13 2010 @@ -221,9 +221,9 @@ if ptr: # init_typeobject() bootstraps with NULL references state.py_objects_r2w[ptr] = w_obj -def make_ref(space, w_obj, steal=False): +def make_ref(space, w_obj): """ - Returns a reference to an intepreter object. + Returns a new reference to an intepreter object. """ if w_obj is None: return lltype.nullptr(PyObject.TO) @@ -232,12 +232,9 @@ try: py_obj = state.py_objects_w2r[w_obj] except KeyError: - assert not steal py_obj = create_ref(space, w_obj) track_reference(space, py_obj, w_obj) - return py_obj - - if not steal: + else: Py_IncRef(space, py_obj) return py_obj Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Thu May 6 14:54:13 2010 @@ -191,11 +191,12 @@ def inherit_special(space, pto, base_pto): # XXX missing: copy basicsize and flags in a magical way flags = rffi.cast(lltype.Signed, pto.c_tp_flags) - base_object_pyo = make_ref(space, space.w_object, steal=True) + base_object_pyo = make_ref(space, space.w_object) base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo) if base_pto != base_object_pto or flags & Py_TPFLAGS_HEAPTYPE: if not pto.c_tp_new: pto.c_tp_new = base_pto.c_tp_new + Py_DecRef(space, base_object_pyo) class PyOLifeline(object): def __init__(self, space, pyo): @@ -411,8 +412,10 @@ if segment != 0: raise OperationError(space.w_SystemError, space.wrap ("accessing non-existent string segment")) - pyref = make_ref(space, w_str, steal=True) + pyref = make_ref(space, w_str) ref[0] = PyString_AsString(space, pyref) + # Stolen reference: the object has better exist somewhere else + Py_DecRef(space, pyref) return space.int_w(space.len(w_str)) def setup_string_buffer_procs(space, pto): @@ -430,7 +433,7 @@ base_pyo = rffi.cast(PyObject, obj_pto.c_tp_base) Py_DecRef(space, obj_pto.c_tp_bases) Py_DecRef(space, obj_pto.c_tp_mro) - Py_DecRef(space, obj_pto.c_tp_cache) # lets do it like cpython + Py_DecRef(space, obj_pto.c_tp_cache) # let's do it like cpython if obj_pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE: if obj_pto.c_tp_as_buffer: lltype.free(obj_pto.c_tp_as_buffer, flavor='raw') @@ -566,7 +569,9 @@ py_type = rffi.cast(PyTypeObjectPtr, py_obj) if not py_type.c_tp_base: - base = make_ref(space, space.w_object, steal=True) + # borrowed reference, but w_object is unlikely to disappear + base = make_ref(space, space.w_object) + Py_DecRef(space, base) py_type.c_tp_base = rffi.cast(PyTypeObjectPtr, base) finish_type_1(space, py_type) From antocuni at codespeak.net Thu May 6 14:54:35 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 6 May 2010 14:54:35 +0200 (CEST) Subject: [pypy-svn] r74413 - in pypy/extradoc/talk/pycon-italy-2010/pypy_demo: . mandelbrot manderlbrot Message-ID: <20100506125435.2C440282BF9@codespeak.net> Author: antocuni Date: Thu May 6 14:54:33 2010 New Revision: 74413 Added: pypy/extradoc/talk/pycon-italy-2010/pypy_demo/mandelbrot/ - copied from r74381, pypy/extradoc/talk/pycon-italy-2010/pypy_demo/manderlbrot/ Removed: pypy/extradoc/talk/pycon-italy-2010/pypy_demo/manderlbrot/ Modified: pypy/extradoc/talk/pycon-italy-2010/pypy_demo/mandelbrot/py_mandel.py pypy/extradoc/talk/pycon-italy-2010/pypy_demo/mandelbrot/views.py pypy/extradoc/talk/pycon-italy-2010/pypy_demo/settings.py pypy/extradoc/talk/pycon-italy-2010/pypy_demo/urls.py Log: typo: s/manderlbrot/mandelbrot Modified: pypy/extradoc/talk/pycon-italy-2010/pypy_demo/mandelbrot/py_mandel.py ============================================================================== --- pypy/extradoc/talk/pycon-italy-2010/pypy_demo/manderlbrot/py_mandel.py (original) +++ pypy/extradoc/talk/pycon-italy-2010/pypy_demo/mandelbrot/py_mandel.py Thu May 6 14:54:33 2010 @@ -10,7 +10,7 @@ # Changing the values below will change the resulting image -def manderlbrot(image_width, image_height): +def mandelbrot(image_width, image_height): start_real=-2.00 start_imag=-1.00 end_real=1.00 Modified: pypy/extradoc/talk/pycon-italy-2010/pypy_demo/mandelbrot/views.py ============================================================================== --- pypy/extradoc/talk/pycon-italy-2010/pypy_demo/manderlbrot/views.py (original) +++ pypy/extradoc/talk/pycon-italy-2010/pypy_demo/mandelbrot/views.py Thu May 6 14:54:33 2010 @@ -6,26 +6,26 @@ def empty(request): return HttpResponse('') -# render a manderlbrot image +# render a mandelbrot image def render(request): w = int(request.GET.get('w', 320)) h = int(request.GET.get('h', 240)) - from py_mandel import manderlbrot - img = manderlbrot(w, h) + from py_mandel import mandelbrot + img = mandelbrot(w, h) return HttpResponse(img, content_type="image/bmp") -# render a manderlbrot image through the execnet pypy child, which is set up +# render a mandelbrot image through the execnet pypy child, which is set up # below def pypy_render(request): w = int(request.GET.get('w', 320)) h = int(request.GET.get('h', 240)) channel = pypy.remote_exec(""" - from py_mandel import manderlbrot + from py_mandel import mandelbrot w, h = channel.receive() - img = manderlbrot(w, h) + img = mandelbrot(w, h) channel.send(img) """) channel.send((w, h)) @@ -45,7 +45,7 @@ pypy.remote_exec(""" import sys import os - os.chdir("manderlbrot") + os.chdir("mandelbrot") sys.path.insert(0, '') """) Modified: pypy/extradoc/talk/pycon-italy-2010/pypy_demo/settings.py ============================================================================== --- pypy/extradoc/talk/pycon-italy-2010/pypy_demo/settings.py (original) +++ pypy/extradoc/talk/pycon-italy-2010/pypy_demo/settings.py Thu May 6 14:54:33 2010 @@ -76,5 +76,5 @@ ## 'django.contrib.contenttypes', ## 'django.contrib.sessions', ## 'django.contrib.sites', - 'pypy_demo.manderlbrot', + 'pypy_demo.mandelbrot', ) Modified: pypy/extradoc/talk/pycon-italy-2010/pypy_demo/urls.py ============================================================================== --- pypy/extradoc/talk/pycon-italy-2010/pypy_demo/urls.py (original) +++ pypy/extradoc/talk/pycon-italy-2010/pypy_demo/urls.py Thu May 6 14:54:33 2010 @@ -6,9 +6,9 @@ urlpatterns = patterns('', # Example: - (r'^render$', 'pypy_demo.manderlbrot.views.render'), - (r'^pypy_render$', 'pypy_demo.manderlbrot.views.pypy_render'), - (r'^empty$', 'pypy_demo.manderlbrot.views.empty'), + (r'^render$', 'pypy_demo.mandelbrot.views.render'), + (r'^pypy_render$', 'pypy_demo.mandelbrot.views.pypy_render'), + (r'^empty$', 'pypy_demo.mandelbrot.views.empty'), # Uncomment the admin/doc line below and add 'django.contrib.admindocs' # to INSTALLED_APPS to enable admin documentation: From antocuni at codespeak.net Thu May 6 15:05:03 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 6 May 2010 15:05:03 +0200 (CEST) Subject: [pypy-svn] r74414 - pypy/extradoc/talk/pycon-italy-2010/talk Message-ID: <20100506130503.C09B2282BF3@codespeak.net> Author: antocuni Date: Thu May 6 15:05:02 2010 New Revision: 74414 Added: pypy/extradoc/talk/pycon-italy-2010/talk/demo-graph.pdf (contents, props changed) Log: a graph showing the performances of the demo app with cpython, pypy and execnet Added: pypy/extradoc/talk/pycon-italy-2010/talk/demo-graph.pdf ============================================================================== Binary file. No diff available. From antocuni at codespeak.net Thu May 6 15:41:33 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 6 May 2010 15:41:33 +0200 (CEST) Subject: [pypy-svn] r74415 - pypy/extradoc/talk/pycon-italy-2010/talk Message-ID: <20100506134133.2AF19282BDD@codespeak.net> Author: antocuni Date: Thu May 6 15:41:31 2010 New Revision: 74415 Modified: pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt Log: (arigato, antocuni) more outline Modified: pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt ============================================================================== --- pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt (original) +++ pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt Thu May 6 15:41:31 2010 @@ -69,12 +69,70 @@ - beta version of cpyext -bPart 2 +Part 2 ------- -JIT +- the performance of Python: + how fast can Python code really be? + (and using how much memory?) + +Overview of implementations +--------------------------- +(Discussing their performance) +- CPython +- Psyco +- Jython, IronPython +- (Stackless) +- PyPy (without and with JIT) +- Unladen Swallow + +What a JIT is +------------- + +XXX + +What a tracing JIT is +--------------------- + +XXX e.g. the structure of TraceMonkey + +The architecture of PyPy +------------------------ + +XXX diagram showing where the JIT is + +XXX diagram detailling the JIT + +- it can be used for any interpreter, not just Python + +Speed of the PyPy JIT +--------------------- + +Python programs that are, or are not, nicely handled +by the JIT: + +- loops, even across many calls, are nicely handled +- interpreter-like constructs are not +- bad support so far for generators and recursion + +The optimizations we get +------------------------ + +(Quick overview only, with examples of programs that benefit) + +- "virtuals", i.e. temporary objects are not constructed +- removed frame handling + +Practical results +----------------------- + +- so far, x86-32 only +- mention again the performance shown by the graphs +- a quick tour of other missing improvements +- reminder: works for *any* Python code (even more + than Psyco) Part 3 From afa at codespeak.net Thu May 6 15:50:08 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 6 May 2010 15:50:08 +0200 (CEST) Subject: [pypy-svn] r74416 - pypy/trunk/pypy/module/cpyext Message-ID: <20100506135008.9B495282BDD@codespeak.net> Author: afa Date: Thu May 6 15:50:07 2010 New Revision: 74416 Modified: pypy/trunk/pypy/module/cpyext/__init__.py Log: fix translation Modified: pypy/trunk/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/__init__.py (original) +++ pypy/trunk/pypy/module/cpyext/__init__.py Thu May 6 15:50:07 2010 @@ -23,13 +23,15 @@ def startup(self, space): state = space.fromcache(State) from pypy.module.cpyext.typeobject import setup_new_method_def + from pypy.module.cpyext.pyobject import RefcountState setup_new_method_def(space) if not we_are_translated(): space.setattr(space.wrap(self), space.wrap('api_lib'), space.wrap(state.api_lib)) else: - state.init_r2w_from_w2r() + refcountstate = space.fromcache(RefcountState) + refcountstate.init_r2w_from_w2r() for func in api.INIT_FUNCTIONS: func(space) From antocuni at codespeak.net Thu May 6 17:14:13 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 6 May 2010 17:14:13 +0200 (CEST) Subject: [pypy-svn] r74417 - pypy/extradoc/talk/pycon-italy-2010/pypy_demo Message-ID: <20100506151413.31FA3282BDD@codespeak.net> Author: antocuni Date: Thu May 6 17:14:11 2010 New Revision: 74417 Added: pypy/extradoc/talk/pycon-italy-2010/pypy_demo/cpython_server (contents, props changed) pypy/extradoc/talk/pycon-italy-2010/pypy_demo/index.html pypy/extradoc/talk/pycon-italy-2010/pypy_demo/index2.html pypy/extradoc/talk/pycon-italy-2010/pypy_demo/pypy_server (contents, props changed) Log: add scripts to start the servers with cpython and pypy, and an html with the links Added: pypy/extradoc/talk/pycon-italy-2010/pypy_demo/cpython_server ============================================================================== --- (empty file) +++ pypy/extradoc/talk/pycon-italy-2010/pypy_demo/cpython_server Thu May 6 17:14:11 2010 @@ -0,0 +1 @@ +PYTHONPATH=/home/antocuni/pypy/misc/django/:/home/antocuni/pypy/misc/execnet/::/home/antocuni/WebFormKit:/home/antocuni/pypy/trunk/:/home/antocuni/pypy/user/antocuni/:/home/antocuni/pypy/user/antocuni/hack/:/home/antocuni/pypy/misc/pg8000/:/home/antocuni/pypy/misc/casalino-carta/:/home/antocuni/site-packages python manage.py runserver Added: pypy/extradoc/talk/pycon-italy-2010/pypy_demo/index.html ============================================================================== --- (empty file) +++ pypy/extradoc/talk/pycon-italy-2010/pypy_demo/index.html Thu May 6 17:14:11 2010 @@ -0,0 +1,10 @@ + + +

Mandelbrot PyPy demo

+ + + Show more + Added: pypy/extradoc/talk/pycon-italy-2010/pypy_demo/index2.html ============================================================================== --- (empty file) +++ pypy/extradoc/talk/pycon-italy-2010/pypy_demo/index2.html Thu May 6 17:14:11 2010 @@ -0,0 +1,11 @@ + + +

Mandelbrot PyPy demo

+ + + Show less + Added: pypy/extradoc/talk/pycon-italy-2010/pypy_demo/pypy_server ============================================================================== --- (empty file) +++ pypy/extradoc/talk/pycon-italy-2010/pypy_demo/pypy_server Thu May 6 17:14:11 2010 @@ -0,0 +1 @@ +PYTHONPATH=/home/antocuni/pypy/misc/django/:/home/antocuni/pypy/misc/execnet/::/home/antocuni/WebFormKit:/home/antocuni/pypy/trunk/:/home/antocuni/pypy/user/antocuni/:/home/antocuni/pypy/user/antocuni/hack/:/home/antocuni/pypy/misc/pg8000/:/home/antocuni/pypy/misc/casalino-carta/:/home/antocuni/site-packages pypy-c manage.py runserver 8001 From afa at codespeak.net Thu May 6 17:45:01 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 6 May 2010 17:45:01 +0200 (CEST) Subject: [pypy-svn] r74418 - pypy/trunk/pypy/module/cpyext Message-ID: <20100506154501.9C6A1282BDD@codespeak.net> Author: afa Date: Thu May 6 17:45:00 2010 New Revision: 74418 Modified: pypy/trunk/pypy/module/cpyext/TODO Log: remove a DONE item, and another too vague task Modified: pypy/trunk/pypy/module/cpyext/TODO ============================================================================== --- pypy/trunk/pypy/module/cpyext/TODO (original) +++ pypy/trunk/pypy/module/cpyext/TODO Thu May 6 17:45:00 2010 @@ -16,8 +16,6 @@ - Fix GIL handling (e.g. after releasing the GIL, GC operations might occur in savethreads). - Fix PyEval_SaveThread: http://paste.pocoo.org/show/206521/ - - Make Numpy work. - - Export pointers where pointers are expected, i.e. kill the "#define PyExc_OSError &PyExc_OSError" hack. - refactor management of py_objects_r2w and py_objects_w2r, this can @@ -29,9 +27,3 @@ - sort out pypy's buffer protocol. PyPy's buffer right now don't support raw memory (except array which supports it in a hackish way), which should be fixed in order to make it nicely work with cpyext. - - - Py_InitModule() does not seem to return the correct module when using dotted import. - for example: "import wx" imports "wx._core_", which calls Py_InitModule("_core_"). - This correcly creates and populates sys.modules["wx._core_"], but returns a new empty - module sys.modules["_core_"]. So functions go in the correct module, but types and - constants go into the wrong one. From afa at codespeak.net Thu May 6 17:55:35 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 6 May 2010 17:55:35 +0200 (CEST) Subject: [pypy-svn] r74419 - pypy/trunk/pypy/module/cpyext Message-ID: <20100506155535.14965282BDE@codespeak.net> Author: afa Date: Thu May 6 17:55:33 2010 New Revision: 74419 Modified: pypy/trunk/pypy/module/cpyext/TODO Log: Remember that "win32 + JIT + cpyext" is a broken combination Modified: pypy/trunk/pypy/module/cpyext/TODO ============================================================================== --- pypy/trunk/pypy/module/cpyext/TODO (original) +++ pypy/trunk/pypy/module/cpyext/TODO Thu May 6 17:55:33 2010 @@ -27,3 +27,6 @@ - sort out pypy's buffer protocol. PyPy's buffer right now don't support raw memory (except array which supports it in a hackish way), which should be fixed in order to make it nicely work with cpyext. + + - Fix Makefile generation on Windows when --shared is used (mandatory when + cpyext is used with -Ojit) From arigo at codespeak.net Thu May 6 17:57:45 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 6 May 2010 17:57:45 +0200 (CEST) Subject: [pypy-svn] r74420 - in pypy/branch/blackhole-improvement/pypy: jit/codewriter jit/codewriter/test rpython/lltypesystem Message-ID: <20100506155745.0CBE0282BEC@codespeak.net> Author: arigo Date: Thu May 6 17:57:44 2010 New Revision: 74420 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/lloperation.py Log: - Encode int_lt(0, x) as int_gt(x, 0) to minimize the number of extra encodings for the instructions. - Preparation work for handling multiple JitCodes. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Thu May 6 17:57:44 2010 @@ -12,17 +12,17 @@ _empty_r = [] _empty_f = [] - def __init__(self, name, cfnptr=None, calldescr=None, called_from=None, - liveness=None, assembler=None): + def __init__(self, name, fnaddr=None, calldescr=None, called_from=None, + assembler=None): self.name = name - #self.cfnptr = cfnptr - #self.calldescr = calldescr - #self.called_from = called_from - self.liveness = liveness - self._assembler = assembler + self.fnaddr = fnaddr + self.calldescr = calldescr + self._called_from = called_from # debugging + self._assembler = assembler # debugging def setup(self, code, constants_i=[], constants_r=[], constants_f=[], - num_regs_i=256, num_regs_r=256, num_regs_f=256): + num_regs_i=256, num_regs_r=256, num_regs_f=256, + liveness=None): self.code = code # if the following lists are empty, use a single shared empty list self.constants_i = constants_i or self._empty_i @@ -32,6 +32,7 @@ self.num_regs_encoded = ((num_regs_i << 18) | (num_regs_r << 9) | (num_regs_f << 0)) + self.liveness = liveness def num_regs_i(self): return self.num_regs_encoded >> 18 @@ -261,15 +262,15 @@ assert self.count_regs['float'] + len(self.constants_f) <= 256 def make_jitcode(self, ssarepr): - jitcode = JitCode(ssarepr.name, liveness=self.liveness, - assembler=self) + jitcode = JitCode(ssarepr.name, assembler=self) jitcode.setup(''.join(self.code), self.constants_i, self.constants_r, self.constants_f, self.count_regs['int'], self.count_regs['ref'], - self.count_regs['float']) + self.count_regs['float'], + liveness=self.liveness) if self._count_jitcodes < 50: # stop if we have a lot of them jitcode._dump = format_assembler(ssarepr) self._count_jitcodes += 1 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py Thu May 6 17:57:44 2010 @@ -1,10 +1,12 @@ from pypy.jit.codewriter import support from pypy.jit.codewriter.regalloc import perform_register_allocation from pypy.jit.codewriter.flatten import flatten_graph, KINDS -from pypy.jit.codewriter.assembler import Assembler +from pypy.jit.codewriter.assembler import Assembler, JitCode from pypy.jit.codewriter.jitter import transform_graph from pypy.jit.codewriter.format import format_assembler from pypy.jit.codewriter.liveness import compute_liveness +from pypy.jit.codewriter.policy import log +from pypy.tool.udir import udir class CodeWriter(object): @@ -17,9 +19,9 @@ """For testing.""" rtyper = support.annotate(func, values, type_system=type_system) graph = rtyper.annotator.translator.graphs[0] - return self.transform_graph_to_jitcode(graph) + return self.transform_graph_to_jitcode(graph, True, True) - def transform_graph_to_jitcode(self, graph, verbose=False): + def transform_graph_to_jitcode(self, graph, portal, verbose): """Transform a graph into a JitCode containing the same bytecode in a different format. Note that the original 'graph' is mangled by the process and should not be used any more. @@ -27,7 +29,7 @@ # # step 1: mangle the graph so that it contains the final instructions # that we want in the JitCode, but still as a control flow graph - transform_graph(graph, self.cpu) + transform_graph(graph, self.cpu, portal) # # step 2a: perform register allocation on it regallocs = {} @@ -43,10 +45,8 @@ # of the shape ("opname", arg1, ..., argN) or (Label(...),). ssarepr = flatten_graph(graph, regallocs) # - # if 'verbose', print the resulting assembler - if verbose: - print graph - print indent(format_assembler(ssarepr), 4) + # print the resulting assembler + self.print_ssa_repr(ssarepr, portal, verbose) # # step 4: "assemble" it into a JitCode, which contains a sequence # of bytes and lists of constants. It's during this step that @@ -57,7 +57,22 @@ def make_jitcodes(self, maingraph, verbose=False): self.portal_graph = maingraph - return self.transform_graph_to_jitcode(maingraph, verbose) + return self.transform_graph_to_jitcode(maingraph, True, verbose) + + def print_ssa_repr(self, ssarepr, portal, verbose): + if verbose: + print '%s:' % (ssarepr.name,) + print indent(format_assembler(ssarepr), 4) + else: + dir = udir.ensure("jitcodes", dir=1) + if portal: + name = "00_portal_runner" + elif ssarepr.name and ssarepr.name != '?': + name = ssarepr.name + else: + name = 'unnamed_%x' % id(ssarepr) + dir.join(name).write(format_assembler(ssarepr)) + log.dot() def indent(s, indent): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Thu May 6 17:57:44 2010 @@ -7,12 +7,12 @@ from pypy.jit.codewriter import support, heaptracker -def transform_graph(graph, cpu=None): +def transform_graph(graph, cpu=None, portal=True): """Transform a control flow graph to make it suitable for being flattened in a JitCode. """ t = Transformer(cpu) - t.transform(graph) + t.transform(graph, portal) class NoOp(Exception): @@ -24,8 +24,9 @@ def __init__(self, cpu=None): self.cpu = cpu - def transform(self, graph): + def transform(self, graph, portal): self.graph = graph + self.portal = portal for block in list(graph.iterblocks()): self.optimize_block(block) @@ -157,10 +158,21 @@ rewrite_op_int_and = _rewrite_symmetric rewrite_op_int_or = _rewrite_symmetric rewrite_op_int_xor = _rewrite_symmetric + rewrite_op_int_lt = _rewrite_symmetric + rewrite_op_int_le = _rewrite_symmetric + rewrite_op_int_gt = _rewrite_symmetric + rewrite_op_int_ge = _rewrite_symmetric rewrite_op_G_int_add_ovf = _rewrite_symmetric rewrite_op_G_int_mul_ovf = _rewrite_symmetric + rewrite_op_float_add = _rewrite_symmetric + rewrite_op_float_mul = _rewrite_symmetric + rewrite_op_float_lt = _rewrite_symmetric + rewrite_op_float_le = _rewrite_symmetric + rewrite_op_float_gt = _rewrite_symmetric + rewrite_op_float_ge = _rewrite_symmetric + def rewrite_op_direct_call(self, op): """Turn 'i0 = direct_call(fn, i1, i2, ref1, ref2)' into e.g. 'i0 = residual_call_ir_i(fn, [i1, i2], [ref1, ref2])'. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Thu May 6 17:57:44 2010 @@ -100,20 +100,32 @@ def test_symmetric(): ops = {'int_add': 'int_add', - 'int_gt': 'int_lt', - 'int_le': 'int_ge'} + 'int_or': 'int_or', + 'int_gt': ('int_gt', 'int_lt'), + 'uint_le': ('int_le', 'int_ge'), + 'char_ne': 'int_ne', + 'char_lt': ('int_lt', 'int_gt'), + 'int_add_nonneg_ovf': 'G_int_add_ovf', + 'uint_xor': 'int_xor', + 'float_mul': 'float_mul', + 'float_gt': ('float_gt', 'float_lt'), + } v3 = varoftype(lltype.Signed) for v1 in [varoftype(lltype.Signed), Constant(42, lltype.Signed)]: for v2 in [varoftype(lltype.Signed), Constant(43, lltype.Signed)]: for name1, name2 in ops.items(): op = SpaceOperation(name1, [v1, v2], v3) op1 = Transformer(FakeCPU()).rewrite_operation(op) + if isinstance(name2, str): + name2 = name2, name2 if isinstance(v1, Constant) and isinstance(v2, Variable): - assert op1.opname == name2 assert op1.args == [v2, v1] assert op1.result == v3 + assert op1.opname == name2[1] else: - assert op1 is op + assert op1.args == [v1, v2] + assert op1.result == v3 + assert op1.opname == name2[0] def test_residual_call(): for RESTYPE in [lltype.Signed, rclass.OBJECTPTR, @@ -146,7 +158,7 @@ op = get_direct_call_op(argtypes, restype) op1 = Transformer(FakeCPU()).rewrite_operation(op) reskind = getkind(restype)[0] - assert op1.opname == 'residual_call_%s_%s' % (expectedkind, reskind) + assert op1.opname == 'G_residual_call_%s_%s' % (expectedkind, reskind) assert op1.result == op.result assert op1.args[0] == op.args[0] FUNC = op.args[0].concretetype.TO @@ -197,7 +209,7 @@ v_result = varoftype(rclass.OBJECT.typeptr) op = SpaceOperation('getfield', [v_parent, c_name], v_result) op1 = Transformer(FakeCPU()).rewrite_operation(op) - assert op1.opname == 'guard_class' + assert op1.opname == 'G_guard_class' assert op1.args == [v_parent] assert op1.result == v_result @@ -261,7 +273,7 @@ op = SpaceOperation('malloc', [Constant(S, lltype.Void), Constant({'flavor': 'gc'}, lltype.Void)], v) op1 = Transformer(FakeCPU()).rewrite_operation(op) - assert op1.opname == 'residual_call_r_r' + assert op1.opname == 'G_residual_call_r_r' assert op1.args[0].value == 'alloc_with_del' # pseudo-function as a str assert list(op1.args[2]) == [] Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/lloperation.py Thu May 6 17:57:44 2010 @@ -263,7 +263,7 @@ 'uint_rshift': LLOp(canfold=True), 'uint_xor': LLOp(canfold=True), - 'float_is_true': LLOp(canfold=True), + 'float_is_true': LLOp(canfold=True), # it really means "x != 0.0" 'float_neg': LLOp(canfold=True), 'float_abs': LLOp(canfold=True), From afa at codespeak.net Thu May 6 18:02:52 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 6 May 2010 18:02:52 +0200 (CEST) Subject: [pypy-svn] r74421 - pypy/trunk/pypy/module/cpyext Message-ID: <20100506160252.2D137282BF2@codespeak.net> Author: afa Date: Thu May 6 18:02:50 2010 New Revision: 74421 Modified: pypy/trunk/pypy/module/cpyext/TODO Log: This has been fixed with r74102 Modified: pypy/trunk/pypy/module/cpyext/TODO ============================================================================== --- pypy/trunk/pypy/module/cpyext/TODO (original) +++ pypy/trunk/pypy/module/cpyext/TODO Thu May 6 18:02:50 2010 @@ -14,7 +14,6 @@ it does is a lltype function pointer, no C code involved. - Fix GIL handling (e.g. after releasing the GIL, GC operations might occur in savethreads). - - Fix PyEval_SaveThread: http://paste.pocoo.org/show/206521/ - Export pointers where pointers are expected, i.e. kill the "#define PyExc_OSError &PyExc_OSError" hack. From antocuni at codespeak.net Thu May 6 18:13:35 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 6 May 2010 18:13:35 +0200 (CEST) Subject: [pypy-svn] r74422 - pypy/extradoc/talk/pycon-italy-2010/talk Message-ID: <20100506161335.CCC13282BF3@codespeak.net> Author: antocuni Date: Thu May 6 18:13:33 2010 New Revision: 74422 Added: pypy/extradoc/talk/pycon-italy-2010/talk/merlinux-logo.jpg (contents, props changed) pypy/extradoc/talk/pycon-italy-2010/talk/pypy-vs-cpython.png (contents, props changed) pypy/extradoc/talk/pycon-italy-2010/talk/pypy-vs-psyco.png (contents, props changed) pypy/extradoc/talk/pycon-italy-2010/talk/wxpython1.png (contents, props changed) pypy/extradoc/talk/pycon-italy-2010/talk/wxpython2.png (contents, props changed) Modified: pypy/extradoc/talk/pycon-italy-2010/talk/beamerdefs.txt pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt pypy/extradoc/talk/pycon-italy-2010/talk/title.latex Log: slides for part 1 Modified: pypy/extradoc/talk/pycon-italy-2010/talk/beamerdefs.txt ============================================================================== --- pypy/extradoc/talk/pycon-italy-2010/talk/beamerdefs.txt (original) +++ pypy/extradoc/talk/pycon-italy-2010/talk/beamerdefs.txt Thu May 6 18:13:33 2010 @@ -20,6 +20,15 @@ } +.. |scriptsize| raw:: latex + + {\scriptsize + +.. |end_scriptsize| raw:: latex + + } + + .. closed bracket .. =========================== Added: pypy/extradoc/talk/pycon-italy-2010/talk/merlinux-logo.jpg ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pycon-italy-2010/talk/pypy-vs-cpython.png ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pycon-italy-2010/talk/pypy-vs-psyco.png ============================================================================== Binary file. No diff available. Modified: pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt ============================================================================== --- pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt (original) +++ pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt Thu May 6 18:13:33 2010 @@ -15,6 +15,25 @@ - Demo: how to use PyPy right now +Part 0: What is PyPy? :-) +-------------------------- + +- Python interpreter written in Python + +- Framework for developing dynamic languages + +- etc. etc. + +|pause| + +- From the user point of view + +- An alternative to CPython + +- with more features! + + + Part 1 ------- @@ -26,38 +45,133 @@ What's new in PyPy 1.2 ----------------------- +- Released on March 12th, 2010 + - Main theme: speed - JIT compiler -- Memory savings - - speed.pypy.org -- ubuntu packages +- Ubuntu packages -Speed ------ -XXX show the performance graphs -(also with psyco) +Speed: PyPy vs CPython +----------------------- + +.. image:: pypy-vs-cpython.png + :scale: 35 + + +Speed: PyPy vs Psyco +----------------------- + +.. image:: pypy-vs-psyco.png + :scale: 35 What works on PyPy ------------------- -XXX +- Pure Python modules should Just Work (TM) + + * django trunk + + * twisted, nevow + + * pylons + + * bittorrent + + * ... + +|pause| + +- lot of standard modules + +|scriptsize| + + * __builtin__ __pypy__ _codecs _lsprof _minimal_curses _random _rawffi + _socket _sre _weakref bz2 cStringIO crypt errno exceptions fcntl gc + itertools marshal math md5 mmap operator parser posix pyexpat select sha + signal struct symbol sys termios thread time token unicodedata zipimport + zlib + + * array binascii cPickle cmath collections ctypes datetime functools grp md5 + pwd pyexpat sha sqlite3 syslog + +|end_scriptsize| + + * **ctypes** What does not work on PyPy -------------------------- -XXX :-) +- Pure Python modules should Just Work (TM) |pause| + + * ... unless they don't :-) + +|pause| + +- Programs that rely on CPython-specific behavior + + * refcounting: ``open('xxx', 'w').write('stuff')`` + + * non-string keys in dict of types (try it!) + + * exact naming of a list comprehension variable + + * exact message matching in exception catching code + + * ... + +|pause| + +- Extension modules |pause| + + * really? + + * drum roll... + cpyext ------ -XXX +- CPython extension modules in PyPy + +- ``pypy-c setup.py build`` + +- still beta + +- not 100% of CPython API is supported + +- not included in PyPy 1.2 + +- Known to work: + + * wxPython (after a patch) + + * _sre + + * PyCrypto + + * PIL + + +wxPython on PyPy (1) +--------------------- + +.. image:: wxpython1.png + :scale: 30 + +wxPython on PyPy (2) +--------------------- + + +.. image:: wxpython2.png + :scale: 30 + PyPy 1.2.1 ---------- @@ -66,6 +180,8 @@ - Many bugfixes + * 26 issues reported after release of 1.2 + - beta version of cpyext Modified: pypy/extradoc/talk/pycon-italy-2010/talk/title.latex ============================================================================== --- pypy/extradoc/talk/pycon-italy-2010/talk/title.latex (original) +++ pypy/extradoc/talk/pycon-italy-2010/talk/title.latex Thu May 6 18:13:33 2010 @@ -1,5 +1,8 @@ \begin{titlepage} \begin{figure}[h] -\includegraphics[width=80px]{../img/py-web.png} +\scalebox{0.8}{\includegraphics[width=80px]{../../img/py-web.png}} +\qquad +\qquad +\scalebox{0.8}{\includegraphics[width=64px,height=64px]{merlinux-logo.jpg}} \end{figure} \end{titlepage} Added: pypy/extradoc/talk/pycon-italy-2010/talk/wxpython1.png ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/pycon-italy-2010/talk/wxpython2.png ============================================================================== Binary file. No diff available. From hpk at codespeak.net Thu May 6 18:18:16 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 6 May 2010 18:18:16 +0200 (CEST) Subject: [pypy-svn] r74423 - in pypy/branch/py12: . lib-python/modified-2.5.2/distutils pypy/jit/metainterp pypy/jit/metainterp/test pypy/module/cpyext pypy/module/cpyext/include pypy/module/cpyext/test pypy/module/posix/test pypy/translator/tool Message-ID: <20100506161816.1C519282BF5@codespeak.net> Author: hpk Date: Thu May 6 18:18:13 2010 New Revision: 74423 Modified: pypy/branch/py12/ (props changed) pypy/branch/py12/lib-python/modified-2.5.2/distutils/msvccompiler.py pypy/branch/py12/pypy/jit/metainterp/pyjitpl.py pypy/branch/py12/pypy/jit/metainterp/test/test_exception.py pypy/branch/py12/pypy/module/cpyext/TODO pypy/branch/py12/pypy/module/cpyext/__init__.py pypy/branch/py12/pypy/module/cpyext/api.py pypy/branch/py12/pypy/module/cpyext/dictobject.py pypy/branch/py12/pypy/module/cpyext/funcobject.py pypy/branch/py12/pypy/module/cpyext/import_.py pypy/branch/py12/pypy/module/cpyext/include/patchlevel.h pypy/branch/py12/pypy/module/cpyext/listobject.py pypy/branch/py12/pypy/module/cpyext/methodobject.py pypy/branch/py12/pypy/module/cpyext/modsupport.py pypy/branch/py12/pypy/module/cpyext/object.py pypy/branch/py12/pypy/module/cpyext/presetup.py pypy/branch/py12/pypy/module/cpyext/pyerrors.py pypy/branch/py12/pypy/module/cpyext/pyobject.py pypy/branch/py12/pypy/module/cpyext/sequence.py pypy/branch/py12/pypy/module/cpyext/state.py pypy/branch/py12/pypy/module/cpyext/stringobject.py pypy/branch/py12/pypy/module/cpyext/stubgen.py pypy/branch/py12/pypy/module/cpyext/stubs.py pypy/branch/py12/pypy/module/cpyext/sysmodule.py pypy/branch/py12/pypy/module/cpyext/test/foo.c pypy/branch/py12/pypy/module/cpyext/test/test_api.py pypy/branch/py12/pypy/module/cpyext/test/test_borrow.py pypy/branch/py12/pypy/module/cpyext/test/test_cpyext.py pypy/branch/py12/pypy/module/cpyext/test/test_getargs.py pypy/branch/py12/pypy/module/cpyext/test/test_import.py pypy/branch/py12/pypy/module/cpyext/test/test_mapping.py pypy/branch/py12/pypy/module/cpyext/test/test_pyerrors.py pypy/branch/py12/pypy/module/cpyext/test/test_thread.py pypy/branch/py12/pypy/module/cpyext/test/test_typeobject.py pypy/branch/py12/pypy/module/cpyext/test/test_weakref.py pypy/branch/py12/pypy/module/cpyext/tupleobject.py pypy/branch/py12/pypy/module/cpyext/typeobject.py pypy/branch/py12/pypy/module/cpyext/unicodeobject.py pypy/branch/py12/pypy/module/cpyext/weakrefobject.py pypy/branch/py12/pypy/module/posix/test/test_posix2.py pypy/branch/py12/pypy/translator/tool/cbuild.py Log: merge trunk Modified: pypy/branch/py12/lib-python/modified-2.5.2/distutils/msvccompiler.py ============================================================================== --- pypy/branch/py12/lib-python/modified-2.5.2/distutils/msvccompiler.py (original) +++ pypy/branch/py12/lib-python/modified-2.5.2/distutils/msvccompiler.py Thu May 6 18:18:13 2010 @@ -165,7 +165,7 @@ For Python 2.3 and up, the version number is included in sys.version. For earlier versions, assume the compiler is MSVC 6. """ - return 8.0 + return 9.0 def normalize_and_reduce_paths(paths): """Return a list of normalized paths with duplicates removed. Modified: pypy/branch/py12/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/py12/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/py12/pypy/jit/metainterp/pyjitpl.py Thu May 6 18:18:13 2010 @@ -1396,13 +1396,13 @@ etype, evalue = self.cpu.get_overflow_error() return self.finishframe_exception( self.cpu.ts.get_exception_box(etype), - self.cpu.ts.get_exc_value_box(evalue)) + self.cpu.ts.get_exc_value_box(evalue).constbox()) def raise_zero_division_error(self): etype, evalue = self.cpu.get_zero_division_error() return self.finishframe_exception( self.cpu.ts.get_exception_box(etype), - self.cpu.ts.get_exc_value_box(evalue)) + self.cpu.ts.get_exc_value_box(evalue).constbox()) def create_empty_history(self): warmrunnerstate = self.staticdata.state Modified: pypy/branch/py12/pypy/jit/metainterp/test/test_exception.py ============================================================================== --- pypy/branch/py12/pypy/jit/metainterp/test/test_exception.py (original) +++ pypy/branch/py12/pypy/jit/metainterp/test/test_exception.py Thu May 6 18:18:13 2010 @@ -556,6 +556,20 @@ optimizer=OPTIMIZER_SIMPLE) assert res == 8 + + def test_overflowerror_escapes(self): + def g(x): + return ovfcheck(x + 1) + def f(x): + try: + return g(x) + except Exception, e: + if isinstance(e, OverflowError): + return -42 + raise + res = self.interp_operations(f, [sys.maxint]) + assert res == -42 + class MyError(Exception): def __init__(self, n): self.n = n Modified: pypy/branch/py12/pypy/module/cpyext/TODO ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/TODO (original) +++ pypy/branch/py12/pypy/module/cpyext/TODO Thu May 6 18:18:13 2010 @@ -14,9 +14,6 @@ it does is a lltype function pointer, no C code involved. - Fix GIL handling (e.g. after releasing the GIL, GC operations might occur in savethreads). - - Fix PyEval_SaveThread: http://paste.pocoo.org/show/206521/ - - - Make Numpy work. - Export pointers where pointers are expected, i.e. kill the "#define PyExc_OSError &PyExc_OSError" hack. @@ -30,8 +27,5 @@ raw memory (except array which supports it in a hackish way), which should be fixed in order to make it nicely work with cpyext. - - Py_InitModule() does not seem to return the correct module when using dotted import. - for example: "import wx" imports "wx._core_", which calls Py_InitModule("_core_"). - This correcly creates and populates sys.modules["wx._core_"], but returns a new empty - module sys.modules["_core_"]. So functions go in the correct module, but types and - constants go into the wrong one. + - Fix Makefile generation on Windows when --shared is used (mandatory when + cpyext is used with -Ojit) Modified: pypy/branch/py12/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/py12/pypy/module/cpyext/__init__.py Thu May 6 18:18:13 2010 @@ -23,19 +23,19 @@ def startup(self, space): state = space.fromcache(State) from pypy.module.cpyext.typeobject import setup_new_method_def + from pypy.module.cpyext.pyobject import RefcountState setup_new_method_def(space) if not we_are_translated(): space.setattr(space.wrap(self), space.wrap('api_lib'), space.wrap(state.api_lib)) else: - state.init_r2w_from_w2r() + refcountstate = space.fromcache(RefcountState) + refcountstate.init_r2w_from_w2r() for func in api.INIT_FUNCTIONS: func(space) state.check_and_raise_exception() - if not we_are_translated(): - state.non_heaptypes[:] = [] # import these modules to register api functions by side-effect import pypy.module.cpyext.thread Modified: pypy/branch/py12/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/api.py (original) +++ pypy/branch/py12/pypy/module/cpyext/api.py Thu May 6 18:18:13 2010 @@ -104,13 +104,11 @@ # class ApiFunction: - def __init__(self, argtypes, restype, callable, - borrowed=False, error=_NOT_SPECIFIED): + def __init__(self, argtypes, restype, callable, error=_NOT_SPECIFIED): self.argtypes = argtypes self.restype = restype self.functype = lltype.Ptr(lltype.FuncType(argtypes, restype)) self.callable = callable - self.borrowed = borrowed if error is not _NOT_SPECIFIED: self.error_value = error @@ -141,7 +139,7 @@ wrapper.relax_sig_check = True return wrapper -def cpython_api(argtypes, restype, borrowed=False, error=_NOT_SPECIFIED, +def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, external=True, name=None): if error is _NOT_SPECIFIED: if restype is PyObject: @@ -157,7 +155,7 @@ else: func_name = name func = func_with_new_name(func, name) - api_function = ApiFunction(argtypes, restype, func, borrowed, error) + api_function = ApiFunction(argtypes, restype, func, error) func.api_func = api_function assert func_name not in FUNCTIONS @@ -176,8 +174,10 @@ def unwrapper(space, *args): from pypy.module.cpyext.pyobject import Py_DecRef from pypy.module.cpyext.pyobject import make_ref, from_ref + from pypy.module.cpyext.pyobject import BorrowPair newargs = () to_decref = [] + assert len(args) == len(api_function.argtypes) for i, (ARG, is_wrapped) in types_names_enum_ui: input_arg = args[i] if is_PyObject(ARG) and not is_wrapped: @@ -203,23 +203,25 @@ newargs += (arg, ) try: try: - return func(space, *newargs) + res = func(space, *newargs) except OperationError, e: if not catch_exception: raise if not hasattr(api_function, "error_value"): raise state = space.fromcache(State) - e.normalize_exception(space) - state.set_exception(e.w_type, e.get_w_value(space)) + state.set_exception(e) if restype is PyObject: return None else: return api_function.error_value + if res is None: + return None + elif isinstance(res, BorrowPair): + return res.w_borrowed + else: + return res finally: - if api_function.borrowed: - state = space.fromcache(State) - state.last_container = 0 for arg in to_decref: Py_DecRef(space, arg) unwrapper.func = func @@ -398,7 +400,7 @@ @specialize.ll() def wrapper(*args): from pypy.module.cpyext.pyobject import make_ref, from_ref - from pypy.module.cpyext.pyobject import add_borrowed_object + from pypy.module.cpyext.pyobject import BorrowPair from pypy.module.cpyext.pyobject import NullPointerException # we hope that malloc removal removes the newtuple() that is # inserted exactly here by the varargs specializer @@ -409,6 +411,7 @@ try: if not we_are_translated() and DEBUG_WRAPPER: print >>sys.stderr, callable, + assert len(args) == len(callable.api_func.argtypes) for i, (typ, is_wrapped) in argtypes_enum_ui: arg = args[i] if typ is PyObject and is_wrapped: @@ -426,11 +429,11 @@ print >>sys.stderr, " DONE" except OperationError, e: failed = True - e.normalize_exception(space) - state.set_exception(e.w_type, e.get_w_value(space)) + state.set_exception(e) except BaseException, e: failed = True - state.set_exception(space.w_SystemError, space.wrap(str(e))) + state.set_exception(OperationError(space.w_SystemError, + space.wrap(str(e)))) if not we_are_translated(): import traceback traceback.print_exc() @@ -445,13 +448,14 @@ retval = error_value elif callable.api_func.restype is PyObject: - borrowed = callable.api_func.borrowed - if not rffi._isllptr(result): - retval = make_ref(space, result, borrowed=borrowed) + if result is None: + retval = make_ref(space, None) + elif isinstance(result, BorrowPair): + retval = result.get_ref(space) + elif not rffi._isllptr(result): + retval = make_ref(space, result) else: retval = result - if borrowed: - add_borrowed_object(space, retval) elif callable.api_func.restype is not lltype.Void: retval = rffi.cast(callable.api_func.restype, result) except NullPointerException: @@ -818,6 +822,7 @@ def generic_cpy_call(space, func, *args): boxed_args = () to_decref = [] + assert len(args) == len(FT.ARGS) for i, ARG in unrolling_arg_types: arg = args[i] if ARG is PyObject: Modified: pypy/branch/py12/pypy/module/cpyext/dictobject.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/dictobject.py (original) +++ pypy/branch/py12/pypy/module/cpyext/dictobject.py Thu May 6 18:18:13 2010 @@ -1,7 +1,7 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, build_type_checkers,\ Py_ssize_t, PyObjectP, CONST_STRING -from pypy.module.cpyext.pyobject import PyObject, register_container +from pypy.module.cpyext.pyobject import PyObject, borrow_from from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.interpreter.error import OperationError @@ -11,14 +11,13 @@ PyDict_Check, PyDict_CheckExact = build_type_checkers("Dict") - at cpython_api([PyObject, PyObject], PyObject, borrowed=True, error=CANNOT_FAIL) + at cpython_api([PyObject, PyObject], PyObject, error=CANNOT_FAIL) def PyDict_GetItem(space, w_dict, w_key): try: w_res = space.getitem(w_dict, w_key) except: return None - register_container(space, w_dict) - return w_res + return borrow_from(w_dict, w_res) @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) def PyDict_SetItem(space, w_dict, w_key, w_obj): @@ -47,7 +46,7 @@ else: PyErr_BadInternalCall(space) - at cpython_api([PyObject, CONST_STRING], PyObject, borrowed=True, error=CANNOT_FAIL) + at cpython_api([PyObject, CONST_STRING], PyObject, error=CANNOT_FAIL) def PyDict_GetItemString(space, w_dict, key): """This is the same as PyDict_GetItem(), but key is specified as a char*, rather than a PyObject*.""" @@ -57,8 +56,7 @@ w_res = None if w_res is None: return None - register_container(space, w_dict) - return w_res + return borrow_from(w_dict, w_res) @cpython_api([PyObject], Py_ssize_t, error=-1) def PyDict_Size(space, w_obj): Modified: pypy/branch/py12/pypy/module/cpyext/funcobject.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/funcobject.py (original) +++ pypy/branch/py12/pypy/module/cpyext/funcobject.py Thu May 6 18:18:13 2010 @@ -3,7 +3,7 @@ PyObjectFields, generic_cpy_call, cpython_api, bootstrap_function, cpython_struct, build_type_checkers) from pypy.module.cpyext.pyobject import ( - PyObject, make_ref, from_ref, Py_DecRef, make_typedescr, register_container) + PyObject, make_ref, from_ref, Py_DecRef, make_typedescr, borrow_from) from pypy.interpreter.function import Function, Method PyFunctionObjectStruct = lltype.ForwardReference() @@ -43,18 +43,16 @@ class which provides the unbound method.""" return Method(space, w_func, w_self, w_cls) - at cpython_api([PyObject], PyObject, borrowed=True) + at cpython_api([PyObject], PyObject) def PyMethod_Function(space, w_method): """Return the function object associated with the method meth.""" assert isinstance(w_method, Method) - register_container(space, w_method) - return w_method.w_function + return borrow_from(w_method, w_method.w_function) - at cpython_api([PyObject], PyObject, borrowed=True) + at cpython_api([PyObject], PyObject) def PyMethod_Class(space, w_method): """Return the class object from which the method meth was created; if this was created from an instance, it will be the class of the instance.""" assert isinstance(w_method, Method) - register_container(space, w_method) - return w_method.w_class + return borrow_from(w_method, w_method.w_class) Modified: pypy/branch/py12/pypy/module/cpyext/import_.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/import_.py (original) +++ pypy/branch/py12/pypy/module/cpyext/import_.py Thu May 6 18:18:13 2010 @@ -30,11 +30,11 @@ else: w_import = space.getattr(w_builtin, space.wrap("__import__")) - # Call the __import__ function with the proper argument list - # Always use absolute import here. - return space.call(w_import, space.newtuple( - [w_name, w_globals, w_globals, - space.newlist([space.wrap("__doc__")])])) + # Call the __import__ function with the proper argument list + # Always use absolute import here. + return space.call_function(w_import, + w_name, w_globals, w_globals, + space.newlist([space.wrap("__doc__")])) @cpython_api([CONST_STRING], PyObject) def PyImport_ImportModule(space, name): Modified: pypy/branch/py12/pypy/module/cpyext/include/patchlevel.h ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/include/patchlevel.h (original) +++ pypy/branch/py12/pypy/module/cpyext/include/patchlevel.h Thu May 6 18:18:13 2010 @@ -28,6 +28,9 @@ /* Version as a string */ #define PY_VERSION "2.5.5" +/* PyPy version as a string */ +#define PYPY_VERSION "1.2.0" + /* Subversion Revision number of this file (not of the repository) */ #define PY_PATCHLEVEL_REVISION "$Revision: 77872 $" Modified: pypy/branch/py12/pypy/module/cpyext/listobject.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/listobject.py (original) +++ pypy/branch/py12/pypy/module/cpyext/listobject.py Thu May 6 18:18:13 2010 @@ -3,7 +3,7 @@ from pypy.module.cpyext.api import (cpython_api, CANNOT_FAIL, Py_ssize_t, build_type_checkers) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall -from pypy.module.cpyext.pyobject import Py_DecRef, PyObject, register_container +from pypy.module.cpyext.pyobject import Py_DecRef, PyObject, borrow_from from pypy.objspace.std.listobject import W_ListObject from pypy.interpreter.error import OperationError @@ -39,20 +39,19 @@ wrappeditems[index] = w_item return 0 - at cpython_api([PyObject, Py_ssize_t], PyObject, borrowed=True) + at cpython_api([PyObject, Py_ssize_t], PyObject) def PyList_GetItem(space, w_list, index): """Return the object at position pos in the list pointed to by p. The position must be positive, indexing from the end of the list is not supported. If pos is out of bounds, return NULL and set an IndexError exception.""" - register_container(space, w_list) if not isinstance(w_list, W_ListObject): PyErr_BadInternalCall(space) wrappeditems = w_list.wrappeditems if index < 0 or index >= len(wrappeditems): raise OperationError(space.w_IndexError, space.wrap( "list index out of range")) - return wrappeditems[index] + return borrow_from(w_list, wrappeditems[index]) @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) Modified: pypy/branch/py12/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/methodobject.py (original) +++ pypy/branch/py12/pypy/module/cpyext/methodobject.py Thu May 6 18:18:13 2010 @@ -13,7 +13,6 @@ generic_cpy_call, cpython_api, PyObject, cpython_struct, METH_KEYWORDS, METH_O, CONST_STRING, METH_CLASS, METH_STATIC, METH_COEXIST, METH_NOARGS, METH_VARARGS, build_type_checkers, PyObjectFields, bootstrap_function) -from pypy.module.cpyext.state import State from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.rlib.objectmodel import we_are_translated from pypy.objspace.std.tupleobject import W_TupleObject Modified: pypy/branch/py12/pypy/module/cpyext/modsupport.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/modsupport.py (original) +++ pypy/branch/py12/pypy/module/cpyext/modsupport.py Thu May 6 18:18:13 2010 @@ -1,14 +1,14 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import cpython_api, cpython_struct, \ METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, CONST_STRING -from pypy.module.cpyext.pyobject import PyObject, register_container +from pypy.module.cpyext.pyobject import PyObject, borrow_from from pypy.interpreter.module import Module from pypy.module.cpyext.methodobject import W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod, PyMethodDef, PyCFunction from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.module.cpyext.state import State from pypy.interpreter.error import OperationError -#@cpython_api([rffi.CCHARP], PyObject, borrowed=True) +#@cpython_api([rffi.CCHARP], PyObject) def PyImport_AddModule(space, name): """Return the module object corresponding to a module name. The name argument may be of the form package.module. First check the modules dictionary if @@ -30,7 +30,7 @@ return w_mod @cpython_api([CONST_STRING, lltype.Ptr(PyMethodDef), CONST_STRING, - PyObject, rffi.INT_real], PyObject, borrowed=False) # we cannot borrow here + PyObject, rffi.INT_real], PyObject) def Py_InitModule4(space, name, methods, doc, w_self, apiver): """ Create a new module object based on a name and table of functions, returning @@ -58,6 +58,7 @@ if doc: space.setattr(w_mod, space.wrap("__doc__"), space.wrap(rffi.charp2str(doc))) + # we cannot borrow here return w_mod @@ -108,13 +109,12 @@ return int(space.is_w(w_type, w_obj_type) or space.is_true(space.issubtype(w_obj_type, w_type))) - at cpython_api([PyObject], PyObject, borrowed=True) + at cpython_api([PyObject], PyObject) def PyModule_GetDict(space, w_mod): if PyModule_Check(space, w_mod): assert isinstance(w_mod, Module) w_dict = w_mod.getdict() - register_container(space, w_mod) - return w_dict + return borrow_from(w_mod, w_dict) else: PyErr_BadInternalCall(space) Modified: pypy/branch/py12/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/object.py (original) +++ pypy/branch/py12/pypy/module/cpyext/object.py Thu May 6 18:18:13 2010 @@ -2,9 +2,9 @@ from pypy.module.cpyext.api import cpython_api, generic_cpy_call, CANNOT_FAIL,\ Py_ssize_t, PyVarObject, Py_TPFLAGS_HEAPTYPE,\ Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT, Py_GE, CONST_STRING -from pypy.module.cpyext.pyobject import PyObject, PyObjectP, make_ref, from_ref -from pypy.module.cpyext.pyobject import Py_IncRef, Py_DecRef -from pypy.module.cpyext.state import State +from pypy.module.cpyext.pyobject import ( + PyObject, PyObjectP, create_ref, from_ref, Py_IncRef, Py_DecRef, + track_reference) from pypy.module.cpyext.typeobject import PyTypeObjectPtr, W_PyCTypeObject from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall from pypy.objspace.std.objectobject import W_ObjectObject @@ -22,7 +22,9 @@ w_type = from_ref(space, rffi.cast(PyObject, type)) if isinstance(w_type, W_PyCTypeObject): w_obj = space.allocate_instance(W_ObjectObject, w_type) - return make_ref(space, w_obj, items=size) + py_obj = create_ref(space, w_obj, items=size) + track_reference(space, py_obj, w_obj) + return py_obj assert False, "Please add more cases in _PyObject_New" @cpython_api([rffi.VOIDP_real], lltype.Void) @@ -145,7 +147,7 @@ This is the equivalent of the Python expression o[key].""" return space.getitem(w_obj, w_key) - at cpython_api([PyObject, PyTypeObjectPtr], PyObject, borrowed=True) + at cpython_api([PyObject, PyTypeObjectPtr], PyObject) def PyObject_Init(space, op, type): """Initialize a newly-allocated object op with its type and initial reference. Returns the initialized object. If type indicates that the @@ -158,7 +160,7 @@ op.c_ob_refcnt = 1 return from_ref(space, op) # XXX will give an exception - at cpython_api([PyVarObject, PyTypeObjectPtr, Py_ssize_t], PyObject, borrowed=True) + at cpython_api([PyVarObject, PyTypeObjectPtr, Py_ssize_t], PyObject) def PyObject_InitVar(space, op, type, size): """This does everything PyObject_Init() does, and also initializes the length information for a variable-size object.""" Modified: pypy/branch/py12/pypy/module/cpyext/presetup.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/presetup.py (original) +++ pypy/branch/py12/pypy/module/cpyext/presetup.py Thu May 6 18:18:13 2010 @@ -4,7 +4,7 @@ Run python setup.py build in your project directory -You can import resulting .so with py.py --allworingmodules +You can import resulting .so with py.py --allworkingmodules """ import sys, os @@ -21,8 +21,10 @@ from pypy.conftest import gettestobjspace from pypy.module.cpyext.api import build_bridge -space = gettestobjspace(usemodules=['cpyext', 'thread']) -build_bridge(space) +usemodules = ['cpyext', 'thread'] +if sys.platform == 'win32': + usemodules.append('_winreg') # necessary in distutils +space = gettestobjspace(usemodules=usemodules) inc_paths = str(udir) Modified: pypy/branch/py12/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/pyerrors.py (original) +++ pypy/branch/py12/pypy/module/cpyext/pyerrors.py Thu May 6 18:18:13 2010 @@ -1,12 +1,13 @@ import os from pypy.rpython.lltypesystem import rffi, lltype -from pypy.interpreter.error import OperationError +from pypy.interpreter.error import OperationError, wrap_oserror from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING from pypy.module.exceptions.interp_exceptions import W_RuntimeWarning from pypy.module.cpyext.pyobject import ( - PyObject, PyObjectP, make_ref, Py_DecRef, register_container) + PyObject, PyObjectP, make_ref, Py_DecRef, borrow_from) from pypy.module.cpyext.state import State +from pypy.module.cpyext.import_ import PyImport_Import from pypy.rlib.rposix import get_errno @cpython_api([PyObject, PyObject], lltype.Void) @@ -14,7 +15,7 @@ """This function is similar to PyErr_SetString() but lets you specify an arbitrary Python object for the "value" of the exception.""" state = space.fromcache(State) - state.set_exception(w_type, w_value) + state.set_exception(OperationError(w_type, w_value)) @cpython_api([PyObject, CONST_STRING], lltype.Void) def PyErr_SetString(space, w_type, message_ptr): @@ -26,11 +27,12 @@ """This is a shorthand for PyErr_SetObject(type, Py_None).""" PyErr_SetObject(space, w_type, space.w_None) - at cpython_api([], PyObject, borrowed=True) + at cpython_api([], PyObject) def PyErr_Occurred(space): state = space.fromcache(State) - register_container(space, lltype.nullptr(PyObject.TO)) - return state.exc_type + if state.operror is None: + return None + return borrow_from(None, state.operror.w_type) @cpython_api([], lltype.Void) def PyErr_Clear(space): @@ -47,10 +49,13 @@ This function is normally only used by code that needs to handle exceptions or by code that needs to save and restore the error indicator temporarily.""" state = space.fromcache(State) - ptype[0] = make_ref(space, state.exc_type, steal=True) - pvalue[0] = make_ref(space, state.exc_type, steal=True) - state.exc_type = None - state.exc_value = None + operror = state.clear_exception() + if operror: + ptype[0] = make_ref(space, operror.w_type) + pvalue[0] = make_ref(space, operror.get_w_value(space)) + else: + ptype[0] = lltype.nullptr(PyObject.TO) + pvalue[0] = lltype.nullptr(PyObject.TO) ptraceback[0] = lltype.nullptr(PyObject.TO) @cpython_api([PyObject, PyObject, PyObject], lltype.Void) @@ -69,10 +74,10 @@ error indicator temporarily; use PyErr_Fetch() to save the current exception state.""" state = space.fromcache(State) - state.exc_type = w_type - state.exc_value = w_value + state.set_exception(OperationError(w_type, w_value)) Py_DecRef(space, w_type) Py_DecRef(space, w_value) + Py_DecRef(space, w_traceback) @cpython_api([], lltype.Void) def PyErr_BadArgument(space): @@ -110,9 +115,7 @@ Return value: always NULL.""" # XXX Doesn't actually do anything with PyErr_CheckSignals. errno = get_errno() - errno_w = space.wrap(errno) - message_w = space.wrap(os.strerror(errno)) - PyErr_SetObject(space, w_type, errno_w, message_w) + raise wrap_oserror(space, OSError(errno, "PyErr_SetFromErrno")) @cpython_api([], rffi.INT_real, error=-1) def PyErr_CheckSignals(space): @@ -185,10 +188,14 @@ For information about warning control, see the documentation for the warnings module and the -W option in the command line documentation. There is no C API for warning control.""" - message = rffi.charp2str(message_ptr) if w_category is None: - w_category = space.gettypeobject(W_RuntimeWarning.typedef) - os.write(2, "WARNING: " + message + "\n") + w_category = space.w_None + w_message = space.wrap(rffi.charp2str(message_ptr)) + w_stacklevel = space.wrap(rffi.cast(lltype.Signed, stacklevel)) + + w_module = PyImport_Import(space, space.wrap("warnings")) + w_warn = space.getattr(w_module, space.wrap("warn")) + space.call_function(w_warn, w_message, w_category, w_stacklevel) return 0 @cpython_api([PyObject, CONST_STRING], rffi.INT_real, error=-1) Modified: pypy/branch/py12/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/py12/pypy/module/cpyext/pyobject.py Thu May 6 18:18:13 2010 @@ -14,7 +14,18 @@ # type description class BaseCpyTypedescr(object): - pass + basestruct = PyObject.TO + + def get_dealloc(self, space): + raise NotImplementedError + def allocate(self, space, w_type, itemcount=0): + raise NotImplementedError + def make_ref(self, space, w_type, w_obj, itemcount=0): + raise NotImplementedError + def attach(self, space, pyobj, w_obj): + raise NotImplementedError + def realize(self, space, ref): + raise NotImplementedError typedescr_cache = {} @@ -135,6 +146,38 @@ #________________________________________________________ # refcounted object support +class RefcountState: + def __init__(self, space): + self.space = space + self.py_objects_w2r = {} # { w_obj -> raw PyObject } + self.py_objects_r2w = {} # { addr of raw PyObject -> w_obj } + self.borrow_mapping = {} # { w_container -> { w_containee -> None } } + self.borrowed_objects = {} # { addr of containee -> None } + self.non_heaptypes_w = [] + + def _freeze_(self): + assert not self.borrowed_objects and not self.borrow_mapping + self.py_objects_r2w.clear() # is not valid anymore after translation + return False + + def init_r2w_from_w2r(self): + from pypy.module.cpyext.api import ADDR + for w_obj, obj in self.py_objects_w2r.items(): + ptr = rffi.cast(ADDR, obj) + self.py_objects_r2w[ptr] = w_obj + + def print_refcounts(self): + print "REFCOUNTS" + for w_obj, obj in self.py_objects_w2r.items(): + print "%r: %i" % (w_obj, obj.c_ob_refcnt) + + def reset_borrowed_references(self): + while self.borrowed_objects: + addr, _ = self.borrowed_objects.popitem() + w_obj = self.py_objects_r2w[addr] + Py_DecRef(self.space, w_obj) + self.borrow_mapping = {} + class NullPointerException(Exception): pass @@ -153,62 +196,71 @@ print >>sys.stderr def create_ref(space, w_obj, items=0): + """ + Allocates a PyObject, and fills its fields with info from the given + intepreter object. + """ w_type = space.type(w_obj) metatypedescr = get_typedescr(w_type.typedef) return metatypedescr.make_ref(space, w_type, w_obj, itemcount=items) -def track_reference(space, py_obj, w_obj, borrowed=False): +def track_reference(space, py_obj, w_obj, replace=False): + """ + Ties together a PyObject and an interpreter object. + """ # XXX looks like a PyObject_GC_TRACK ptr = rffi.cast(ADDR, py_obj) + state = space.fromcache(RefcountState) if DEBUG_REFCOUNT: debug_refcount("MAKREF", py_obj, w_obj) - state = space.fromcache(State) + if not replace: + assert w_obj not in state.py_objects_w2r + assert ptr not in state.py_objects_r2w + assert ptr not in state.borrowed_objects state.py_objects_w2r[w_obj] = py_obj - state.py_objects_r2w[ptr] = w_obj - if borrowed and ptr not in state.borrowed_objects: - state.borrowed_objects[ptr] = None + if ptr: # init_typeobject() bootstraps with NULL references + state.py_objects_r2w[ptr] = w_obj -def make_ref(space, w_obj, borrowed=False, steal=False, items=0): +def make_ref(space, w_obj): + """ + Returns a new reference to an intepreter object. + """ if w_obj is None: return lltype.nullptr(PyObject.TO) assert isinstance(w_obj, W_Root) - state = space.fromcache(State) + state = space.fromcache(RefcountState) try: py_obj = state.py_objects_w2r[w_obj] except KeyError: - assert not steal - py_obj = create_ref(space, w_obj, items) - track_reference(space, py_obj, w_obj, borrowed=borrowed) - return py_obj - - if not steal: - if borrowed: - py_obj_addr = rffi.cast(ADDR, py_obj) - if py_obj_addr not in state.borrowed_objects: - Py_IncRef(space, py_obj) - state.borrowed_objects[py_obj_addr] = None - else: - Py_IncRef(space, py_obj) + py_obj = create_ref(space, w_obj) + track_reference(space, py_obj, w_obj) + else: + Py_IncRef(space, py_obj) return py_obj -def from_ref(space, ref, recurse=False): +def from_ref(space, ref): + """ + Finds the interpreter object corresponding to the given reference. If the + object is not yet realized (see stringobject.py), creates it. + """ assert lltype.typeOf(ref) == PyObject if not ref: return None - state = space.fromcache(State) + state = space.fromcache(RefcountState) ptr = rffi.cast(ADDR, ref) try: return state.py_objects_r2w[ptr] except KeyError: - if recurse: - raise InvalidPointerException(str(ref)) + pass # This reference is not yet a real interpreter object. # Realize it. ref_type = rffi.cast(PyObject, ref.c_ob_type) - w_type = from_ref(space, ref_type, True) + if ref_type == ref: + raise InvalidPointerException(str(ref)) + w_type = from_ref(space, ref_type) assert isinstance(w_type, W_TypeObject) return get_typedescr(w_type.instancetypedef).realize(space, ref) @@ -225,12 +277,8 @@ if DEBUG_REFCOUNT: debug_refcount("DECREF", obj, obj.c_ob_refcnt, frame_stackdepth=3) if obj.c_ob_refcnt == 0: - state = space.fromcache(State) + state = space.fromcache(RefcountState) ptr = rffi.cast(ADDR, obj) - try: - del state.borrowed_objects[ptr] - except KeyError: - pass if ptr not in state.py_objects_r2w: w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type)) if space.is_w(w_type, space.w_str) or space.is_w(w_type, space.w_unicode): @@ -245,22 +293,8 @@ if not space.is_w(w_typetype, space.gettypeobject(W_PyCTypeObject.typedef)): _Py_Dealloc(space, obj) del state.py_objects_w2r[w_obj] - if ptr in state.borrow_mapping: # move to lifeline __del__ - for containee in state.borrow_mapping[ptr]: - w_containee = state.py_objects_r2w.get(containee, None) - if w_containee is not None: - containee = state.py_objects_w2r[w_containee] - Py_DecRef(space, w_containee) - containee_ptr = rffi.cast(ADDR, containee) - try: - del state.borrowed_objects[containee_ptr] - except KeyError: - pass - else: - if DEBUG_REFCOUNT: - print >>sys.stderr, "Borrowed object is already gone:", \ - hex(containee) - del state.borrow_mapping[ptr] + # if the object was a container for borrowed references + delete_borrower(space, w_obj) else: if not we_are_translated() and obj.c_ob_refcnt < 0: message = "Negative refcount for obj %s with type %s" % ( @@ -291,31 +325,72 @@ #___________________________________________________________ # Support for borrowed references - at cpython_api([PyObject], lltype.Void, external=False) -def register_container(space, container): - state = space.fromcache(State) - if not container: # self-managed - container_ptr = -1 +def make_borrowed_ref(space, w_container, w_borrowed): + """ + Create a borrowed reference, which will live as long as the container + has a living reference (as a PyObject!) + """ + ref = make_ref(space, w_borrowed) + if not ref: + return ref + + # state.borrowed_objects owns the reference + state = space.fromcache(RefcountState) + obj_ptr = rffi.cast(ADDR, ref) + if obj_ptr not in state.borrowed_objects: + state.borrowed_objects[obj_ptr] = None else: - container_ptr = rffi.cast(ADDR, container) - assert not state.last_container, "Last container was not fetched" - state.last_container = container_ptr - -def add_borrowed_object(space, obj): - state = space.fromcache(State) - container_ptr = state.last_container - state.last_container = 0 - if not obj: - return - if not container_ptr: - raise NullPointerException - if container_ptr == -1: + Py_DecRef(space, ref) # already in borrowed list + + if w_container is None: # self-managed + return ref + borrowees = state.borrow_mapping.setdefault(w_container, {}) + borrowees[w_borrowed] = None + return ref + +class BorrowPair: + """ + Delays the creation of a borrowed reference. + """ + def __init__(self, w_container, w_borrowed): + self.w_container = w_container + self.w_borrowed = w_borrowed + + def get_ref(self, space): + return make_borrowed_ref(space, self.w_container, self.w_borrowed) + +def borrow_from(container, borrowed): + return BorrowPair(container, borrowed) + +def forget_borrowee(space, w_obj): + "De-register an object from the list of borrowed references" + state = space.fromcache(RefcountState) + ref = state.py_objects_w2r.get(w_obj, lltype.nullptr(PyObject.TO)) + if not ref: + if DEBUG_REFCOUNT: + print >>sys.stderr, "Borrowed object is already gone:", \ + hex(containee) return - borrowees = state.borrow_mapping.get(container_ptr, None) - if borrowees is None: - state.borrow_mapping[container_ptr] = borrowees = {} - obj_ptr = rffi.cast(ADDR, obj) - borrowees[obj_ptr] = None + + containee_ptr = rffi.cast(ADDR, ref) + try: + del state.borrowed_objects[containee_ptr] + except KeyError: + pass + else: + Py_DecRef(space, ref) + +def delete_borrower(space, w_obj): + """ + Called when a potential container for borrowed references has lost its + last reference. Removes the borrowed references it contains. + """ + state = space.fromcache(RefcountState) + if w_obj in state.borrow_mapping: # move to lifeline __del__ + for w_containee in state.borrow_mapping[w_obj]: + forget_borrowee(space, w_containee) + del state.borrow_mapping[w_obj] + #___________________________________________________________ Modified: pypy/branch/py12/pypy/module/cpyext/sequence.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/sequence.py (original) +++ pypy/branch/py12/pypy/module/cpyext/sequence.py Thu May 6 18:18:13 2010 @@ -2,7 +2,7 @@ from pypy.interpreter.error import OperationError from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, CONST_STRING, Py_ssize_t) -from pypy.module.cpyext.pyobject import PyObject, register_container +from pypy.module.cpyext.pyobject import PyObject, borrow_from from pypy.rpython.lltypesystem import rffi, lltype from pypy.objspace.std import listobject, tupleobject @@ -40,7 +40,7 @@ except OperationError: raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m))) - at cpython_api([PyObject, Py_ssize_t], PyObject, borrowed=True) + at cpython_api([PyObject, Py_ssize_t], PyObject) def PySequence_Fast_GET_ITEM(space, w_obj, index): """Return the ith element of o, assuming that o was returned by PySequence_Fast(), o is not NULL, and that i is within bounds. @@ -50,8 +50,7 @@ else: assert isinstance(w_obj, tupleobject.W_TupleObject) w_res = w_obj.wrappeditems[index] - register_container(space, w_obj) - return w_res + return borrow_from(w_obj, w_res) @cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL) def PySequence_Fast_GET_SIZE(space, w_obj): Modified: pypy/branch/py12/pypy/module/cpyext/state.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/state.py (original) +++ pypy/branch/py12/pypy/module/cpyext/state.py Thu May 6 18:18:13 2010 @@ -12,14 +12,7 @@ def reset(self): from pypy.module.cpyext.modsupport import PyMethodDef - self.py_objects_w2r = {} # { w_obj -> raw PyObject } - self.py_objects_r2w = {} # { addr of raw PyObject -> w_obj } - self.borrow_mapping = {} # { addr of container -> { addr of containee -> None } } - self.borrowed_objects = {} # { addr of containee -> None } - self.non_heaptypes = [] # list of wrapped objects - self.last_container = 0 # addr of last container - self.exc_type = None - self.exc_value = None + self.operror = None self.new_method_def = lltype.nullptr(PyMethodDef) # When importing a package, use this to keep track of its name. This is @@ -31,48 +24,30 @@ # already read it by that time. self.package_context = None - - def _freeze_(self): - assert not self.borrowed_objects and not self.borrow_mapping - self.py_objects_r2w.clear() # is not valid anymore after translation - return False - - def init_r2w_from_w2r(self): - from pypy.module.cpyext.api import ADDR - for w_obj, obj in self.py_objects_w2r.items(): - ptr = rffi.cast(ADDR, obj) - self.py_objects_r2w[ptr] = w_obj - - def set_exception(self, w_type, w_value): + def set_exception(self, operror): self.clear_exception() - self.exc_type = w_type - self.exc_value = w_value + self.operror = operror def clear_exception(self): - from pypy.module.cpyext.pyobject import Py_DecRef, make_ref + """Clear the current exception state, and return the operror. + Also frees the borrowed reference returned by PyErr_Occurred() + """ from pypy.module.cpyext.api import ADDR # handling of borrowed objects, remove when we have # a weakkeydict - exc_type = make_ref(self.space, self.exc_type, borrowed=True) - if exc_type: - Py_DecRef(self.space, exc_type) - containee_ptr = rffi.cast(ADDR, exc_type) - del self.borrowed_objects[containee_ptr] - self.exc_type = None - self.exc_value = None + operror = self.operror + if operror is not None: + # we have no explicit container + from pypy.module.cpyext.pyobject import forget_borrowee + forget_borrowee(self.space, operror.w_type) + self.operror = None + return operror def check_and_raise_exception(self, always=False): - exc_value = self.exc_value - exc_type = self.exc_type - if exc_type is not None or exc_value is not None: + operror = self.operror + if operror: self.clear_exception() - op_err = OperationError(exc_type, exc_value) - raise op_err + raise operror if always: raise OperationError(self.space.w_SystemError, self.space.wrap( "Function returned an error result without setting an exception")) - - def print_refcounts(self): - print "REFCOUNTS" - for w_obj, obj in self.py_objects_w2r.items(): - print "%r: %i" % (w_obj, obj.c_ob_refcnt) Modified: pypy/branch/py12/pypy/module/cpyext/stringobject.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/stringobject.py (original) +++ pypy/branch/py12/pypy/module/cpyext/stringobject.py Thu May 6 18:18:13 2010 @@ -1,11 +1,53 @@ from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import ( - cpython_api, bootstrap_function, PyVarObjectFields, Py_ssize_t, - cpython_struct, PyObjectFields, ADDR, CONST_STRING, CANNOT_FAIL, - build_type_checkers, PyObjectP, PyTypeObjectPtr, generic_cpy_call) -from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref, Py_DecRef, make_typedescr -from pypy.module.cpyext.state import State + cpython_api, cpython_struct, bootstrap_function, build_type_checkers, + PyObjectFields, Py_ssize_t, CONST_STRING) +from pypy.module.cpyext.pyobject import ( + PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference, + make_typedescr, get_typedescr) + +## +## Implementation of PyStringObject +## ================================ +## +## The problem +## ----------- +## +## PyString_AsString() must returns a (non-movable) pointer to the underlying +## buffer, whereas pypy strings are movable. C code may temporarily store +## this address and use it, as long as it owns a reference to the PyObject. +## There is no "release" function to specify that the pointer is not needed +## any more. +## +## Also, the pointer may be used to fill the initial value of string. This is +## valid only when the string was just allocated, and is not used elsewhere. +## +## Solution +## -------- +## +## PyStringObject contains two additional members: the size and a pointer to a +## char buffer; it may be NULL. +## +## - A string allocated by pypy will be converted into a PyStringObject with a +## NULL buffer. The first time PyString_AsString() is called, memory is +## allocated (with flavor='raw') and content is copied. +## +## - A string allocated with PyString_FromStringAndSize(NULL, size) will +## allocate a PyStringObject structure, and a buffer with the specified +## size, but the reference won't be stored in the global map; there is no +## corresponding object in pypy. When from_ref() or Py_INCREF() is called, +## the pypy string is created, and added to the global map of tracked +## objects. The buffer is then supposed to be immutable. +## +## - _PyString_Resize() works only on not-yet-pypy'd strings, and returns a +## similar object. +## +## - PyString_Size() doesn't need to force the object. +## +## - There could be an (expensive!) check in from_ref() that the buffer still +## corresponds to the pypy gc-managed string. +## PyStringObjectStruct = lltype.ForwardReference() PyStringObject = lltype.Ptr(PyStringObjectStruct) @@ -15,6 +57,7 @@ @bootstrap_function def init_stringobject(space): + "Type description of PyStringObject" make_typedescr(space.w_str.instancetypedef, basestruct=PyStringObject.TO, attach=string_attach, @@ -24,40 +67,53 @@ PyString_Check, PyString_CheckExact = build_type_checkers("String", "w_str") def new_empty_str(space, length): - py_str = lltype.malloc(PyStringObject.TO, flavor='raw') - py_str.c_ob_refcnt = 1 - + """ + Allocatse a PyStringObject and its buffer, but without a corresponding + interpreter object. The buffer may be mutated, until string_realize() is + called. + """ + typedescr = get_typedescr(space.w_str.instancetypedef) + py_obj = typedescr.allocate(space, space.w_str) + py_str = rffi.cast(PyStringObject, py_obj) + buflen = length + 1 + py_str.c_size = length py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, buflen, flavor='raw', zero=True) - py_str.c_size = length - py_str.c_ob_type = rffi.cast(PyTypeObjectPtr, make_ref(space, space.w_str)) return py_str def string_attach(space, py_obj, w_obj): + """ + Fills a newly allocated PyStringObject with the given string object. The + buffer must not be modified. + """ py_str = rffi.cast(PyStringObject, py_obj) py_str.c_size = len(space.str_w(w_obj)) py_str.c_buffer = lltype.nullptr(rffi.CCHARP.TO) -def string_realize(space, ref): - state = space.fromcache(State) - ref = rffi.cast(PyStringObject, ref) - s = rffi.charpsize2str(ref.c_buffer, ref.c_size) - ref = rffi.cast(PyObject, ref) - w_str = space.wrap(s) - state.py_objects_w2r[w_str] = ref - ptr = rffi.cast(ADDR, ref) - state.py_objects_r2w[ptr] = w_str - return w_str +def string_realize(space, py_obj): + """ + Creates the string in the interpreter. The PyStringObject buffer must not + be modified after this call. + """ + py_str = rffi.cast(PyStringObject, py_obj) + s = rffi.charpsize2str(py_str.c_buffer, py_str.c_size) + w_obj = space.wrap(s) + track_reference(space, py_obj, w_obj) + return w_obj @cpython_api([PyObject], lltype.Void, external=False) def string_dealloc(space, py_obj): + """Frees allocated PyStringObject resources. + """ py_str = rffi.cast(PyStringObject, py_obj) if py_str.c_buffer: lltype.free(py_str.c_buffer, flavor="raw") from pypy.module.cpyext.object import PyObject_dealloc PyObject_dealloc(space, py_obj) +#_______________________________________________________________________ + @cpython_api([CONST_STRING, Py_ssize_t], PyObject) def PyString_FromStringAndSize(space, char_p, length): if char_p: Modified: pypy/branch/py12/pypy/module/cpyext/stubgen.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/stubgen.py (original) +++ pypy/branch/py12/pypy/module/cpyext/stubgen.py Thu May 6 18:18:13 2010 @@ -7,9 +7,10 @@ TEMPLATE = """ - at cpython_api([%(paramtypes)s], %(rettype)s%(borrows)s) + at cpython_api([%(paramtypes)s], %(rettype)s) def %(functionname)s(%(params)s): %(docstring)s raise NotImplementedError + %(borrows)s """ C_TYPE_TO_PYPY_TYPE = { @@ -77,7 +78,7 @@ if entry.result_refs is None: docstring += "\nReturn value: always NULL." else: - borrows = (", borrowed=True", "")[entry.result_refs] + borrows = ("borrow_from()", "")[entry.result_refs] docstring = "\n ".join(docstring.splitlines()) if docstring: docstring = ' """%s"""\n' % (docstring,) Modified: pypy/branch/py12/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/py12/pypy/module/cpyext/stubs.py Thu May 6 18:18:13 2010 @@ -32,16 +32,17 @@ def _PyObject_Del(space, op): raise NotImplementedError - at cpython_api([rffi.CCHARP, PyMethodDef], PyObject, borrowed=True) + at cpython_api([rffi.CCHARP, PyMethodDef], PyObject) def Py_InitModule(space, name, methods): """Create a new module object based on a name and table of functions, returning the new module object. Older versions of Python did not support NULL as the value for the methods argument.""" + borrow_from() raise NotImplementedError - at cpython_api([rffi.CCHARP, PyMethodDef, rffi.CCHARP], PyObject, borrowed=True) + at cpython_api([rffi.CCHARP, PyMethodDef, rffi.CCHARP], PyObject) def Py_InitModule3(space, name, methods, doc): """Create a new module object based on a name and table of functions, returning the new module object. If doc is non-NULL, it will be used @@ -49,6 +50,7 @@ Older versions of Python did not support NULL as the value for the methods argument.""" + borrow_from() raise NotImplementedError @cpython_api([PyObject, rffi.CCHARP, va_list], rffi.INT_real, error=0) @@ -337,10 +339,11 @@ """Return the contents of the cell cell.""" raise NotImplementedError - at cpython_api([PyObject], PyObject, borrowed=True) + at cpython_api([PyObject], PyObject) def PyCell_GET(space, cell): """Return the contents of the cell cell, but without checking that cell is non-NULL and a cell object.""" + borrow_from() raise NotImplementedError @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) @@ -1059,9 +1062,10 @@ raised if the end of the file is reached immediately.""" raise NotImplementedError - at cpython_api([PyObject], PyObject, borrowed=True) + at cpython_api([PyObject], PyObject) def PyFile_Name(space, p): """Return the name of the file specified by p as a string object.""" + borrow_from() raise NotImplementedError @cpython_api([PyFileObject, rffi.INT_real], lltype.Void) @@ -1184,27 +1188,31 @@ object, the argument defaults and closure are set to NULL.""" raise NotImplementedError - at cpython_api([PyObject], PyObject, borrowed=True) + at cpython_api([PyObject], PyObject) def PyFunction_GetCode(space, op): """Return the code object associated with the function object op.""" + borrow_from() raise NotImplementedError - at cpython_api([PyObject], PyObject, borrowed=True) + at cpython_api([PyObject], PyObject) def PyFunction_GetGlobals(space, op): """Return the globals dictionary associated with the function object op.""" + borrow_from() raise NotImplementedError - at cpython_api([PyObject], PyObject, borrowed=True) + at cpython_api([PyObject], PyObject) def PyFunction_GetModule(space, op): """Return the __module__ attribute of the function object op. This is normally a string containing the module name, but can be set to any other object by Python code.""" + borrow_from() raise NotImplementedError - at cpython_api([PyObject], PyObject, borrowed=True) + at cpython_api([PyObject], PyObject) def PyFunction_GetDefaults(space, op): """Return the argument default values of the function object op. This can be a tuple of arguments or NULL.""" + borrow_from() raise NotImplementedError @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) @@ -1215,10 +1223,11 @@ Raises SystemError and returns -1 on failure.""" raise NotImplementedError - at cpython_api([PyObject], PyObject, borrowed=True) + at cpython_api([PyObject], PyObject) def PyFunction_GetClosure(space, op): """Return the closure associated with the function object op. This can be NULL or a tuple of cell objects.""" + borrow_from() raise NotImplementedError @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) @@ -1352,7 +1361,7 @@ with an exception set on failure (the module still exists in this case).""" raise NotImplementedError - at cpython_api([rffi.CCHARP], PyObject, borrowed=True) + at cpython_api([rffi.CCHARP], PyObject) def PyImport_AddModule(space, name): """Return the module object corresponding to a module name. The name argument may be of the form package.module. First check the modules dictionary if @@ -1363,6 +1372,7 @@ loaded, you will get an empty module object. Use PyImport_ImportModule() or one of its variants to import a module. Package structures implied by a dotted name for name are not created if not already present.""" + borrow_from() raise NotImplementedError @cpython_api([rffi.CCHARP, PyObject], PyObject) @@ -1398,10 +1408,11 @@ of the bytecode file, in little-endian byte order.""" raise NotImplementedError - at cpython_api([], PyObject, borrowed=True) + at cpython_api([], PyObject) def PyImport_GetModuleDict(space, ): """Return the dictionary used for the module administration (a.k.a. sys.modules). Note that this is a per-interpreter variable.""" + borrow_from() raise NotImplementedError @cpython_api([PyObject], PyObject) @@ -1870,7 +1881,7 @@ the caller needn't check for NULL).""" raise NotImplementedError - at cpython_api([], PyObject, borrowed=True) + at cpython_api([], PyObject) def PyThreadState_GetDict(space, ): """Return a dictionary in which extensions can store thread-specific state information. Each extension should use a unique key to use to store state in @@ -1880,6 +1891,7 @@ Previously this could only be called when a current thread is active, and NULL meant that an exception was raised.""" + borrow_from() raise NotImplementedError @cpython_api([lltype.Signed, PyObject], rffi.INT_real, error=CANNOT_FAIL) @@ -2103,12 +2115,13 @@ """ raise NotImplementedError - at cpython_api([PyObject, Py_ssize_t], PyObject, borrowed=True) + at cpython_api([PyObject, Py_ssize_t], PyObject) def PyList_GET_ITEM(space, list, i): """Macro form of PyList_GetItem() without error checking. This macro used an int for i. This might require changes in your code for properly supporting 64-bit systems.""" + borrow_from() raise NotImplementedError @cpython_api([PyObject, Py_ssize_t, PyObject], lltype.Void) @@ -2293,15 +2306,17 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject], PyObject, borrowed=True) + at cpython_api([PyObject], PyObject) def PyMethod_Self(space, meth): """Return the instance associated with the method meth if it is bound, otherwise return NULL.""" + borrow_from() raise NotImplementedError - at cpython_api([PyObject], PyObject, borrowed=True) + at cpython_api([PyObject], PyObject) def PyMethod_GET_SELF(space, meth): """Macro version of PyMethod_Self() which avoids error checking.""" + borrow_from() raise NotImplementedError @cpython_api([], rffi.INT_real, error=CANNOT_FAIL) @@ -2442,28 +2457,32 @@ is active then NULL is returned but PyErr_Occurred() will return false.""" raise NotImplementedError - at cpython_api([], PyObject, borrowed=True) + at cpython_api([], PyObject) def PyEval_GetBuiltins(space, ): """Return a dictionary of the builtins in the current execution frame, or the interpreter of the thread state if no frame is currently executing.""" + borrow_from() raise NotImplementedError - at cpython_api([], PyObject, borrowed=True) + at cpython_api([], PyObject) def PyEval_GetLocals(space, ): """Return a dictionary of the local variables in the current execution frame, or NULL if no frame is currently executing.""" + borrow_from() raise NotImplementedError - at cpython_api([], PyObject, borrowed=True) + at cpython_api([], PyObject) def PyEval_GetGlobals(space, ): """Return a dictionary of the global variables in the current execution frame, or NULL if no frame is currently executing.""" + borrow_from() raise NotImplementedError - at cpython_api([], PyFrameObject, borrowed=True) + at cpython_api([], PyFrameObject) def PyEval_GetFrame(space, ): """Return the current thread state's frame, which is NULL if no frame is currently executing.""" + borrow_from() raise NotImplementedError @cpython_api([PyFrameObject], rffi.INT_real, error=CANNOT_FAIL) @@ -2958,12 +2977,13 @@ the cleanup function, no Python APIs should be called by func.""" raise NotImplementedError - at cpython_api([PyObject, Py_ssize_t], PyObject, borrowed=True) + at cpython_api([PyObject, Py_ssize_t], PyObject) def PyTuple_GET_ITEM(space, p, pos): """Like PyTuple_GetItem(), but does no checking of its arguments. This function used an int type for pos. This might require changes in your code for properly supporting 64-bit systems.""" + borrow_from() raise NotImplementedError @cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject) @@ -3856,9 +3876,10 @@ """ raise NotImplementedError - at cpython_api([PyObject], PyObject, borrowed=True) + at cpython_api([PyObject], PyObject) def PyWeakref_GET_OBJECT(space, ref): """Similar to PyWeakref_GetObject(), but implemented as a macro that does no error checking. """ + borrow_from() raise NotImplementedError Modified: pypy/branch/py12/pypy/module/cpyext/sysmodule.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/sysmodule.py (original) +++ pypy/branch/py12/pypy/module/cpyext/sysmodule.py Thu May 6 18:18:13 2010 @@ -1,17 +1,16 @@ from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import CANNOT_FAIL, cpython_api, CONST_STRING -from pypy.module.cpyext.pyobject import PyObject, register_container +from pypy.module.cpyext.pyobject import PyObject, borrow_from - at cpython_api([CONST_STRING], PyObject, borrowed=True, error=CANNOT_FAIL) + at cpython_api([CONST_STRING], PyObject, error=CANNOT_FAIL) def PySys_GetObject(space, name): """Return the object name from the sys module or NULL if it does not exist, without setting an exception.""" name = rffi.charp2str(name) w_dict = space.sys.getdict() w_obj = space.finditem_str(w_dict, name) - register_container(space, w_dict) - return w_obj + return borrow_from(w_dict, w_obj) @cpython_api([CONST_STRING, PyObject], rffi.INT_real, error=-1) def PySys_SetObject(space, name, w_obj): Modified: pypy/branch/py12/pypy/module/cpyext/test/foo.c ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/test/foo.c (original) +++ pypy/branch/py12/pypy/module/cpyext/test/foo.c Thu May 6 18:18:13 2010 @@ -2,8 +2,8 @@ #include "structmember.h" typedef struct { - PyObject_HEAD - int foo; /* the context holder */ + PyObject_HEAD + int foo; /* the context holder */ PyObject *foo_object; char *foo_string; char foo_string_inplace[5]; @@ -14,17 +14,17 @@ static fooobject * newfooobject(void) { - fooobject *foop; + fooobject *foop; - foop = PyObject_New(fooobject, &footype); - if (foop == NULL) - return NULL; - - foop->foo = 42; - foop->foo_object = NULL; - foop->foo_string = "Hello from PyPy"; - strncpy(foop->foo_string_inplace, "spam", 5); - return foop; + foop = PyObject_New(fooobject, &footype); + if (foop == NULL) + return NULL; + + foop->foo = 42; + foop->foo_object = NULL; + foop->foo_string = "Hello from PyPy"; + strncpy(foop->foo_string_inplace, "spam", 5); + return foop; } @@ -33,7 +33,7 @@ static void foo_dealloc(fooobject *foop) { - PyObject_Del(foop); + PyObject_Del(foop); } @@ -42,14 +42,14 @@ static PyObject * foo_copy(fooobject *self) { - fooobject *foop; + fooobject *foop; - if ((foop = newfooobject()) == NULL) - return NULL; + if ((foop = newfooobject()) == NULL) + return NULL; - foop->foo = self->foo; + foop->foo = self->foo; - return (PyObject *)foop; + return (PyObject *)foop; } static PyObject * @@ -61,9 +61,9 @@ static PyMethodDef foo_methods[] = { - {"copy", (PyCFunction)foo_copy, METH_NOARGS, NULL}, - {"unset_string_member", (PyCFunction)foo_unset, METH_NOARGS, NULL}, - {NULL, NULL} /* sentinel */ + {"copy", (PyCFunction)foo_copy, METH_NOARGS, NULL}, + {"unset_string_member", (PyCFunction)foo_unset, METH_NOARGS, NULL}, + {NULL, NULL} /* sentinel */ }; static PyObject * @@ -126,37 +126,37 @@ }; static PyTypeObject footype = { - PyVarObject_HEAD_INIT(NULL, 0) - "foo.foo", /*tp_name*/ - sizeof(fooobject), /*tp_size*/ - 0, /*tp_itemsize*/ - /* methods */ - (destructor)foo_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare*/ - foo_repr, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - foo_call, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT, /*tp_flags*/ - 0, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - foo_methods, /*tp_methods*/ - foo_members, /*tp_members*/ - foo_getseters, /*tp_getset*/ + PyVarObject_HEAD_INIT(NULL, 0) + "foo.foo", /*tp_name*/ + sizeof(fooobject), /*tp_size*/ + 0, /*tp_itemsize*/ + /* methods */ + (destructor)foo_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + foo_repr, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + foo_call, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + foo_methods, /*tp_methods*/ + foo_members, /*tp_members*/ + foo_getseters, /*tp_getset*/ }; typedef struct { @@ -175,6 +175,12 @@ {NULL} /* Sentinel */ }; +static PyObject * +Fuu_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds) +{ + return subtype->tp_alloc(subtype, 0); +} + PyTypeObject FuuType = { PyObject_HEAD_INIT(NULL) 0, @@ -224,7 +230,7 @@ 0, /*tp_init*/ 0, /*tp_alloc will be set to PyType_GenericAlloc in module init*/ - 0, /*tp_new*/ + Fuu_new, /*tp_new*/ 0, /*tp_free Low-level free-memory routine */ 0, /*tp_is_gc For PyObject_IS_GC */ 0, /*tp_bases*/ @@ -240,21 +246,20 @@ static PyObject * foo_new(PyObject *self, PyObject *args) { - fooobject *foop; + fooobject *foop; - if ((foop = newfooobject()) == NULL) { - return NULL; - } - - return (PyObject *)foop; + if ((foop = newfooobject()) == NULL) { + return NULL; + } + + return (PyObject *)foop; } - /* List of functions exported by this module */ static PyMethodDef foo_functions[] = { - {"new", (PyCFunction)foo_new, METH_NOARGS, NULL}, - {NULL, NULL} /* Sentinel */ + {"new", (PyCFunction)foo_new, METH_NOARGS, NULL}, + {NULL, NULL} /* Sentinel */ }; @@ -262,7 +267,7 @@ void initfoo(void) { - PyObject *m, *d; + PyObject *m, *d; footype.ob_type = &PyType_Type; @@ -274,14 +279,14 @@ return; if (PyType_Ready(&FuuType) < 0) return; - m = Py_InitModule("foo", foo_functions); - if (m == NULL) - return; - d = PyModule_GetDict(m); - if (d) { - if (PyDict_SetItemString(d, "fooType", (PyObject *)&footype) < 0) + m = Py_InitModule("foo", foo_functions); + if (m == NULL) + return; + d = PyModule_GetDict(m); + if (d) { + if (PyDict_SetItemString(d, "fooType", (PyObject *)&footype) < 0) return; PyDict_SetItemString(d, "FuuType", (PyObject *) &FuuType); } - /* No need to check the error here, the caller will do that */ + /* No need to check the error here, the caller will do that */ } Modified: pypy/branch/py12/pypy/module/cpyext/test/test_api.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/test/test_api.py (original) +++ pypy/branch/py12/pypy/module/cpyext/test/test_api.py Thu May 6 18:18:13 2010 @@ -7,6 +7,7 @@ PyObject = api.PyObject from pypy.interpreter.error import OperationError from pypy.module.cpyext.state import State +import os @api.cpython_api([PyObject], lltype.Void) def PyPy_GetWrapped(space, w_arg): @@ -17,8 +18,17 @@ class BaseApiTest(LeakCheckingTest): def setup_class(cls): - cls.space = gettestobjspace(usemodules=['cpyext', 'thread']) - cls.space.getbuiltinmodule("cpyext") + cls.space = space = gettestobjspace(usemodules=['cpyext', 'thread']) + + # warm up reference counts: + # - the posix module allocates a HCRYPTPROV on Windows + # - writing to stderr allocates a file lock + space.getbuiltinmodule("cpyext") + space.getbuiltinmodule(os.name) + space.call_function(space.getattr(space.sys.get("stderr"), + space.wrap("write")), + space.wrap("")) + class CAPI: def __getattr__(self, name): return getattr(cls.space, name) @@ -30,10 +40,10 @@ raise Exception("%s is not callable" % (f,)) f(*args) state = space.fromcache(State) - tp = state.exc_type - if not tp: + operror = state.operror + if not operror: raise Exception("DID NOT RAISE") - if getattr(space, 'w_' + expected_exc.__name__) is not tp: + if getattr(space, 'w_' + expected_exc.__name__) is not operror.w_type: raise Exception("Wrong exception") state.clear_exception() Modified: pypy/branch/py12/pypy/module/cpyext/test/test_borrow.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/test/test_borrow.py (original) +++ pypy/branch/py12/pypy/module/cpyext/test/test_borrow.py Thu May 6 18:18:13 2010 @@ -1,21 +1,17 @@ import py from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.test.test_api import BaseApiTest -from pypy.module.cpyext.state import State -from pypy.module.cpyext.pyobject import make_ref, add_borrowed_object +from pypy.module.cpyext.pyobject import make_ref, borrow_from, RefcountState class TestBorrowing(BaseApiTest): def test_borrowing(self, space, api): - state = space.fromcache(State) w_int = space.wrap(1) w_tuple = space.newtuple([w_int]) api.Py_IncRef(w_tuple) - api.register_container(w_tuple) - one_pyo = make_ref(space, w_int, borrowed=True) - add_borrowed_object(space, one_pyo) - print state.borrowed_objects + one_pyo = borrow_from(w_tuple, w_int).get_ref(space) api.Py_DecRef(w_tuple) + state = space.fromcache(RefcountState) state.print_refcounts() py.test.raises(AssertionError, api.Py_DecRef, one_pyo) Modified: pypy/branch/py12/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/py12/pypy/module/cpyext/test/test_cpyext.py Thu May 6 18:18:13 2010 @@ -12,6 +12,7 @@ from pypy.tool.udir import udir from pypy.module.cpyext import api, typeobject from pypy.module.cpyext.state import State +from pypy.module.cpyext.pyobject import RefcountState from pypy.module.cpyext.pyobject import Py_DecRef, InvalidPointerException from pypy.translator.goal import autopath from pypy.lib.identity_dict import identity_dict @@ -71,7 +72,7 @@ return str(pydname) def freeze_refcnts(self): - state = self.space.fromcache(State) + state = self.space.fromcache(RefcountState) self.frozen_refcounts = {} for w_obj, obj in state.py_objects_w2r.iteritems(): self.frozen_refcounts[w_obj] = obj.c_ob_refcnt @@ -85,7 +86,7 @@ import gc leaking = False - state = self.space.fromcache(State) + state = self.space.fromcache(RefcountState) gc.collect() lost_objects_w = identity_dict() lost_objects_w.update((key, None) for key in self.frozen_refcounts.keys()) @@ -136,6 +137,8 @@ cls.space.getbuiltinmodule("cpyext") from pypy.module.imp.importing import importhook importhook(cls.space, "os") # warm up reference counts + state = cls.space.fromcache(RefcountState) + state.non_heaptypes_w[:] = [] def compile_module(self, name, **kwds): """ @@ -258,16 +261,11 @@ def teardown_method(self, func): for name in self.imported_module_names: self.unimport_module(name) - state = self.space.fromcache(State) - for w_obj in state.non_heaptypes: - Py_DecRef(self.space, w_obj) - state.non_heaptypes[:] = [] - while state.borrowed_objects: - addr = state.borrowed_objects.keys()[0] - w_obj = state.py_objects_r2w[addr] + state = self.space.fromcache(RefcountState) + for w_obj in state.non_heaptypes_w: Py_DecRef(self.space, w_obj) - state.borrowed_objects = {} - state.borrow_mapping = {} + state.non_heaptypes_w[:] = [] + state.reset_borrowed_references() if self.check_and_print_leaks(): assert False, "Test leaks or loses object(s)." Modified: pypy/branch/py12/pypy/module/cpyext/test/test_getargs.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/test/test_getargs.py (original) +++ pypy/branch/py12/pypy/module/cpyext/test/test_getargs.py Thu May 6 18:18:13 2010 @@ -47,9 +47,11 @@ if (!PyArg_ParseTuple(args, "O|O", &a, &b)) { return NULL; } - if (!b) { + if (b) + Py_INCREF(b); + else b = PyInt_FromLong(42); - } + /* return an owned reference */ return b; ''')]) assert mod.oneargint(1) == 1 Modified: pypy/branch/py12/pypy/module/cpyext/test/test_import.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/test/test_import.py (original) +++ pypy/branch/py12/pypy/module/cpyext/test/test_import.py Thu May 6 18:18:13 2010 @@ -2,11 +2,6 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase class TestImport(BaseApiTest): - def setup_method(self, func): - from pypy.module.imp.importing import importhook - importhook(self.space, "os") # warm up reference counts - BaseApiTest.setup_method(self, func) - def test_import(self, space, api): pdb = api.PyImport_Import(space.wrap("pdb")) assert pdb Modified: pypy/branch/py12/pypy/module/cpyext/test/test_mapping.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/test/test_mapping.py (original) +++ pypy/branch/py12/pypy/module/cpyext/test/test_mapping.py Thu May 6 18:18:13 2010 @@ -26,7 +26,7 @@ key = rffi.str2charp("key") api.PyMapping_SetItemString(w_d, key, space.wrap(42)) assert 42 == space.unwrap( - api.PyMapping_GetItemString(w_d, key, space.wrap(42))) + api.PyMapping_GetItemString(w_d, key)) rffi.free_charp(key) def test_haskey(self, space, api): Modified: pypy/branch/py12/pypy/module/cpyext/test/test_pyerrors.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/test/test_pyerrors.py (original) +++ pypy/branch/py12/pypy/module/cpyext/test/test_pyerrors.py Thu May 6 18:18:13 2010 @@ -50,37 +50,40 @@ api.PyErr_SetObject(space.w_ValueError, space.wrap("a value")) assert api.PyErr_Occurred() is space.w_ValueError state = space.fromcache(State) - assert space.eq_w(state.exc_value, space.wrap("a value")) + assert space.eq_w(state.operror.get_w_value(space), + space.wrap("a value")) api.PyErr_Clear() def test_SetNone(self, space, api): api.PyErr_SetNone(space.w_KeyError) state = space.fromcache(State) - assert space.eq_w(state.exc_type, space.w_KeyError) - assert space.eq_w(state.exc_value, space.w_None) + assert space.eq_w(state.operror.w_type, space.w_KeyError) + assert space.eq_w(state.operror.get_w_value(space), space.w_None) api.PyErr_Clear() api.PyErr_NoMemory() - assert space.eq_w(state.exc_type, space.w_MemoryError) + assert space.eq_w(state.operror.w_type, space.w_MemoryError) api.PyErr_Clear() def test_BadArgument(self, space, api): api.PyErr_BadArgument() state = space.fromcache(State) - assert space.eq_w(state.exc_type, space.w_TypeError) + assert space.eq_w(state.operror.w_type, space.w_TypeError) api.PyErr_Clear() + def test_Warning(self, space, api, capfd): + message = rffi.str2charp("this is a warning") + api.PyErr_WarnEx(None, message, 1) + out, err = capfd.readouterr() + assert ": UserWarning: this is a warning" in err + rffi.free_charp(message) + class AppTestFetch(AppTestCpythonExtensionBase): def setup_class(cls): AppTestCpythonExtensionBase.setup_class.im_func(cls) space = cls.space - def set_errno(num): - ll2ctypes.TLS.errno = num - - cls.w_set_errno = space.wrap(interp2app(set_errno, unwrap_spec=[int])) - def test_occurred(self): module = self.import_extension('foo', [ ("check_error", "METH_NOARGS", @@ -92,7 +95,7 @@ ''' ), ]) - module.check_error() + assert module.check_error() def test_fetch_and_restore(self): module = self.import_extension('foo', [ @@ -106,8 +109,6 @@ return NULL; if (type != PyExc_TypeError) Py_RETURN_FALSE; - if (val->ob_type != type) - Py_RETURN_FALSE; PyErr_Restore(type, val, tb); if (!PyErr_Occurred()) Py_RETURN_FALSE; @@ -116,26 +117,22 @@ ''' ), ]) - module.check_error() + assert module.check_error() def test_SetFromErrno(self): - skip("The test does not set the errno in a way which " - "untranslated pypy can actually notice") - - import errno + import errno, os module = self.import_extension('foo', [ ("set_from_errno", "METH_NOARGS", ''' + errno = EBADF; PyErr_SetFromErrno(PyExc_OSError); return NULL; '''), - ]) + ], + prologue="#include ") try: - self.set_errno(errno.EBADF) module.set_from_errno() except OSError, e: assert e.errno == errno.EBADF - assert e.message == os.strerror(errno.EBADF) - - + assert e.strerror == os.strerror(errno.EBADF) Modified: pypy/branch/py12/pypy/module/cpyext/test/test_thread.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/test/test_thread.py (original) +++ pypy/branch/py12/pypy/module/cpyext/test/test_thread.py Thu May 6 18:18:13 2010 @@ -11,7 +11,7 @@ def test_get_thread_ident(self, space, api): results = [] def some_thread(): - res = api.PyThread_get_thread_ident(space) + res = api.PyThread_get_thread_ident() results.append((res, thread.get_ident())) some_thread() Modified: pypy/branch/py12/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/py12/pypy/module/cpyext/test/test_typeobject.py Thu May 6 18:18:13 2010 @@ -1,6 +1,8 @@ +from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref +from pypy.module.cpyext.typeobject import PyTypeObjectPtr import py import sys @@ -19,7 +21,7 @@ print "type of obj has type", type(type(obj)) print "type of type of obj has type", type(type(type(obj))) - def test_typeobject2(self): + def test_typeobject_method_descriptor(self): module = self.import_module(name='foo') obj = module.new() obj2 = obj.copy() @@ -37,7 +39,7 @@ assert obj.foo == 42 assert obj.int_member == obj.foo - def test_typeobject3(self): + def test_typeobject_data_member(self): module = self.import_module(name='foo') obj = module.new() obj.int_member = 23 @@ -53,7 +55,7 @@ assert module.fooType.broken_member.__doc__ is None assert module.fooType.object_member.__doc__ == "A Python object." - def test_typeobject4(self): + def test_typeobject_object_member(self): module = self.import_module(name='foo') obj = module.new() assert obj.object_member is None @@ -70,7 +72,7 @@ del obj.object_member_ex raises(AttributeError, "del obj.object_member_ex") - def test_typeobject5(self): + def test_typeobject_string_member(self): module = self.import_module(name='foo') obj = module.new() assert obj.string_member == "Hello from PyPy" @@ -87,6 +89,14 @@ raises(TypeError, "obj.char_member = 'spam'") raises(TypeError, "obj.char_member = 42") + def test_new(self): + skip("skip") + module = self.import_module(name='foo') + obj = module.new() + # call __new__ + newobj = module.FuuType(u"xyz") + assert newobj == u"xyz" + a = module.fooType assert "cannot create" in raises(TypeError, "a()").value.message class bar(module.fooType): @@ -119,6 +129,20 @@ re._cache_repl.clear() class TestTypes(BaseApiTest): + def test_type_attributes(self, space, api): + w_class = space.appexec([], """(): + class A(object): + pass + return A + """) + ref = make_ref(space, w_class) + + py_type = rffi.cast(PyTypeObjectPtr, ref) + assert py_type.c_tp_alloc + assert from_ref(space, py_type.c_tp_mro).wrappeditems is w_class.mro_w + + api.Py_DecRef(ref) + def test_multiple_inheritance(self, space, api): w_class = space.appexec([], """(): class A(object): Modified: pypy/branch/py12/pypy/module/cpyext/test/test_weakref.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/test/test_weakref.py (original) +++ pypy/branch/py12/pypy/module/cpyext/test/test_weakref.py Thu May 6 18:18:13 2010 @@ -7,8 +7,23 @@ w_ref = api.PyWeakref_NewRef(w_obj, space.w_None) assert w_ref is not None assert space.is_w(api.PyWeakref_GetObject(w_ref), w_obj) + assert space.is_w(api.PyWeakref_LockObject(w_ref), w_obj) w_obj = space.newtuple([]) assert api.PyWeakref_NewRef(w_obj, space.w_None) is None assert api.PyErr_Occurred() is space.w_TypeError api.PyErr_Clear() + + def test_weakref_lockobject(self, space, api): + # some new weakrefable object + w_obj = space.call_function(space.w_type, space.wrap("newtype"), + space.newtuple([]), space.newdict()) + assert w_obj is not None + + w_ref = api.PyWeakref_NewRef(w_obj, space.w_None) + assert w_obj is not None + + assert space.is_w(api.PyWeakref_LockObject(w_ref), w_obj) + del w_obj + import gc; gc.collect() + assert space.is_w(api.PyWeakref_LockObject(w_ref), space.w_None) Modified: pypy/branch/py12/pypy/module/cpyext/tupleobject.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/tupleobject.py (original) +++ pypy/branch/py12/pypy/module/cpyext/tupleobject.py Thu May 6 18:18:13 2010 @@ -2,7 +2,7 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL, build_type_checkers) -from pypy.module.cpyext.pyobject import PyObject, Py_DecRef, register_container +from pypy.module.cpyext.pyobject import PyObject, Py_DecRef, borrow_from from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.objspace.std.tupleobject import W_TupleObject @@ -23,14 +23,13 @@ Py_DecRef(space, w_obj) # SetItem steals a reference! return 0 - at cpython_api([PyObject, Py_ssize_t], PyObject, borrowed=True) + at cpython_api([PyObject, Py_ssize_t], PyObject) def PyTuple_GetItem(space, w_t, pos): if not PyTuple_Check(space, w_t): PyErr_BadInternalCall(space) assert isinstance(w_t, W_TupleObject) w_obj = w_t.wrappeditems[pos] - register_container(space, w_t) - return w_obj + return borrow_from(w_t, w_obj) @cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL) def PyTuple_GET_SIZE(space, w_t): Modified: pypy/branch/py12/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/py12/pypy/module/cpyext/typeobject.py Thu May 6 18:18:13 2010 @@ -6,18 +6,18 @@ from pypy.rlib.rweakref import RWeakKeyDictionary from pypy.interpreter.gateway import ObjSpace, W_Root, Arguments from pypy.interpreter.gateway import interp2app, unwrap_spec -from pypy.interpreter.baseobjspace import Wrappable, DescrMismatch +from pypy.interpreter.baseobjspace import DescrMismatch from pypy.objspace.std.typeobject import W_TypeObject, _CPYTYPE, call__Type from pypy.objspace.std.typetype import _precheck_for_new -from pypy.objspace.std.objectobject import W_ObjectObject from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.module.cpyext.api import cpython_api, cpython_struct, bootstrap_function, \ - PyVarObjectFields, Py_ssize_t, Py_TPFLAGS_READYING, generic_cpy_call, \ - Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE, ADDR, \ - Py_TPFLAGS_HAVE_CLASS, METH_VARARGS, METH_KEYWORDS, \ - CANNOT_FAIL, PyBufferProcs, build_type_checkers -from pypy.module.cpyext.pyobject import PyObject, make_ref, create_ref, from_ref -from pypy.module.cpyext.pyobject import get_typedescr, make_typedescr, track_reference +from pypy.module.cpyext.api import ( + cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, + generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING, + Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL, + PyBufferProcs, build_type_checkers) +from pypy.module.cpyext.pyobject import ( + PyObject, make_ref, create_ref, from_ref, get_typedescr, make_typedescr, + track_reference, RefcountState) from pypy.interpreter.module import Module from pypy.interpreter.function import FunctionWithFixedCode, StaticMethod from pypy.module.cpyext import structmemberdefs @@ -106,7 +106,6 @@ def update_all_slots(space, w_obj, pto): # XXX fill slots in pto - state = space.fromcache(State) for method_name, slot_name, slot_func, _, _, _ in slotdefs: w_descr = space.lookup(w_obj, method_name) if w_descr is None: @@ -135,7 +134,6 @@ def add_operators(space, dict_w, pto): # XXX support PyObject_HashNotImplemented - state = space.fromcache(State) for method_name, slot_name, _, wrapper_func, wrapper_func_kwds, doc in slotdefs: if method_name in dict_w: continue @@ -158,11 +156,11 @@ add_tp_new_wrapper(space, dict_w, pto) @cpython_api([PyObject, PyObject, PyObject], PyObject, external=False) -def tp_new_wrapper(space, w_self, w_args, w_kwds): # XXX untested code - args_w = space.listview(w_args)[:] - args_w.insert(0, w_self) - w_args_new = space.newlist(args_w) - return space.call(space.lookup(space.w_type, "__new__"), w_args_new, w_kwds) +def tp_new_wrapper(space, w_self, w_args, w_kwds): + fn = rffi.cast(PyTypeObjectPtr, make_ref(space, w_self)).c_tp_new + pyo = make_ref(space, w_self) + pto = rffi.cast(PyTypeObjectPtr, pyo) + return generic_cpy_call(space, fn, pto, w_args, w_kwds) @specialize.memo() def get_new_method_def(space): @@ -193,11 +191,12 @@ def inherit_special(space, pto, base_pto): # XXX missing: copy basicsize and flags in a magical way flags = rffi.cast(lltype.Signed, pto.c_tp_flags) - base_object_pyo = make_ref(space, space.w_object, steal=True) + base_object_pyo = make_ref(space, space.w_object) base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo) if base_pto != base_object_pto or flags & Py_TPFLAGS_HEAPTYPE: if not pto.c_tp_new: pto.c_tp_new = base_pto.c_tp_new + Py_DecRef(space, base_object_pyo) class PyOLifeline(object): def __init__(self, space, pyo): @@ -219,12 +218,14 @@ class GettersAndSetters: def getter(self, space, w_self): + assert isinstance(self, W_GetSetPropertyEx) check_descr(space, w_self, self.w_type) return generic_cpy_call( space, self.getset.c_get, w_self, self.getset.c_closure) def setter(self, space, w_self, w_value): + assert isinstance(self, W_GetSetPropertyEx) check_descr(space, w_self, self.w_type) res = generic_cpy_call( space, self.getset.c_set, w_self, w_value, @@ -234,14 +235,17 @@ state.check_and_raise_exception() def member_getter(self, space, w_self): + assert isinstance(self, W_MemberDescr) check_descr(space, w_self, self.w_type) return PyMember_GetOne(space, w_self, self.member) def member_delete(self, space, w_self): + assert isinstance(self, W_MemberDescr) check_descr(space, w_self, self.w_type) PyMember_SetOne(space, w_self, self.member, None) def member_setter(self, space, w_self, w_value): + assert isinstance(self, W_MemberDescr) check_descr(space, w_self, self.w_type) PyMember_SetOne(space, w_self, self.member, w_value) @@ -335,11 +339,10 @@ # - object.tp_bases is a tuple # - tuple.tp_bases is a tuple - # insert null placeholders to please make_ref() - state = space.fromcache(State) - state.py_objects_w2r[space.w_type] = lltype.nullptr(PyObject.TO) - state.py_objects_w2r[space.w_object] = lltype.nullptr(PyObject.TO) - state.py_objects_w2r[space.w_tuple] = lltype.nullptr(PyObject.TO) + # insert null placeholders to please create_ref() + track_reference(space, lltype.nullptr(PyObject.TO), space.w_type) + track_reference(space, lltype.nullptr(PyObject.TO), space.w_object) + track_reference(space, lltype.nullptr(PyObject.TO), space.w_tuple) # create the objects py_type = create_ref(space, space.w_type) @@ -362,9 +365,9 @@ pto_tuple.c_tp_bases.c_ob_type = pto_tuple # Restore the mapping - track_reference(space, py_type, space.w_type) - track_reference(space, py_object, space.w_object) - track_reference(space, py_tuple, space.w_tuple) + track_reference(space, py_type, space.w_type, replace=True) + track_reference(space, py_object, space.w_object, replace=True) + track_reference(space, py_tuple, space.w_tuple, replace=True) @cpython_api([PyObject], lltype.Void, external=False) @@ -409,8 +412,10 @@ if segment != 0: raise OperationError(space.w_SystemError, space.wrap ("accessing non-existent string segment")) - pyref = make_ref(space, w_str, steal=True) + pyref = make_ref(space, w_str) ref[0] = PyString_AsString(space, pyref) + # Stolen reference: the object has better exist somewhere else + Py_DecRef(space, pyref) return space.int_w(space.len(w_str)) def setup_string_buffer_procs(space, pto): @@ -423,12 +428,12 @@ @cpython_api([PyObject], lltype.Void, external=False) def type_dealloc(space, obj): - state = space.fromcache(State) obj_pto = rffi.cast(PyTypeObjectPtr, obj) type_pto = obj.c_ob_type base_pyo = rffi.cast(PyObject, obj_pto.c_tp_base) Py_DecRef(space, obj_pto.c_tp_bases) - Py_DecRef(space, obj_pto.c_tp_cache) # lets do it like cpython + Py_DecRef(space, obj_pto.c_tp_mro) + Py_DecRef(space, obj_pto.c_tp_cache) # let's do it like cpython if obj_pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE: if obj_pto.c_tp_as_buffer: lltype.free(obj_pto.c_tp_as_buffer, flavor='raw') @@ -441,7 +446,9 @@ def type_attach(space, py_obj, w_type): - """ Allocates a PyTypeObject from a w_type which must be a PyPy type. """ + """ + Fills a newly allocated PyTypeObject from an existing type. + """ from pypy.module.cpyext.object import PyObject_Del assert isinstance(w_type, W_TypeObject) @@ -471,9 +478,8 @@ w_base = best_base(space, w_type.bases_w) pto.c_tp_base = rffi.cast(PyTypeObjectPtr, make_ref(space, w_base)) - pto.c_tp_bases = lltype.nullptr(PyObject.TO) - PyPyType_Ready(space, pto, w_type) - + finish_type_1(space, pto) + finish_type_2(space, pto, w_type) pto.c_tp_basicsize = rffi.sizeof(typedescr.basestruct) if pto.c_tp_base: @@ -485,11 +491,21 @@ if space.is_w(w_type, space.w_object): pto.c_tp_new = rffi.cast(newfunc, 1) update_all_slots(space, w_type, pto) + pto.c_tp_flags |= Py_TPFLAGS_READY return pto @cpython_api([PyTypeObjectPtr], rffi.INT_real, error=-1) def PyType_Ready(space, pto): - return PyPyType_Ready(space, pto, None) + if pto.c_tp_flags & Py_TPFLAGS_READY: + return 0 + assert pto.c_tp_flags & Py_TPFLAGS_READYING == 0 + pto.c_tp_flags |= Py_TPFLAGS_READYING + try: + type_realize(space, rffi.cast(PyObject, pto)) + pto.c_tp_flags |= Py_TPFLAGS_READY + finally: + pto.c_tp_flags &= ~Py_TPFLAGS_READYING + return 0 def solid_base(space, w_type): typedef = w_type.instancetypedef @@ -541,62 +557,68 @@ finally: Py_DecRef(space, base_pyo) -def type_realize(space, ref): - PyPyType_Ready(space, rffi.cast(PyTypeObjectPtr, ref), None) - return from_ref(space, ref, True) +def type_realize(space, py_obj): + """ + Creates an interpreter type from a PyTypeObject structure. + """ + # missing: + # setting __doc__ if not defined and tp_doc defined + # inheriting tp_as_* slots + # unsupported: + # tp_mro, tp_subclasses + py_type = rffi.cast(PyTypeObjectPtr, py_obj) + + if not py_type.c_tp_base: + # borrowed reference, but w_object is unlikely to disappear + base = make_ref(space, space.w_object) + Py_DecRef(space, base) + py_type.c_tp_base = rffi.cast(PyTypeObjectPtr, base) + + finish_type_1(space, py_type) + + w_obj = space.allocate_instance( + W_PyCTypeObject, + space.gettypeobject(W_PyCTypeObject.typedef)) + track_reference(space, py_obj, w_obj) + w_obj.__init__(space, py_type) + w_obj.ready() -def PyPyType_Ready(space, pto, w_obj): - try: - pto.c_tp_dict = lltype.nullptr(PyObject.TO) # not supported - if pto.c_tp_flags & Py_TPFLAGS_READY: - return 0 - assert pto.c_tp_flags & Py_TPFLAGS_READYING == 0 - pto.c_tp_flags |= Py_TPFLAGS_READYING - base = pto.c_tp_base + finish_type_2(space, py_type, w_obj) + + state = space.fromcache(RefcountState) + state.non_heaptypes_w.append(w_obj) + + return w_obj + +def finish_type_1(space, pto): + """ + Sets up tp_bases, necessary before creating the interpreter type. + """ + pto.c_tp_dict = lltype.nullptr(PyObject.TO) # not supported + + base = pto.c_tp_base + base_pyo = rffi.cast(PyObject, pto.c_tp_base) + if base and not base.c_tp_flags & Py_TPFLAGS_READY: + type_realize(space, rffi.cast(PyObject, base_pyo)) + if base and not pto.c_ob_type: # will be filled later + pto.c_ob_type = base.c_ob_type + if not pto.c_tp_bases: if not base: - base_pyo = make_ref(space, space.w_object, steal=True) - base = pto.c_tp_base = rffi.cast(PyTypeObjectPtr, base_pyo) + bases = space.newtuple([]) else: - base_pyo = rffi.cast(PyObject, base) - if base and not base.c_tp_flags & Py_TPFLAGS_READY: - PyPyType_Ready(space, base, None) - if base and not pto.c_ob_type: # will be filled later - pto.c_ob_type = base.c_ob_type - if not pto.c_tp_bases: - if not base: - bases = space.newtuple([]) - else: - bases = space.newtuple([from_ref(space, base_pyo)]) - pto.c_tp_bases = make_ref(space, bases) - if w_obj is None: - PyPyType_Register(space, pto) - if base: - inherit_special(space, pto, base) - for w_base in space.fixedview(from_ref(space, pto.c_tp_bases)): - inherit_slots(space, pto, w_base) - # missing: - # setting __doc__ if not defined and tp_doc defined - # inheriting tp_as_* slots - # unsupported: - # tp_mro, tp_subclasses - finally: - pto.c_tp_flags &= ~Py_TPFLAGS_READYING - pto.c_tp_flags = (pto.c_tp_flags & ~Py_TPFLAGS_READYING) | Py_TPFLAGS_READY - return 0 + bases = space.newtuple([from_ref(space, base_pyo)]) + pto.c_tp_bases = make_ref(space, bases) -def PyPyType_Register(space, pto): - state = space.fromcache(State) - ptr = rffi.cast(ADDR, pto) - if ptr not in state.py_objects_r2w: - w_obj = space.allocate_instance(W_PyCTypeObject, - space.gettypeobject(W_PyCTypeObject.typedef)) - state.non_heaptypes.append(w_obj) - pyo = rffi.cast(PyObject, pto) - state.py_objects_r2w[ptr] = w_obj - state.py_objects_w2r[w_obj] = pyo - w_obj.__init__(space, pto) - w_obj.ready() - return 1 +def finish_type_2(space, pto, w_obj): + """ + Sets up other attributes, when the interpreter type has been created. + """ + pto.c_tp_mro = make_ref(space, space.newtuple(w_obj.mro_w)) + base = pto.c_tp_base + if base: + inherit_special(space, pto, base) + for w_base in space.fixedview(from_ref(space, pto.c_tp_bases)): + inherit_slots(space, pto, w_base) @cpython_api([PyTypeObjectPtr, PyTypeObjectPtr], rffi.INT_real, error=CANNOT_FAIL) def PyType_IsSubtype(space, a, b): Modified: pypy/branch/py12/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/py12/pypy/module/cpyext/unicodeobject.py Thu May 6 18:18:13 2010 @@ -3,15 +3,17 @@ from pypy.module.unicodedata import unicodedb_4_1_0 as unicodedb from pypy.module.cpyext.api import ( CANNOT_FAIL, Py_ssize_t, build_type_checkers, cpython_api, - bootstrap_function, generic_cpy_call, PyObjectFields, - cpython_struct, CONST_STRING, CONST_WSTRING) + bootstrap_function, PyObjectFields, cpython_struct, CONST_STRING, + CONST_WSTRING) from pypy.module.cpyext.pyerrors import PyErr_BadArgument -from pypy.module.cpyext.pyobject import PyObject, from_ref, make_ref, Py_DecRef, make_typedescr -from pypy.module.cpyext.stringobject import PyString_Check +from pypy.module.cpyext.pyobject import PyObject, from_ref, make_typedescr from pypy.module.sys.interp_encoding import setdefaultencoding from pypy.objspace.std import unicodeobject, unicodetype import sys +## See comment in stringobject.py. PyUnicode_FromUnicode(NULL, size) is not +## yet supported. + PyUnicodeObjectStruct = lltype.ForwardReference() PyUnicodeObject = lltype.Ptr(PyUnicodeObjectStruct) PyUnicodeObjectFields = (PyObjectFields + @@ -35,6 +37,7 @@ Py_UNICODE = lltype.UniChar def unicode_attach(space, py_obj, w_obj): + "Fills a newly allocated PyUnicodeObject with a unicode string" py_unicode = rffi.cast(PyUnicodeObject, py_obj) py_unicode.c_size = len(space.unicode_w(w_obj)) py_unicode.c_buffer = lltype.nullptr(rffi.CWCHARP.TO) @@ -214,8 +217,7 @@ if not wchar_p: raise NotImplementedError s = rffi.wcharpsize2unicode(wchar_p, length) - ptr = make_ref(space, space.wrap(s)) - return ptr + return space.wrap(s) @cpython_api([CONST_WSTRING, Py_ssize_t], PyObject) def PyUnicode_FromWideChar(space, wchar_p, length): Modified: pypy/branch/py12/pypy/module/cpyext/weakrefobject.py ============================================================================== --- pypy/branch/py12/pypy/module/cpyext/weakrefobject.py (original) +++ pypy/branch/py12/pypy/module/cpyext/weakrefobject.py Thu May 6 18:18:13 2010 @@ -1,5 +1,5 @@ from pypy.module.cpyext.api import cpython_api -from pypy.module.cpyext.pyobject import PyObject, register_container +from pypy.module.cpyext.pyobject import PyObject, borrow_from from pypy.module._weakref.interp__weakref import W_Weakref @cpython_api([PyObject, PyObject], PyObject) @@ -16,11 +16,17 @@ w_weakref = space.gettypeobject(W_Weakref.typedef) return space.call_function(w_weakref, w_obj, w_callback) - at cpython_api([PyObject], PyObject, borrowed=True) + at cpython_api([PyObject], PyObject) def PyWeakref_GetObject(space, w_ref): - """Return the referenced object from a weak reference, ref. If the referent is - no longer live, returns None. + """Return the referenced object from a weak reference. If the referent is + no longer live, returns None. This function returns a borrowed reference. + """ + return borrow_from(w_ref, space.call_function(w_ref)) + + at cpython_api([PyObject], PyObject) +def PyWeakref_LockObject(space, w_ref): + """Return the referenced object from a weak reference. If the referent is + no longer live, returns None. This function returns a new reference. """ - register_container(space, w_ref) return space.call_function(w_ref) Modified: pypy/branch/py12/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/branch/py12/pypy/module/posix/test/test_posix2.py (original) +++ pypy/branch/py12/pypy/module/posix/test/test_posix2.py Thu May 6 18:18:13 2010 @@ -180,16 +180,18 @@ assert 0 def test_filename_exception(self): - for fn in [self.posix.unlink, self.posix.remove, - self.posix.chdir, self.posix.mkdir, self.posix.rmdir, - self.posix.listdir, self.posix.readlink, - self.posix.chroot]: - try: - fn('qowieuqw/oeiu') - except OSError, e: - assert e.filename == 'qowieuqw/oeiu' - else: - assert 0 + for fname in ['unlink', 'remove', + 'chdir', 'mkdir', 'rmdir', + 'listdir', 'readlink', + 'chroot']: + if hasattr(self.posix, fname): + func = getattr(self.posix, fname) + try: + func('qowieuqw/oeiu') + except OSError, e: + assert e.filename == 'qowieuqw/oeiu' + else: + assert 0 def test_chmod_exception(self): try: @@ -200,12 +202,13 @@ assert 0 def test_chown_exception(self): - try: - self.posix.chown('qowieuqw/oeiu', 0, 0) - except OSError, e: - assert e.filename == 'qowieuqw/oeiu' - else: - assert 0 + if hasattr(self.posix, 'chown'): + try: + self.posix.chown('qowieuqw/oeiu', 0, 0) + except OSError, e: + assert e.filename == 'qowieuqw/oeiu' + else: + assert 0 def test_utime_exception(self): for arg in [None, (0, 0)]: Modified: pypy/branch/py12/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/branch/py12/pypy/translator/tool/cbuild.py (original) +++ pypy/branch/py12/pypy/translator/tool/cbuild.py Thu May 6 18:18:13 2010 @@ -270,14 +270,20 @@ if not file_name.check(): break + # XXX this logic should be moved to translator/platform/*.py + d = self._copy_attributes() f = file_name.open("w") - f.write("{\n") - for sym in self.export_symbols: - f.write("%s;\n" % (sym,)) - f.write("};") + if host.name.startswith('darwin'): + for sym in self.export_symbols: + f.write("_%s\n" % (sym,)) + d['link_extra'] += ("-Wl,-exported_symbols_list,"+str(file_name), ) + else: + f.write("{\n") + for sym in self.export_symbols: + f.write("%s;\n" % (sym,)) + f.write("};") + d['link_extra'] += ("-Wl,--dynamic-list=" + str(file_name), ) f.close() - d = self._copy_attributes() - d['link_extra'] += ("-Wl,--dynamic-list=" + str(file_name), ) d['export_symbols'] = () return ExternalCompilationInfo(**d) From hpk at codespeak.net Thu May 6 18:20:35 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 6 May 2010 18:20:35 +0200 (CEST) Subject: [pypy-svn] r74424 - in pypy/branch/py12/py: . _plugin _test Message-ID: <20100506162035.43691282BEF@codespeak.net> Author: hpk Date: Thu May 6 18:20:33 2010 New Revision: 74424 Removed: pypy/branch/py12/py/__init__.py.orig pypy/branch/py12/py/_plugin/pytest_pytester.py.orig pypy/branch/py12/py/_plugin/pytest_terminal.py.orig pypy/branch/py12/py/_test/.pluginmanager.py.swp Log: remove wrongly added files From antocuni at codespeak.net Thu May 6 18:45:48 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 6 May 2010 18:45:48 +0200 (CEST) Subject: [pypy-svn] r74425 - pypy/extradoc/talk/pycon-italy-2010/talk Message-ID: <20100506164548.6123F282BF6@codespeak.net> Author: antocuni Date: Thu May 6 18:45:46 2010 New Revision: 74425 Modified: pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt Log: add the slides about part 3 Modified: pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt ============================================================================== --- pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt (original) +++ pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt Thu May 6 18:45:46 2010 @@ -39,8 +39,6 @@ - What's new and status update -- Mandelbrot demo - What's new in PyPy 1.2 ----------------------- @@ -69,6 +67,20 @@ .. image:: pypy-vs-psyco.png :scale: 35 +Speed: Demo +----------- + +- Django application + +- Mandelbrot fractal + + * fished randomly on the net :-) + +- Run both on CPython and PyPy + + * django trunk! + + What works on PyPy ------------------- @@ -254,5 +266,129 @@ Part 3 ------ -Demo explained +How to use PyPy right now + + +Mandelbrot demo +--------------- + +- Works purely on PyPy + +- Not always the case + + * missing extension modules (cpyext mitigates the problem) + + * libraries that rely on CPython details + + * ... + +- clear performance-critical part + + +CPython and PyPy side by side +------------------------------ + +- CPython: runs the main application + +- PyPy: subprocess, runs only the hotspots + +- How do they communicate? + +- execnet + + * **The Ring of Python**, Holger Krekel, 9:00 + + * oups, too late :-) + + +Rendering (1) +--------------- + +|example<| Mandelbrot |>| +|small| +:: + + def render(request): + w = int(request.GET.get('w', 320)) + h = int(request.GET.get('h', 240)) + + from py_mandel import mandelbrot + img = mandelbrot(w, h) + + return HttpResponse(img, content_type="image/bmp") + +|end_small| +|end_example| + + +Rendering (2) +------------- + +|example<| Mandelbrot on PyPy |>| +|small| +:: + + def pypy_render(request): + w = int(request.GET.get('w', 320)) + h = int(request.GET.get('h', 240)) + + channel = pypy.remote_exec(""" + from py_mandel import mandelbrot + w, h = channel.receive() + img = mandelbrot(w, h) + channel.send(img) + """) + channel.send((w, h)) + img = channel.receive() + + return HttpResponse(img, content_type="image/bmp") + +|end_small| +|end_example| + +execnet setup +------------- + +|example<| At startup |>| +|small| +:: + + import execnet + mygroup = execnet.Group() + pypy = mygroup.makegateway("popen//python=pypy-c") + pypy.remote_exec(""" + import sys + import os + os.chdir("mandelbrot") + sys.path.insert(0, '') + """) + +|end_small| +|end_example| + + +Demo +---- + + +Benchmarks +---------- + +.. image:: demo-graph.pdf + :scale: 45 + + +Contact / Q&A +-------------- + +* Antonio Cuni: at http://merlinux.eu + +* Armin Rigo: arigo (at) tunes.org + +* Links: + + - PyPy: http://www.pypy.org + + - PyPy speed center: http://speed.pypy.org/ + - Blog: http://morepypy.blogspot.com From hpk at codespeak.net Thu May 6 19:07:00 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 6 May 2010 19:07:00 +0200 (CEST) Subject: [pypy-svn] r74426 - in pypy/trunk: . lib-python py py/_plugin py/bin py/bin/win32 pypy pypy/interpreter/test pypy/jit/backend/cli/test pypy/jit/backend/llvm/test pypy/jit/tl/test pypy/lib/app_test/ctypes_tests pypy/lib/distributed pypy/lib/distributed/test pypy/module/_demo/test pypy/module/_sre/test pypy/module/gc/test pypy/module/imp/test pypy/module/oracle/test pypy/module/select pypy/rlib/rsdl/test pypy/rpython/numpy/test pypy/tool pypy/tool/pytest pypy/tool/pytest/test pypy/tool/test pypy/translator/benchmark pypy/translator/platform pypy/translator/platform/test Message-ID: <20100506170700.A508D282BF9@codespeak.net> Author: hpk Date: Thu May 6 19:06:57 2010 New Revision: 74426 Added: pypy/trunk/py/ - copied from r74323, pypy/branch/py12/py/ pypy/trunk/py/__init__.py - copied unchanged from r74323, pypy/branch/py12/py/__init__.py pypy/trunk/py/apipkg.py - copied unchanged from r74323, pypy/branch/py12/py/apipkg.py pypy/trunk/py/bin/ (props changed) - copied from r74323, pypy/branch/py12/py/bin/ pypy/trunk/py/bin/_findpy.py - copied unchanged from r74323, pypy/branch/py12/py/bin/_findpy.py pypy/trunk/py/bin/env.cmd - copied unchanged from r74323, pypy/branch/py12/py/bin/env.cmd pypy/trunk/py/bin/env.py - copied unchanged from r74323, pypy/branch/py12/py/bin/env.py pypy/trunk/py/bin/py.cleanup - copied unchanged from r74323, pypy/branch/py12/py/bin/py.cleanup pypy/trunk/py/bin/py.convert_unittest - copied unchanged from r74323, pypy/branch/py12/py/bin/py.convert_unittest pypy/trunk/py/bin/py.countloc - copied unchanged from r74323, pypy/branch/py12/py/bin/py.countloc pypy/trunk/py/bin/py.lookup - copied unchanged from r74323, pypy/branch/py12/py/bin/py.lookup pypy/trunk/py/bin/py.svnwcrevert - copied unchanged from r74323, pypy/branch/py12/py/bin/py.svnwcrevert pypy/trunk/py/bin/py.test - copied unchanged from r74323, pypy/branch/py12/py/bin/py.test pypy/trunk/py/bin/py.which - copied unchanged from r74323, pypy/branch/py12/py/bin/py.which pypy/trunk/py/bin/win32/ - copied from r74323, pypy/branch/py12/py/bin/win32/ pypy/trunk/py/bin/win32/py.cleanup.cmd - copied unchanged from r74323, pypy/branch/py12/py/bin/win32/py.cleanup.cmd pypy/trunk/py/bin/win32/py.convert_unittest.cmd - copied unchanged from r74323, pypy/branch/py12/py/bin/win32/py.convert_unittest.cmd pypy/trunk/py/bin/win32/py.countloc.cmd - copied unchanged from r74323, pypy/branch/py12/py/bin/win32/py.countloc.cmd pypy/trunk/py/bin/win32/py.lookup.cmd - copied unchanged from r74323, pypy/branch/py12/py/bin/win32/py.lookup.cmd pypy/trunk/py/bin/win32/py.svnwcrevert.cmd - copied unchanged from r74323, pypy/branch/py12/py/bin/win32/py.svnwcrevert.cmd pypy/trunk/py/bin/win32/py.test.cmd - copied unchanged from r74323, pypy/branch/py12/py/bin/win32/py.test.cmd pypy/trunk/py/bin/win32/py.which.cmd - copied unchanged from r74323, pypy/branch/py12/py/bin/win32/py.which.cmd pypy/trunk/pypy/module/_demo/test/__init__.py - copied unchanged from r74323, pypy/branch/py12/pypy/module/_demo/test/__init__.py pypy/trunk/pypy/module/oracle/test/conftest.py - copied unchanged from r74323, pypy/branch/py12/pypy/module/oracle/test/conftest.py pypy/trunk/pypy/tool/pytest/test/test_pytestsupport.py - copied unchanged from r74422, pypy/branch/py12/pypy/tool/pytest/test/test_pytestsupport.py Removed: pypy/trunk/pypy/rpython/numpy/test/conftest.py pypy/trunk/pypy/tool/statistic_over_time.py pypy/trunk/pypy/tool/test/test_pytestsupport.py Modified: pypy/trunk/ (props changed) pypy/trunk/lib-python/conftest.py pypy/trunk/py/_plugin/pytest_skipping.py pypy/trunk/py/_plugin/pytest_terminal.py pypy/trunk/pypy/conftest.py pypy/trunk/pypy/interpreter/test/test_zzpickle_and_slow.py pypy/trunk/pypy/jit/backend/cli/test/conftest.py pypy/trunk/pypy/jit/backend/llvm/test/conftest.py pypy/trunk/pypy/jit/tl/test/test_pypyjit.py pypy/trunk/pypy/lib/app_test/ctypes_tests/conftest.py pypy/trunk/pypy/lib/app_test/ctypes_tests/support.py pypy/trunk/pypy/lib/distributed/socklayer.py pypy/trunk/pypy/lib/distributed/test/test_distributed.py pypy/trunk/pypy/module/_sre/test/test_app_sre.py pypy/trunk/pypy/module/gc/test/test_gc.py pypy/trunk/pypy/module/imp/test/test_import.py pypy/trunk/pypy/module/oracle/test/test_connect.py pypy/trunk/pypy/module/select/conftest.py pypy/trunk/pypy/rlib/rsdl/test/conftest.py pypy/trunk/pypy/rpython/numpy/test/test_array.py pypy/trunk/pypy/tool/ansi_mandelbrot.py pypy/trunk/pypy/tool/ansi_print.py pypy/trunk/pypy/tool/genstatistic.py pypy/trunk/pypy/tool/pytest/appsupport.py pypy/trunk/pypy/tool/pytest/test/test_appsupport.py pypy/trunk/pypy/translator/benchmark/conftest.py pypy/trunk/pypy/translator/platform/__init__.py pypy/trunk/pypy/translator/platform/test/test_platform.py Log: merge py12 branch - incorporates and updates pypy tests to use py.test-1.3.0 * introduces spaceconfig attribute to specify which space to use for a certain test class. * removes deprecated usage of py.test features * removes internal imports of py lib * various little test functionality related cleanups note: the merging was going odd with svn throwing assertion errors and other issues - so i merged in two steps - hope this worked fine. Modified: pypy/trunk/lib-python/conftest.py ============================================================================== --- pypy/trunk/lib-python/conftest.py (original) +++ pypy/trunk/lib-python/conftest.py Thu May 6 19:06:57 2010 @@ -531,7 +531,8 @@ l.append(RunFileExternal(name, parent=self, regrtest=regrtest)) return l -Directory = RegrDirectory +def pytest_collect_directory(parent, path): + return RegrDirectory(path, parent) class RunFileExternal(py.test.collect.File): def __init__(self, name, parent, regrtest): Modified: pypy/trunk/py/_plugin/pytest_skipping.py ============================================================================== --- pypy/branch/py12/py/_plugin/pytest_skipping.py (original) +++ pypy/trunk/py/_plugin/pytest_skipping.py Thu May 6 19:06:57 2010 @@ -10,9 +10,8 @@ If a test fails under all conditions then it's probably better to mark your test as 'xfail'. -By passing ``--report=xfailed,skipped`` to the terminal reporter -you will see summary information on skips and xfail-run tests -at the end of a test run. +By passing ``-rxs`` to the terminal reporter you will see extra +summary information on skips and xfail-run tests at the end of a test run. .. _skipif: @@ -122,6 +121,45 @@ import py +class MarkEvaluator: + def __init__(self, item, name): + self.item = item + self.name = name + self.holder = getattr(item.obj, name, None) + + def __bool__(self): + return bool(self.holder) + __nonzero__ = __bool__ + + def istrue(self): + if self.holder: + d = {'os': py.std.os, 'sys': py.std.sys, 'config': self.item.config} + self.result = True + for expr in self.holder.args: + self.expr = expr + if isinstance(expr, str): + result = cached_eval(self.item.config, expr, d) + else: + result = expr + if not result: + self.result = False + self.expr = expr + break + return getattr(self, 'result', False) + + def get(self, attr, default=None): + return self.holder.kwargs.get(attr, default) + + def getexplanation(self): + expl = self.get('reason', None) + if not expl: + if not hasattr(self, 'expr'): + return "" + else: + return "condition: " + self.expr + return expl + + def pytest_runtest_setup(item): expr, result = evalexpression(item, 'skipif') if result: @@ -155,19 +193,41 @@ # called by the terminalreporter instance/plugin def pytest_terminal_summary(terminalreporter): - show_xfailed(terminalreporter) - show_skipped(terminalreporter) - -def show_xfailed(terminalreporter): tr = terminalreporter - xfailed = tr.stats.get("xfailed") + if not tr.reportchars: + #for name in "xfailed skipped failed xpassed": + # if not tr.stats.get(name, 0): + # tr.write_line("HINT: use '-r' option to see extra " + # "summary info about tests") + # break + return + + lines = [] + for char in tr.reportchars: + if char == "x": + show_xfailed(terminalreporter, lines) + elif char == "X": + show_xpassed(terminalreporter, lines) + elif char == "f": + show_failed(terminalreporter, lines) + elif char == "s": + show_skipped(terminalreporter, lines) + if lines: + tr._tw.sep("=", "short test summary info") + for line in lines: + tr._tw.line(line) + +def show_failed(terminalreporter, lines): + tw = terminalreporter._tw + failed = terminalreporter.stats.get("failed") + if failed: + for rep in failed: + pos = terminalreporter.gettestid(rep.item) + lines.append("FAIL %s" %(pos, )) + +def show_xfailed(terminalreporter, lines): + xfailed = terminalreporter.stats.get("xfailed") if xfailed: - if not tr.hasopt('xfailed'): - tr.write_line( - "%d expected failures, use --report=xfailed for more info" % - len(xfailed)) - return - tr.write_sep("_", "expected failures") for rep in xfailed: entry = rep.longrepr.reprcrash modpath = rep.item.getmodpath(includemodule=True) @@ -176,15 +236,15 @@ i = reason.find("\n") if i != -1: reason = reason[:i] - tr._tw.line("%s %s" %(pos, reason)) + lines.append("XFAIL %s %s" %(pos, reason)) +def show_xpassed(terminalreporter, lines): xpassed = terminalreporter.stats.get("xpassed") if xpassed: - tr.write_sep("_", "UNEXPECTEDLY PASSING TESTS") for rep in xpassed: - fspath, lineno, modpath = rep.item.reportinfo() - pos = "%s %s:%d: unexpectedly passing" %(modpath, fspath, lineno) - tr._tw.line(pos) + pos = terminalreporter.gettestid(rep.item) + reason = rep.keywords['xfail'] + lines.append("XPASS %s %s" %(pos, reason)) def evalexpression(item, keyword): @@ -227,17 +287,20 @@ l.append((len(events),) + key) return l -def show_skipped(terminalreporter): +def show_skipped(terminalreporter, lines): tr = terminalreporter skipped = tr.stats.get('skipped', []) if skipped: - if not tr.hasopt('skipped'): - tr.write_line( - "%d skipped tests, use --report=skipped for more info" % - len(skipped)) - return + #if not tr.hasopt('skipped'): + # tr.write_line( + # "%d skipped tests, specify -rs for more info" % + # len(skipped)) + # return fskips = folded_skips(skipped) if fskips: - tr.write_sep("_", "skipped test summary") + #tr.write_sep("_", "skipped test summary") for num, fspath, lineno, reason in fskips: - tr._tw.line("%s:%d: [%d] %s" %(fspath, lineno, num, reason)) + if reason.startswith("Skipped: "): + reason = reason[9:] + lines.append("SKIP [%d] %s:%d: %s" % + (num, fspath, lineno, reason)) Modified: pypy/trunk/py/_plugin/pytest_terminal.py ============================================================================== --- pypy/branch/py12/py/_plugin/pytest_terminal.py (original) +++ pypy/trunk/py/_plugin/pytest_terminal.py Thu May 6 19:06:57 2010 @@ -12,12 +12,16 @@ group = parser.getgroup("terminal reporting", "reporting", after="general") group._addoption('-v', '--verbose', action="count", dest="verbose", default=0, help="increase verbosity."), + group._addoption('-r', + action="store", dest="reportchars", default=None, metavar="chars", + help="show extra test summary info as specified by chars (f)ailed, " + "(s)skipped, (x)failed, (X)passed.") group._addoption('-l', '--showlocals', - action="store_true", dest="showlocals", default=False, - help="show locals in tracebacks (disabled by default).") - group.addoption('--report', - action="store", dest="report", default=None, metavar="opts", - help="show more info, valid: skipped,xfailed") + action="store_true", dest="showlocals", default=False, + help="show locals in tracebacks (disabled by default).") + group._addoption('--report', + action="store", dest="report", default=None, metavar="opts", + help="(deprecated, use -r)") group._addoption('--tb', metavar="style", action="store", dest="tbstyle", default='long', type="choice", choices=['long', 'short', 'no', 'line'], @@ -47,17 +51,25 @@ setattr(reporter._tw, name, getattr(config, attr)) config.pluginmanager.register(reporter, 'terminalreporter') -def getreportopt(optvalue): - d = {} +def getreportopt(config): + reportopts = "" + optvalue = config.getvalue("report") if optvalue: - for setting in optvalue.split(","): - setting = setting.strip() - val = True - if setting.startswith("no"): - val = False - setting = setting[2:] - d[setting] = val - return d + py.builtin.print_("DEPRECATED: use -r instead of --report option.", + file=py.std.sys.stderr) + if optvalue: + for setting in optvalue.split(","): + setting = setting.strip() + if setting == "skipped": + reportopts += "s" + elif setting == "xfailed": + reportopts += "x" + reportchars = config.getvalue("reportchars") + if reportchars: + for char in reportchars: + if char not in reportopts: + reportopts += char + return reportopts class TerminalReporter: def __init__(self, config, file=None): @@ -69,10 +81,11 @@ self._tw = py.io.TerminalWriter(file) self.currentfspath = None self.gateway2info = {} - self._reportopt = getreportopt(config.getvalue('report')) + self.reportchars = getreportopt(config) - def hasopt(self, name): - return self._reportopt.get(name, False) + def hasopt(self, char): + char = {'xfailed': 'x', 'skipped': 's'}.get(char,char) + return char in self.reportchars def write_fspath_result(self, fspath, res): fspath = self.curdir.bestrelpath(fspath) Modified: pypy/trunk/pypy/conftest.py ============================================================================== --- pypy/trunk/pypy/conftest.py (original) +++ pypy/trunk/pypy/conftest.py Thu May 6 19:06:57 2010 @@ -43,7 +43,8 @@ help="set up tests to use specified platform as compile/run target") def pytest_funcarg__space(request): - return gettestobjspace() + spaceconfig = getattr(request.cls, 'spaceconfig', {}) + return gettestobjspace(**spaceconfig) _SPACECACHE={} def gettestobjspace(name=None, **kwds): @@ -93,6 +94,14 @@ space.eq_w = appsupport.eq_w.__get__(space) return space +def pytest_runtest_setup(item): + if isinstance(item, PyPyTestFunction): + appclass = item.getparent(PyPyClassCollector) + if appclass is not None: + spaceconfig = getattr(appclass.obj, 'spaceconfig', None) + if spaceconfig: + appclass.obj.space = gettestobjspace(**spaceconfig) + class TinyObjSpace(object): def __init__(self, **kwds): import sys @@ -196,7 +205,10 @@ if not hasattr(__builtin__, helper): setattr(__builtin__, helper, getattr(py.test, helper)) -class Module(py.test.collect.Module): +def pytest_pycollect_makemodule(path, parent): + return PyPyModule(path, parent) + +class PyPyModule(py.test.collect.Module): """ we take care of collecting classes both at app level and at interp-level (because we need to stick a space at the class) ourselves. @@ -204,7 +216,7 @@ def __init__(self, *args, **kwargs): if hasattr(sys, 'pypy_objspaceclass'): option.conf_iocapture = "sys" # pypy cannot do FD-based - super(Module, self).__init__(*args, **kwargs) + super(PyPyModule, self).__init__(*args, **kwargs) def accept_regular_test(self): if option.runappdirect: @@ -235,7 +247,7 @@ def setup(self): # stick py.test raise in module globals -- carefully ensure_pytest_builtin_helpers() - super(Module, self).setup() + super(PyPyModule, self).setup() # if hasattr(mod, 'objspacename'): # mod.space = getttestobjspace(mod.objspacename) @@ -405,11 +417,17 @@ class PyPyClassCollector(py.test.collect.Class): def setup(self): cls = self.obj - cls.space = LazyObjSpaceGetter() + if not hasattr(cls, 'spaceconfig'): + cls.space = LazyObjSpaceGetter() + else: + assert hasattr(cls, 'space') # set by pytest_runtest_setup super(PyPyClassCollector, self).setup() + +class IntInstanceCollector(py.test.collect.Instance): + Function = IntTestFunction class IntClassCollector(PyPyClassCollector): - Function = IntTestFunction + Instance = IntInstanceCollector def _haskeyword(self, keyword): return keyword == 'interplevel' or \ @@ -512,9 +530,5 @@ py.test.skip("pexpect not found") -class Directory(py.test.collect.Directory): - - def recfilter(self, path): - # disable recursion in symlinked subdirectories - return (py.test.collect.Directory.recfilter(self, path) - and path.check(link=0)) +def pytest_ignore_collect(path): + return path.check(link=1) Modified: pypy/trunk/pypy/interpreter/test/test_zzpickle_and_slow.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_zzpickle_and_slow.py (original) +++ pypy/trunk/pypy/interpreter/test/test_zzpickle_and_slow.py Thu May 6 19:06:57 2010 @@ -73,10 +73,8 @@ space.wrap('restore_top_frame')) class AppTestInterpObjectPickling: - + pytestmark = py.test.mark.skipif("config.option.runappdirect") def setup_class(cls): - if conftest.option.runappdirect: - py.test.skip("not for py.test -A") _attach_helpers(cls.space) def teardown_class(cls): Modified: pypy/trunk/pypy/jit/backend/cli/test/conftest.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/test/conftest.py (original) +++ pypy/trunk/pypy/jit/backend/cli/test/conftest.py Thu May 6 19:06:57 2010 @@ -1,5 +1,4 @@ import py -class Directory(py.test.collect.Directory): - def collect(self): - py.test.skip("CLI backend tests skipped for now") +def pytest_collect_directory(path): + py.test.skip("CLI backend tests skipped for now") Modified: pypy/trunk/pypy/jit/backend/llvm/test/conftest.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llvm/test/conftest.py (original) +++ pypy/trunk/pypy/jit/backend/llvm/test/conftest.py Thu May 6 19:06:57 2010 @@ -1,5 +1,4 @@ import py -class Directory(py.test.collect.Directory): - def collect(self): - py.test.skip("llvm backend tests skipped for now") +def pytest_collect_directory(): + py.test.skip("llvm backend tests skipped for now") Modified: pypy/trunk/pypy/jit/tl/test/test_pypyjit.py ============================================================================== --- pypy/trunk/pypy/jit/tl/test/test_pypyjit.py (original) +++ pypy/trunk/pypy/jit/tl/test/test_pypyjit.py Thu May 6 19:06:57 2010 @@ -21,7 +21,7 @@ def check_crasher(func_name): try: JIT_EXECUTABLE.sysexec(CRASH_FILE, func_name) - except py.impl.process.cmdexec.ExecutionFailed, e: + except py.process.cmdexec.Error, e: print "stderr" print "------" print e.err Modified: pypy/trunk/pypy/lib/app_test/ctypes_tests/conftest.py ============================================================================== --- pypy/trunk/pypy/lib/app_test/ctypes_tests/conftest.py (original) +++ pypy/trunk/pypy/lib/app_test/ctypes_tests/conftest.py Thu May 6 19:06:57 2010 @@ -1,11 +1,9 @@ import py import sys -class Directory(py.test.collect.Directory): - def collect(self): - if '__pypy__' not in sys.builtin_module_names: - py.test.skip("these tests are meant to be run on top of pypy-c") - return super(Directory, self).collect() +def pytest_collect_directory(): + if '__pypy__' not in sys.builtin_module_names: + py.test.skip("these tests are meant to be run on top of pypy-c") def compile_so_file(): from pypy.translator.platform import platform Modified: pypy/trunk/pypy/lib/app_test/ctypes_tests/support.py ============================================================================== --- pypy/trunk/pypy/lib/app_test/ctypes_tests/support.py (original) +++ pypy/trunk/pypy/lib/app_test/ctypes_tests/support.py Thu May 6 19:06:57 2010 @@ -1,36 +1,29 @@ import py import ctypes -if ctypes.__version__ < "1.0.2": - py.test.skip("we expect a ctypes implementation with ver >= 1.0.2") +py.test.importorskip("ctypes", "1.0.2") + +try: + import _rawffi +except ImportError: + _rawffi = None class WhiteBoxTests: def setup_class(cls): - try: - import _rawffi - except ImportError: - py.test.skip("these tests are white-box tests for pypy _rawffi based ctypes impl") + if _rawffi: + py.test.skip("white-box tests for pypy _rawffi based ctypes impl") class BaseCTypesTestChecker: def setup_class(cls): - try: - import _rawffi - except ImportError: - pass - else: + if _rawffi: import gc for _ in range(4): gc.collect() cls.old_num = _rawffi._num_of_allocated_objects() def teardown_class(cls): - #return - try: - import _rawffi - except ImportError: - pass - else: + if hasattr(cls, 'old_num'): import gc for _ in range(4): gc.collect() Modified: pypy/trunk/pypy/lib/distributed/socklayer.py ============================================================================== --- pypy/trunk/pypy/lib/distributed/socklayer.py (original) +++ pypy/trunk/pypy/lib/distributed/socklayer.py Thu May 6 19:06:57 2010 @@ -1,6 +1,8 @@ import py from socket import socket + +XXX needs import adaptation as 'green' is removed from py lib for years from py.impl.green.msgstruct import decodemessage, message from socket import socket, AF_INET, SOCK_STREAM import marshal Modified: pypy/trunk/pypy/lib/distributed/test/test_distributed.py ============================================================================== --- pypy/trunk/pypy/lib/distributed/test/test_distributed.py (original) +++ pypy/trunk/pypy/lib/distributed/test/test_distributed.py Thu May 6 19:06:57 2010 @@ -94,9 +94,11 @@ assert len(item) == 11 class AppTestDistributedTasklets(object): + spaceconfig = {"objspace.std.withtproxy": True, + "objspace.usemodules._stackless": True} def setup_class(cls): - cls.space = gettestobjspace(**{"objspace.std.withtproxy": True, - "usemodules":("_stackless",)}) + #cls.space = gettestobjspace(**{"objspace.std.withtproxy": True, + # "usemodules":("_stackless",)}) cls.w_test_env = cls.space.appexec([], """(): from distributed import test_env return test_env Modified: pypy/trunk/pypy/module/_sre/test/test_app_sre.py ============================================================================== --- pypy/trunk/pypy/module/_sre/test/test_app_sre.py (original) +++ pypy/trunk/pypy/module/_sre/test/test_app_sre.py Thu May 6 19:06:57 2010 @@ -1,9 +1,9 @@ """Regular expression tests specific to _sre.py and accumulated during TDD.""" import autopath +import py from py.test import raises, skip from pypy.interpreter.gateway import app2interp_temp from pypy.conftest import gettestobjspace, option -from py.impl.test.outcome import Skipped def init_globals_hack(space): space.appexec([space.wrap(autopath.this_dir)], """(this_dir): @@ -288,7 +288,7 @@ # This imports support_test_sre as the global "s" try: cls.space = gettestobjspace(usemodules=('_locale',)) - except Skipped: + except py.test.skip.Exception: cls.space = gettestobjspace(usemodules=('_rawffi',)) init_globals_hack(cls.space) @@ -548,7 +548,7 @@ def setup_class(cls): try: cls.space = gettestobjspace(usemodules=('_locale',)) - except Skipped: + except py.test.skip.Exception: cls.space = gettestobjspace(usemodules=('_rawffi',)) # This imports support_test_sre as the global "s" init_globals_hack(cls.space) Modified: pypy/trunk/pypy/module/gc/test/test_gc.py ============================================================================== --- pypy/trunk/pypy/module/gc/test/test_gc.py (original) +++ pypy/trunk/pypy/module/gc/test/test_gc.py Thu May 6 19:06:57 2010 @@ -1,4 +1,5 @@ from pypy.conftest import gettestobjspace +import py class AppTestGC(object): def test_collect(self): @@ -76,9 +77,10 @@ assert gc.isenabled() class AppTestGcDumpHeap(object): + pytestmark = py.test.mark.xfail(run=False) + def setup_class(cls): import py - py.test.skip("Disabled") from pypy.tool.udir import udir from pypy.rlib import rgc class X(object): Modified: pypy/trunk/pypy/module/imp/test/test_import.py ============================================================================== --- pypy/trunk/pypy/module/imp/test/test_import.py (original) +++ pypy/trunk/pypy/module/imp/test/test_import.py Thu May 6 19:06:57 2010 @@ -920,16 +920,15 @@ sys.path_hooks.pop() class AppTestNoPycFile(object): - usepycfiles = False - lonepycfiles = False - + spaceconfig = { + "objspace.usepycfiles": False, + "objspace.lonepycfiles": False + } def setup_class(cls): - cls.space = gettestobjspace(**{ - "objspace.usepycfiles": cls.usepycfiles, - "objspace.lonepycfiles": cls.lonepycfiles, - }) - cls.w_usepycfiles = cls.space.wrap(cls.usepycfiles) - cls.w_lonepycfiles = cls.space.wrap(cls.lonepycfiles) + usepycfiles = cls.spaceconfig['objspace.usepycfiles'] + lonepycfiles = cls.spaceconfig['objspace.lonepycfiles'] + cls.w_usepycfiles = cls.space.wrap(usepycfiles) + cls.w_lonepycfiles = cls.space.wrap(lonepycfiles) cls.saved_modules = _setup(cls.space) def teardown_class(cls): @@ -950,9 +949,13 @@ assert lone.__file__.endswith('lone.pyc') class AppTestNoLonePycFile(AppTestNoPycFile): - usepycfiles = True - lonepycfiles = False + spaceconfig = { + "objspace.usepycfiles": True, + "objspace.lonepycfiles": False + } class AppTestLonePycFile(AppTestNoPycFile): - usepycfiles = True - lonepycfiles = True + spaceconfig = { + "objspace.usepycfiles": True, + "objspace.lonepycfiles": True + } Modified: pypy/trunk/pypy/module/oracle/test/test_connect.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_connect.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_connect.py Thu May 6 19:06:57 2010 @@ -7,11 +7,6 @@ @classmethod def setup_class(cls): - try: - from pypy.module.oracle import roci - except ImportError: - py.test.skip("Oracle client not available") - space = gettestobjspace(usemodules=('oracle',)) cls.space = space space.setitem(space.builtin.w_dict, space.wrap('oracle'), Modified: pypy/trunk/pypy/module/select/conftest.py ============================================================================== --- pypy/trunk/pypy/module/select/conftest.py (original) +++ pypy/trunk/pypy/module/select/conftest.py Thu May 6 19:06:57 2010 @@ -1,10 +1,4 @@ import py -class Directory(py.test.collect.Directory): - - def run(self): - try: - import ctypes - except ImportError: - py.test.skip("these tests need ctypes installed") - return super(Directory, self).run() +def pytest_collect_directory(): + py.test.importorskip("ctypes") Modified: pypy/trunk/pypy/rlib/rsdl/test/conftest.py ============================================================================== --- pypy/trunk/pypy/rlib/rsdl/test/conftest.py (original) +++ pypy/trunk/pypy/rlib/rsdl/test/conftest.py Thu May 6 19:06:57 2010 @@ -1,10 +1,8 @@ from pypy.rlib.rsdl.eci import check_sdl_installation, SDLNotInstalled import py -class Directory(py.test.collect.Directory): - def collect(self): - try: - check_sdl_installation() - except SDLNotInstalled, e: - py.test.skip("SDL not installed(?): %s" % (e,)) - return py.test.collect.Directory.collect(self) +def pytest_collect_directory(): + try: + check_sdl_installation() + except SDLNotInstalled, e: + py.test.skip("SDL not installed(?): %s" % (e,)) Modified: pypy/trunk/pypy/rpython/numpy/test/test_array.py ============================================================================== --- pypy/trunk/pypy/rpython/numpy/test/test_array.py (original) +++ pypy/trunk/pypy/rpython/numpy/test/test_array.py Thu May 6 19:06:57 2010 @@ -3,6 +3,7 @@ """ import py +numpy = py.test.importorskip("numpy") import pypy.rpython.numpy.implementation from pypy.annotation import model as annmodel from pypy.annotation.model import SomeObject, SomeInteger, SomeChar, SomeTuple @@ -15,13 +16,6 @@ from pypy.rpython.lltypesystem import rffi from pypy.rpython.rint import IntegerRepr -def setup_module(mod): - try: - import numpy - except ImportError: - py.test.skip("numpy not found") - mod.numpy = numpy - from pypy.rpython.numpy.rarray import ArrayRepr from pypy.rpython.numpy.aarray import SomeArray Modified: pypy/trunk/pypy/tool/ansi_mandelbrot.py ============================================================================== --- pypy/trunk/pypy/tool/ansi_mandelbrot.py (original) +++ pypy/trunk/pypy/tool/ansi_mandelbrot.py Thu May 6 19:06:57 2010 @@ -1,6 +1,6 @@ import sys -from py.impl.io.terminalwriter import ansi_print, get_terminal_width +from py.io import ansi_print, get_terminal_width """ Black 0;30 Dark Gray 1;30 Modified: pypy/trunk/pypy/tool/ansi_print.py ============================================================================== --- pypy/trunk/pypy/tool/ansi_print.py (original) +++ pypy/trunk/pypy/tool/ansi_print.py Thu May 6 19:06:57 2010 @@ -3,8 +3,7 @@ """ import sys - -from py.impl.io.terminalwriter import ansi_print +from py.io import ansi_print from pypy.tool.ansi_mandelbrot import Driver class AnsiLog: Modified: pypy/trunk/pypy/tool/genstatistic.py ============================================================================== --- pypy/trunk/pypy/tool/genstatistic.py (original) +++ pypy/trunk/pypy/tool/genstatistic.py Thu May 6 19:06:57 2010 @@ -1,7 +1,7 @@ import autopath import py -from py.impl.misc.cmdline import countloc +from py._cmdline import pycountloc as countloc from py.xml import raw pypydir = py.path.local(autopath.pypydir) Modified: pypy/trunk/pypy/tool/pytest/appsupport.py ============================================================================== --- pypy/trunk/pypy/tool/pytest/appsupport.py (original) +++ pypy/trunk/pypy/tool/pytest/appsupport.py Thu May 6 19:06:57 2010 @@ -1,10 +1,7 @@ import autopath import py -import py.impl.code.assertion -from py.impl.code import _assertionold as exprinfo from pypy.interpreter import gateway from pypy.interpreter.error import OperationError -from py.impl.test.outcome import ExceptionFailure # ____________________________________________________________ @@ -158,7 +155,7 @@ source = None from pypy import conftest if source and not conftest.option.nomagic: - msg = exprinfo.interpret(source, runner, should_fail=True) + msg = py.code._reinterpret_old(source, runner, should_fail=True) space.setattr(w_self, space.wrap('args'), space.newtuple([space.wrap(msg)])) w_msg = space.wrap(msg) @@ -254,7 +251,7 @@ if not value.match(space, w_ExpectedException): raise type, value, tb return excinfo - except ExceptionFailure, e: + except py.test.raises.Exception, e: e.tbindex = getattr(e, 'tbindex', -1) - 1 raise Modified: pypy/trunk/pypy/tool/pytest/test/test_appsupport.py ============================================================================== --- pypy/trunk/pypy/tool/pytest/test/test_appsupport.py (original) +++ pypy/trunk/pypy/tool/pytest/test/test_appsupport.py Thu May 6 19:06:57 2010 @@ -1,4 +1,74 @@ +import sys +import py +import pypy +pytest_plugins = "pytester" + +def setpypyconftest(testdir): + path = str(py.path.local(pypy.__file__).dirpath().dirpath()) + testdir.makeconftest(""" + import sys + sys.path.insert(0, %r) + from pypy.conftest import * + """ % path) + +def test_pypy_collection(testdir): + testdir.makepyfile(""" + def test_func(): + pass + class TestClassInt: + def test_method(self, space): + pass + class AppTestClass: + def test_method(self): + pass + """) + setpypyconftest(testdir) + result = testdir.runpytest("--collectonly") + assert result.ret == 0 + result.stdout.fnmatch_lines([ + "*IntTestFunction*test_func*", + "*IntClassCollector*TestClassInt*", + "*IntTestFunction*test_method*", + "*AppClassCollector*AppTestClass*", + "*AppTestMethod*", + ]) + +class TestSpaceConfig: + def test_applevel_skipped_on_cpython_and_spaceconfig(self, testdir): + setpypyconftest(testdir) + testdir.makepyfile(""" + class AppTestClass: + spaceconfig = {"objspace.usemodules._stackless": True} + def setup_class(cls): + assert 0 + def test_applevel(self): + pass + """) + result = testdir.runpytest("-A") + assert result.ret == 0 + if hasattr(sys, 'pypy_translation_info') and \ + sys.pypy_translation_info.get('objspace.usemodules._stackless'): + result.stdout.fnmatch_lines(["*1 error*"]) + else: + # setup_class didn't get called, otherwise it would error + result.stdout.fnmatch_lines(["*1 skipped*"]) + + def test_interp_spaceconfig(self, testdir): + setpypyconftest(testdir) + p = testdir.makepyfile(""" + class TestClass: + spaceconfig = {"objspace.usemodules._stackless": False} + def setup_class(cls): + assert not cls.space.config.objspace.usemodules._stackless + def test_interp(self, space): + assert self.space is space + def test_interp2(self, space): + assert self.space is space + """) + result = testdir.runpytest(p) + assert result.ret == 0 + result.stdout.fnmatch_lines(["*2 passed*"]) def app_test_raises(): info = raises(TypeError, id) Modified: pypy/trunk/pypy/translator/benchmark/conftest.py ============================================================================== --- pypy/trunk/pypy/translator/benchmark/conftest.py (original) +++ pypy/trunk/pypy/translator/benchmark/conftest.py Thu May 6 19:06:57 2010 @@ -1,7 +1,4 @@ import py -class Directory(py.test.collect.Directory): - - def recfilter(self, path): - # exclude the subdirectories added by 'svn co' from benchmarks.py - return path.check(basename='test') +def pytest_ignore_collect(path): + return path.basename == "test" Modified: pypy/trunk/pypy/translator/platform/__init__.py ============================================================================== --- pypy/trunk/pypy/translator/platform/__init__.py (original) +++ pypy/trunk/pypy/translator/platform/__init__.py Thu May 6 19:06:57 2010 @@ -6,7 +6,6 @@ import sys, py, os from pypy.tool.ansi_print import ansi_log -from py.impl.code.code import safe_repr log = py.log.Producer("platform") py.log.setconsumer("platform", ansi_log) @@ -19,9 +18,9 @@ def __repr__(self): if self.err: - return "" % safe_repr(self.err) + return "" % py.io.saferepr(self.err) else: - return "" % safe_repr(self.out) + return "" % py.io.saferepr(self.out) __str__ = __repr__ Modified: pypy/trunk/pypy/translator/platform/test/test_platform.py ============================================================================== --- pypy/trunk/pypy/translator/platform/test/test_platform.py (original) +++ pypy/trunk/pypy/translator/platform/test/test_platform.py Thu May 6 19:06:57 2010 @@ -5,6 +5,14 @@ from pypy.translator.platform import host from pypy.translator.tool.cbuild import ExternalCompilationInfo +def test_compilationerror_repr(): + # compilation error output/stdout may be large, make sure + # repr creates a limited version + c = CompilationError('', '*'*1000) + assert len(repr(c)) < 500 + c = CompilationError('*'*1000, '') + assert len(repr(c)) < 500 + class TestPlatform(object): platform = host strict_on_stderr = True From afa at codespeak.net Thu May 6 19:28:10 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 6 May 2010 19:28:10 +0200 (CEST) Subject: [pypy-svn] r74427 - pypy/trunk/pypy/module/cpyext/include Message-ID: <20100506172810.DB6CD282BFA@codespeak.net> Author: afa Date: Thu May 6 19:28:09 2010 New Revision: 74427 Modified: pypy/trunk/pypy/module/cpyext/include/pycobject.h Log: Fix compilation error on PyCObject_Check(*x) Modified: pypy/trunk/pypy/module/cpyext/include/pycobject.h ============================================================================== --- pypy/trunk/pypy/module/cpyext/include/pycobject.h (original) +++ pypy/trunk/pypy/module/cpyext/include/pycobject.h Thu May 6 19:28:09 2010 @@ -6,7 +6,7 @@ PyAPI_DATA(PyTypeObject) PyCObject_Type; -#define PyCObject_Check(op) (op->ob_type == &PyCObject_Type) +#define PyCObject_Check(op) ((op)->ob_type == &PyCObject_Type) /* Create a PyCObject from a pointer to a C object and an optional destructor function. If the second argument is non-null, then it From afa at codespeak.net Thu May 6 21:41:45 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 6 May 2010 21:41:45 +0200 (CEST) Subject: [pypy-svn] r74428 - pypy/trunk/pypy/module/cpyext Message-ID: <20100506194145.1A43A282B90@codespeak.net> Author: afa Date: Thu May 6 21:41:41 2010 New Revision: 74428 Modified: pypy/trunk/pypy/module/cpyext/api.py Log: This exception is not supposed to happen anymore Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Thu May 6 21:41:41 2010 @@ -458,8 +458,6 @@ retval = result elif callable.api_func.restype is not lltype.Void: retval = rffi.cast(callable.api_func.restype, result) - except NullPointerException: - print "Container not registered by %s" % callable.__name__ except Exception, e: if not we_are_translated(): import traceback From fijal at codespeak.net Fri May 7 01:15:25 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 7 May 2010 01:15:25 +0200 (CEST) Subject: [pypy-svn] r74429 - pypy/trunk/pypy/module/cpyext Message-ID: <20100506231525.0FA04282B90@codespeak.net> Author: fijal Date: Fri May 7 01:15:24 2010 New Revision: 74429 Modified: pypy/trunk/pypy/module/cpyext/api.py pypy/trunk/pypy/module/cpyext/pyobject.py Log: Nobody ever raises this exception, remove it alltogether Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Fri May 7 01:15:24 2010 @@ -401,7 +401,6 @@ def wrapper(*args): from pypy.module.cpyext.pyobject import make_ref, from_ref from pypy.module.cpyext.pyobject import BorrowPair - from pypy.module.cpyext.pyobject import NullPointerException # we hope that malloc removal removes the newtuple() that is # inserted exactly here by the varargs specializer llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Fri May 7 01:15:24 2010 @@ -178,9 +178,6 @@ Py_DecRef(self.space, w_obj) self.borrow_mapping = {} -class NullPointerException(Exception): - pass - class InvalidPointerException(Exception): pass From agaynor at codespeak.net Fri May 7 04:21:37 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Fri, 7 May 2010 04:21:37 +0200 (CEST) Subject: [pypy-svn] r74430 - pypy/trunk/pypy/module/cpyext Message-ID: <20100507022137.07884282B90@codespeak.net> Author: agaynor Date: Fri May 7 04:21:35 2010 New Revision: 74430 Modified: pypy/trunk/pypy/module/cpyext/api.py Log: Added Cell type to cpyext, untested. Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Fri May 7 04:21:35 2010 @@ -17,6 +17,7 @@ from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.gateway import ObjSpace, unwrap_spec +from pypy.interpreter.nestedscope import Cell from pypy.rlib.entrypoint import entrypoint from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.objectmodel import specialize @@ -305,6 +306,7 @@ "BaseObject": "space.w_object", 'None': 'space.type(space.w_None)', 'NotImplemented': 'space.type(space.w_NotImplemented)', + 'Cell': 'space.gettypeobject(Cell.typedef)', }.items(): GLOBALS['Py%s_Type#' % (cpyname, )] = ('PyTypeObject*', pypyexpr) From afa at codespeak.net Fri May 7 14:13:36 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 7 May 2010 14:13:36 +0200 (CEST) Subject: [pypy-svn] r74431 - pypy/trunk/pypy/module/cpyext Message-ID: <20100507121336.C39C9282B90@codespeak.net> Author: afa Date: Fri May 7 14:13:34 2010 New Revision: 74431 Modified: pypy/trunk/pypy/module/cpyext/api.py Log: In tests, use repr(e) when a function raises an interp-level exception. Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Fri May 7 14:13:34 2010 @@ -433,11 +433,14 @@ state.set_exception(e) except BaseException, e: failed = True - state.set_exception(OperationError(space.w_SystemError, - space.wrap(str(e)))) if not we_are_translated(): + message = repr(e) import traceback traceback.print_exc() + else: + message = str(e) + state.set_exception(OperationError(space.w_SystemError, + space.wrap(message))) else: failed = False From afa at codespeak.net Fri May 7 14:17:47 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 7 May 2010 14:17:47 +0200 (CEST) Subject: [pypy-svn] r74432 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100507121747.07521282B90@codespeak.net> Author: afa Date: Fri May 7 14:17:46 2010 New Revision: 74432 Modified: pypy/trunk/pypy/module/cpyext/stubs.py pypy/trunk/pypy/module/cpyext/test/foo.c pypy/trunk/pypy/module/cpyext/test/test_typeobject.py pypy/trunk/pypy/module/cpyext/typeobject.py Log: Implement and test tp_new_wrapper, it works at least when tp_new=PyType_GenericNew Modified: pypy/trunk/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/stubs.py (original) +++ pypy/trunk/pypy/module/cpyext/stubs.py Fri May 7 14:17:46 2010 @@ -3058,10 +3058,6 @@ """ raise NotImplementedError - at cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject) -def PyType_GenericNew(space, type, args, kwds): - raise NotImplementedError - @cpython_api([], rffi.INT_real, error=CANNOT_FAIL) def PyUnicode_ClearFreeList(space, ): """Clear the free list. Return the total number of freed items. Modified: pypy/trunk/pypy/module/cpyext/test/foo.c ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/foo.c (original) +++ pypy/trunk/pypy/module/cpyext/test/foo.c Fri May 7 14:17:46 2010 @@ -175,12 +175,6 @@ {NULL} /* Sentinel */ }; -static PyObject * -Fuu_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds) -{ - return subtype->tp_alloc(subtype, 0); -} - PyTypeObject FuuType = { PyObject_HEAD_INIT(NULL) 0, @@ -230,7 +224,7 @@ 0, /*tp_init*/ 0, /*tp_alloc will be set to PyType_GenericAlloc in module init*/ - Fuu_new, /*tp_new*/ + 0, /*tp_new*/ 0, /*tp_free Low-level free-memory routine */ 0, /*tp_is_gc For PyObject_IS_GC */ 0, /*tp_bases*/ @@ -269,10 +263,8 @@ { PyObject *m, *d; - footype.ob_type = &PyType_Type; + footype.tp_new = PyType_GenericNew; - /* Workaround for quirk in Visual Studio, see - */ FuuType.tp_base = &PyUnicode_Type; if (PyType_Ready(&footype) < 0) Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Fri May 7 14:17:46 2010 @@ -90,19 +90,18 @@ raises(TypeError, "obj.char_member = 42") def test_new(self): - skip("skip") module = self.import_module(name='foo') obj = module.new() # call __new__ newobj = module.FuuType(u"xyz") assert newobj == u"xyz" + assert isinstance(newobj, module.FuuType) - a = module.fooType - assert "cannot create" in raises(TypeError, "a()").value.message + assert isinstance(module.fooType(), module.fooType) class bar(module.fooType): - def baz(self): - return self - assert "cannot create" in raises(TypeError, "bar()").value.message + pass + assert isinstance(bar(), bar) + fuu = module.FuuType class fuu2(fuu): def baz(self): Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Fri May 7 14:17:46 2010 @@ -156,11 +156,24 @@ add_tp_new_wrapper(space, dict_w, pto) @cpython_api([PyObject, PyObject, PyObject], PyObject, external=False) -def tp_new_wrapper(space, w_self, w_args, w_kwds): - fn = rffi.cast(PyTypeObjectPtr, make_ref(space, w_self)).c_tp_new - pyo = make_ref(space, w_self) - pto = rffi.cast(PyTypeObjectPtr, pyo) - return generic_cpy_call(space, fn, pto, w_args, w_kwds) +def tp_new_wrapper(space, self, w_args, w_kwds): + tp_new = rffi.cast(PyTypeObjectPtr, self).c_tp_new + + # Check that the user doesn't do something silly and unsafe like + # object.__new__(dict). To do this, we check that the most + # derived base that's not a heap type is this type. + # XXX do it + + args_w = space.fixedview(w_args) + w_subtype = args_w[0] + w_args = space.newtuple(args_w[1:]) + + subtype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_subtype)) + try: + obj = generic_cpy_call(space, tp_new, subtype, w_args, w_kwds) + finally: + Py_DecRef(space, w_subtype) + return obj @specialize.memo() def get_new_method_def(space): @@ -630,8 +643,12 @@ @cpython_api([PyTypeObjectPtr, Py_ssize_t], PyObject) def PyType_GenericAlloc(space, type, nitems): - """This function used an int type for nitems. This might require - changes in your code for properly supporting 64-bit systems.""" from pypy.module.cpyext.object import _PyObject_NewVar return _PyObject_NewVar(space, type, nitems) + at cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject) +def PyType_GenericNew(space, type, w_args, w_kwds): + return generic_cpy_call( + space, type.c_tp_alloc, type, 0) + + From fijal at codespeak.net Fri May 7 18:56:13 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 7 May 2010 18:56:13 +0200 (CEST) Subject: [pypy-svn] r74436 - pypy/trunk/pypy/module/cpyext Message-ID: <20100507165613.E21F6282B90@codespeak.net> Author: fijal Date: Fri May 7 18:56:12 2010 New Revision: 74436 Modified: pypy/trunk/pypy/module/cpyext/api.py Log: Revert 74430. Random, untested changes are not welcomed here. Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Fri May 7 18:56:12 2010 @@ -17,7 +17,6 @@ from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.gateway import ObjSpace, unwrap_spec -from pypy.interpreter.nestedscope import Cell from pypy.rlib.entrypoint import entrypoint from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.objectmodel import specialize @@ -306,7 +305,6 @@ "BaseObject": "space.w_object", 'None': 'space.type(space.w_None)', 'NotImplemented': 'space.type(space.w_NotImplemented)', - 'Cell': 'space.gettypeobject(Cell.typedef)', }.items(): GLOBALS['Py%s_Type#' % (cpyname, )] = ('PyTypeObject*', pypyexpr) From agaynor at codespeak.net Sat May 8 02:40:23 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sat, 8 May 2010 02:40:23 +0200 (CEST) Subject: [pypy-svn] r74437 - pypy/trunk/pypy/module/cpyext/include Message-ID: <20100508004023.1D7252A2B40@codespeak.net> Author: agaynor Date: Sat May 8 02:40:21 2010 New Revision: 74437 Added: pypy/trunk/pypy/module/cpyext/include/datetime.h Log: Add a dummy datetime.h to cpyext for modules that import it. Added: pypy/trunk/pypy/module/cpyext/include/datetime.h ============================================================================== From agaynor at codespeak.net Sat May 8 05:06:56 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sat, 8 May 2010 05:06:56 +0200 (CEST) Subject: [pypy-svn] r74438 - in pypy/trunk/pypy/module/cpyext: . include Message-ID: <20100508030656.9BBB6282B90@codespeak.net> Author: agaynor Date: Sat May 8 05:06:55 2010 New Revision: 74438 Modified: pypy/trunk/pypy/module/cpyext/include/object.h pypy/trunk/pypy/module/cpyext/object.py Log: Added _PyObject_GC_Del as an alias for PyObject_GC_Del as it is in CPython. Modified: pypy/trunk/pypy/module/cpyext/include/object.h ============================================================================== --- pypy/trunk/pypy/module/cpyext/include/object.h (original) +++ pypy/trunk/pypy/module/cpyext/include/object.h Sat May 8 05:06:55 2010 @@ -481,6 +481,7 @@ /* PyPy internal ----------------------------------- */ int PyPyType_Register(PyTypeObject *); #define PyObject_Length PyObject_Size +#define _PyObject_GC_Del PyObject_GC_Del #ifdef __cplusplus Modified: pypy/trunk/pypy/module/cpyext/object.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/object.py (original) +++ pypy/trunk/pypy/module/cpyext/object.py Sat May 8 05:06:55 2010 @@ -46,7 +46,7 @@ @cpython_api([rffi.VOIDP_real], lltype.Void) def PyObject_GC_Del(space, obj): PyObject_Del(space, obj) - + @cpython_api([PyObject], lltype.Void) def PyObject_GC_Track(space, op): """Adds the object op to the set of container objects tracked by the From arigo at codespeak.net Sat May 8 12:05:47 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 8 May 2010 12:05:47 +0200 (CEST) Subject: [pypy-svn] r74439 - pypy/extradoc/talk/pycon-italy-2010/talk Message-ID: <20100508100547.1E364282B90@codespeak.net> Author: arigo Date: Sat May 8 12:05:45 2010 New Revision: 74439 Added: pypy/extradoc/talk/pycon-italy-2010/talk/overview2.png - copied, changed from r74422, pypy/extradoc/talk/pycon2008/overview2.png Modified: pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt Log: (antocuni, arigo) Part 2 of the talk. Copied: pypy/extradoc/talk/pycon-italy-2010/talk/overview2.png (from r74422, pypy/extradoc/talk/pycon2008/overview2.png) ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt ============================================================================== --- pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt (original) +++ pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt Sat May 8 12:05:45 2010 @@ -197,43 +197,96 @@ - beta version of cpyext -Part 2 -------- +Part 2: Just-in-Time compilation +--------------------------------- + + Snakes never crawled so fast -- the performance of Python: - how fast can Python code really be? - (and using how much memory?) - Overview of implementations --------------------------- -(Discussing their performance) - - CPython +- Stackless - Psyco -- Jython, IronPython -- (Stackless) +- Jython +- IronPython - PyPy (without and with JIT) - Unladen Swallow -What a JIT is +Demo +---- + +Features +-------- + +- it just works +- it may give good speed-ups (better than Psyco) +- may have a few bugs left (less than Psyco) |pause| +- *it is not a hack (unlike Psyco)* + +|pause| + +- PyPy also has excellent memory usage + + * half that of CPython for a program + using several hunderds MBs + + +What is a JIT ------------- -XXX +* CPython compiles the program source into *bytecodes* + +* without a JIT, the bytecodes are then interpreted + +* with a JIT, the bytecodes are further translated to + machine code (assembler) + + +What is a JIT (2) +----------------- -What a tracing JIT is +The translation can be: + +* syntactic: translate the whole functions into machine code + + - "the obvious way" + - e.g. Pyrex/Cython, Unladen Swallow + - not good performance, or needs tricks |pause| + +* semantic: translate bits of the function just-in-time + + - only used parts + - exploit runtime information (e.g. types) + - Psyco, PyPy + + +What is a tracing JIT --------------------- -XXX e.g. the structure of TraceMonkey +* start by interpreting normally +* find loops as they are executed +* turn them into machine code +* 80% of the time is spent in 20% of the code + +What is a tracing JIT (history) +------------------------------- + +* tracing assembler (Dynamo, ~2000) +* tracing Java (~2005) +* tracing JavaScript (~2008) + +|pause| + +* PyPy is a "tracing JIT generator" + The architecture of PyPy ------------------------ -XXX diagram showing where the JIT is +.. image:: overview2.png + :scale: 60 -XXX diagram detailling the JIT - -- it can be used for any interpreter, not just Python Speed of the PyPy JIT --------------------- @@ -242,25 +295,91 @@ by the JIT: - loops, even across many calls, are nicely handled -- interpreter-like constructs are not + +- loops with very many taken paths are not + + * e.g. Python programs that look like interpreters + * typical in tracing JITs + - bad support so far for generators and recursion + The optimizations we get ------------------------ -(Quick overview only, with examples of programs that benefit) +- ``!=`` optimizations we *wrote* :-) -- "virtuals", i.e. temporary objects are not constructed - removed frame handling + - local variables are in CPU registers or on the C stack + - but ``sys._getframe()`` works correctly + +- "virtuals": temporary objects are not constructed + +- attribute and method lookups, etc. + + +Example +------------- + +|column1| + +|example<| |>| + +:: + + def g(a, b): + if a < 0: # 2 + return -1 + return a - b # 3 + + def f(x): + total = 0 + # 1 + for i in range(x): + d = g(i, x) + total += d # 4 + +|end_example| + +|pause| + +|column2| + +|example<| |>| + +:: + + ADD EAX, 1 + CMP EAX, EBX + JNL + CMP EAX, 0 + JL + MOV ECX, EAX + SUB ECX, EBX + JO + ADD EDX, ECX + JO + JMP + +|end_example| + +|end_columns| + Practical results ----------------------- +- fast :-) + - so far, x86-32 only -- mention again the performance shown by the graphs -- a quick tour of other missing improvements -- reminder: works for *any* Python code (even more - than Psyco) + +- relatively easy to maintain (or port to x86-64, etc.) + +- reminder: works transparently for *any* Python code + + * or any language (Prolog JIT :-)) + +- viable alternative to CPython Part 3 From arigo at codespeak.net Sat May 8 12:11:08 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 8 May 2010 12:11:08 +0200 (CEST) Subject: [pypy-svn] r74440 - pypy/extradoc/talk/pycon-italy-2010/talk Message-ID: <20100508101108.D80BC282B90@codespeak.net> Author: arigo Date: Sat May 8 12:11:07 2010 New Revision: 74440 Modified: pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt Log: Try to simplify the URL. Modified: pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt ============================================================================== --- pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt (original) +++ pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt Sat May 8 12:11:07 2010 @@ -506,7 +506,7 @@ * Links: - - PyPy: http://www.pypy.org + - PyPy: http://pypy.org/ - PyPy speed center: http://speed.pypy.org/ From arigo at codespeak.net Sat May 8 12:42:12 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 8 May 2010 12:42:12 +0200 (CEST) Subject: [pypy-svn] r74441 - pypy/extradoc/talk/pycon-italy-2010/talk Message-ID: <20100508104212.BB36D282B90@codespeak.net> Author: arigo Date: Sat May 8 12:42:11 2010 New Revision: 74441 Modified: pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt Log: Tweaks. Modified: pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt ============================================================================== --- pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt (original) +++ pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt Sat May 8 12:42:11 2010 @@ -310,11 +310,15 @@ - ``!=`` optimizations we *wrote* :-) - removed frame handling + - local variables are in CPU registers or on the C stack - but ``sys._getframe()`` works correctly - "virtuals": temporary objects are not constructed + - ``e = a + b + c + d`` + - and much more complex examples + - attribute and method lookups, etc. @@ -328,7 +332,7 @@ :: def g(a, b): - if a < 0: # 2 + if a < 5: # 2 return -1 return a - b # 3 @@ -377,7 +381,7 @@ - reminder: works transparently for *any* Python code - * or any language (Prolog JIT :-)) + * or any language (Prolog JIT :-) at PPDP 2010) - viable alternative to CPython From afa at codespeak.net Sat May 8 14:34:41 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 8 May 2010 14:34:41 +0200 (CEST) Subject: [pypy-svn] r74442 - pypy/trunk/pypy/module/cpyext Message-ID: <20100508123441.DAAC736C226@codespeak.net> Author: afa Date: Sat May 8 14:34:40 2010 New Revision: 74442 Modified: pypy/trunk/pypy/module/cpyext/pyobject.py Log: A few comments Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Sat May 8 14:34:40 2010 @@ -153,6 +153,7 @@ self.py_objects_r2w = {} # { addr of raw PyObject -> w_obj } self.borrow_mapping = {} # { w_container -> { w_containee -> None } } self.borrowed_objects = {} # { addr of containee -> None } + # For tests self.non_heaptypes_w = [] def _freeze_(self): @@ -161,6 +162,7 @@ return False def init_r2w_from_w2r(self): + """Rebuilds the dict py_objects_r2w on startup""" from pypy.module.cpyext.api import ADDR for w_obj, obj in self.py_objects_w2r.items(): ptr = rffi.cast(ADDR, obj) From afa at codespeak.net Sat May 8 14:38:30 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 8 May 2010 14:38:30 +0200 (CEST) Subject: [pypy-svn] r74443 - in pypy/trunk/pypy/module/cpyext: . include test Message-ID: <20100508123830.0BF68282B90@codespeak.net> Author: afa Date: Sat May 8 14:38:29 2010 New Revision: 74443 Added: pypy/trunk/pypy/module/cpyext/include/complexobject.h Modified: pypy/trunk/pypy/module/cpyext/complexobject.py pypy/trunk/pypy/module/cpyext/include/Python.h pypy/trunk/pypy/module/cpyext/stubs.py pypy/trunk/pypy/module/cpyext/test/test_complexobject.py Log: Implement PyComplex_AsCComplex, it's an inlined function around a cpyext method because lltype does not handle functions returning structures. Modified: pypy/trunk/pypy/module/cpyext/complexobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/complexobject.py (original) +++ pypy/trunk/pypy/module/cpyext/complexobject.py Sat May 8 14:38:29 2010 @@ -1,9 +1,18 @@ -from pypy.rpython.lltypesystem import lltype -from pypy.module.cpyext.api import cpython_api, PyObject, build_type_checkers +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.module.cpyext.api import ( + cpython_api, cpython_struct, PyObject, build_type_checkers) +from pypy.module.cpyext.pyerrors import PyErr_BadArgument +from pypy.module.cpyext.floatobject import PyFloat_AsDouble from pypy.objspace.std.complexobject import W_ComplexObject +from pypy.interpreter.error import OperationError PyComplex_Check, PyComplex_CheckExact = build_type_checkers("Complex") +Py_complex_t = lltype.ForwardReference() +Py_complex_ptr = lltype.Ptr(Py_complex_t) +Py_complex_fields = (("real", rffi.DOUBLE), ("imag", rffi.DOUBLE)) +cpython_struct("Py_complex", Py_complex_fields, Py_complex_t) + @cpython_api([lltype.Float, lltype.Float], PyObject) def PyComplex_FromDoubles(space, real, imag): return space.newcomplex(real, imag) @@ -25,3 +34,31 @@ # CPython also accepts anything return 0.0 +# lltype does not handle functions returning a structure. This implements a +# helper function, which takes as argument a reference to the return value. + at cpython_api([PyObject, Py_complex_ptr], lltype.Void, error=None) +def _PyComplex_AsCComplex(space, w_obj, result): + """Return the Py_complex value of the complex number op. + + If op is not a Python complex number object but has a __complex__() + method, this method will first be called to convert op to a Python complex + number object.""" + # return -1 on failure + result.c_real = -1.0 + result.c_imag = 0.0 + if not PyComplex_Check(space, w_obj): + try: + w_obj = space.call_method(w_obj, "__complex__") + except: + # if the above did not work, interpret obj as a float giving the + # real part of the result, and fill in the imaginary part as 0. + result.c_real = PyFloat_AsDouble(space, w_obj) # -1 on failure + return + + if not PyComplex_Check(space, w_obj): + raise OperationError(space.w_TypeError, space.wrap( + "__complex__ should return a complex object")) + + assert isinstance(w_obj, W_ComplexObject) + result.c_real = w_obj.realval + result.c_imag = w_obj.imagval Modified: pypy/trunk/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/trunk/pypy/module/cpyext/include/Python.h (original) +++ pypy/trunk/pypy/module/cpyext/include/Python.h Sat May 8 14:38:29 2010 @@ -11,6 +11,7 @@ # define Py_DEPRECATED(VERSION_UNUSED) __attribute__((__deprecated__)) # define PyAPI_FUNC(RTYPE) RTYPE # define PyAPI_DATA(RTYPE) extern RTYPE +# define Py_LOCAL_INLINE(type) static inline type #else # define MS_WIN32 1 # include @@ -23,6 +24,7 @@ # define PyAPI_FUNC(RTYPE) __declspec(dllimport) RTYPE # define PyAPI_DATA(RTYPE) extern __declspec(dllimport) RTYPE # endif +# define Py_LOCAL_INLINE(type) static __inline type __fastcall #endif #define Py_ssize_t long @@ -101,6 +103,7 @@ #include "boolobject.h" #include "floatobject.h" +#include "complexobject.h" #include "methodobject.h" #include "funcobject.h" Added: pypy/trunk/pypy/module/cpyext/include/complexobject.h ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/cpyext/include/complexobject.h Sat May 8 14:38:29 2010 @@ -0,0 +1,27 @@ +/* Complex object interface */ + +#ifndef Py_COMPLEXOBJECT_H +#define Py_COMPLEXOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct Py_complex_t { + double real; + double imag; +} Py_complex; + +/* generated function */ +PyAPI_FUNC(void) _PyComplex_AsCComplex(PyObject *, Py_complex *); + +_inline static Py_complex PyComplex_AsCComplex(PyObject *obj) +{ + Py_complex result; + _PyComplex_AsCComplex(obj, &result); + return result; +} + +#ifdef __cplusplus +} +#endif +#endif /* !Py_COMPLEXOBJECT_H */ Modified: pypy/trunk/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/stubs.py (original) +++ pypy/trunk/pypy/module/cpyext/stubs.py Sat May 8 14:38:29 2010 @@ -450,15 +450,6 @@ """Create a new Python complex number object from a C Py_complex value.""" raise NotImplementedError - at cpython_api([PyObject], Py_complex) -def PyComplex_AsCComplex(space, op): - """Return the Py_complex value of the complex number op. - - If op is not a Python complex number object but has a __complex__() - method, this method will first be called to convert op to a Python complex - number object.""" - raise NotImplementedError - @cpython_api([rffi.CCHARP, rffi.CCHARPP, PyObject], rffi.DOUBLE, error=-1.0) def PyOS_string_to_double(space, s, endptr, overflow_exception): """Convert a string s to a double, raising a Python Modified: pypy/trunk/pypy/module/cpyext/test/test_complexobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_complexobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_complexobject.py Sat May 8 14:38:29 2010 @@ -1,3 +1,4 @@ +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.test.test_api import BaseApiTest class TestComplexObject(BaseApiTest): @@ -17,3 +18,16 @@ assert api.PyComplex_RealAsDouble(space.w_None) == -1.0 assert api.PyErr_Occurred() api.PyErr_Clear() + +class AppTestCComplex(AppTestCpythonExtensionBase): + def test_AsCComplex(self): + module = self.import_extension('foo', [ + ("as_tuple", "METH_O", + """ + Py_complex c = PyComplex_AsCComplex(args); + if (PyErr_Occurred()) return NULL; + return Py_BuildValue("dd", c.real, c.imag); + """)]) + assert module.as_tuple(12-34j) == (12, -34) + assert module.as_tuple(-3.14) == (-3.14, 0.0) + raises(TypeError, module.as_tuple, "12") From afa at codespeak.net Sat May 8 14:44:16 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 8 May 2010 14:44:16 +0200 (CEST) Subject: [pypy-svn] r74444 - pypy/trunk/pypy/module/cpyext/include Message-ID: <20100508124416.1D8AB282B90@codespeak.net> Author: afa Date: Sat May 8 14:44:14 2010 New Revision: 74444 Modified: pypy/trunk/pypy/module/cpyext/include/complexobject.h Log: Actually use the macro we've defined Modified: pypy/trunk/pypy/module/cpyext/include/complexobject.h ============================================================================== --- pypy/trunk/pypy/module/cpyext/include/complexobject.h (original) +++ pypy/trunk/pypy/module/cpyext/include/complexobject.h Sat May 8 14:44:14 2010 @@ -14,7 +14,7 @@ /* generated function */ PyAPI_FUNC(void) _PyComplex_AsCComplex(PyObject *, Py_complex *); -_inline static Py_complex PyComplex_AsCComplex(PyObject *obj) +Py_LOCAL_INLINE(Py_complex) PyComplex_AsCComplex(PyObject *obj) { Py_complex result; _PyComplex_AsCComplex(obj, &result); From arigo at codespeak.net Sat May 8 14:57:28 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 8 May 2010 14:57:28 +0200 (CEST) Subject: [pypy-svn] r74445 - pypy/extradoc/talk/pycon-italy-2010/talk Message-ID: <20100508125728.7DDD1282B90@codespeak.net> Author: arigo Date: Sat May 8 14:57:27 2010 New Revision: 74445 Modified: pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt Log: 26 + mine reported today :-( Modified: pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt ============================================================================== --- pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt (original) +++ pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt Sat May 8 14:57:27 2010 @@ -192,7 +192,7 @@ - Many bugfixes - * 26 issues reported after release of 1.2 + * 27 issues reported after release of 1.2 - beta version of cpyext From arigo at codespeak.net Sat May 8 16:12:59 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 8 May 2010 16:12:59 +0200 (CEST) Subject: [pypy-svn] r74446 - pypy/extradoc/talk/pycon-italy-2010/talk Message-ID: <20100508141259.439C7282B90@codespeak.net> Author: arigo Date: Sat May 8 16:12:57 2010 New Revision: 74446 Modified: pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt Log: Tweak. Modified: pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt ============================================================================== --- pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt (original) +++ pypy/extradoc/talk/pycon-italy-2010/talk/talk.txt Sat May 8 16:12:57 2010 @@ -221,7 +221,7 @@ - it just works - it may give good speed-ups (better than Psyco) -- may have a few bugs left (less than Psyco) |pause| +- it may have a few bugs left (Psyco too) |pause| - *it is not a hack (unlike Psyco)* |pause| From agaynor at codespeak.net Sun May 9 00:43:01 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sun, 9 May 2010 00:43:01 +0200 (CEST) Subject: [pypy-svn] r74447 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100508224301.13E07282B90@codespeak.net> Author: agaynor Date: Sun May 9 00:42:59 2010 New Revision: 74447 Modified: pypy/trunk/pypy/module/cpyext/test/foo.c pypy/trunk/pypy/module/cpyext/test/test_typeobject.py pypy/trunk/pypy/module/cpyext/typeobject.py Log: type.__call__ now correctly calls tp_init correctly. Modified: pypy/trunk/pypy/module/cpyext/test/foo.c ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/foo.c (original) +++ pypy/trunk/pypy/module/cpyext/test/foo.c Sun May 9 00:42:59 2010 @@ -161,17 +161,28 @@ typedef struct { PyUnicodeObject HEAD; + int val; } FuuObject; + +void Fuu_init(FuuObject *self, PyObject *args, PyObject *kwargs) { + self->val = 42; +} + static PyObject * Fuu_escape(PyTypeObject* type, PyObject *args) { Py_RETURN_TRUE; } +static PyObject * +Fuu_get_val(FuuObject *self) { + return PyInt_FromLong(self->val); +} static PyMethodDef Fuu_methods[] = { {"escape", (PyCFunction) Fuu_escape, METH_VARARGS, NULL}, + {"get_val", (PyCFunction) Fuu_get_val, METH_NOARGS, NULL}, {NULL} /* Sentinel */ }; @@ -222,7 +233,7 @@ 0, /*tp_descr_set*/ 0, /*tp_dictoffset*/ - 0, /*tp_init*/ + Fuu_init, /*tp_init*/ 0, /*tp_alloc will be set to PyType_GenericAlloc in module init*/ 0, /*tp_new*/ 0, /*tp_free Low-level free-memory routine */ Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Sun May 9 00:42:59 2010 @@ -108,6 +108,11 @@ return self assert fuu2(u"abc").baz().escape() raises(TypeError, module.fooType.object_member.__get__, 1) + + def test_init(self): + module = self.import_module(name="foo") + newobj = module.FuuType() + assert newobj.get_val() == 42 def test_sre(self): module = self.import_module(name='_sre') Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Sun May 9 00:42:59 2010 @@ -274,7 +274,20 @@ w_kw = space.newdict() for key, w_obj in kw_w.items(): space.setitem(w_kw, space.wrap(key), w_obj) - return generic_cpy_call(space, tp_new, pto, w_args, w_kw) + w_obj = generic_cpy_call(space, tp_new, pto, w_args, w_kw) + if w_obj: + w_obj_type = space.type(w_obj) + if not int(space.is_w(w_obj_type, w_type) or + space.is_true(space.issubtype(w_obj_type, w_type))): + return w_obj + pyo = make_ref(space, w_type) + pto = rffi.cast(PyTypeObjectPtr, pyo) + try: + if pto.c_tp_init: + generic_cpy_call(space, pto.c_tp_init, w_obj, w_args, w_kw) + return w_obj + finally: + Py_DecRef(space, pyo) else: raise operationerrfmt(space.w_TypeError, "cannot create '%s' instances", w_type.getname(space, '?')) From agaynor at codespeak.net Sun May 9 08:45:18 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sun, 9 May 2010 08:45:18 +0200 (CEST) Subject: [pypy-svn] r74448 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100509064518.6D34E282B90@codespeak.net> Author: agaynor Date: Sun May 9 08:45:15 2010 New Revision: 74448 Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py pypy/trunk/pypy/module/cpyext/test/test_typeobject.py pypy/trunk/pypy/module/cpyext/typeobject.py Log: Make __init__ on subclasses of cpyext types works. Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/slotdefs.py (original) +++ pypy/trunk/pypy/module/cpyext/slotdefs.py Sun May 9 08:45:15 2010 @@ -5,7 +5,8 @@ PyObject from pypy.module.cpyext.typeobjectdefs import unaryfunc, wrapperfunc,\ ternaryfunc, PyTypeObjectPtr, binaryfunc, getattrfunc, lenfunc,\ - ssizeargfunc, ssizessizeargfunc, ssizeobjargproc, iternextfunc + ssizeargfunc, ssizessizeargfunc, ssizeobjargproc, iternextfunc,\ + initproc from pypy.module.cpyext.pyobject import from_ref from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.state import State @@ -26,6 +27,11 @@ raise operationerrfmt(space.w_TypeError, "expected %d arguments, got %d", n, PyTuple_GET_SIZE(space, ob)) +def wrap_init(space, w_self, w_args, func, w_kwargs): + func_init = rffi.cast(initproc, func) + generic_cpy_call(space, func_init, w_self, w_args, w_kwargs) + return None + def wrap_unaryfunc(space, w_self, w_args, func): func_unary = rffi.cast(unaryfunc, func) check_num_args(space, w_args, 0) Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Sun May 9 08:45:15 2010 @@ -113,6 +113,15 @@ module = self.import_module(name="foo") newobj = module.FuuType() assert newobj.get_val() == 42 + + class Fuu2(module.FuuType): + def __init__(self): + self.foobar = 32 + super(Fuu2, self).__init__() + + newobj = Fuu2() + assert newobj.get_val() == 42 + assert newobj.foobar == 32 def test_sre(self): module = self.import_module(name='_sre') Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Sun May 9 08:45:15 2010 @@ -285,6 +285,9 @@ try: if pto.c_tp_init: generic_cpy_call(space, pto.c_tp_init, w_obj, w_args, w_kw) + else: + w_descr = space.lookup(w_obj, '__init__') + w_result = space.get_and_call_args(w_descr, w_obj, __args__) return w_obj finally: Py_DecRef(space, pyo) From afa at codespeak.net Sun May 9 09:33:48 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sun, 9 May 2010 09:33:48 +0200 (CEST) Subject: [pypy-svn] r74449 - pypy/trunk/pypy/module/cpyext Message-ID: <20100509073348.A7382282B90@codespeak.net> Author: afa Date: Sun May 9 09:33:47 2010 New Revision: 74449 Modified: pypy/trunk/pypy/module/cpyext/typeobject.py Log: Check for errors first, this dedents code a bit Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Sun May 9 09:33:47 2010 @@ -268,32 +268,32 @@ pto = rffi.cast(PyTypeObjectPtr, pyo) tp_new = pto.c_tp_new try: - if tp_new: - args_w, kw_w = __args__.unpack() - w_args = space.newtuple(args_w) - w_kw = space.newdict() - for key, w_obj in kw_w.items(): - space.setitem(w_kw, space.wrap(key), w_obj) - w_obj = generic_cpy_call(space, tp_new, pto, w_args, w_kw) - if w_obj: - w_obj_type = space.type(w_obj) - if not int(space.is_w(w_obj_type, w_type) or - space.is_true(space.issubtype(w_obj_type, w_type))): - return w_obj - pyo = make_ref(space, w_type) - pto = rffi.cast(PyTypeObjectPtr, pyo) - try: - if pto.c_tp_init: - generic_cpy_call(space, pto.c_tp_init, w_obj, w_args, w_kw) - else: - w_descr = space.lookup(w_obj, '__init__') - w_result = space.get_and_call_args(w_descr, w_obj, __args__) - return w_obj - finally: - Py_DecRef(space, pyo) - else: + if not tp_new: raise operationerrfmt(space.w_TypeError, "cannot create '%s' instances", w_type.getname(space, '?')) + + args_w, kw_w = __args__.unpack() + w_args = space.newtuple(args_w) + w_kw = space.newdict() + for key, w_obj in kw_w.items(): + space.setitem(w_kw, space.wrap(key), w_obj) + w_obj = generic_cpy_call(space, tp_new, pto, w_args, w_kw) + if w_obj: + w_obj_type = space.type(w_obj) + if not int(space.is_w(w_obj_type, w_type) or + space.is_true(space.issubtype(w_obj_type, w_type))): + return w_obj + pyo = make_ref(space, w_type) + pto = rffi.cast(PyTypeObjectPtr, pyo) + try: + if pto.c_tp_init: + generic_cpy_call(space, pto.c_tp_init, w_obj, w_args, w_kw) + else: + w_descr = space.lookup(w_obj, '__init__') + w_result = space.get_and_call_args(w_descr, w_obj, __args__) + return w_obj + finally: + Py_DecRef(space, pyo) finally: Py_DecRef(space, pyo) else: From afa at codespeak.net Sun May 9 09:43:12 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sun, 9 May 2010 09:43:12 +0200 (CEST) Subject: [pypy-svn] r74450 - pypy/trunk/pypy/module/cpyext Message-ID: <20100509074312.146F5282B90@codespeak.net> Author: afa Date: Sun May 9 09:43:10 2010 New Revision: 74450 Modified: pypy/trunk/pypy/module/cpyext/typeobject.py Log: No need to check for the result of generic_cpy_call(): it conveniently raises exceptions. More code simplification, and probably a fix in DecRef when tp_new returns an object of a different type. Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Sun May 9 09:43:10 2010 @@ -278,24 +278,27 @@ for key, w_obj in kw_w.items(): space.setitem(w_kw, space.wrap(key), w_obj) w_obj = generic_cpy_call(space, tp_new, pto, w_args, w_kw) - if w_obj: - w_obj_type = space.type(w_obj) - if not int(space.is_w(w_obj_type, w_type) or - space.is_true(space.issubtype(w_obj_type, w_type))): - return w_obj - pyo = make_ref(space, w_type) - pto = rffi.cast(PyTypeObjectPtr, pyo) - try: - if pto.c_tp_init: - generic_cpy_call(space, pto.c_tp_init, w_obj, w_args, w_kw) - else: - w_descr = space.lookup(w_obj, '__init__') - w_result = space.get_and_call_args(w_descr, w_obj, __args__) - return w_obj - finally: - Py_DecRef(space, pyo) finally: Py_DecRef(space, pyo) + + # call tp_init on the result + w_obj_type = space.type(w_obj) + if not int(space.is_w(w_obj_type, w_type) or + space.is_true(space.issubtype(w_obj_type, w_type))): + return w_obj + + pyo = make_ref(space, w_type) + pto = rffi.cast(PyTypeObjectPtr, pyo) + try: + if pto.c_tp_init: + generic_cpy_call(space, pto.c_tp_init, w_obj, w_args, w_kw) + else: + w_descr = space.lookup(w_obj, '__init__') + space.get_and_call_args(w_descr, w_obj, __args__) + finally: + Py_DecRef(space, pyo) + + return w_obj else: w_type = _precheck_for_new(space, w_type) return call__Type(space, w_type, __args__) From afa at codespeak.net Sun May 9 10:00:18 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sun, 9 May 2010 10:00:18 +0200 (CEST) Subject: [pypy-svn] r74451 - pypy/trunk/pypy/module/cpyext Message-ID: <20100509080018.EAC46282B90@codespeak.net> Author: afa Date: Sun May 9 10:00:17 2010 New Revision: 74451 Modified: pypy/trunk/pypy/module/cpyext/typeobject.py Log: Dedent more code, and add a few comments and XXX Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Sun May 9 10:00:17 2010 @@ -263,46 +263,51 @@ PyMember_SetOne(space, w_self, self.member, w_value) def c_type_descr__call__(space, w_type, __args__): - if isinstance(w_type, W_PyCTypeObject): - pyo = make_ref(space, w_type) - pto = rffi.cast(PyTypeObjectPtr, pyo) - tp_new = pto.c_tp_new - try: - if not tp_new: - raise operationerrfmt(space.w_TypeError, - "cannot create '%s' instances", w_type.getname(space, '?')) - - args_w, kw_w = __args__.unpack() - w_args = space.newtuple(args_w) - w_kw = space.newdict() - for key, w_obj in kw_w.items(): - space.setitem(w_kw, space.wrap(key), w_obj) - w_obj = generic_cpy_call(space, tp_new, pto, w_args, w_kw) - finally: - Py_DecRef(space, pyo) - - # call tp_init on the result - w_obj_type = space.type(w_obj) - if not int(space.is_w(w_obj_type, w_type) or - space.is_true(space.issubtype(w_obj_type, w_type))): - return w_obj - - pyo = make_ref(space, w_type) - pto = rffi.cast(PyTypeObjectPtr, pyo) - try: - if pto.c_tp_init: - generic_cpy_call(space, pto.c_tp_init, w_obj, w_args, w_kw) - else: - w_descr = space.lookup(w_obj, '__init__') - space.get_and_call_args(w_descr, w_obj, __args__) - finally: - Py_DecRef(space, pyo) - - return w_obj - else: + if not isinstance(w_type, W_PyCTypeObject): + # XXX is this possible? w_type = _precheck_for_new(space, w_type) return call__Type(space, w_type, __args__) + pyo = make_ref(space, w_type) + pto = rffi.cast(PyTypeObjectPtr, pyo) + tp_new = pto.c_tp_new + try: + if not tp_new: + raise operationerrfmt(space.w_TypeError, + "cannot create '%s' instances", w_type.getname(space, '?')) + + args_w, kw_w = __args__.unpack() + w_args = space.newtuple(args_w) + w_kw = space.newdict() + for key, w_obj in kw_w.items(): + space.setitem(w_kw, space.wrap(key), w_obj) + w_obj = generic_cpy_call(space, tp_new, pto, w_args, w_kw) + finally: + Py_DecRef(space, pyo) + + # If the returned object is not an instance of type, + # it won't be initialized. + w_obj_type = space.type(w_obj) + if not (space.is_w(w_obj_type, w_type) or + space.is_true(space.issubtype(w_obj_type, w_type))): + return w_obj + + # call tp_init + # XXX 1 w_obj_type should be used instead + pyo = make_ref(space, w_type) + pto = rffi.cast(PyTypeObjectPtr, pyo) + try: + if pto.c_tp_init: + generic_cpy_call(space, pto.c_tp_init, w_obj, w_args, w_kw) + else: + # XXX 2 this should not necessary: tp_init should be inherited + w_descr = space.lookup(w_obj, '__init__') + space.get_and_call_args(w_descr, w_obj, __args__) + finally: + Py_DecRef(space, pyo) + + return w_obj + def c_type_descr__new__(space, w_typetype, w_name, w_bases, w_dict): # copied from typetype.descr__new__, XXX missing logic: metaclass resolving w_typetype = _precheck_for_new(space, w_typetype) From afa at codespeak.net Sun May 9 11:10:03 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sun, 9 May 2010 11:10:03 +0200 (CEST) Subject: [pypy-svn] r74452 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100509091003.E0F28282B90@codespeak.net> Author: afa Date: Sun May 9 11:10:01 2010 New Revision: 74452 Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py pypy/trunk/pypy/module/cpyext/test/foo.c pypy/trunk/pypy/module/cpyext/test/test_typeobject.py pypy/trunk/pypy/module/cpyext/typeobject.py Log: Implement slot_tp_init, and inherit tp_init from base class. Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/slotdefs.py (original) +++ pypy/trunk/pypy/module/cpyext/slotdefs.py Sun May 9 11:10:01 2010 @@ -11,6 +11,7 @@ from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.state import State from pypy.interpreter.error import OperationError, operationerrfmt +from pypy.interpreter.argument import Arguments from pypy.rlib.unroll import unrolling_iterable space = None @@ -116,6 +117,13 @@ w_args_new = space.newtuple(args_w) return space.call(w_func, w_args_new, w_kwds) + at cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) +def slot_tp_init(space, w_self, w_args, w_kwds): + w_descr = space.lookup(w_self, '__init__') + args = Arguments.frompacked(space, w_args, w_kwds) + space.get_and_call_args(w_descr, w_self, args) + return 0 + PyWrapperFlag_KEYWORDS = 1 # adopted from typeobject.c Modified: pypy/trunk/pypy/module/cpyext/test/foo.c ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/foo.c (original) +++ pypy/trunk/pypy/module/cpyext/test/foo.c Sun May 9 11:10:01 2010 @@ -245,6 +245,65 @@ 0 /*tp_weaklist*/ }; +PyTypeObject Fuu2Type = { + PyObject_HEAD_INIT(NULL) + 0, + "foo.fuu2", + sizeof(FuuObject), + 0, + 0, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_CHECKTYPES, /*tp_flags*/ + 0, /*tp_doc*/ + + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + + /* Attribute descriptor and subclassing stuff */ + + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + + 0, /*tp_init*/ + 0, /*tp_alloc will be set to PyType_GenericAlloc in module init*/ + 0, /*tp_new*/ + 0, /*tp_free Low-level free-memory routine */ + 0, /*tp_is_gc For PyObject_IS_GC */ + 0, /*tp_bases*/ + 0, /*tp_mro method resolution order */ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0 /*tp_weaklist*/ +}; + /* foo functions */ @@ -277,11 +336,14 @@ footype.tp_new = PyType_GenericNew; FuuType.tp_base = &PyUnicode_Type; + Fuu2Type.tp_base = &FuuType; if (PyType_Ready(&footype) < 0) return; if (PyType_Ready(&FuuType) < 0) return; + if (PyType_Ready(&Fuu2Type) < 0) + return; m = Py_InitModule("foo", foo_functions); if (m == NULL) return; @@ -290,6 +352,7 @@ if (PyDict_SetItemString(d, "fooType", (PyObject *)&footype) < 0) return; PyDict_SetItemString(d, "FuuType", (PyObject *) &FuuType); + PyDict_SetItemString(d, "Fuu2Type", (PyObject *) &Fuu2Type); } /* No need to check the error here, the caller will do that */ } Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Sun May 9 11:10:01 2010 @@ -113,7 +113,12 @@ module = self.import_module(name="foo") newobj = module.FuuType() assert newobj.get_val() == 42 - + + # this subtype should inherit tp_init + newobj = module.Fuu2Type() + assert newobj.get_val() == 42 + + # this subclass redefines __init__ class Fuu2(module.FuuType): def __init__(self): self.foobar = 32 Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Sun May 9 11:10:01 2010 @@ -293,16 +293,13 @@ return w_obj # call tp_init - # XXX 1 w_obj_type should be used instead - pyo = make_ref(space, w_type) + pyo = make_ref(space, w_obj_type) pto = rffi.cast(PyTypeObjectPtr, pyo) try: if pto.c_tp_init: - generic_cpy_call(space, pto.c_tp_init, w_obj, w_args, w_kw) - else: - # XXX 2 this should not necessary: tp_init should be inherited - w_descr = space.lookup(w_obj, '__init__') - space.get_and_call_args(w_descr, w_obj, __args__) + if generic_cpy_call(space, pto.c_tp_init, w_obj, w_args, w_kw) < 0: + state = space.fromcache(State) + state.check_and_raise_exception() finally: Py_DecRef(space, pyo) @@ -586,6 +583,8 @@ base = rffi.cast(PyTypeObjectPtr, base_pyo) if not pto.c_tp_dealloc: pto.c_tp_dealloc = base.c_tp_dealloc + if not pto.c_tp_init: + pto.c_tp_init = base.c_tp_init if not pto.c_tp_alloc: pto.c_tp_alloc = base.c_tp_alloc # XXX check for correct GC flags! From arigo at codespeak.net Sun May 9 14:22:57 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 9 May 2010 14:22:57 +0200 (CEST) Subject: [pypy-svn] r74453 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter codewriter/test metainterp Message-ID: <20100509122257.ED08B282B90@codespeak.net> Author: arigo Date: Sun May 9 14:22:54 2010 New Revision: 74453 Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py - copied, changed from r74439, pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitcode.py (contents, props changed) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: Move JitCode to its own module. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Sun May 9 14:22:54 2010 @@ -1,107 +1,10 @@ -from pypy.jit.metainterp.history import AbstractValue, AbstractDescr, getkind +from pypy.jit.metainterp.history import AbstractDescr, getkind from pypy.jit.codewriter.flatten import Register, Label, TLabel, KINDS -from pypy.jit.codewriter.flatten import ListOfKind, SwitchDictDescr +from pypy.jit.codewriter.flatten import ListOfKind from pypy.jit.codewriter.format import format_assembler +from pypy.jit.codewriter.jitcode import SwitchDictDescr, JitCode from pypy.objspace.flow.model import Constant from pypy.rpython.lltypesystem import lltype, llmemory -from pypy.rlib.objectmodel import we_are_translated - - -class JitCode(AbstractValue): - _empty_i = [] - _empty_r = [] - _empty_f = [] - - def __init__(self, name, fnaddr=None, calldescr=None, called_from=None, - assembler=None): - self.name = name - self.fnaddr = fnaddr - self.calldescr = calldescr - self._called_from = called_from # debugging - self._assembler = assembler # debugging - - def setup(self, code, constants_i=[], constants_r=[], constants_f=[], - num_regs_i=256, num_regs_r=256, num_regs_f=256, - liveness=None): - self.code = code - # if the following lists are empty, use a single shared empty list - self.constants_i = constants_i or self._empty_i - self.constants_r = constants_r or self._empty_r - self.constants_f = constants_f or self._empty_f - # encode the three num_regs into a single integer - self.num_regs_encoded = ((num_regs_i << 18) | - (num_regs_r << 9) | - (num_regs_f << 0)) - self.liveness = liveness - - def num_regs_i(self): - return self.num_regs_encoded >> 18 - - def num_regs_r(self): - return (self.num_regs_encoded >> 9) & 0x1FF - - def num_regs_f(self): - return self.num_regs_encoded & 0x1FF - - def has_liveness_info(self, pc): - return pc in self.liveness - - def enumerate_live_vars(self, pc, callback, arg, - registers_i, registers_r, registers_f): - # 'pc' gives a position in this bytecode. This invokes - # 'callback' for each variable that is live across the - # instruction which starts at 'pc'. (It excludes the arguments - # of that instruction which are no longer used afterwards, and - # excludes the return value of that instruction.) More precisely, - # this invokes 'callback(arg, box, index)' where 'box' comes from one - # of the three lists of registers and 'index' is 0, 1, 2... - # If the callback returns a box, then it is stored back. - if not we_are_translated() and pc not in self.liveness: - self._missing_liveness(pc) - live_i, live_r, live_f = self.liveness[pc] # XXX compactify!! - index = 0 - for c in live_i: - newbox = callback(arg, registers_i[ord(c)], index) - index += 1 - if newbox is not None: - registers_i[ord(c)] = newbox - for c in live_r: - newbox = callback(arg, registers_r[ord(c)], index) - index += 1 - if newbox is not None: - registers_r[ord(c)] = newbox - for c in live_f: - newbox = callback(arg, registers_f[ord(c)], index) - index += 1 - if newbox is not None: - registers_f[ord(c)] = newbox - return index - enumerate_live_vars._annspecialcase_ = 'specialize:arg(2)' - - def _live_vars(self, pc): - # for testing only - class Names: - def __init__(self, kind): - self.kind = kind - def __getitem__(self, index): - return '%%%s%d' % (self.kind, index) - def callback(lst, reg, index): - lst.append(reg) - lst = [] - self.enumerate_live_vars(pc, callback, lst, - Names('i'), Names('r'), Names('f')) - lst.sort() - return ' '.join(lst) - - def _missing_liveness(self, pc): - opcode = ord(self.code[pc]) - insn = 'insn %d' % opcode - if self._assembler is not None: - for name, code in self._assembler.insns.items(): - if code == opcode: - insn = name - raise KeyError("missing liveness[%d], corresponding to %r" % ( - pc, insn)) class Assembler(object): @@ -110,7 +13,6 @@ self.insns = {} self.descrs = [] self._descr_dict = {} - self._count_jitcodes = 0 def assemble(self, ssarepr): self.setup() @@ -271,7 +173,7 @@ self.count_regs['ref'], self.count_regs['float'], liveness=self.liveness) - if self._count_jitcodes < 50: # stop if we have a lot of them - jitcode._dump = format_assembler(ssarepr) - self._count_jitcodes += 1 + #if self._count_jitcodes < 50: # stop if we have a lot of them + # jitcode._dump = format_assembler(ssarepr) + #self._count_jitcodes += 1 return jitcode Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Sun May 9 14:22:54 2010 @@ -44,9 +44,6 @@ def __iter__(self): return iter(self.content) -class SwitchDictDescr(AbstractDescr): - "Get a 'dict' attribute mapping integer values to bytecode positions." - KINDS = ['int', 'ref', 'float'] # ____________________________________________________________ @@ -222,6 +219,7 @@ if len(switches) >= 5 and kind == 'int': # A large switch on an integer, implementable efficiently # with the help of a SwitchDictDescr + from pypy.jit.codewriter.jitcode import SwitchDictDescr switchdict = SwitchDictDescr() switchdict._labels = [] self.emitline('switch', self.getcolor(block.exitswitch), Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py Sun May 9 14:22:54 2010 @@ -2,7 +2,8 @@ from pypy.objspace.flow.model import Constant from pypy.rpython.lltypesystem import lltype from pypy.jit.codewriter.flatten import SSARepr, Label, TLabel, Register -from pypy.jit.codewriter.flatten import ListOfKind, SwitchDictDescr +from pypy.jit.codewriter.flatten import ListOfKind +from pypy.jit.codewriter.jitcode import SwitchDictDescr from pypy.jit.metainterp.history import AbstractDescr Copied: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py (from r74439, pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py) ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py Sun May 9 14:22:54 2010 @@ -1,9 +1,4 @@ -from pypy.jit.metainterp.history import AbstractValue, AbstractDescr, getkind -from pypy.jit.codewriter.flatten import Register, Label, TLabel, KINDS -from pypy.jit.codewriter.flatten import ListOfKind, SwitchDictDescr -from pypy.jit.codewriter.format import format_assembler -from pypy.objspace.flow.model import Constant -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.jit.metainterp.history import AbstractValue, AbstractDescr from pypy.rlib.objectmodel import we_are_translated @@ -20,7 +15,7 @@ self._called_from = called_from # debugging self._assembler = assembler # debugging - def setup(self, code, constants_i=[], constants_r=[], constants_f=[], + def setup(self, code='', constants_i=[], constants_r=[], constants_f=[], num_regs_i=256, num_regs_r=256, num_regs_f=256, liveness=None): self.code = code @@ -104,174 +99,5 @@ pc, insn)) -class Assembler(object): - - def __init__(self): - self.insns = {} - self.descrs = [] - self._descr_dict = {} - self._count_jitcodes = 0 - - def assemble(self, ssarepr): - self.setup() - for insn in ssarepr.insns: - self.write_insn(insn) - self.fix_labels() - self.check_result() - return self.make_jitcode(ssarepr) - - def setup(self): - self.code = [] - self.constants_dict = {} - self.constants_i = [] - self.constants_r = [] - self.constants_f = [] - self.label_positions = {} - self.tlabel_positions = [] - self.switchdictdescrs = [] - self.count_regs = dict.fromkeys(KINDS, 0) - self.liveness = {} - - def emit_reg(self, reg): - if reg.index >= self.count_regs[reg.kind]: - self.count_regs[reg.kind] = reg.index + 1 - self.code.append(chr(reg.index)) - - def emit_const(self, const, kind, allow_short=False): - if const not in self.constants_dict: - value = const.value - TYPE = lltype.typeOf(value) - if kind == 'int': - if isinstance(TYPE, lltype.Ptr): - assert TYPE.TO._gckind == 'raw' - value = llmemory.cast_ptr_to_adr(value) - TYPE = llmemory.Address - if TYPE == llmemory.Address: - value = llmemory.cast_adr_to_int(value) - else: - value = lltype.cast_primitive(lltype.Signed, value) - if allow_short and -128 <= value <= 127: # xxx symbolic - # emit the constant as a small integer - self.code.append(chr(value & 0xFF)) - return True - constants = self.constants_i - elif kind == 'ref': - value = lltype.cast_opaque_ptr(llmemory.GCREF, value) - constants = self.constants_r - elif kind == 'float': - assert TYPE == lltype.Float - constants = self.constants_f - else: - raise NotImplementedError(const) - constants.append(value) - self.constants_dict[const] = 256 - len(constants) - # emit the constant normally, as one byte that is an index in the - # list of constants - self.code.append(chr(self.constants_dict[const])) - return False - - def write_insn(self, insn): - if isinstance(insn[0], Label): - self.label_positions[insn[0].name] = len(self.code) - return - if insn[0] == '-live-': - self.liveness[len(self.code)] = ( - self.get_liveness_info(insn, 'int'), - self.get_liveness_info(insn, 'ref'), - self.get_liveness_info(insn, 'float')) - return - startposition = len(self.code) - self.code.append("temporary placeholder") - # - argcodes = [] - for x in insn[1:]: - if isinstance(x, Register): - self.emit_reg(x) - argcodes.append(x.kind[0]) - elif isinstance(x, Constant): - kind = getkind(x.concretetype) - is_short = self.emit_const(x, kind, allow_short=True) - if is_short: - argcodes.append('c') - else: - argcodes.append(kind[0]) - elif isinstance(x, TLabel): - self.tlabel_positions.append((x.name, len(self.code))) - self.code.append("temp 1") - self.code.append("temp 2") - argcodes.append('L') - elif isinstance(x, ListOfKind): - itemkind = x.kind - lst = list(x) - assert len(lst) <= 255, "list too long!" - self.code.append(chr(len(lst))) - for item in lst: - if isinstance(item, Register): - assert itemkind == item.kind - self.emit_reg(item) - elif isinstance(item, Constant): - assert itemkind == getkind(item.concretetype) - self.emit_const(item, itemkind) - else: - raise NotImplementedError("found in ListOfKind(): %r" - % (item,)) - argcodes.append(itemkind[0].upper()) - elif isinstance(x, AbstractDescr): - if x not in self._descr_dict: - self._descr_dict[x] = len(self.descrs) - self.descrs.append(x) - if isinstance(x, SwitchDictDescr): - self.switchdictdescrs.append(x) - num = self._descr_dict[x] - assert 0 <= num <= 0xFFFF, "too many AbstractDescrs!" - self.code.append(chr(num & 0xFF)) - self.code.append(chr(num >> 8)) - argcodes.append('d') - else: - raise NotImplementedError(x) - # - opname = insn[0] - if opname.startswith('G_'): opname = opname[2:] - key = opname + '/' + ''.join(argcodes) - num = self.insns.setdefault(key, len(self.insns)) - self.code[startposition] = chr(num) - - def get_liveness_info(self, insn, kind): - lives = [chr(reg.index) for reg in insn[1:] if reg.kind == kind] - lives.sort() - return ''.join(lives) - - def fix_labels(self): - for name, pos in self.tlabel_positions: - assert self.code[pos ] == "temp 1" - assert self.code[pos+1] == "temp 2" - target = self.label_positions[name] - assert 0 <= target <= 0xFFFF - self.code[pos ] = chr(target & 0xFF) - self.code[pos+1] = chr(target >> 8) - for descr in self.switchdictdescrs: - descr.dict = {} - for key, switchlabel in descr._labels: - target = self.label_positions[switchlabel.name] - descr.dict[key] = target - - def check_result(self): - # Limitation of the number of registers, from the single-byte encoding - assert self.count_regs['int'] + len(self.constants_i) <= 256 - assert self.count_regs['ref'] + len(self.constants_r) <= 256 - assert self.count_regs['float'] + len(self.constants_f) <= 256 - - def make_jitcode(self, ssarepr): - jitcode = JitCode(ssarepr.name, assembler=self) - jitcode.setup(''.join(self.code), - self.constants_i, - self.constants_r, - self.constants_f, - self.count_regs['int'], - self.count_regs['ref'], - self.count_regs['float'], - liveness=self.liveness) - if self._count_jitcodes < 50: # stop if we have a lot of them - jitcode._dump = format_assembler(ssarepr) - self._count_jitcodes += 1 - return jitcode +class SwitchDictDescr(AbstractDescr): + "Get a 'dict' attribute mapping integer values to bytecode positions." Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Sun May 9 14:22:54 2010 @@ -470,9 +470,8 @@ ('uint_lshift', 'int_lshift'), ('uint_rshift', 'int_rshift'), ('uint_xor', 'int_xor'), - - ]: + assert _old not in locals() exec py.code.Source(''' def rewrite_op_%s(self, op): op1 = SpaceOperation(%r, op.args, op.result) Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitcode.py ============================================================================== --- (empty file) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitcode.py Sun May 9 14:22:54 2010 @@ -0,0 +1,35 @@ +from pypy.jit.codewriter.jitcode import JitCode + + +def test_num_regs(): + j = JitCode("test") + j.setup(num_regs_i=12, num_regs_r=34, num_regs_f=56) + assert j.num_regs_i() == 12 + assert j.num_regs_r() == 34 + assert j.num_regs_f() == 56 + j.setup(num_regs_i=0, num_regs_r=0, num_regs_f=0) + assert j.num_regs_i() == 0 + assert j.num_regs_r() == 0 + assert j.num_regs_f() == 0 + j.setup(num_regs_i=256, num_regs_r=256, num_regs_f=256) + assert j.num_regs_i() == 256 + assert j.num_regs_r() == 256 + assert j.num_regs_f() == 256 + +def test_liveness(): + j = JitCode("test") + j.setup(liveness={5: (" A", "b", "CD")}) + assert j.has_liveness_info(5) + assert not j.has_liveness_info(4) + # + seen = [] + def callback(arg, value, index): + assert arg == "foo" + seen.append((value, index)) + # + total = j.enumerate_live_vars(5, callback, "foo", + {ord(" "): "i10", ord("A"): "i20"}, + {ord("b"): "r30"}, + {ord("C"): "f40", ord("D"): "f50"}) + assert total == 5 + assert seen == [("i10", 0), ("i20", 1), ("r30", 2), ("f40", 3), ("f50", 4)] Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Sun May 9 14:22:54 2010 @@ -5,7 +5,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory, rclass from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.llinterp import LLException -from pypy.jit.codewriter.flatten import SwitchDictDescr +from pypy.jit.codewriter.jitcode import SwitchDictDescr def arguments(*argtypes, **kwds): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Sun May 9 14:22:54 2010 @@ -21,8 +21,7 @@ from pypy.rlib.objectmodel import specialize from pypy.rlib.jit import DEBUG_OFF, DEBUG_PROFILE, DEBUG_STEPS, DEBUG_DETAILED from pypy.jit.metainterp.compile import GiveUp -from pypy.jit.codewriter.assembler import JitCode -from pypy.jit.codewriter.flatten import SwitchDictDescr +from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr # ____________________________________________________________ From benjamin at codespeak.net Mon May 10 04:33:06 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 10 May 2010 04:33:06 +0200 (CEST) Subject: [pypy-svn] r74454 - pypy/trunk/pypy/rpython Message-ID: <20100510023306.E46D236C22A@codespeak.net> Author: benjamin Date: Mon May 10 04:33:04 2010 New Revision: 74454 Modified: pypy/trunk/pypy/rpython/annlowlevel.py Log: compare classes with is not == Modified: pypy/trunk/pypy/rpython/annlowlevel.py ============================================================================== --- pypy/trunk/pypy/rpython/annlowlevel.py (original) +++ pypy/trunk/pypy/rpython/annlowlevel.py Mon May 10 04:33:04 2010 @@ -17,7 +17,7 @@ def __init__(self, val): self.val = val def __eq__(self, other): - return self.__class__ == other.__class__ and self.val == other.val + return self.__class__ is other.__class__ and self.val == other.val def __ne__(self, other): return not (self == other) def __hash__(self): From afa at codespeak.net Mon May 10 09:33:09 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 10 May 2010 09:33:09 +0200 (CEST) Subject: [pypy-svn] r74455 - in pypy/trunk/pypy/module/cpyext: include src Message-ID: <20100510073309.77B13282BF5@codespeak.net> Author: afa Date: Mon May 10 09:33:08 2010 New Revision: 74455 Modified: pypy/trunk/pypy/module/cpyext/include/Python.h pypy/trunk/pypy/module/cpyext/src/pythonrun.c Log: Python.h defines both MS_WIN32 and MS_WINDOWS Modified: pypy/trunk/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/trunk/pypy/module/cpyext/include/Python.h (original) +++ pypy/trunk/pypy/module/cpyext/include/Python.h Mon May 10 09:33:08 2010 @@ -14,6 +14,7 @@ # define Py_LOCAL_INLINE(type) static inline type #else # define MS_WIN32 1 +# define MS_WINDOWS 1 # include # include # define Py_DEPRECATED(VERSION_UNUSED) Modified: pypy/trunk/pypy/module/cpyext/src/pythonrun.c ============================================================================== --- pypy/trunk/pypy/module/cpyext/src/pythonrun.c (original) +++ pypy/trunk/pypy/module/cpyext/src/pythonrun.c Mon May 10 09:33:08 2010 @@ -1,6 +1,11 @@ #include "Python.h" +#ifdef MS_WINDOWS +#include "malloc.h" /* for alloca */ +#include "windows.h" +#endif + void Py_FatalError(const char *msg) { From arigo at codespeak.net Mon May 10 11:56:52 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 10 May 2010 11:56:52 +0200 (CEST) Subject: [pypy-svn] r74457 - in pypy/branch/blackhole-improvement/pypy: jit/codewriter jit/codewriter/test objspace/flow Message-ID: <20100510095652.AAB7F282BF5@codespeak.net> Author: arigo Date: Mon May 10 11:56:50 2010 New Revision: 74457 Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py - copied, changed from r74453, pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py - copied, changed from r74453, pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_policy.py (contents, props changed) Removed: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_call.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py pypy/branch/blackhole-improvement/pypy/objspace/flow/model.py Log: Rename 'jitter' into 'jtransform'. Add policy tests. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Mon May 10 11:56:50 2010 @@ -13,14 +13,24 @@ self.insns = {} self.descrs = [] self._descr_dict = {} + self._count_jitcodes = 0 - def assemble(self, ssarepr): + def assemble(self, ssarepr, jitcode=None): + """Take the 'ssarepr' representation of the code and assemble + it inside the 'jitcode'. If jitcode is None, make a new one. + """ self.setup() for insn in ssarepr.insns: self.write_insn(insn) self.fix_labels() self.check_result() - return self.make_jitcode(ssarepr) + if jitcode is None: + jitcode = JitCode(ssarepr.name) + self.make_jitcode(jitcode) + if self._count_jitcodes < 20: # stop if we have a lot of them + jitcode._dump = format_assembler(ssarepr) + self._count_jitcodes += 1 + return jitcode def setup(self): self.code = [] @@ -163,8 +173,7 @@ assert self.count_regs['ref'] + len(self.constants_r) <= 256 assert self.count_regs['float'] + len(self.constants_f) <= 256 - def make_jitcode(self, ssarepr): - jitcode = JitCode(ssarepr.name, assembler=self) + def make_jitcode(self, jitcode): jitcode.setup(''.join(self.code), self.constants_i, self.constants_r, @@ -172,8 +181,5 @@ self.count_regs['int'], self.count_regs['ref'], self.count_regs['float'], - liveness=self.liveness) - #if self._count_jitcodes < 50: # stop if we have a lot of them - # jitcode._dump = format_assembler(ssarepr) - #self._count_jitcodes += 1 - return jitcode + liveness=self.liveness, + assembler=self) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py Mon May 10 11:56:50 2010 @@ -5,19 +5,23 @@ from pypy.jit.codewriter import support from pypy.translator.simplify import get_funcobj +from pypy.jit.codewriter.jitcode import JitCode class CallControl(object): - def __init__(self, rtyper=None, portal_runner_obj=None): + def __init__(self, rtyper=None, portal_graph=None): self.rtyper = rtyper - self.portal_runner_obj = portal_runner_obj + self.portal_graph = portal_graph + self.jitcodes = {} # map {graph: jitcode} + self.unfinished_graphs = [] # list of graphs with pending jitcodes - def find_all_graphs(self, portal_graph, policy): + def find_all_graphs(self, policy): def is_candidate(graph): return policy.look_inside_graph(graph) - - todo = [portal_graph] + + assert self.portal_graph is not None + todo = [self.portal_graph] candidate_graphs = set(todo) def callers(): @@ -94,9 +98,9 @@ funcobj = get_funcobj(funcptr) if getattr(funcobj, 'graph', None) is None: return 'residual' - if funcobj is self.portal_runner_obj: - return 'recursive' targetgraph = funcobj.graph + if targetgraph is self.portal_graph: + return 'recursive' if (hasattr(targetgraph, 'func') and hasattr(targetgraph.func, 'oopspec')): return 'builtin' @@ -111,3 +115,18 @@ def is_candidate(self, graph): # used only after find_all_graphs() return graph in self.candidate_graphs + + def enum_pending_graphs(self): + while self.unfinished_graphs: + graph = self.unfinished_graphs.pop() + yield graph, self.jitcodes[graph] + + def get_jitcode(self, graph, called_from=None): + # 'called_from' is only one of the callers, used for debugging. + try: + return self.jitcodes[graph] + except KeyError: + jitcode = JitCode(graph.name, called_from=called_from) + self.jitcodes[graph] = jitcode + self.unfinished_graphs.append(graph) + return jitcode Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py Mon May 10 11:56:50 2010 @@ -2,14 +2,17 @@ from pypy.jit.codewriter.regalloc import perform_register_allocation from pypy.jit.codewriter.flatten import flatten_graph, KINDS from pypy.jit.codewriter.assembler import Assembler, JitCode -from pypy.jit.codewriter.jitter import transform_graph +from pypy.jit.codewriter.jtransform import transform_graph from pypy.jit.codewriter.format import format_assembler from pypy.jit.codewriter.liveness import compute_liveness +from pypy.jit.codewriter.call import CallControl from pypy.jit.codewriter.policy import log +from pypy.objspace.flow.model import copygraph from pypy.tool.udir import udir class CodeWriter(object): + callcontrol = None # for tests def __init__(self, cpu=None): self.cpu = cpu @@ -19,17 +22,19 @@ """For testing.""" rtyper = support.annotate(func, values, type_system=type_system) graph = rtyper.annotator.translator.graphs[0] - return self.transform_graph_to_jitcode(graph, True, True) + jitcode = JitCode("test") + self.transform_graph_to_jitcode(graph, jitcode, True, True) + return jitcode - def transform_graph_to_jitcode(self, graph, portal, verbose): + def transform_graph_to_jitcode(self, graph, jitcode, portal, verbose): """Transform a graph into a JitCode containing the same bytecode - in a different format. Note that the original 'graph' is mangled - by the process and should not be used any more. + in a different format. """ + graph = copygraph(graph, shallowvars=True) # # step 1: mangle the graph so that it contains the final instructions # that we want in the JitCode, but still as a control flow graph - transform_graph(graph, self.cpu, portal) + transform_graph(graph, self.cpu, self.callcontrol, portal) # # step 2a: perform register allocation on it regallocs = {} @@ -52,12 +57,16 @@ # of bytes and lists of constants. It's during this step that # constants are cast to their normalized type (Signed, GCREF or # Float). - jitcode = self.assembler.assemble(ssarepr) - return jitcode + self.assembler.assemble(ssarepr, jitcode) - def make_jitcodes(self, maingraph, verbose=False): - self.portal_graph = maingraph - return self.transform_graph_to_jitcode(maingraph, True, verbose) + def make_jitcodes(self, maingraph, policy, verbose=False): + self.callcontrol = CallControl(self.cpu.rtyper, maingraph) + self.callcontrol.find_all_graphs(policy) + mainjitcode = self.callcontrol.get_jitcode(maingraph) + for graph, jitcode in self.callcontrol.enum_pending_graphs(): + self.transform_graph_to_jitcode(graph, jitcode, + graph is maingraph, verbose) + return mainjitcode def print_ssa_repr(self, ssarepr, portal, verbose): if verbose: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py Mon May 10 11:56:50 2010 @@ -1,23 +1,21 @@ -from pypy.jit.metainterp.history import AbstractValue, AbstractDescr +from pypy.jit.metainterp.history import AbstractDescr from pypy.rlib.objectmodel import we_are_translated -class JitCode(AbstractValue): +class JitCode(AbstractDescr): _empty_i = [] _empty_r = [] _empty_f = [] - def __init__(self, name, fnaddr=None, calldescr=None, called_from=None, - assembler=None): + def __init__(self, name, fnaddr=None, calldescr=None, called_from=None): self.name = name self.fnaddr = fnaddr self.calldescr = calldescr self._called_from = called_from # debugging - self._assembler = assembler # debugging def setup(self, code='', constants_i=[], constants_r=[], constants_f=[], num_regs_i=256, num_regs_r=256, num_regs_f=256, - liveness=None): + liveness=None, assembler=None): self.code = code # if the following lists are empty, use a single shared empty list self.constants_i = constants_i or self._empty_i @@ -28,6 +26,7 @@ (num_regs_r << 9) | (num_regs_f << 0)) self.liveness = liveness + self._assembler = assembler # debugging def num_regs_i(self): return self.num_regs_encoded >> 18 @@ -98,6 +97,13 @@ raise KeyError("missing liveness[%d], corresponding to %r" % ( pc, insn)) + def __repr__(self): + return '' % self.name + class SwitchDictDescr(AbstractDescr): "Get a 'dict' attribute mapping integer values to bytecode positions." + + def __repr__(self): + dict = getattr(self, 'dict', '?') + return '' % (dict,) Copied: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (from r74453, pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py) ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Mon May 10 11:56:50 2010 @@ -7,11 +7,11 @@ from pypy.jit.codewriter import support, heaptracker -def transform_graph(graph, cpu=None, portal=True): +def transform_graph(graph, cpu=None, callcontrol=None, portal=True): """Transform a control flow graph to make it suitable for being flattened in a JitCode. """ - t = Transformer(cpu) + t = Transformer(cpu, callcontrol) t.transform(graph, portal) @@ -21,8 +21,9 @@ class Transformer(object): - def __init__(self, cpu=None): + def __init__(self, cpu=None, callcontrol=None): self.cpu = cpu + self.callcontrol = callcontrol def transform(self, graph, portal): self.graph = graph @@ -173,10 +174,21 @@ rewrite_op_float_gt = _rewrite_symmetric rewrite_op_float_ge = _rewrite_symmetric + # ---------- + # Various kinds of calls + def rewrite_op_direct_call(self, op): + kind = self.callcontrol.guess_call_kind(op) + return getattr(self, 'handle_%s_call' % kind)(op) + + def rewrite_op_indirect_call(self, op): + kind = self.callcontrol.guess_call_kind(op) + return getattr(self, 'handle_%s_indirect_call' % kind)(op) + + def rewrite_call(self, op, namebase, initialargs): """Turn 'i0 = direct_call(fn, i1, i2, ref1, ref2)' - into e.g. 'i0 = residual_call_ir_i(fn, [i1, i2], [ref1, ref2])'. - The name is one of 'residual_call_{r,ir,irf}_{i,r,f,v}'.""" + into 'i0 = xxx_call_ir_i(fn, descr, [i1,i2], [ref1,ref2])'. + The name is one of '{residual,direct}_call_{r,ir,irf}_{i,r,f,v}'.""" args_i = [] args_r = [] args_f = [] @@ -190,16 +202,8 @@ if 'r' in kinds: sublists.append(ListOfKind('ref', args_r)) if 'f' in kinds: sublists.append(ListOfKind('float', args_f)) reskind = getkind(op.result.concretetype)[0] - FUNC = op.args[0].concretetype.TO - NONVOIDARGS = tuple([ARG for ARG in FUNC.ARGS if ARG != lltype.Void]) - calldescr = self.cpu.calldescrof(FUNC, NONVOIDARGS, FUNC.RESULT) - return SpaceOperation('G_residual_call_%s_%s' % (kinds, reskind), - [op.args[0], calldescr] + sublists, - op.result) - - def rewrite_op_indirect_call(self, op): - op1 = SpaceOperation('direct_call', op.args[:-1], op.result) - return self.rewrite_op_direct_call(op1) + return SpaceOperation('G_%s_call_%s_%s' % (namebase, kinds, reskind), + initialargs + sublists, op.result) def add_in_correct_list(self, v, lst_i, lst_r, lst_f): kind = getkind(v.concretetype) @@ -210,6 +214,25 @@ else: raise AssertionError(kind) lst.append(v) + def handle_residual_call(self, op): + """A direct_call turns into the operation 'residual_call_xxx' if it + is calling a function that we don't want to JIT. The initial args + of 'residual_call_xxx' are the constant function to call, and its + calldescr.""" + FUNC = op.args[0].concretetype.TO + NONVOIDARGS = tuple([ARG for ARG in FUNC.ARGS if ARG != lltype.Void]) + calldescr = self.cpu.calldescrof(FUNC, NONVOIDARGS, FUNC.RESULT) + return self.rewrite_call(op, 'residual', [op.args[0], calldescr]) + + def handle_regular_call(self, op): + """A direct_call turns into the operation 'inline_call_xxx' if it + is calling a function that we want to JIT. The initial arg of + 'inline_call_xxx' is the JitCode of the called function.""" + [targetgraph] = self.callcontrol.graphs_from(op) + jitcode = self.callcontrol.get_jitcode(targetgraph, + called_from=self.graph) + return self.rewrite_call(op, 'inline', [jitcode]) + def _do_builtin_call(self, op, oopspec_name=None, args=None, extra=None): if oopspec_name is None: oopspec_name = op.opname if args is None: args = op.args @@ -230,6 +253,9 @@ rewrite_op_int_lshift_ovf = _do_builtin_call rewrite_op_gc_identityhash = _do_builtin_call + # ---------- + # getfield/setfield/mallocs etc. + def rewrite_op_hint(self, op): hints = op.args[1].value if hints.get('promote') and op.args[0].concretetype is not lltype.Void: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py Mon May 10 11:56:50 2010 @@ -55,6 +55,10 @@ # t.view() return rtyper +def getgraph(func, values): + rtyper = annotate(func, values) + return rtyper.annotator.translator.graphs[0] + def split_before_jit_merge_point(graph, portalblock, portalopindex): """Find the block with 'jit_merge_point' and split just before, making sure the input args are in the canonical order. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_call.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_call.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_call.py Mon May 10 11:56:50 2010 @@ -4,7 +4,10 @@ from pypy.rlib import jit from pypy.jit.codewriter.call import CallControl from pypy.jit.codewriter import support -from pypy.jit.codewriter.policy import JitPolicy + +class FakePolicy: + def look_inside_graph(self, graph): + return True def test_graphs_from_direct_call(): @@ -49,106 +52,30 @@ # ____________________________________________________________ def test_find_all_graphs(): - def f(x): - if x < 0: - return f(-x) - return x + 1 - @jit.purefunction def g(x): return x + 2 - @jit.dont_look_inside - def h(x): - return x + 3 - def i(x): - return f(x) * g(x) * h(x) - - rtyper = support.annotate(i, [7]) - cc = CallControl() - jitpolicy = JitPolicy() - res = cc.find_all_graphs(rtyper.annotator.translator.graphs[0], - jitpolicy) - translator = rtyper.annotator.translator - - funcs = set([graph.func for graph in res]) - assert funcs == set([i, f]) - -def test_find_all_graphs_without_floats(): - def g(x): - return int(x * 12.5) def f(x): return g(x) + 1 rtyper = support.annotate(f, [7]) - cc = CallControl() - jitpolicy = JitPolicy() - jitpolicy.set_supports_floats(True) - translator = rtyper.annotator.translator - res = cc.find_all_graphs(translator.graphs[0], jitpolicy) + cc = CallControl(portal_graph=rtyper.annotator.translator.graphs[0]) + res = cc.find_all_graphs(FakePolicy()) funcs = set([graph.func for graph in res]) assert funcs == set([f, g]) - cc = CallControl() - jitpolicy.set_supports_floats(False) - res = cc.find_all_graphs(translator.graphs[0], jitpolicy) - funcs = [graph.func for graph in res] - assert funcs == [f] - -def test_find_all_graphs_loops(): +def test_find_all_graphs_without_g(): def g(x): - i = 0 - while i < x: - i += 1 - return i - @jit.unroll_safe - def h(x): - i = 0 - while i < x: - i += 1 - return i - + return x + 2 def f(x): - i = 0 - while i < x*x: - i += g(x) + h(x) - return i - + return g(x) + 1 rtyper = support.annotate(f, [7]) - cc = CallControl() - jitpolicy = JitPolicy() - translator = rtyper.annotator.translator - res = cc.find_all_graphs(translator.graphs[0], jitpolicy) - funcs = set([graph.func for graph in res]) - assert funcs == set([f, h]) - -def test_unroll_safe_and_inline(): - @jit.unroll_safe - def h(x): - i = 0 - while i < x: - i += 1 - return i - h._always_inline_ = True - - def g(x): - return h(x) - - rtyper = support.annotate(g, [7]) - cc = CallControl() - jitpolicy = JitPolicy() - translator = rtyper.annotator.translator - res = cc.find_all_graphs(translator.graphs[0], jitpolicy) - funcs = set([graph.func for graph in res]) - assert funcs == set([g, h]) - -def test_find_all_graphs_str_join(): - def i(x, y): - return "hello".join([str(x), str(y), "bye"]) - - rtyper = support.annotate(i, [7, 100]) - cc = CallControl() - jitpolicy = JitPolicy() - translator = rtyper.annotator.translator - # does not explode - cc.find_all_graphs(translator.graphs[0], jitpolicy) + cc = CallControl(portal_graph=rtyper.annotator.translator.graphs[0]) + class CustomFakePolicy: + def look_inside_graph(self, graph): + assert graph.name == 'g' + return False + res = cc.find_all_graphs(CustomFakePolicy()) + funcs = [graph.func for graph in res] + assert funcs == [f] # ____________________________________________________________ @@ -157,7 +84,7 @@ graph = object() g = object() g1 = object() - cc = CallControl(portal_runner_obj=portal_runner_obj) + cc = CallControl(portal_graph=portal_runner_obj.graph) cc.candidate_graphs = [g, g1] op = SpaceOperation('direct_call', [Constant(portal_runner_obj)], @@ -206,3 +133,15 @@ res = cc.graphs_from(op) assert res is None assert cc.guess_call_kind(op) == 'residual' + +# ____________________________________________________________ + +def test_get_jitcode(): + cc = CallControl() + class somegraph: + name = "foo" + jitcode = cc.get_jitcode(somegraph) + assert jitcode is cc.get_jitcode(somegraph) # caching + assert jitcode.name == "foo" + pending = list(cc.enum_pending_graphs()) + assert pending == [(somegraph, jitcode)] Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py Mon May 10 11:56:50 2010 @@ -1,5 +1,13 @@ import py from pypy.jit.codewriter.codewriter import CodeWriter +from pypy.jit.codewriter import support + +class FakeCPU: + rtyper = None + +class FakePolicy: + def look_inside_graph(self, graph): + return True def test_loop(): @@ -27,6 +35,23 @@ for i in range(1, len(jitcode.code)): py.test.raises(KeyError, jitcode._live_vars, i) +def test_call(): + def ggg(x): + return x * 2 + def fff(a, b): + return ggg(b) - ggg(a) + rtyper = support.annotate(fff, [35, 42]) + maingraph = rtyper.annotator.translator.graphs[0] + cw = CodeWriter(FakeCPU()) + jitcode = cw.make_jitcodes(maingraph, FakePolicy(), verbose=True) + print jitcode._dump + [jitcode2] = cw.assembler.descrs + print jitcode2._dump + assert jitcode is not jitcode2 + assert jitcode.name == 'fff' + assert jitcode2.name == 'ggg' + assert 'ggg' in jitcode._dump + def test_integration(): from pypy.jit.metainterp.blackhole import BlackholeInterpBuilder def f(a, b): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Mon May 10 11:56:50 2010 @@ -43,6 +43,10 @@ def fielddescrof(self, STRUCT, name): return FakeDescr() +class FakePolicy: + def guess_call_kind(self, op): + return 'residual' + def fake_regallocs(): return {'int': FakeRegAlloc(), 'ref': FakeRegAlloc(), @@ -76,8 +80,8 @@ transform=False, liveness=False): graphs = self.make_graphs(func, args) if transform: - from pypy.jit.codewriter.jitter import transform_graph - transform_graph(graphs[0], FakeCPU()) + from pypy.jit.codewriter.jtransform import transform_graph + transform_graph(graphs[0], FakeCPU(), FakePolicy()) if liveness: from pypy.jit.codewriter.liveness import compute_liveness compute_liveness(graphs[0]) Copied: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py (from r74453, pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py) ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Mon May 10 11:56:50 2010 @@ -1,7 +1,7 @@ import random from pypy.objspace.flow.model import FunctionGraph, Block, Link from pypy.objspace.flow.model import SpaceOperation, Variable, Constant -from pypy.jit.codewriter.jitter import Transformer +from pypy.jit.codewriter.jtransform import Transformer from pypy.jit.metainterp.history import getkind from pypy.rpython.lltypesystem import lltype, rclass, rstr from pypy.translator.unsimplify import varoftype @@ -27,6 +27,20 @@ def __init__(self, exitcase): self.exitcase = self.llexitcase = exitcase +class FakeResidualCallControl: + def guess_call_kind(self, op): + return 'residual' + +class FakeRegularCallControl: + def guess_call_kind(self, op): + return 'regular' + def graphs_from(self, op): + return ['somegraph'] + def get_jitcode(self, graph, called_from=None): + assert graph == 'somegraph' + return 'somejitcode' + + def test_optimize_goto_if_not(): v1 = Variable() v2 = Variable() @@ -127,7 +141,7 @@ assert op1.result == v3 assert op1.opname == name2[0] -def test_residual_call(): +def test_calls(): for RESTYPE in [lltype.Signed, rclass.OBJECTPTR, lltype.Float, lltype.Void]: for with_void in [False, True]: @@ -144,6 +158,7 @@ elif with_i: expectedkind = 'ir' # integers and references else: expectedkind = 'r' # only references yield residual_call_test, ARGS, RESTYPE, expectedkind + yield direct_call_test, ARGS, RESTYPE, expectedkind def get_direct_call_op(argtypes, restype): FUNC = lltype.FuncType(argtypes, restype) @@ -156,7 +171,8 @@ def residual_call_test(argtypes, restype, expectedkind): op = get_direct_call_op(argtypes, restype) - op1 = Transformer(FakeCPU()).rewrite_operation(op) + tr = Transformer(FakeCPU(), FakeResidualCallControl()) + op1 = tr.rewrite_operation(op) reskind = getkind(restype)[0] assert op1.opname == 'G_residual_call_%s_%s' % (expectedkind, reskind) assert op1.result == op.result @@ -173,6 +189,24 @@ kind = getkind(v.concretetype) assert kind == 'void' or kind[0] in expectedkind +def direct_call_test(argtypes, restype, expectedkind): + op = get_direct_call_op(argtypes, restype) + tr = Transformer(FakeCPU(), FakeRegularCallControl()) + tr.graph = 'someinitialgraph' + op1 = tr.rewrite_operation(op) + reskind = getkind(restype)[0] + assert op1.opname == 'G_inline_call_%s_%s' % (expectedkind, reskind) + assert op1.result == op.result + assert op1.args[0] == 'somejitcode' + assert len(op1.args) == 1 + len(expectedkind) + for sublist, kind1 in zip(op1.args[1:], expectedkind): + assert sublist.kind.startswith(kind1) + assert list(sublist) == [v for v in op.args[1:] + if getkind(v.concretetype) == sublist.kind] + for v in op.args[1:]: + kind = getkind(v.concretetype) + assert kind == 'void' or kind[0] in expectedkind + def test_getfield(): # XXX a more compact encoding would be possible, something along # the lines of getfield_gc_r %r0, $offset, %r1 @@ -272,7 +306,8 @@ v = varoftype(lltype.Ptr(S)) op = SpaceOperation('malloc', [Constant(S, lltype.Void), Constant({'flavor': 'gc'}, lltype.Void)], v) - op1 = Transformer(FakeCPU()).rewrite_operation(op) + tr = Transformer(FakeCPU(), FakeResidualCallControl()) + op1 = tr.rewrite_operation(op) assert op1.opname == 'G_residual_call_r_r' assert op1.args[0].value == 'alloc_with_del' # pseudo-function as a str assert list(op1.args[2]) == [] Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_policy.py ============================================================================== --- (empty file) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_policy.py Mon May 10 11:56:50 2010 @@ -0,0 +1,98 @@ +from pypy.jit.codewriter.policy import contains_unsupported_variable_type +from pypy.jit.codewriter.policy import JitPolicy +from pypy.jit.codewriter import support +from pypy.rlib.rarithmetic import r_singlefloat +from pypy.rlib import jit + +def test_contains_unsupported_variable_type(): + def f(x): + return x + graph = support.getgraph(f, [5]) + assert not contains_unsupported_variable_type(graph, True) + assert not contains_unsupported_variable_type(graph, False) + # + graph = support.getgraph(f, [5.5]) + assert not contains_unsupported_variable_type(graph, True) + assert contains_unsupported_variable_type(graph, False) + # + graph = support.getgraph(f, [r_singlefloat(5.5)]) + assert contains_unsupported_variable_type(graph, True) + assert contains_unsupported_variable_type(graph, False) + + +def test_regular_function(): + graph = support.getgraph(lambda x: x+3, [5]) + assert JitPolicy().look_inside_graph(graph) + +def test_without_floats(): + graph = support.getgraph(lambda x: x+3.2, [5.4]) + policy = JitPolicy() + policy.set_supports_floats(True) + assert policy.look_inside_graph(graph) + policy = JitPolicy() + policy.set_supports_floats(False) + assert not policy.look_inside_graph(graph) + +def test_purefunction(): + @jit.purefunction + def g(x): + return x + 2 + graph = support.getgraph(g, [5]) + assert not JitPolicy().look_inside_graph(graph) + +def test_dont_look_inside(): + @jit.dont_look_inside + def h(x): + return x + 3 + graph = support.getgraph(h, [5]) + assert not JitPolicy().look_inside_graph(graph) + +def test_loops(): + def g(x): + i = 0 + while i < x: + i += 1 + return i + graph = support.getgraph(g, [5]) + assert not JitPolicy().look_inside_graph(graph) + +def test_unroll_safe(): + @jit.unroll_safe + def h(x): + i = 0 + while i < x: + i += 1 + return i + graph = support.getgraph(h, [5]) + assert JitPolicy().look_inside_graph(graph) + +def test_unroll_safe_and_inline(): + @jit.unroll_safe + def h(x): + i = 0 + while i < x: + i += 1 + return i + h._always_inline_ = True + + def g(x): + return h(x) + + graph = support.getgraph(h, [5]) + assert JitPolicy().look_inside_graph(graph) + +def test_str_join(): + def f(x, y): + return "hello".join([str(x), str(y), "bye"]) + + graph = support.getgraph(f, [5, 83]) + for block in graph.iterblocks(): + for op in block.operations: + if op.opname == 'direct_call': + funcptr = op.args[0].value + called_graph = funcptr._obj.graph + if JitPolicy().look_inside_graph(called_graph): + # the calls to join() and str() should be residual + mod = called_graph.func.__module__ + assert (mod == 'pypy.rpython.rlist' or + mod == 'pypy.rpython.lltypesystem.rlist') Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Mon May 10 11:56:50 2010 @@ -3,7 +3,6 @@ from pypy.jit.codewriter.regalloc import perform_register_allocation from pypy.jit.codewriter.flatten import flatten_graph, ListOfKind from pypy.jit.codewriter.format import format_assembler -from pypy.jit.codewriter.jitter import transform_graph from pypy.objspace.flow.model import Variable, Constant, SpaceOperation from pypy.objspace.flow.model import FunctionGraph, Block, Link from pypy.objspace.flow.model import c_last_exception @@ -22,6 +21,7 @@ # graphs must first be transformed by jitter.py before they can be # subjected to register allocation and flattening. if transform: + from pypy.jit.codewriter.jtransform import transform_graph transform_graph(graph) regalloc = perform_register_allocation(graph, 'int') ssarepr = flatten_graph(graph, {'int': regalloc}) Modified: pypy/branch/blackhole-improvement/pypy/objspace/flow/model.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/objspace/flow/model.py (original) +++ pypy/branch/blackhole-improvement/pypy/objspace/flow/model.py Mon May 10 11:56:50 2010 @@ -414,13 +414,14 @@ lst.append(link) return result -def copygraph(graph, shallow=False, varmap={}): +def copygraph(graph, shallow=False, varmap={}, shallowvars=False): "Make a copy of a flow graph." blockmap = {} varmap = varmap.copy() + shallowvars = shallowvars or shallow def copyvar(v): - if shallow: + if shallowvars: return v try: return varmap[v] From arigo at codespeak.net Mon May 10 12:19:32 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 10 May 2010 12:19:32 +0200 (CEST) Subject: [pypy-svn] r74458 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100510101932.28BBD282BF5@codespeak.net> Author: arigo Date: Mon May 10 12:19:30 2010 New Revision: 74458 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Log: Copy the code about choosing the kind of residual_call operation based on the magic attributes of the function and the raise analysis. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py Mon May 10 12:19:30 2010 @@ -15,6 +15,15 @@ self.portal_graph = portal_graph self.jitcodes = {} # map {graph: jitcode} self.unfinished_graphs = [] # list of graphs with pending jitcodes + if rtyper is not None: + from pypy.jit.metainterp.effectinfo import VirtualizableAnalyzer + from pypy.translator.backendopt.canraise import RaiseAnalyzer + from pypy.translator.backendopt.writeanalyze import \ + ReadWriteAnalyzer + translator = rtyper.annotator.translator + self.raise_analyzer = RaiseAnalyzer(translator) + self.readwrite_analyzer = ReadWriteAnalyzer(translator) + self.virtualizable_analyzer = VirtualizableAnalyzer(translator) def find_all_graphs(self, policy): def is_candidate(graph): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Mon May 10 12:19:30 2010 @@ -48,10 +48,10 @@ # ____________________________________________________________ -def flatten_graph(graph, regallocs): +def flatten_graph(graph, regallocs, _include_all_exc_links=False): """Flatten the graph into an SSARepr, with already-computed register allocations. 'regallocs' in a dict {kind: RegAlloc}.""" - flattener = GraphFlattener(graph, regallocs) + flattener = GraphFlattener(graph, regallocs, _include_all_exc_links) flattener.enforce_input_args() flattener.generate_ssa_form() return flattener.ssarepr @@ -59,9 +59,10 @@ class GraphFlattener(object): - def __init__(self, graph, regallocs): + def __init__(self, graph, regallocs, _include_all_exc_links=False): self.graph = graph self.regallocs = regallocs + self._include_all_exc_links = _include_all_exc_links self.registers = {} if graph: name = graph.name @@ -156,6 +157,12 @@ if lastopname == '-live-': lastopname = block.operations[-2].opname assert block.exits[0].exitcase is None # is this always True? + # + if not self._include_all_exc_links: + if not lastopname.startswith('G_'): # cannot actually raise + self.make_link(block.exits[0]) + return + # self.emitline('catch_exception', TLabel(block.exits[0])) self.make_link(block.exits[0]) self.emitline(Label(block.exits[0])) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Mon May 10 12:19:30 2010 @@ -5,6 +5,7 @@ from pypy.objspace.flow.model import Block, Link, c_last_exception from pypy.jit.codewriter.flatten import ListOfKind from pypy.jit.codewriter import support, heaptracker +from pypy.translator.simplify import get_funcobj def transform_graph(graph, cpu=None, callcontrol=None, portal=True): @@ -202,7 +203,7 @@ if 'r' in kinds: sublists.append(ListOfKind('ref', args_r)) if 'f' in kinds: sublists.append(ListOfKind('float', args_f)) reskind = getkind(op.result.concretetype)[0] - return SpaceOperation('G_%s_call_%s_%s' % (namebase, kinds, reskind), + return SpaceOperation('%s_%s_%s' % (namebase, kinds, reskind), initialargs + sublists, op.result) def add_in_correct_list(self, v, lst_i, lst_r, lst_f): @@ -222,7 +223,34 @@ FUNC = op.args[0].concretetype.TO NONVOIDARGS = tuple([ARG for ARG in FUNC.ARGS if ARG != lltype.Void]) calldescr = self.cpu.calldescrof(FUNC, NONVOIDARGS, FUNC.RESULT) - return self.rewrite_call(op, 'residual', [op.args[0], calldescr]) + # + pure = False + loopinvariant = False + if op.opname == "direct_call": + func = getattr(get_funcobj(op.args[0].value), '_callable', None) + pure = getattr(func, "_pure_function_", False) + loopinvariant = getattr(func, "_jit_loop_invariant_", False) + if pure or loopinvariant: + effectinfo = calldescr.get_extra_info() + assert (effectinfo is not None and + not effectinfo.forces_virtual_or_virtualizable) + try: + canraise = self.callcontrol.raise_analyzer.can_raise(op) + except lltype.DelayedPointer: + canraise = True # if we need to look into the delayed ptr that is + # the portal, then it's certainly going to raise + if loopinvariant: + name = 'G_residual_call_loopinvariant' + assert not non_void_args, ("arguments not supported for " + "loop-invariant function!") + elif pure: + # XXX check what to do about exceptions (also MemoryError?) + name = 'residual_call_pure' + elif canraise: + name = 'G_residual_call' + else: + name = 'residual_call' + return self.rewrite_call(op, name, [op.args[0], calldescr]) def handle_regular_call(self, op): """A direct_call turns into the operation 'inline_call_xxx' if it @@ -231,7 +259,7 @@ [targetgraph] = self.callcontrol.graphs_from(op) jitcode = self.callcontrol.get_jitcode(targetgraph, called_from=self.graph) - return self.rewrite_call(op, 'inline', [jitcode]) + return self.rewrite_call(op, 'G_inline_call', [jitcode]) def _do_builtin_call(self, op, oopspec_name=None, args=None, extra=None): if oopspec_name is None: oopspec_name = op.opname Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Mon May 10 12:19:30 2010 @@ -43,9 +43,17 @@ def fielddescrof(self, STRUCT, name): return FakeDescr() -class FakePolicy: +class FakeRaiseAnalyzer: + def can_raise(self, op): + try: + return 'cannot_raise' not in op.args[0].value._obj.graph.name + except AttributeError: + return True + +class FakeCallControl: def guess_call_kind(self, op): return 'residual' + raise_analyzer = FakeRaiseAnalyzer() def fake_regallocs(): return {'int': FakeRegAlloc(), @@ -81,11 +89,12 @@ graphs = self.make_graphs(func, args) if transform: from pypy.jit.codewriter.jtransform import transform_graph - transform_graph(graphs[0], FakeCPU(), FakePolicy()) + transform_graph(graphs[0], FakeCPU(), FakeCallControl()) if liveness: from pypy.jit.codewriter.liveness import compute_liveness compute_liveness(graphs[0]) - ssarepr = flatten_graph(graphs[0], fake_regallocs()) + ssarepr = flatten_graph(graphs[0], fake_regallocs(), + _include_all_exc_links=not transform) assert_format(ssarepr, expected) def test_simple(self): @@ -416,14 +425,14 @@ def test_residual_call_nonraising(self): @dont_look_inside - def g(i, j): + def cannot_raise(i, j): return i + j def f(i, j): try: - return g(i, j) + return cannot_raise(i, j) except Exception: return 42 + j self.encoding_test(f, [7, 2], """ - residual_call_ir_i $<* fn g>, , I[%i0, %i1], R[], %i2 + residual_call_ir_i $<* fn cannot_raise>, , I[%i0, %i1], R[], %i2 int_return %i2 """, transform=True, liveness=True) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Mon May 10 12:19:30 2010 @@ -27,9 +27,14 @@ def __init__(self, exitcase): self.exitcase = self.llexitcase = exitcase +class FakeRaiseAnalyzer: + def can_raise(self, op): + return True + class FakeResidualCallControl: def guess_call_kind(self, op): return 'residual' + raise_analyzer = FakeRaiseAnalyzer() class FakeRegularCallControl: def guess_call_kind(self, op): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Mon May 10 12:19:30 2010 @@ -163,7 +163,7 @@ v4 = Variable(); v4.concretetype = rclass.CLASSTYPE block = Block([]) block.operations = [ - SpaceOperation('res_call', [], v1), + SpaceOperation('G_res_call', [], v1), ] graph = FunctionGraph('f', block, v4) exclink = Link([v2], graph.returnblock) @@ -175,7 +175,7 @@ exclink) # self.check_assembler(graph, """ - res_call %i0 + G_res_call %i0 catch_exception L1 int_return %i0 L1: From arigo at codespeak.net Mon May 10 12:50:09 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 10 May 2010 12:50:09 +0200 (CEST) Subject: [pypy-svn] r74459 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter codewriter/test metainterp metainterp/test Message-ID: <20100510105009.6AB6D282BF5@codespeak.net> Author: arigo Date: Mon May 10 12:50:07 2010 New Revision: 74459 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_call.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py Log: Have test_basic.test_direct_call() half-passing again. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Mon May 10 12:50:07 2010 @@ -43,6 +43,7 @@ self.switchdictdescrs = [] self.count_regs = dict.fromkeys(KINDS, 0) self.liveness = {} + self.startpoints = set() def emit_reg(self, reg): if reg.index >= self.count_regs[reg.kind]: @@ -147,6 +148,7 @@ key = opname + '/' + ''.join(argcodes) num = self.insns.setdefault(key, len(self.insns)) self.code[startposition] = chr(num) + self.startpoints.add(startposition) def get_liveness_info(self, insn, kind): lives = [chr(reg.index) for reg in insn[1:] if reg.kind == kind] @@ -182,4 +184,5 @@ self.count_regs['ref'], self.count_regs['float'], liveness=self.liveness, - assembler=self) + assembler=self, + startpoints=self.startpoints) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py Mon May 10 12:50:07 2010 @@ -4,23 +4,25 @@ # from pypy.jit.codewriter import support -from pypy.translator.simplify import get_funcobj from pypy.jit.codewriter.jitcode import JitCode +from pypy.translator.simplify import get_funcobj, get_functype +from pypy.rpython.lltypesystem import lltype, llmemory class CallControl(object): - def __init__(self, rtyper=None, portal_graph=None): - self.rtyper = rtyper + def __init__(self, cpu=None, portal_graph=None): + self.cpu = cpu self.portal_graph = portal_graph self.jitcodes = {} # map {graph: jitcode} self.unfinished_graphs = [] # list of graphs with pending jitcodes - if rtyper is not None: + if cpu is not None: from pypy.jit.metainterp.effectinfo import VirtualizableAnalyzer from pypy.translator.backendopt.canraise import RaiseAnalyzer from pypy.translator.backendopt.writeanalyze import \ ReadWriteAnalyzer - translator = rtyper.annotator.translator + self.rtyper = cpu.rtyper + translator = self.rtyper.annotator.translator self.raise_analyzer = RaiseAnalyzer(translator) self.readwrite_analyzer = ReadWriteAnalyzer(translator) self.virtualizable_analyzer = VirtualizableAnalyzer(translator) @@ -135,7 +137,21 @@ try: return self.jitcodes[graph] except KeyError: - jitcode = JitCode(graph.name, called_from=called_from) + fnaddr, calldescr = self.get_jitcode_calldescr(graph) + jitcode = JitCode(graph.name, fnaddr, calldescr, + called_from=called_from) self.jitcodes[graph] = jitcode self.unfinished_graphs.append(graph) return jitcode + + def get_jitcode_calldescr(self, graph): + fnptr = self.rtyper.getcallable(graph) + FUNC = get_functype(lltype.typeOf(fnptr)) + if self.rtyper.type_system.name == 'ootypesystem': + XXX + else: + fnaddr = llmemory.cast_ptr_to_adr(fnptr) + NON_VOID_ARGS = [ARG for ARG in FUNC.ARGS if ARG is not lltype.Void] + calldescr = self.cpu.calldescrof(FUNC, tuple(NON_VOID_ARGS), + FUNC.RESULT) + return (fnaddr, calldescr) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py Mon May 10 12:50:07 2010 @@ -60,7 +60,8 @@ self.assembler.assemble(ssarepr, jitcode) def make_jitcodes(self, maingraph, policy, verbose=False): - self.callcontrol = CallControl(self.cpu.rtyper, maingraph) + self.portal_graph = maingraph + self.callcontrol = CallControl(self.cpu, maingraph) self.callcontrol.find_all_graphs(policy) mainjitcode = self.callcontrol.get_jitcode(maingraph) for graph, jitcode in self.callcontrol.enum_pending_graphs(): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py Mon May 10 12:50:07 2010 @@ -1,5 +1,6 @@ from pypy.jit.metainterp.history import AbstractDescr from pypy.rlib.objectmodel import we_are_translated +from pypy.rpython.lltypesystem import llmemory class JitCode(AbstractDescr): @@ -15,7 +16,7 @@ def setup(self, code='', constants_i=[], constants_r=[], constants_f=[], num_regs_i=256, num_regs_r=256, num_regs_f=256, - liveness=None, assembler=None): + liveness=None, assembler=None, startpoints=None): self.code = code # if the following lists are empty, use a single shared empty list self.constants_i = constants_i or self._empty_i @@ -27,6 +28,10 @@ (num_regs_f << 0)) self.liveness = liveness self._assembler = assembler # debugging + self._startpoints = startpoints # debugging + + def get_fnaddr_as_int(self): + return llmemory.cast_adr_to_int(self.fnaddr) def num_regs_i(self): return self.num_regs_encoded >> 18 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_call.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_call.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_call.py Mon May 10 12:50:07 2010 @@ -137,7 +137,8 @@ # ____________________________________________________________ def test_get_jitcode(): - cc = CallControl() + from pypy.jit.codewriter.test.test_codewriter import FakeCPU + cc = CallControl(FakeCPU()) class somegraph: name = "foo" jitcode = cc.get_jitcode(somegraph) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py Mon May 10 12:50:07 2010 @@ -1,9 +1,21 @@ import py from pypy.jit.codewriter.codewriter import CodeWriter from pypy.jit.codewriter import support +from pypy.rpython.lltypesystem import lltype, llmemory + +class FakeRTyper: + class annotator: + translator = None + class type_system: + name = 'lltypesystem' + def getcallable(self, graph): + F = lltype.FuncType([], lltype.Signed) + return lltype.functionptr(F, 'bar') class FakeCPU: - rtyper = None + rtyper = FakeRTyper() + def calldescrof(self, FUNC, ARGS, RESULT): + return ('calldescr', FUNC, ARGS, RESULT) class FakePolicy: def look_inside_graph(self, graph): @@ -51,6 +63,8 @@ assert jitcode.name == 'fff' assert jitcode2.name == 'ggg' assert 'ggg' in jitcode._dump + assert lltype.typeOf(jitcode2.fnaddr) == llmemory.Address + assert jitcode2.calldescr[0] == 'calldescr' def test_integration(): from pypy.jit.metainterp.blackhole import BlackholeInterpBuilder Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Mon May 10 12:50:07 2010 @@ -5,7 +5,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory, rclass from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.llinterp import LLException -from pypy.jit.codewriter.jitcode import SwitchDictDescr +from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr def arguments(*argtypes, **kwds): @@ -80,6 +80,10 @@ # def dispatch_loop(self, code, position): while True: + if not we_are_translated(): + assert position in self._current_jitcode._startpoints, ( + "the current position %d is in the middle of " + "an instruction!" % position) opcode = ord(code[position]) position += 1 for i, func in all_funcs: @@ -148,11 +152,13 @@ value = self.cpu elif argtype == 'pc': value = position - elif argtype == 'd': + elif argtype == 'd' or argtype == 'j': assert argcodes[next_argcode] == 'd' next_argcode = next_argcode + 1 index = ord(code[position]) | (ord(code[position+1])<<8) value = self.descrs[index] + if argtype == 'j': + assert isinstance(value, JitCode) position += 2 else: raise AssertionError("bad argtype: %r" % (argtype,)) @@ -262,6 +268,8 @@ self.registers_f[index] = value def run(self, jitcode, position): + if not we_are_translated(): + self._current_jitcode = jitcode self.copy_constants(self.registers_i, jitcode.constants_i) self.copy_constants(self.registers_r, jitcode.constants_r) self.copy_constants(self.registers_f, jitcode.constants_f) @@ -751,6 +759,16 @@ def bhimpl_residual_call_irf_v(cpu, func, calldescr,args_i,args_r,args_f): cpu.bh_call_v(func, calldescr, args_i, args_r, args_f) + @arguments("cpu", "j", "R", returns="i") + def bhimpl_inline_call_r_i(cpu, jitcode, args_r): + return cpu.bh_call_i(jitcode.get_fnaddr_as_int(), jitcode.calldescr, + None, args_r, None) + + @arguments("cpu", "j", "I", "R", returns="i") + def bhimpl_inline_call_ir_i(cpu, jitcode, args_i, args_r): + return cpu.bh_call_i(jitcode.get_fnaddr_as_int(), jitcode.calldescr, + args_i, args_r, None) + @arguments("cpu", "d", "i", returns="r") def bhimpl_new_array(cpu, arraydescr, length): return cpu.bh_new_array(arraydescr, length) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Mon May 10 12:50:07 2010 @@ -5,7 +5,7 @@ from pypy.jit.metainterp.warmspot import ll_meta_interp, get_stats from pypy.jit.backend.llgraph import runner from pypy.jit.metainterp import pyjitpl, history -from pypy.jit.metainterp.policy import JitPolicy, StopAtXPolicy +from pypy.jit.codewriter.policy import JitPolicy, StopAtXPolicy from pypy import conftest from pypy.rlib.rarithmetic import ovfcheck from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper @@ -21,7 +21,7 @@ stats = history.Stats() cpu = CPUClass(rtyper, stats, None, False) cw = codewriter.CodeWriter(cpu) - mainjitcode = cw.make_jitcodes(graphs[0], verbose=True) + mainjitcode = cw.make_jitcodes(graphs[0], JitPolicy(), verbose=True) return cw, mainjitcode def _run_with_blackhole(cw, mainjitcode, args): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py Mon May 10 12:50:07 2010 @@ -18,10 +18,10 @@ from pypy.jit.metainterp import history, pyjitpl, gc from pypy.jit.metainterp.pyjitpl import MetaInterpStaticData, MetaInterp -from pypy.jit.metainterp.policy import JitPolicy from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper from pypy.jit.metainterp.jitprof import Profiler, EmptyProfiler from pypy.jit.codewriter import support +from pypy.jit.codewriter.policy import JitPolicy from pypy.rlib.jit import DEBUG_STEPS, DEBUG_DETAILED, DEBUG_OFF, DEBUG_PROFILE # ____________________________________________________________ From arigo at codespeak.net Mon May 10 14:35:27 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 10 May 2010 14:35:27 +0200 (CEST) Subject: [pypy-svn] r74460 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter codewriter/test metainterp Message-ID: <20100510123527.0F411282BFC@codespeak.net> Author: arigo Date: Mon May 10 14:35:25 2010 New Revision: 74460 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: Support 'inline_call' in the blackhole and the pyjitpl interpreters. Start simplifying the handling of pc in pyjitpl. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Mon May 10 14:35:25 2010 @@ -43,6 +43,7 @@ self.switchdictdescrs = [] self.count_regs = dict.fromkeys(KINDS, 0) self.liveness = {} + self.nextlive = None self.startpoints = set() def emit_reg(self, reg): @@ -88,7 +89,8 @@ self.label_positions[insn[0].name] = len(self.code) return if insn[0] == '-live-': - self.liveness[len(self.code)] = ( + assert self.nextlive is None + self.nextlive = ( self.get_liveness_info(insn, 'int'), self.get_liveness_info(insn, 'ref'), self.get_liveness_info(insn, 'float')) @@ -149,6 +151,10 @@ num = self.insns.setdefault(key, len(self.insns)) self.code[startposition] = chr(num) self.startpoints.add(startposition) + # + if self.nextlive is not None: + self.liveness[len(self.code)] = self.nextlive + self.nextlive = None def get_liveness_info(self, insn, kind): lives = [chr(reg.index) for reg in insn[1:] if reg.kind == kind] @@ -184,5 +190,4 @@ self.count_regs['ref'], self.count_regs['float'], liveness=self.liveness, - assembler=self, startpoints=self.startpoints) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py Mon May 10 14:35:25 2010 @@ -16,7 +16,7 @@ def setup(self, code='', constants_i=[], constants_r=[], constants_f=[], num_regs_i=256, num_regs_r=256, num_regs_f=256, - liveness=None, assembler=None, startpoints=None): + liveness=None, startpoints=None): self.code = code # if the following lists are empty, use a single shared empty list self.constants_i = constants_i or self._empty_i @@ -24,10 +24,9 @@ self.constants_f = constants_f or self._empty_f # encode the three num_regs into a single integer self.num_regs_encoded = ((num_regs_i << 18) | - (num_regs_r << 9) | - (num_regs_f << 0)) + (num_regs_f << 9) | + (num_regs_r << 0)) self.liveness = liveness - self._assembler = assembler # debugging self._startpoints = startpoints # debugging def get_fnaddr_as_int(self): @@ -36,10 +35,10 @@ def num_regs_i(self): return self.num_regs_encoded >> 18 - def num_regs_r(self): + def num_regs_f(self): return (self.num_regs_encoded >> 9) & 0x1FF - def num_regs_f(self): + def num_regs_r(self): return self.num_regs_encoded & 0x1FF def has_liveness_info(self, pc): @@ -49,7 +48,7 @@ registers_i, registers_r, registers_f): # 'pc' gives a position in this bytecode. This invokes # 'callback' for each variable that is live across the - # instruction which starts at 'pc'. (It excludes the arguments + # instruction which ends at 'pc'. (It excludes the arguments # of that instruction which are no longer used afterwards, and # excludes the return value of that instruction.) More precisely, # this invokes 'callback(arg, box, index)' where 'box' comes from one @@ -93,14 +92,7 @@ return ' '.join(lst) def _missing_liveness(self, pc): - opcode = ord(self.code[pc]) - insn = 'insn %d' % opcode - if self._assembler is not None: - for name, code in self._assembler.insns.items(): - if code == opcode: - insn = name - raise KeyError("missing liveness[%d], corresponding to %r" % ( - pc, insn)) + raise KeyError("missing liveness[%d]" % (pc,)) def __repr__(self): return '' % self.name Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py Mon May 10 14:35:25 2010 @@ -174,20 +174,21 @@ ] assembler = Assembler() jitcode = assembler.assemble(ssarepr) - assert jitcode.code == ("\x00\x00\x0A\x01" - "\x00\x00\x03\x02" - "\x01\x01\x02\x03" - "\x00\x00\x06\x04" - "\x01\x03\x04\x05" - "\x02\x05") + assert jitcode.code == ("\x00\x00\x0A\x01" # ends at 4 + "\x00\x00\x03\x02" # ends at 8 + "\x01\x01\x02\x03" # ends at 12 + "\x00\x00\x06\x04" # ends at 16 + "\x01\x03\x04\x05" # ends at 20 + "\x02\x05") assert assembler.insns == {'int_add/ici': 0, 'int_mul/iii': 1, 'int_return/i': 2} - py.test.raises(KeyError, jitcode._live_vars, 1) + py.test.raises(KeyError, jitcode._live_vars, 0) py.test.raises(KeyError, jitcode._live_vars, 3) - py.test.raises(KeyError, jitcode._live_vars, 20) - assert jitcode._live_vars(0) == '%i0' - assert jitcode._live_vars(4) == '%i0 %i1' - assert jitcode._live_vars(8) == '%i0' - assert jitcode._live_vars(12) == '%i3' - assert jitcode._live_vars(16) == '' + py.test.raises(KeyError, jitcode._live_vars, 5) + py.test.raises(KeyError, jitcode._live_vars, 24) + assert jitcode._live_vars(4) == '%i0' + assert jitcode._live_vars(8) == '%i0 %i1' + assert jitcode._live_vars(12) == '%i0' + assert jitcode._live_vars(16) == '%i3' + assert jitcode._live_vars(20) == '' Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py Mon May 10 14:35:25 2010 @@ -30,7 +30,7 @@ return b cw = CodeWriter() jitcode = cw.transform_func_to_jitcode(f, [5, 6]) - assert jitcode.code == ("\x00\x10\x00\x00\x00" + assert jitcode.code == ("\x00\x10\x00\x00\x00" # ends at 5 "\x01\x01\x00\x01" "\x02\x00\x01\x00" "\x03\x00\x00" @@ -43,9 +43,10 @@ assert jitcode.num_regs_i() == 2 assert jitcode.num_regs_r() == 0 assert jitcode.num_regs_f() == 0 - assert jitcode._live_vars(0) == '%i0 %i1' - for i in range(1, len(jitcode.code)): - py.test.raises(KeyError, jitcode._live_vars, i) + assert jitcode._live_vars(5) == '%i0 %i1' + for i in range(len(jitcode.code)+1): + if i != 5: + py.test.raises(KeyError, jitcode._live_vars, i) def test_call(): def ggg(x): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Mon May 10 14:35:25 2010 @@ -763,11 +763,52 @@ def bhimpl_inline_call_r_i(cpu, jitcode, args_r): return cpu.bh_call_i(jitcode.get_fnaddr_as_int(), jitcode.calldescr, None, args_r, None) + @arguments("cpu", "j", "R", returns="r") + def bhimpl_inline_call_r_r(cpu, jitcode, args_r): + return cpu.bh_call_r(jitcode.get_fnaddr_as_int(), jitcode.calldescr, + None, args_r, None) + @arguments("cpu", "j", "R", returns="f") + def bhimpl_inline_call_r_f(cpu, jitcode, args_r): + return cpu.bh_call_f(jitcode.get_fnaddr_as_int(), jitcode.calldescr, + None, args_r, None) + @arguments("cpu", "j", "R") + def bhimpl_inline_call_r_v(cpu, jitcode, args_r): + return cpu.bh_call_v(jitcode.get_fnaddr_as_int(), jitcode.calldescr, + None, args_r, None) @arguments("cpu", "j", "I", "R", returns="i") def bhimpl_inline_call_ir_i(cpu, jitcode, args_i, args_r): return cpu.bh_call_i(jitcode.get_fnaddr_as_int(), jitcode.calldescr, args_i, args_r, None) + @arguments("cpu", "j", "I", "R", returns="r") + def bhimpl_inline_call_ir_r(cpu, jitcode, args_i, args_r): + return cpu.bh_call_r(jitcode.get_fnaddr_as_int(), jitcode.calldescr, + args_i, args_r, None) + @arguments("cpu", "j", "I", "R", returns="f") + def bhimpl_inline_call_ir_f(cpu, jitcode, args_i, args_r): + return cpu.bh_call_f(jitcode.get_fnaddr_as_int(), jitcode.calldescr, + args_i, args_r, None) + @arguments("cpu", "j", "I", "R") + def bhimpl_inline_call_ir_v(cpu, jitcode, args_i, args_r): + return cpu.bh_call_v(jitcode.get_fnaddr_as_int(), jitcode.calldescr, + args_i, args_r, None) + + @arguments("cpu", "j", "I", "R", "F", returns="i") + def bhimpl_inline_call_irf_i(cpu, jitcode, args_i, args_r, args_f): + return cpu.bh_call_i(jitcode.get_fnaddr_as_int(), jitcode.calldescr, + args_i, args_r, args_f) + @arguments("cpu", "j", "I", "R", "F", returns="r") + def bhimpl_inline_call_irf_r(cpu, jitcode, args_i, args_r, args_f): + return cpu.bh_call_r(jitcode.get_fnaddr_as_int(), jitcode.calldescr, + args_i, args_r, args_f) + @arguments("cpu", "j", "I", "R", "F", returns="f") + def bhimpl_inline_call_irf_f(cpu, jitcode, args_i, args_r, args_f): + return cpu.bh_call_f(jitcode.get_fnaddr_as_int(), jitcode.calldescr, + args_i, args_r, args_f) + @arguments("cpu", "j", "I", "R", "F") + def bhimpl_inline_call_irf_v(cpu, jitcode, args_i, args_r, args_f): + return cpu.bh_call_v(jitcode.get_fnaddr_as_int(), jitcode.calldescr, + args_i, args_r, args_f) @arguments("cpu", "d", "i", returns="r") def bhimpl_new_array(cpu, arraydescr, length): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Mon May 10 14:35:25 2010 @@ -50,6 +50,8 @@ parent_resumedata_snapshot = None parent_resumedata_frame_info_list = None + env = property(lambda: xxx, lambda x: xxx) # no read/write! + def __init__(self, metainterp): self.metainterp = metainterp self.registers_i = [None] * 256 @@ -133,6 +135,17 @@ if not we_are_translated(): assert oldbox not in registers[count:] + def make_result_of_lastop(self, resultbox): + target_index = ord(self.bytecode[self.pc-1]) + if resultbox.type == history.INT: + self.registers_i[target_index] = resultbox + elif resultbox.type == history.REF: + self.registers_r[target_index] = resultbox + elif resultbox.type == history.FLOAT: + self.registers_f[target_index] = resultbox + else: + raise AssertionError("bad result box type") + # ------------------------------ for _opimpl in ['int_add', 'int_sub', 'int_mul', 'int_floordiv', 'int_mod', @@ -222,23 +235,24 @@ def opimpl_goto(self, target): self.pc = target - @arguments("orgpc", "label", "box") - def opimpl_goto_if_not(self, pc, target, box): + @arguments("label", "box") + def opimpl_goto_if_not(self, target, box): switchcase = box.getint() if switchcase: opnum = rop.GUARD_TRUE else: - self.pc = target opnum = rop.GUARD_FALSE - self.generate_guard(pc, opnum, box) + self.generate_guard(opnum, box) + if not switchcase: + self.pc = target for _opimpl in ['int_lt', 'int_le', 'int_eq', 'int_ne', 'int_gt', 'int_ge', ]: exec py.code.Source(''' - @arguments("orgpc", "label", "box", "box") - def opimpl_goto_if_not_%s(self, pc, target, b1, b2): + @arguments("label", "box", "box") + def opimpl_goto_if_not_%s(self, target, b1, b2): condbox = self.execute(rop.%s, b1, b2) - self.opimpl_goto_if_not(pc, target, condbox) + self.opimpl_goto_if_not(target, condbox) ''' % (_opimpl, _opimpl.upper())).compile() def follow_jump(self): @@ -606,25 +620,38 @@ result = vinfo.get_array_length(virtualizable, arrayindex) self.make_result_box(ConstInt(result)) - def perform_call(self, jitcode, varargs, greenkey=None): - # when tracing, this bytecode causes the subfunction to be entered - f = self.metainterp.newframe(jitcode, greenkey) - f.setup_call(varargs) - return True - - @XXX #arguments("bytecode", "varargs") - def opimpl_call(self, callee, varargs): - return self.perform_call(callee, varargs) - - @arguments("orgpc", "box", "descr", "boxes") - def _opimpl_residual_call1(self, pc, funcbox, calldescr, argboxes): - return self.do_residual_call(funcbox, calldescr, argboxes, exc_at=pc) - @arguments("orgpc", "box", "descr", "boxes2") - def _opimpl_residual_call2(self, pc, funcbox, calldescr, argboxes): - return self.do_residual_call(funcbox, calldescr, argboxes, exc_at=pc) - @arguments("orgpc", "box", "descr", "boxes3") - def _opimpl_residual_call3(self, pc, funcbox, calldescr, argboxes): - return self.do_residual_call(funcbox, calldescr, argboxes, exc_at=pc) + @arguments("jitcode", "boxes") + def _opimpl_inline_call1(self, jitcode, argboxes): + self.metainterp.perform_call(jitcode, argboxes) + @arguments("jitcode", "boxes2") + def _opimpl_inline_call2(self, jitcode, argboxes): + self.metainterp.perform_call(jitcode, argboxes) + @arguments("jitcode", "boxes3") + def _opimpl_inline_call3(self, jitcode, argboxes): + self.metainterp.perform_call(jitcode, argboxes) + + opimpl_inline_call_r_i = _opimpl_inline_call1 + opimpl_inline_call_r_r = _opimpl_inline_call1 + opimpl_inline_call_r_f = _opimpl_inline_call1 + opimpl_inline_call_r_v = _opimpl_inline_call1 + opimpl_inline_call_ir_i = _opimpl_inline_call2 + opimpl_inline_call_ir_r = _opimpl_inline_call2 + opimpl_inline_call_ir_f = _opimpl_inline_call2 + opimpl_inline_call_ir_v = _opimpl_inline_call2 + opimpl_inline_call_irf_i = _opimpl_inline_call3 + opimpl_inline_call_irf_r = _opimpl_inline_call3 + opimpl_inline_call_irf_f = _opimpl_inline_call3 + opimpl_inline_call_irf_v = _opimpl_inline_call3 + + @arguments("box", "descr", "boxes") + def _opimpl_residual_call1(self, funcbox, calldescr, argboxes): + return self.do_residual_call(funcbox, calldescr, argboxes, exc=True) + @arguments("box", "descr", "boxes2") + def _opimpl_residual_call2(self, funcbox, calldescr, argboxes): + return self.do_residual_call(funcbox, calldescr, argboxes, exc=True) + @arguments("box", "descr", "boxes3") + def _opimpl_residual_call3(self, funcbox, calldescr, argboxes): + return self.do_residual_call(funcbox, calldescr, argboxes, exc=True) opimpl_residual_call_r_i = _opimpl_residual_call1 opimpl_residual_call_r_r = _opimpl_residual_call1 @@ -931,10 +958,20 @@ # ------------------------------ def setup_call(self, argboxes): - if not we_are_translated(): - check_args(*argboxes) self.pc = 0 - self.env = argboxes + count_i = count_r = count_f = 0 + for box in argboxes: + if box.type == history.INT: + self.registers_i[count_i] = box + count_i += 1 + elif box.type == history.REF: + self.registers_r[count_r] = box + count_r += 1 + elif box.type == history.FLOAT: + self.registers_f[count_f] = box + count_f += 1 + else: + raise AssertionError(box.type) def setup_resume_at_op(self, pc, env): if not we_are_translated(): @@ -962,14 +999,12 @@ except ChangeFrame: pass - def generate_guard(self, pc, opnum, box=None, extraargs=[]): + def generate_guard(self, opnum, box=None, extraargs=[]): if isinstance(box, Const): # no need for a guard return metainterp = self.metainterp if metainterp.is_blackholing(): return - saved_pc = self.pc - self.pc = pc if box is not None: moreargs = [box] + extraargs else: @@ -992,7 +1027,6 @@ self.metainterp.staticdata.profiler.count_ops(opnum, GUARDS) # count metainterp.attach_debug_info(guard_op) - self.pc = saved_pc return guard_op def implement_guard_value(self, pc, box): @@ -1019,16 +1053,16 @@ return self.metainterp.execute_and_record(opnum, descr, *argboxes) @specialize.arg(1) - def execute_varargs(self, opnum, argboxes, descr, exc_at): + def execute_varargs(self, opnum, argboxes, descr, exc): resbox = self.metainterp.execute_and_record_varargs(opnum, argboxes, descr=descr) - if exc_at >= 0: - self.metainterp.handle_possible_exception(exc_at) + if exc: + self.metainterp.handle_possible_exception(self.pc) else: self.metainterp.assert_no_exception() return resbox - def do_residual_call(self, funcbox, descr, argboxes, exc_at): + def do_residual_call(self, funcbox, descr, argboxes, exc): allboxes = [funcbox] + argboxes effectinfo = descr.get_extra_info() if 0:# XXX effectinfo is None or effectinfo.forces_virtual_or_virtualizable: @@ -1046,7 +1080,7 @@ else: return self.metainterp.assert_no_exception() else: - return self.execute_varargs(rop.CALL, allboxes, descr, exc_at) + return self.execute_varargs(rop.CALL, allboxes, descr, exc) # ____________________________________________________________ @@ -1238,6 +1272,12 @@ def is_blackholing(self): return False # XXX get rid of this method + def perform_call(self, jitcode, boxes, greenkey=None): + # when tracing, this bytecode causes the subfunction to be entered + f = self.newframe(jitcode, greenkey) + f.setup_call(boxes) + raise ChangeFrame + def newframe(self, jitcode, greenkey=None): if jitcode is self.staticdata.portal_code: self.in_recursion += 1 @@ -1268,7 +1308,7 @@ self.popframe() if self.framestack: if resultbox is not None: - self.framestack[-1].make_result_box(resultbox) + self.framestack[-1].make_result_of_lastop(resultbox) raise ChangeFrame else: if not self.is_blackholing(): @@ -1715,20 +1755,7 @@ # ----- make a new frame ----- self.framestack = [] f = self.newframe(self.staticdata.portal_code) - f.pc = 0 - count_i = count_r = count_f = 0 - for box in original_boxes: - if box.type == history.INT: - f.registers_i[count_i] = box - count_i += 1 - elif box.type == history.REF: - f.registers_r[count_r] = box - count_r += 1 - elif box.type == history.FLOAT: - f.registers_f[count_f] = box - count_f += 1 - else: - raise AssertionError(box.type) + f.setup_call(original_boxes) self.virtualref_boxes = [] self.initialize_virtualizable(original_boxes) return original_boxes @@ -2056,11 +2083,13 @@ else: raise AssertionError("bad argcode") position += 1 - elif argtype == "descr": + elif argtype == "descr" or argtype == "jitcode": assert argcodes[next_argcode] == 'd' next_argcode = next_argcode + 1 index = ord(code[position]) | (ord(code[position+1])<<8) value = self.metainterp.staticdata.opcode_descrs[index] + if argtype == "jitcode": + assert isinstance(value, JitCode) position += 2 elif argtype == "label": assert argcodes[next_argcode] == 'L' @@ -2110,25 +2139,27 @@ assert num_return_args == 0 or num_return_args == 1 self.pc = position + num_return_args # - resultbox = unboundmethod(self, *args) - # - if num_return_args == 0: - assert resultbox is None - else: - assert resultbox is not None - result_argcode = argcodes[next_argcode] - target_index = ord(code[position]) - if result_argcode == 'i': - assert resultbox.type == history.INT - self.registers_i[target_index] = resultbox - elif result_argcode == 'r': - assert resultbox.type == history.REF - self.registers_r[target_index] = resultbox - elif result_argcode == 'f': - assert resultbox.type == history.FLOAT - self.registers_f[target_index] = resultbox + if not we_are_translated(): + print '\tjitcode:', name, list(args), + try: + resultbox = unboundmethod(self, *args) + except Exception, e: + print '-> %s!' % e.__class__.__name__ + raise + if num_return_args == 0: + print + assert resultbox is None else: - raise AssertionError("bad result argcode") + print '->', resultbox + result_argcode = argcodes[next_argcode] + assert resultbox.type == {'i': history.INT, + 'r': history.REF, + 'f': history.FLOAT}[result_argcode] + else: + resultbox = unboundmethod(self, *args) + # + if resultbox is not None: + self.make_result_of_lastop(resultbox) # unboundmethod = getattr(MIFrame, 'opimpl_' + name).im_func argtypes = unrolling_iterable(unboundmethod.argtypes) From arigo at codespeak.net Mon May 10 14:50:55 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 10 May 2010 14:50:55 +0200 (CEST) Subject: [pypy-svn] r74461 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100510125055.2600D282BFC@codespeak.net> Author: arigo Date: Mon May 10 14:50:53 2010 New Revision: 74461 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Log: Copy and clean up (for now) 'getcalldescr'. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py Mon May 10 14:50:53 2010 @@ -8,6 +8,11 @@ from pypy.translator.simplify import get_funcobj, get_functype from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.translator.backendopt.canraise import RaiseAnalyzer +from pypy.translator.backendopt.writeanalyze import ReadWriteAnalyzer +from pypy.jit.metainterp.effectinfo import VirtualizableAnalyzer +from pypy.jit.metainterp.effectinfo import effectinfo_from_writeanalyze + class CallControl(object): @@ -17,10 +22,6 @@ self.jitcodes = {} # map {graph: jitcode} self.unfinished_graphs = [] # list of graphs with pending jitcodes if cpu is not None: - from pypy.jit.metainterp.effectinfo import VirtualizableAnalyzer - from pypy.translator.backendopt.canraise import RaiseAnalyzer - from pypy.translator.backendopt.writeanalyze import \ - ReadWriteAnalyzer self.rtyper = cpu.rtyper translator = self.rtyper.annotator.translator self.raise_analyzer = RaiseAnalyzer(translator) @@ -155,3 +156,26 @@ calldescr = self.cpu.calldescrof(FUNC, tuple(NON_VOID_ARGS), FUNC.RESULT) return (fnaddr, calldescr) + + def getcalldescr(self, op): + NON_VOID_ARGS = [x.concretetype for x in op.args[1:] + if x.concretetype is not lltype.Void] + RESULT = op.result.concretetype + # check the number and type of arguments + FUNC = get_functype(op.args[0].concretetype) + ARGS = FUNC.ARGS + assert NON_VOID_ARGS == [T for T in ARGS if T is not lltype.Void] + assert RESULT == FUNC.RESULT + # ok + effectinfo = effectinfo_from_writeanalyze( + self.readwrite_analyzer.analyze(op), + self.cpu, + self.virtualizable_analyzer.analyze(op)) + calldescr = self.cpu.calldescrof(FUNC, tuple(NON_VOID_ARGS), RESULT, + effectinfo) + try: + canraise = self.raise_analyzer.can_raise(op) + except lltype.DelayedPointer: + canraise = True # if we need to look into the delayed ptr that is + # the portal, then it's certainly going to raise + return calldescr, canraise Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Mon May 10 14:50:53 2010 @@ -220,9 +220,7 @@ is calling a function that we don't want to JIT. The initial args of 'residual_call_xxx' are the constant function to call, and its calldescr.""" - FUNC = op.args[0].concretetype.TO - NONVOIDARGS = tuple([ARG for ARG in FUNC.ARGS if ARG != lltype.Void]) - calldescr = self.cpu.calldescrof(FUNC, NONVOIDARGS, FUNC.RESULT) + calldescr, canraise = self.callcontrol.getcalldescr(op) # pure = False loopinvariant = False @@ -234,11 +232,6 @@ effectinfo = calldescr.get_extra_info() assert (effectinfo is not None and not effectinfo.forces_virtual_or_virtualizable) - try: - canraise = self.callcontrol.raise_analyzer.can_raise(op) - except lltype.DelayedPointer: - canraise = True # if we need to look into the delayed ptr that is - # the portal, then it's certainly going to raise if loopinvariant: name = 'G_residual_call_loopinvariant' assert not non_void_args, ("arguments not supported for " Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Mon May 10 14:50:53 2010 @@ -43,17 +43,15 @@ def fielddescrof(self, STRUCT, name): return FakeDescr() -class FakeRaiseAnalyzer: - def can_raise(self, op): - try: - return 'cannot_raise' not in op.args[0].value._obj.graph.name - except AttributeError: - return True - class FakeCallControl: def guess_call_kind(self, op): return 'residual' - raise_analyzer = FakeRaiseAnalyzer() + def getcalldescr(self, op): + try: + can_raise = 'cannot_raise' not in op.args[0].value._obj.graph.name + except AttributeError: + can_raise = True + return FakeDescr(), can_raise def fake_regallocs(): return {'int': FakeRegAlloc(), Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Mon May 10 14:50:53 2010 @@ -27,14 +27,11 @@ def __init__(self, exitcase): self.exitcase = self.llexitcase = exitcase -class FakeRaiseAnalyzer: - def can_raise(self, op): - return True - class FakeResidualCallControl: def guess_call_kind(self, op): return 'residual' - raise_analyzer = FakeRaiseAnalyzer() + def getcalldescr(self, op): + return 'calldescr', True class FakeRegularCallControl: def guess_call_kind(self, op): @@ -182,9 +179,7 @@ assert op1.opname == 'G_residual_call_%s_%s' % (expectedkind, reskind) assert op1.result == op.result assert op1.args[0] == op.args[0] - FUNC = op.args[0].concretetype.TO - NONVOIDARGS = tuple([ARG for ARG in FUNC.ARGS if ARG != lltype.Void]) - assert op1.args[1] == ('calldescr', FUNC, NONVOIDARGS, FUNC.RESULT) + assert op1.args[1] == 'calldescr' assert len(op1.args) == 2 + len(expectedkind) for sublist, kind1 in zip(op1.args[2:], expectedkind): assert sublist.kind.startswith(kind1) From arigo at codespeak.net Mon May 10 14:55:43 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 10 May 2010 14:55:43 +0200 (CEST) Subject: [pypy-svn] r74462 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100510125543.7D2D9282BFC@codespeak.net> Author: arigo Date: Mon May 10 14:55:42 2010 New Revision: 74462 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Log: getinteriorfield. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Mon May 10 14:55:42 2010 @@ -430,6 +430,18 @@ sizedescr = self.cpu.sizeof(STRUCT) return SpaceOperation('new', [sizedescr], op.result) + def rewrite_op_getinteriorfield(self, op): + # only supports strings and unicodes + assert len(op.args) == 3 + assert op.args[1].value == 'chars' + optype = op.args[0].concretetype + if optype == lltype.Ptr(rstr.STR): + opname = "strgetitem" + else: + assert optype == lltype.Ptr(rstr.UNICODE) + opname = "unicodegetitem" + return SpaceOperation(opname, [op.args[0], op.args[2]], op.result) + def _rewrite_equality(self, op, opname): arg0, arg1 = op.args if isinstance(arg0, Constant) and not arg0.value: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Mon May 10 14:55:42 2010 @@ -404,3 +404,27 @@ op1 = Transformer().rewrite_operation(op) assert op1.opname == 'int_is_true' assert op1.args == [v1] + +def test_str_getinteriorfield(): + v = varoftype(lltype.Ptr(rstr.STR)) + v_index = varoftype(lltype.Signed) + v_result = varoftype(lltype.Char) + op = SpaceOperation('getinteriorfield', + [v, Constant('chars', lltype.Void), v_index], + v_result) + op1 = Transformer().rewrite_operation(op) + assert op1.opname == 'strgetitem' + assert op1.args == [v, v_index] + assert op1.result == v_result + +def test_unicode_getinteriorfield(): + v = varoftype(lltype.Ptr(rstr.UNICODE)) + v_index = varoftype(lltype.Signed) + v_result = varoftype(lltype.UniChar) + op = SpaceOperation('getinteriorfield', + [v, Constant('chars', lltype.Void), v_index], + v_result) + op1 = Transformer().rewrite_operation(op) + assert op1.opname == 'unicodegetitem' + assert op1.args == [v, v_index] + assert op1.result == v_result From arigo at codespeak.net Mon May 10 14:57:30 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 10 May 2010 14:57:30 +0200 (CEST) Subject: [pypy-svn] r74463 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100510125730.24B2B282BFC@codespeak.net> Author: arigo Date: Mon May 10 14:57:28 2010 New Revision: 74463 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Log: getinteriorarraysize. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Mon May 10 14:57:28 2010 @@ -430,6 +430,18 @@ sizedescr = self.cpu.sizeof(STRUCT) return SpaceOperation('new', [sizedescr], op.result) + def rewrite_op_getinteriorarraysize(self, op): + # only supports strings and unicodes + assert len(op.args) == 2 + assert op.args[1].value == 'chars' + optype = op.args[0].concretetype + if optype == lltype.Ptr(rstr.STR): + opname = "strlen" + else: + assert optype == lltype.Ptr(rstr.UNICODE) + opname = "unicodelen" + return SpaceOperation(opname, [op.args[0]], op.result) + def rewrite_op_getinteriorfield(self, op): # only supports strings and unicodes assert len(op.args) == 3 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Mon May 10 14:57:28 2010 @@ -405,6 +405,28 @@ assert op1.opname == 'int_is_true' assert op1.args == [v1] +def test_str_getinteriorarraysize(): + v = varoftype(lltype.Ptr(rstr.STR)) + v_result = varoftype(lltype.Signed) + op = SpaceOperation('getinteriorarraysize', + [v, Constant('chars', lltype.Void)], + v_result) + op1 = Transformer().rewrite_operation(op) + assert op1.opname == 'strlen' + assert op1.args == [v] + assert op1.result == v_result + +def test_unicode_getinteriorarraysize(): + v = varoftype(lltype.Ptr(rstr.UNICODE)) + v_result = varoftype(lltype.Signed) + op = SpaceOperation('getinteriorarraysize', + [v, Constant('chars', lltype.Void)], + v_result) + op1 = Transformer().rewrite_operation(op) + assert op1.opname == 'unicodelen' + assert op1.args == [v] + assert op1.result == v_result + def test_str_getinteriorfield(): v = varoftype(lltype.Ptr(rstr.STR)) v_index = varoftype(lltype.Signed) From arigo at codespeak.net Mon May 10 14:59:42 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 10 May 2010 14:59:42 +0200 (CEST) Subject: [pypy-svn] r74464 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100510125942.E81CC282BFC@codespeak.net> Author: arigo Date: Mon May 10 14:59:41 2010 New Revision: 74464 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Log: setinteriorfield. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Mon May 10 14:59:41 2010 @@ -454,6 +454,19 @@ opname = "unicodegetitem" return SpaceOperation(opname, [op.args[0], op.args[2]], op.result) + def rewrite_op_setinteriorfield(self, op): + # only supports strings and unicodes + assert len(op.args) == 4 + assert op.args[1].value == 'chars' + optype = op.args[0].concretetype + if optype == lltype.Ptr(rstr.STR): + opname = "strsetitem" + else: + assert optype == lltype.Ptr(rstr.UNICODE) + opname = "unicodesetitem" + return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]], + op.result) + def _rewrite_equality(self, op, opname): arg0, arg1 = op.args if isinstance(arg0, Constant) and not arg0.value: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Mon May 10 14:59:41 2010 @@ -450,3 +450,29 @@ assert op1.opname == 'unicodegetitem' assert op1.args == [v, v_index] assert op1.result == v_result + +def test_str_setinteriorfield(): + v = varoftype(lltype.Ptr(rstr.STR)) + v_index = varoftype(lltype.Signed) + v_newchr = varoftype(lltype.Char) + v_void = varoftype(lltype.Void) + op = SpaceOperation('setinteriorfield', + [v, Constant('chars', lltype.Void), v_index, v_newchr], + v_void) + op1 = Transformer().rewrite_operation(op) + assert op1.opname == 'strsetitem' + assert op1.args == [v, v_index, v_newchr] + assert op1.result == v_void + +def test_unicode_setinteriorfield(): + v = varoftype(lltype.Ptr(rstr.UNICODE)) + v_index = varoftype(lltype.Signed) + v_newchr = varoftype(lltype.UniChar) + v_void = varoftype(lltype.Void) + op = SpaceOperation('setinteriorfield', + [v, Constant('chars', lltype.Void), v_index, v_newchr], + v_void) + op1 = Transformer().rewrite_operation(op) + assert op1.opname == 'unicodesetitem' + assert op1.args == [v, v_index, v_newchr] + assert op1.result == v_void From arigo at codespeak.net Mon May 10 15:15:13 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 10 May 2010 15:15:13 +0200 (CEST) Subject: [pypy-svn] r74465 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100510131513.76DF5282BF5@codespeak.net> Author: arigo Date: Mon May 10 15:15:11 2010 New Revision: 74465 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Log: Constant-fold 'if _we_are_jitted' and 'if malloc_zero_filled'. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Mon May 10 15:15:11 2010 @@ -6,6 +6,8 @@ from pypy.jit.codewriter.flatten import ListOfKind from pypy.jit.codewriter import support, heaptracker from pypy.translator.simplify import get_funcobj +from pypy.rlib import objectmodel +from pypy.rlib.jit import _we_are_jitted def transform_graph(graph, cpu=None, callcontrol=None, portal=True): @@ -50,9 +52,16 @@ if op is op_raising_exception: self.killed_exception_raising_operation(block) else: - op2 = self.do_renaming(rename, op1) - newoperations.append(op2) + if isinstance(op1, SpaceOperation): + op2 = self.do_renaming(rename, op1) + newoperations.append(op2) + elif isinstance(op1, Constant): + rename[op.result] = op1 + else: + raise TypeError(repr(op1)) block.operations = newoperations + block.exitswitch = rename.get(block.exitswitch, block.exitswitch) + self.follow_constant_exit(block) self.optimize_goto_if_not(block) for link in block.exits: self.do_renaming_on_link(rename, link) @@ -85,6 +94,19 @@ # ---------- + def follow_constant_exit(self, block): + v = block.exitswitch + if isinstance(v, Constant) and v != c_last_exception: + llvalue = v.value + for link in block.exits: + if link.llexitcase == llvalue: + break + else: + assert link.exitcase == 'default' + block.exitswitch = None + link.exitcase = link.llexitcase = None + block.recloseblock(link) + def optimize_goto_if_not(self, block): """Replace code like 'v = int_gt(x,y); exitswitch = v' with just 'exitswitch = ('int_gt',x,y)'.""" @@ -574,6 +596,19 @@ op.result) return self.rewrite_operation(op1) + def rewrite_op_int_is_true(self, op): + if isinstance(op.args[0], Constant): + value = op.args[0].value + if value is objectmodel.malloc_zero_filled: + value = True + elif value is _we_are_jitted: + value = True + else: + raise AssertionError("don't know the truth value of %r" + % (value,)) + return Constant(value, lltype.Bool) + return op + # ____________________________________________________________ def _with_prefix(prefix): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Mon May 10 15:15:11 2010 @@ -9,6 +9,7 @@ from pypy.translator.unsimplify import varoftype from pypy.rlib.rarithmetic import ovfcheck from pypy.rlib.jit import dont_look_inside +from pypy.rlib.jit import _we_are_jitted class FakeRegAlloc: @@ -434,3 +435,13 @@ residual_call_ir_i $<* fn cannot_raise>, , I[%i0, %i1], R[], %i2 int_return %i2 """, transform=True, liveness=True) + + def test_we_are_jitted(self): + def f(x): + if _we_are_jitted: + return 2 + else: + return 3 + x + self.encoding_test(f, [5], """ + int_return $2 + """, transform=True) From arigo at codespeak.net Mon May 10 16:06:37 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 10 May 2010 16:06:37 +0200 (CEST) Subject: [pypy-svn] r74466 - pypy/trunk/pypy/jit/metainterp Message-ID: <20100510140637.5F2B0282BF5@codespeak.net> Author: arigo Date: Mon May 10 16:06:35 2010 New Revision: 74466 Modified: pypy/trunk/pypy/jit/metainterp/policy.py Log: Kill unused code. Modified: pypy/trunk/pypy/jit/metainterp/policy.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/policy.py (original) +++ pypy/trunk/pypy/jit/metainterp/policy.py Mon May 10 16:06:35 2010 @@ -40,9 +40,6 @@ # string builder interface if mod == 'pypy.rpython.lltypesystem.rbuilder': return True - # rweakvaluedict implementation - if mod == 'pypy.rlib.rweakrefimpl': - return True return False From arigo at codespeak.net Mon May 10 16:30:17 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 10 May 2010 16:30:17 +0200 (CEST) Subject: [pypy-svn] r74469 - pypy/trunk/pypy/module/pypyjit/test Message-ID: <20100510143017.4E6A1282BF5@codespeak.net> Author: arigo Date: Mon May 10 16:30:15 2010 New Revision: 74469 Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Log: issue529 in-progress Add a test that fails. Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Mon May 10 16:30:15 2010 @@ -603,6 +603,18 @@ # XXX I would like here to say that it's 0, but unfortunately # call that can raise is not exchanged into getarrayitem_gc + def test_overflow_checking(self): + self.run_source(''' + def main(): + def f(a,b): + if a < 0: return -1 + return a-b + total = 0 + for i in range(100000): + total += f(i, 5) + return total + ''', 100, ([], 4999450000L)) + class AppTestJIT(PyPyCJITTests): def setup_class(cls): if not option.runappdirect: From arigo at codespeak.net Mon May 10 16:45:38 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 10 May 2010 16:45:38 +0200 (CEST) Subject: [pypy-svn] r74470 - in pypy/trunk/pypy/jit: backend/x86 metainterp Message-ID: <20100510144538.7751A282BF5@codespeak.net> Author: arigo Date: Mon May 10 16:45:36 2010 New Revision: 74470 Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/regalloc.py pypy/trunk/pypy/jit/metainterp/optimizeopt.py Log: issue529 testing Change the backend to only accept int_xxx_ovf operations immediately followed by guard_(no)_overflow. A corresponding fix in the front-end to keep these two operations together. Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Mon May 10 16:45:36 2010 @@ -629,10 +629,6 @@ genop_float_mul = _binaryop('MULSD', True) genop_float_truediv = _binaryop('DIVSD') - genop_int_mul_ovf = genop_int_mul - genop_int_sub_ovf = genop_int_sub - genop_int_add_ovf = genop_int_add - genop_int_lt = _cmpop("L", "G") genop_int_le = _cmpop("LE", "GE") genop_int_eq = _cmpop("E", "E") @@ -969,13 +965,27 @@ self.mc.MOV(heap(self.cpu.pos_exc_value()), imm(0)) return addr - def genop_guard_guard_no_overflow(self, ign_1, guard_op, addr, - locs, resloc): - return self.implement_guard(addr, self.mc.JO) - - def genop_guard_guard_overflow(self, ign_1, guard_op, addr, - locs, resloc): - return self.implement_guard(addr, self.mc.JNO) + def _gen_guard_overflow(self, guard_op, addr): + guard_opnum = guard_op.opnum + if guard_opnum == rop.GUARD_NO_OVERFLOW: + return self.implement_guard(addr, self.mc.JO) + elif guard_opnum == rop.GUARD_OVERFLOW: + return self.implement_guard(addr, self.mc.JNO) + else: + print "int_xxx_ovf followed by", guard_op.getopname() + raise AssertionError + + def genop_guard_int_add_ovf(self, op, guard_op, addr, arglocs, result_loc): + self.genop_int_add(op, arglocs, result_loc) + return self._gen_guard_overflow(guard_op, addr) + + def genop_guard_int_sub_ovf(self, op, guard_op, addr, arglocs, result_loc): + self.genop_int_sub(op, arglocs, result_loc) + return self._gen_guard_overflow(guard_op, addr) + + def genop_guard_int_mul_ovf(self, op, guard_op, addr, arglocs, result_loc): + self.genop_int_mul(op, arglocs, result_loc) + return self._gen_guard_overflow(guard_op, addr) def genop_guard_guard_false(self, ign_1, guard_op, addr, locs, ign_2): loc = locs[0] Modified: pypy/trunk/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regalloc.py Mon May 10 16:45:36 2010 @@ -322,6 +322,12 @@ assert operations[i + 1].opnum == rop.GUARD_NOT_FORCED return True if not op.is_comparison(): + if op.is_ovf(): + if (operations[i + 1].opnum != rop.GUARD_NO_OVERFLOW and + operations[i + 1].opnum != rop.GUARD_OVERFLOW): + print "int_xxx_ovf not followed by guard_(no)_overflow" + raise AssertionError + return True return False if (operations[i + 1].opnum != rop.GUARD_TRUE and operations[i + 1].opnum != rop.GUARD_FALSE): @@ -467,9 +473,13 @@ consider_int_or = _consider_binop consider_int_xor = _consider_binop - consider_int_mul_ovf = _consider_binop - consider_int_sub_ovf = _consider_binop - consider_int_add_ovf = _consider_binop + def _consider_binop_with_guard(self, op, guard_op): + loc, argloc = self._consider_binop_part(op) + self.perform_with_guard(op, guard_op, [loc, argloc], loc) + + consider_int_mul_ovf = _consider_binop_with_guard + consider_int_sub_ovf = _consider_binop_with_guard + consider_int_add_ovf = _consider_binop_with_guard def consider_int_neg(self, op): res = self.rm.force_result_in_reg(op.result, op.args[0]) @@ -1000,6 +1010,7 @@ name = name[len('consider_'):] num = getattr(rop, name.upper()) if (ResOperation(num, [], None).is_comparison() + or ResOperation(num, [], None).is_ovf() or num == rop.CALL_MAY_FORCE or num == rop.CALL_ASSEMBLER): oplist_with_guard[num] = value oplist[num] = add_none_argument(value) Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Mon May 10 16:45:36 2010 @@ -1111,7 +1111,10 @@ prevop = newoperations[-2] # - is_comparison() for cases like "int_eq/setfield_gc/guard_true" # - CALL_MAY_FORCE: "call_may_force/setfield_gc/guard_not_forced" - if ((prevop.is_comparison() or prevop.opnum == rop.CALL_MAY_FORCE) + # - is_ovf(): "int_add_ovf/setfield_gc/guard_no_overflow" + opnum = prevop.opnum + if ((prevop.is_comparison() or opnum == rop.CALL_MAY_FORCE + or prevop.is_ovf()) and prevop.result not in lastop.args): newoperations[-2] = lastop newoperations[-1] = prevop From afa at codespeak.net Mon May 10 17:45:56 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 10 May 2010 17:45:56 +0200 (CEST) Subject: [pypy-svn] r74471 - pypy/trunk/pypy/module/cpyext Message-ID: <20100510154556.E8AB2282BF5@codespeak.net> Author: afa Date: Mon May 10 17:45:55 2010 New Revision: 74471 Modified: pypy/trunk/pypy/module/cpyext/typeobject.py Log: Translation fix Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Mon May 10 17:45:55 2010 @@ -297,7 +297,8 @@ pto = rffi.cast(PyTypeObjectPtr, pyo) try: if pto.c_tp_init: - if generic_cpy_call(space, pto.c_tp_init, w_obj, w_args, w_kw) < 0: + res = generic_cpy_call(space, pto.c_tp_init, w_obj, w_args, w_kw) + if rffi.cast(lltype.Signed, res) < 0: state = space.fromcache(State) state.check_and_raise_exception() finally: From fijal at codespeak.net Mon May 10 21:41:48 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 10 May 2010 21:41:48 +0200 (CEST) Subject: [pypy-svn] r74472 - pypy/trunk/pypy/rpython/lltypesystem Message-ID: <20100510194148.7B401282BF0@codespeak.net> Author: fijal Date: Mon May 10 21:41:44 2010 New Revision: 74472 Modified: pypy/trunk/pypy/rpython/lltypesystem/rlist.py Log: I think we don't consider it any more Modified: pypy/trunk/pypy/rpython/lltypesystem/rlist.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rlist.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rlist.py Mon May 10 21:41:44 2010 @@ -206,7 +206,6 @@ new_allocated = ovfcheck(newsize + some) except OverflowError: raise MemoryError - # XXX consider to have a real realloc # new_allocated is a bit more than newsize, enough to ensure an amortized # linear complexity for e.g. repeated usage of l.append(). items = l.items From fijal at codespeak.net Tue May 11 01:00:07 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 11 May 2010 01:00:07 +0200 (CEST) Subject: [pypy-svn] r74475 - pypy/branch/chunked-list/pypy/rpython/lltypesystem Message-ID: <20100510230007.D8199282BF0@codespeak.net> Author: fijal Date: Tue May 11 01:00:05 2010 New Revision: 74475 Modified: pypy/branch/chunked-list/pypy/rpython/lltypesystem/rlist.py Log: A dual check for length Modified: pypy/branch/chunked-list/pypy/rpython/lltypesystem/rlist.py ============================================================================== --- pypy/branch/chunked-list/pypy/rpython/lltypesystem/rlist.py (original) +++ pypy/branch/chunked-list/pypy/rpython/lltypesystem/rlist.py Tue May 11 01:00:05 2010 @@ -303,6 +303,8 @@ return l.items def ll_getitem_fast(l, index): + if l.length == 0: + raise IndexError ll_assert(index < l.length, "getitem out of bounds") return l.ll_items()[index] ll_getitem_fast.oopspec = 'list.getitem(l, index)' From hpk at codespeak.net Tue May 11 11:33:33 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 11 May 2010 11:33:33 +0200 (CEST) Subject: [pypy-svn] r74478 - pypy/trunk/py/_test Message-ID: <20100511093333.AF899282BF0@codespeak.net> Author: hpk Date: Tue May 11 11:33:31 2010 New Revision: 74478 Removed: pypy/trunk/py/_test/.pluginmanager.py.swp Log: remove bogus .swp file From agaynor at codespeak.net Tue May 11 11:46:58 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Tue, 11 May 2010 11:46:58 +0200 (CEST) Subject: [pypy-svn] r74479 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100511094658.2E4F5282BF0@codespeak.net> Author: agaynor Date: Tue May 11 11:46:57 2010 New Revision: 74479 Modified: pypy/trunk/pypy/module/cpyext/stubs.py pypy/trunk/pypy/module/cpyext/test/test_tupleobject.py pypy/trunk/pypy/module/cpyext/tupleobject.py Log: Implement _PyTuple_Resize for cpyext. Modified: pypy/trunk/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/stubs.py (original) +++ pypy/trunk/pypy/module/cpyext/stubs.py Tue May 11 11:46:57 2010 @@ -2997,25 +2997,6 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObjectP, Py_ssize_t], rffi.INT_real, error=-1) -def _PyTuple_Resize(space, p, newsize): - """Can be used to resize a tuple. newsize will be the new length of the tuple. - Because tuples are supposed to be immutable, this should only be used if there - is only one reference to the object. Do not use this if the tuple may already - be known to some other part of the code. The tuple will always grow or shrink - at the end. Think of this as destroying the old tuple and creating a new one, - only more efficiently. Returns 0 on success. Client code should never - assume that the resulting value of *p will be the same as before calling - this function. If the object referenced by *p is replaced, the original - *p is destroyed. On failure, returns -1 and sets *p to NULL, and - raises MemoryError or SystemError. - - Removed unused third parameter, last_is_sticky. - - This function used an int type for newsize. This might - require changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([], rffi.INT_real, error=CANNOT_FAIL) def PyTuple_ClearFreeList(space, ): """Clear the free list. Return the total number of freed items. Modified: pypy/trunk/pypy/module/cpyext/test/test_tupleobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_tupleobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_tupleobject.py Tue May 11 11:46:57 2010 @@ -1,6 +1,8 @@ import py +from pypy.module.cpyext.pyobject import PyObject, PyObjectP, make_ref, from_ref from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.rpython.lltypesystem import rffi, lltype class TestTupleObject(BaseApiTest): def test_tupleobject(self, space, api): @@ -11,3 +13,18 @@ assert api.PyTuple_GET_SIZE(atuple) == 3 raises(TypeError, api.PyTuple_Size(space.newlist([]))) api.PyErr_Clear() + + def test_tuple_resize(self, space, api): + py_tuple = api.PyTuple_New(3) + ar = lltype.malloc(PyObjectP.TO, 1, flavor='raw') + ar[0] = rffi.cast(PyObject, make_ref(space, py_tuple)) + api._PyTuple_Resize(ar, 2) + py_tuple = from_ref(space, ar[0]) + assert len(py_tuple.wrappeditems) == 2 + + api._PyTuple_Resize(ar, 10) + py_tuple = from_ref(space, ar[0]) + assert len(py_tuple.wrappeditems) == 10 + + api.Py_DecRef(ar[0]) + lltype.free(ar, flavor='raw') Modified: pypy/trunk/pypy/module/cpyext/tupleobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/tupleobject.py (original) +++ pypy/trunk/pypy/module/cpyext/tupleobject.py Tue May 11 11:46:57 2010 @@ -2,7 +2,8 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL, build_type_checkers) -from pypy.module.cpyext.pyobject import PyObject, Py_DecRef, borrow_from +from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, Py_DecRef, + borrow_from, make_ref, from_ref) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.objspace.std.tupleobject import W_TupleObject @@ -45,3 +46,29 @@ raise OperationError(space.w_TypeError, space.wrap("expected tuple object")) return PyTuple_GET_SIZE(space, ref) + + + at cpython_api([PyObjectP, Py_ssize_t], rffi.INT_real, error=-1) +def _PyTuple_Resize(space, ref, newsize): + """Can be used to resize a tuple. newsize will be the new length of the tuple. + Because tuples are supposed to be immutable, this should only be used if there + is only one reference to the object. Do not use this if the tuple may already + be known to some other part of the code. The tuple will always grow or shrink + at the end. Think of this as destroying the old tuple and creating a new one, + only more efficiently. Returns 0 on success. Client code should never + assume that the resulting value of *p will be the same as before calling + this function. If the object referenced by *p is replaced, the original + *p is destroyed. On failure, returns -1 and sets *p to NULL, and + raises MemoryError or SystemError.""" + py_tuple = from_ref(space, ref[0]) + py_newtuple = PyTuple_New(space, newsize) + + to_cp = newsize + oldsize = len(py_tuple.wrappeditems) + if oldsize < newsize: + to_cp = oldsize + for i in range(to_cp): + py_newtuple.wrappeditems[i] = py_tuple.wrappeditems[i] + Py_DecRef(space, ref[0]) + ref[0] = make_ref(space, py_newtuple) + return 0 From hpk at codespeak.net Tue May 11 11:48:34 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 11 May 2010 11:48:34 +0200 (CEST) Subject: [pypy-svn] r74480 - in pypy/trunk/pypy: jit/backend/x86/test translator/c/gcc/test translator/jvm Message-ID: <20100511094834.BB964282BF0@codespeak.net> Author: hpk Date: Tue May 11 11:48:33 2010 New Revision: 74480 Added: pypy/trunk/pypy/translator/c/gcc/test/conftest.py - copied, changed from r74476, pypy/trunk/pypy/jit/backend/x86/test/conftest.py Modified: pypy/trunk/pypy/jit/backend/x86/test/conftest.py pypy/trunk/pypy/translator/jvm/conftest.py Log: fix 64-bit skipping - The old conftest.py Module or Directory classes will not be detected anymore now that the new hooks are in use. Modified: pypy/trunk/pypy/jit/backend/x86/test/conftest.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/conftest.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/conftest.py Tue May 11 11:48:33 2010 @@ -1,9 +1,8 @@ import py from pypy.jit.backend import detect_cpu -class Module(py.test.collect.Module): - def collect(self): - cpu = detect_cpu.autodetect() - if cpu != 'x86': - py.test.skip("x86 directory skipped: cpu is %r" % (cpu,)) - return super(Module, self).collect() +cpu = detect_cpu.autodetect() +def pytest_runtest_setup(item): + if cpu != 'x86': + py.test.skip("x86 directory skipped: cpu is %r" % (cpu,)) + Copied: pypy/trunk/pypy/translator/c/gcc/test/conftest.py (from r74476, pypy/trunk/pypy/jit/backend/x86/test/conftest.py) ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/conftest.py (original) +++ pypy/trunk/pypy/translator/c/gcc/test/conftest.py Tue May 11 11:48:33 2010 @@ -1,9 +1,8 @@ import py from pypy.jit.backend import detect_cpu -class Module(py.test.collect.Module): - def collect(self): - cpu = detect_cpu.autodetect() - if cpu != 'x86': - py.test.skip("x86 directory skipped: cpu is %r" % (cpu,)) - return super(Module, self).collect() +cpu = detect_cpu.autodetect() +def pytest_runtest_setup(item): + if cpu != 'x86': + py.test.skip("x86 directory skipped: cpu is %r" % (cpu,)) + Modified: pypy/trunk/pypy/translator/jvm/conftest.py ============================================================================== --- pypy/trunk/pypy/translator/jvm/conftest.py (original) +++ pypy/trunk/pypy/translator/jvm/conftest.py Tue May 11 11:48:33 2010 @@ -1,11 +1,9 @@ import py, sys -class Module(py.test.collect.Module): - def collect(self): - if sys.maxint > 2147483647: # 64bit platform - py.test.skip("jvm backend on 64bit unsupported") - return super(Module, self).collect() +def pytest_runtest_setup(item): + if sys.maxint > 2147483647: # 64bit platform + py.test.skip("jvm backend on 64bit unsupported") def pytest_addoption(parser): group = parser.getgroup("pypy-jvm options") From agaynor at codespeak.net Tue May 11 12:00:19 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Tue, 11 May 2010 12:00:19 +0200 (CEST) Subject: [pypy-svn] r74481 - pypy/trunk/pypy/module/cpyext Message-ID: <20100511100019.0A4A9282BF0@codespeak.net> Author: agaynor Date: Tue May 11 12:00:18 2010 New Revision: 74481 Modified: pypy/trunk/pypy/module/cpyext/tupleobject.py Log: A little error checkint for _PyTuple_Resize, shoudl help with translating. Modified: pypy/trunk/pypy/module/cpyext/tupleobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/tupleobject.py (original) +++ pypy/trunk/pypy/module/cpyext/tupleobject.py Tue May 11 12:00:18 2010 @@ -61,6 +61,9 @@ *p is destroyed. On failure, returns -1 and sets *p to NULL, and raises MemoryError or SystemError.""" py_tuple = from_ref(space, ref[0]) + if not PyTuple_Check(space, py_tuple): + PyErr_BadInternalCall(space) + assert isinstance(py_tuple, W_TupleObject) py_newtuple = PyTuple_New(space, newsize) to_cp = newsize From arigo at codespeak.net Wed May 12 09:18:31 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 12 May 2010 09:18:31 +0200 (CEST) Subject: [pypy-svn] r74494 - in pypy/branch/blackhole-improvement/pypy/jit: backend/llgraph codewriter codewriter/test metainterp metainterp/test Message-ID: <20100512071831.1A0AA282BDE@codespeak.net> Author: arigo Date: Wed May 12 09:18:28 2010 New Revision: 74494 Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/effectinfo.py - copied, changed from r74456, pypy/branch/blackhole-improvement/pypy/jit/metainterp/effectinfo.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_effectinfo.py - copied, changed from r74456, pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_effectinfo.py Removed: pypy/branch/blackhole-improvement/pypy/jit/metainterp/effectinfo.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_effectinfo.py Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/typesystem.py Log: General progress. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py Wed May 12 09:18:28 2010 @@ -1185,19 +1185,19 @@ array = array._obj.container return array.getlength() -def do_strlen(_, string): +def do_strlen(string): str = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string) return len(str.chars) -def do_strgetitem(_, string, index): +def do_strgetitem(string, index): str = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string) return ord(str.chars[index]) -def do_unicodelen(_, string): +def do_unicodelen(string): uni = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string) return len(uni.chars) -def do_unicodegetitem(_, string, index): +def do_unicodegetitem(string, index): uni = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string) return ord(uni.chars[index]) @@ -1311,18 +1311,18 @@ newvalue = cast_from_ptr(FIELDTYPE, newvalue) setattr(ptr, fieldname, newvalue) -def do_newstr(_, length): +def do_newstr(length): x = rstr.mallocstr(length) return cast_to_ptr(x) -def do_newunicode(_, length): +def do_newunicode(length): return cast_to_ptr(rstr.mallocunicode(length)) -def do_strsetitem(_, string, index, newvalue): +def do_strsetitem(string, index, newvalue): str = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string) str.chars[index] = chr(newvalue) -def do_unicodesetitem(_, string, index, newvalue): +def do_unicodesetitem(string, index, newvalue): uni = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string) uni.chars[index] = unichr(newvalue) Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py Wed May 12 09:18:28 2010 @@ -315,23 +315,17 @@ ## array = arraybox.getref_base() ## return history.BoxInt(llimpl.do_arraylen_gc(arraydescr, array)) -## def do_strlen(self, stringbox): -## string = stringbox.getref_base() -## return history.BoxInt(llimpl.do_strlen(0, string)) + def bh_strlen(self, string): + return llimpl.do_strlen(string) -## def do_strgetitem(self, stringbox, indexbox): -## string = stringbox.getref_base() -## index = indexbox.getint() -## return history.BoxInt(llimpl.do_strgetitem(0, string, index)) + def bh_strgetitem(self, string, index): + return llimpl.do_strgetitem(string, index) -## def do_unicodelen(self, stringbox): -## string = stringbox.getref_base() -## return history.BoxInt(llimpl.do_unicodelen(0, string)) + def bh_unicodelen(self, string): + return llimpl.do_unicodelen(string) -## def do_unicodegetitem(self, stringbox, indexbox): -## string = stringbox.getref_base() -## index = indexbox.getint() -## return history.BoxInt(llimpl.do_unicodegetitem(0, string, index)) + def bh_unicodegetitem(self, string, index): + return llimpl.do_unicodegetitem(string, index) ## def do_getarrayitem_gc(self, arraybox, indexbox, arraydescr): ## assert isinstance(arraydescr, Descr) @@ -509,25 +503,17 @@ ## def do_same_as(self, box1): ## return box1.clonebox() -## def do_newstr(self, lengthbox): -## length = lengthbox.getint() -## return history.BoxPtr(llimpl.do_newstr(0, length)) - -## def do_newunicode(self, lengthbox): -## length = lengthbox.getint() -## return history.BoxPtr(llimpl.do_newunicode(0, length)) + def bh_newstr(self, length): + return llimpl.do_newstr(length) -## def do_strsetitem(self, stringbox, indexbox, newvaluebox): -## string = stringbox.getref_base() -## index = indexbox.getint() -## newvalue = newvaluebox.getint() -## llimpl.do_strsetitem(0, string, index, newvalue) + def bh_newunicode(self, length): + return llimpl.do_newunicode(length) -## def do_unicodesetitem(self, stringbox, indexbox, newvaluebox): -## string = stringbox.getref_base() -## index = indexbox.getint() -## newvalue = newvaluebox.getint() -## llimpl.do_unicodesetitem(0, string, index, newvalue) + def bh_strsetitem(self, string, index, newvalue): + llimpl.do_strsetitem(string, index, newvalue) + + def bh_unicodesetitem(self, string, index, newvalue): + llimpl.do_unicodesetitem(string, index, newvalue) def bh_call_i(self, func, calldescr, args_i, args_r, args_f): self._prepare_call(INT, calldescr, args_i, args_r, args_f) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Wed May 12 09:18:28 2010 @@ -1,6 +1,6 @@ from pypy.jit.metainterp.history import AbstractDescr, getkind from pypy.jit.codewriter.flatten import Register, Label, TLabel, KINDS -from pypy.jit.codewriter.flatten import ListOfKind +from pypy.jit.codewriter.flatten import ListOfKind, IndirectCallTargets from pypy.jit.codewriter.format import format_assembler from pypy.jit.codewriter.jitcode import SwitchDictDescr, JitCode from pypy.objspace.flow.model import Constant @@ -12,6 +12,7 @@ def __init__(self): self.insns = {} self.descrs = [] + self.indirectcalltargets = set() # set of JitCodes self._descr_dict = {} self._count_jitcodes = 0 @@ -26,6 +27,7 @@ self.check_result() if jitcode is None: jitcode = JitCode(ssarepr.name) + jitcode._ssarepr = ssarepr self.make_jitcode(jitcode) if self._count_jitcodes < 20: # stop if we have a lot of them jitcode._dump = format_assembler(ssarepr) @@ -142,6 +144,8 @@ self.code.append(chr(num & 0xFF)) self.code.append(chr(num >> 8)) argcodes.append('d') + elif isinstance(x, IndirectCallTargets): + self.indirectcalltargets.update(x.lst) else: raise NotImplementedError(x) # Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py Wed May 12 09:18:28 2010 @@ -5,13 +5,13 @@ from pypy.jit.codewriter import support from pypy.jit.codewriter.jitcode import JitCode +from pypy.jit.codewriter.effectinfo import VirtualizableAnalyzer +from pypy.jit.codewriter.effectinfo import effectinfo_from_writeanalyze +from pypy.jit.codewriter import effectinfo as ef from pypy.translator.simplify import get_funcobj, get_functype from pypy.rpython.lltypesystem import lltype, llmemory - from pypy.translator.backendopt.canraise import RaiseAnalyzer from pypy.translator.backendopt.writeanalyze import ReadWriteAnalyzer -from pypy.jit.metainterp.effectinfo import VirtualizableAnalyzer -from pypy.jit.metainterp.effectinfo import effectinfo_from_writeanalyze class CallControl(object): @@ -146,6 +146,12 @@ return jitcode def get_jitcode_calldescr(self, graph): + """Return the calldescr that describes calls to the 'graph'. + This returns a calldescr that is appropriate to attach to the + jitcode corresponding to 'graph'. It has no extra effectinfo, + because it is not needed there; it is only used by the blackhole + interp to really do the call corresponding to 'inline_call' ops. + """ fnptr = self.rtyper.getcallable(graph) FUNC = get_functype(lltype.typeOf(fnptr)) if self.rtyper.type_system.name == 'ootypesystem': @@ -158,6 +164,13 @@ return (fnaddr, calldescr) def getcalldescr(self, op): + """Return the calldescr that describes all calls done by 'op'. + This returns a calldescr that we can put in the corresponding + call operation in the calling jitcode. It gets an effectinfo + describing the effect of the call: which field types it may + change, whether it can force virtualizables, whether it can + raise, etc. + """ NON_VOID_ARGS = [x.concretetype for x in op.args[1:] if x.concretetype is not lltype.Void] RESULT = op.result.concretetype @@ -167,15 +180,46 @@ assert NON_VOID_ARGS == [T for T in ARGS if T is not lltype.Void] assert RESULT == FUNC.RESULT # ok + # get the 'pure' and 'loopinvariant' flags from the function object + pure = False + loopinvariant = False + if op.opname == "direct_call": + func = getattr(get_funcobj(op.args[0].value), '_callable', None) + pure = getattr(func, "_pure_function_", False) + loopinvariant = getattr(func, "_jit_loop_invariant_", False) + if loopinvariant: + assert not NON_VOID_ARGS, ("arguments not supported for " + "loop-invariant function!") + # build the extraeffect + if self.virtualizable_analyzer.analyze(op): + extraeffect = ef.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE + elif loopinvariant: + extraeffect = ef.EF_LOOPINVARIANT + elif pure: + # XXX check what to do about exceptions (also MemoryError?) + extraeffect = ef.EF_PURE + elif self._canraise(op): + extraeffect = ef.EF_CAN_RAISE + else: + extraeffect = ef.EF_CANNOT_RAISE + # effectinfo = effectinfo_from_writeanalyze( - self.readwrite_analyzer.analyze(op), - self.cpu, - self.virtualizable_analyzer.analyze(op)) - calldescr = self.cpu.calldescrof(FUNC, tuple(NON_VOID_ARGS), RESULT, - effectinfo) + self.readwrite_analyzer.analyze(op), self.cpu, extraeffect) + # + if pure or loopinvariant: + assert effectinfo is not None + assert extraeffect != ef.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE + # + return self.cpu.calldescrof(FUNC, tuple(NON_VOID_ARGS), RESULT, + effectinfo) + + def _canraise(self, op): try: - canraise = self.raise_analyzer.can_raise(op) + return self.raise_analyzer.can_raise(op) except lltype.DelayedPointer: - canraise = True # if we need to look into the delayed ptr that is - # the portal, then it's certainly going to raise - return calldescr, canraise + return True # if we need to look into the delayed ptr that is + # the portal, then it's certainly going to raise + + def calldescr_canraise(self, calldescr): + effectinfo = calldescr.get_extra_info() + return effectinfo is None or effectinfo.extraeffect >= ef.EF_CAN_RAISE Copied: pypy/branch/blackhole-improvement/pypy/jit/codewriter/effectinfo.py (from r74456, pypy/branch/blackhole-improvement/pypy/jit/metainterp/effectinfo.py) ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/effectinfo.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/effectinfo.py Wed May 12 09:18:28 2010 @@ -4,28 +4,35 @@ from pypy.rpython.ootypesystem import ootype from pypy.translator.backendopt.graphanalyze import BoolGraphAnalyzer +# the 'extraeffect' field is one of the following values: +EF_PURE = 0 # pure function (and cannot raise) +EF_CANNOT_RAISE = 1 # a function which cannot raise +EF_CAN_RAISE = 2 # normal function (can raise) +EF_LOOPINVARIANT = 3 # special: call it only once per loop +EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE = 4 # can raise and force virtualizables + + class EffectInfo(object): _cache = {} def __new__(cls, readonly_descrs_fields, write_descrs_fields, write_descrs_arrays, - forces_virtual_or_virtualizable=False): + extraeffect=EF_CAN_RAISE): key = (frozenset(readonly_descrs_fields), frozenset(write_descrs_fields), frozenset(write_descrs_arrays), - forces_virtual_or_virtualizable) + extraeffect) if key in cls._cache: return cls._cache[key] result = object.__new__(cls) result.readonly_descrs_fields = readonly_descrs_fields result.write_descrs_fields = write_descrs_fields result.write_descrs_arrays = write_descrs_arrays - result.forces_virtual_or_virtualizable= forces_virtual_or_virtualizable + result.extraeffect = extraeffect cls._cache[key] = result return result -def effectinfo_from_writeanalyze(effects, cpu, - forces_virtual_or_virtualizable=False): +def effectinfo_from_writeanalyze(effects, cpu, extraeffect=EF_CAN_RAISE): from pypy.translator.backendopt.writeanalyze import top_set if effects is top_set: return None @@ -62,7 +69,7 @@ return EffectInfo(readonly_descrs_fields, write_descrs_fields, write_descrs_arrays, - forces_virtual_or_virtualizable) + extraeffect) def consider_struct(TYPE, fieldname): if fieldType(TYPE, fieldname) is lltype.Void: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Wed May 12 09:18:28 2010 @@ -44,6 +44,12 @@ def __iter__(self): return iter(self.content) +class IndirectCallTargets(object): + def __init__(self, lst): + self.lst = lst # list of JitCodes + def __repr__(self): + return '' + KINDS = ['int', 'ref', 'float'] # ____________________________________________________________ @@ -194,7 +200,7 @@ linkfalse, linktrue = linktrue, linkfalse opname = 'goto_if_not' if isinstance(block.exitswitch, tuple): - # special case produced by jitter.optimize_goto_if_not() + # special case produced by jtransform.optimize_goto_if_not() if block.exitswitch[0] != 'int_is_true': opname = 'goto_if_not_' + block.exitswitch[0] opargs = block.exitswitch[1:] @@ -303,6 +309,8 @@ v = ListOfKind(v.kind, lst) elif isinstance(v, AbstractDescr): pass + elif isinstance(v, IndirectCallTargets): + pass else: raise NotImplementedError(type(v)) args.append(v) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py Wed May 12 09:18:28 2010 @@ -2,7 +2,7 @@ from pypy.objspace.flow.model import Constant from pypy.rpython.lltypesystem import lltype from pypy.jit.codewriter.flatten import SSARepr, Label, TLabel, Register -from pypy.jit.codewriter.flatten import ListOfKind +from pypy.jit.codewriter.flatten import ListOfKind, IndirectCallTargets from pypy.jit.codewriter.jitcode import SwitchDictDescr from pypy.jit.metainterp.history import AbstractDescr @@ -28,7 +28,7 @@ return '' % ( ', '.join(['%s:%s' % (key, getlabelname(lbl)) for key, lbl in x._labels])) - elif isinstance(x, AbstractDescr): + elif isinstance(x, (AbstractDescr, IndirectCallTargets)): return '%r' % (x,) else: return '' % (x,) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py Wed May 12 09:18:28 2010 @@ -13,6 +13,7 @@ self.fnaddr = fnaddr self.calldescr = calldescr self._called_from = called_from # debugging + self._ssarepr = None # debugging def setup(self, code='', constants_i=[], constants_r=[], constants_f=[], num_regs_i=256, num_regs_r=256, num_regs_f=256, @@ -92,11 +93,21 @@ return ' '.join(lst) def _missing_liveness(self, pc): - raise KeyError("missing liveness[%d]" % (pc,)) + raise MissingLiveness("missing liveness[%d]\n%s" % (pc, self.dump())) + + def dump(self): + if self._ssarepr is None: + return '' + else: + from pypy.jit.codewriter.format import format_assembler + return format_assembler(self._ssarepr) def __repr__(self): return '' % self.name +class MissingLiveness(Exception): + pass + class SwitchDictDescr(AbstractDescr): "Get a 'dict' attribute mapping integer values to bytecode positions." Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Wed May 12 09:18:28 2010 @@ -3,9 +3,8 @@ from pypy.jit.metainterp.history import getkind from pypy.objspace.flow.model import SpaceOperation, Variable, Constant from pypy.objspace.flow.model import Block, Link, c_last_exception -from pypy.jit.codewriter.flatten import ListOfKind +from pypy.jit.codewriter.flatten import ListOfKind, IndirectCallTargets from pypy.jit.codewriter import support, heaptracker -from pypy.translator.simplify import get_funcobj from pypy.rlib import objectmodel from pypy.rlib.jit import _we_are_jitted @@ -217,14 +216,14 @@ args_f = [] for v in op.args[1:]: self.add_in_correct_list(v, args_i, args_r, args_f) - if args_f: kinds = 'irf' + reskind = getkind(op.result.concretetype)[0] + if args_f or reskind == 'f': kinds = 'irf' elif args_i: kinds = 'ir' else: kinds = 'r' sublists = [] if 'i' in kinds: sublists.append(ListOfKind('int', args_i)) if 'r' in kinds: sublists.append(ListOfKind('ref', args_r)) if 'f' in kinds: sublists.append(ListOfKind('float', args_f)) - reskind = getkind(op.result.concretetype)[0] return SpaceOperation('%s_%s_%s' % (namebase, kinds, reskind), initialargs + sublists, op.result) @@ -237,35 +236,16 @@ else: raise AssertionError(kind) lst.append(v) - def handle_residual_call(self, op): + def handle_residual_call(self, op, extraargs=[]): """A direct_call turns into the operation 'residual_call_xxx' if it is calling a function that we don't want to JIT. The initial args - of 'residual_call_xxx' are the constant function to call, and its - calldescr.""" - calldescr, canraise = self.callcontrol.getcalldescr(op) - # - pure = False - loopinvariant = False - if op.opname == "direct_call": - func = getattr(get_funcobj(op.args[0].value), '_callable', None) - pure = getattr(func, "_pure_function_", False) - loopinvariant = getattr(func, "_jit_loop_invariant_", False) - if pure or loopinvariant: - effectinfo = calldescr.get_extra_info() - assert (effectinfo is not None and - not effectinfo.forces_virtual_or_virtualizable) - if loopinvariant: - name = 'G_residual_call_loopinvariant' - assert not non_void_args, ("arguments not supported for " - "loop-invariant function!") - elif pure: - # XXX check what to do about exceptions (also MemoryError?) - name = 'residual_call_pure' - elif canraise: + of 'residual_call_xxx' are the function to call, and its calldescr.""" + calldescr = self.callcontrol.getcalldescr(op) + if self.callcontrol.calldescr_canraise(calldescr): name = 'G_residual_call' else: name = 'residual_call' - return self.rewrite_call(op, name, [op.args[0], calldescr]) + return self.rewrite_call(op, name, [op.args[0], calldescr] + extraargs) def handle_regular_call(self, op): """A direct_call turns into the operation 'inline_call_xxx' if it @@ -276,6 +256,17 @@ called_from=self.graph) return self.rewrite_call(op, 'G_inline_call', [jitcode]) + handle_residual_indirect_call = handle_residual_call + + def handle_regular_indirect_call(self, op): + """An indirect call where at least one target has a JitCode.""" + lst = [] + for targetgraph in self.callcontrol.graphs_from(op): + jitcode = self.callcontrol.get_jitcode(targetgraph, + called_from=self.graph) + lst.append(jitcode) + return self.handle_residual_call(op, [IndirectCallTargets(lst)]) + def _do_builtin_call(self, op, oopspec_name=None, args=None, extra=None): if oopspec_name is None: oopspec_name = op.opname if args is None: args = op.args Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py Wed May 12 09:18:28 2010 @@ -2,7 +2,7 @@ # Some instruction require liveness information (the ones that can end up -# in generate_guard() in pyjitpl.py); jitter.py prefixes these opnames +# in generate_guard() in pyjitpl.py); jtransform.py prefixes these opnames # with a 'G_'. Additionally, boolean and general switches in the flow graph # will turn in 'goto_if_not_*' operations, which also require liveness info. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py Wed May 12 09:18:28 2010 @@ -1,7 +1,8 @@ import py, struct from pypy.jit.codewriter.assembler import Assembler from pypy.jit.codewriter.flatten import SSARepr, Label, TLabel, Register -from pypy.jit.codewriter.flatten import ListOfKind +from pypy.jit.codewriter.flatten import ListOfKind, IndirectCallTargets +from pypy.jit.codewriter.jitcode import MissingLiveness from pypy.jit.metainterp.history import AbstractDescr from pypy.objspace.flow.model import Constant from pypy.rpython.lltypesystem import lltype, llmemory @@ -138,6 +139,16 @@ assert assembler.insns == {'foobar/d': 0} assert assembler.descrs == descrs[::-1] +def test_assemble_indirect_call(): + lst1 = ["somejitcode1", "somejitcode2"] + lst2 = ["somejitcode1", "somejitcode3"] + ssarepr = SSARepr("test") + ssarepr.insns = [('foobar', IndirectCallTargets(lst1)), + ('foobar', IndirectCallTargets(lst2))] + assembler = Assembler() + assembler.assemble(ssarepr) + assert assembler.indirectcalltargets == set(lst1).union(lst2) + def test_num_regs(): assembler = Assembler() ssarepr = SSARepr("test") @@ -183,10 +194,10 @@ assert assembler.insns == {'int_add/ici': 0, 'int_mul/iii': 1, 'int_return/i': 2} - py.test.raises(KeyError, jitcode._live_vars, 0) - py.test.raises(KeyError, jitcode._live_vars, 3) - py.test.raises(KeyError, jitcode._live_vars, 5) - py.test.raises(KeyError, jitcode._live_vars, 24) + py.test.raises(MissingLiveness, jitcode._live_vars, 0) + py.test.raises(MissingLiveness, jitcode._live_vars, 3) + py.test.raises(MissingLiveness, jitcode._live_vars, 5) + py.test.raises(MissingLiveness, jitcode._live_vars, 24) assert jitcode._live_vars(4) == '%i0' assert jitcode._live_vars(8) == '%i0 %i1' assert jitcode._live_vars(12) == '%i0' Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py Wed May 12 09:18:28 2010 @@ -44,9 +44,11 @@ assert jitcode.num_regs_r() == 0 assert jitcode.num_regs_f() == 0 assert jitcode._live_vars(5) == '%i0 %i1' + # + from pypy.jit.codewriter.jitcode import MissingLiveness for i in range(len(jitcode.code)+1): if i != 5: - py.test.raises(KeyError, jitcode._live_vars, i) + py.test.raises(MissingLiveness, jitcode._live_vars, i) def test_call(): def ggg(x): Copied: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_effectinfo.py (from r74456, pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_effectinfo.py) ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_effectinfo.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_effectinfo.py Wed May 12 09:18:28 2010 @@ -1,7 +1,7 @@ from pypy.rpython.lltypesystem.rclass import OBJECT from pypy.rpython.lltypesystem import lltype from pypy.rpython.ootypesystem import ootype -from pypy.jit.metainterp.effectinfo import effectinfo_from_writeanalyze +from pypy.jit.codewriter.effectinfo import effectinfo_from_writeanalyze class FakeCPU: def fielddescrof(self, T, fieldname): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Wed May 12 09:18:28 2010 @@ -45,14 +45,18 @@ return FakeDescr() class FakeCallControl: + _descr_cannot_raise = FakeDescr() def guess_call_kind(self, op): return 'residual' def getcalldescr(self, op): try: - can_raise = 'cannot_raise' not in op.args[0].value._obj.graph.name + if 'cannot_raise' in op.args[0].value._obj.graph.name: + return self._descr_cannot_raise except AttributeError: - can_raise = True - return FakeDescr(), can_raise + pass + return FakeDescr() + def calldescr_canraise(self, calldescr): + return calldescr is not self._descr_cannot_raise def fake_regallocs(): return {'int': FakeRegAlloc(), Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Wed May 12 09:18:28 2010 @@ -31,7 +31,9 @@ def guess_call_kind(self, op): return 'residual' def getcalldescr(self, op): - return 'calldescr', True + return 'calldescr' + def calldescr_canraise(self, calldescr): + return True class FakeRegularCallControl: def guess_call_kind(self, op): @@ -42,6 +44,19 @@ assert graph == 'somegraph' return 'somejitcode' +class FakeIndirectCallControl: + def guess_call_kind(self, op): + return 'regular' + def graphs_from(self, op): + return ['somegraph1', 'somegraph2'] + def getcalldescr(self, op): + return 'calldescr' + def get_jitcode(self, graph, called_from=None): + assert graph in ('somegraph1', 'somegraph2') + return 'somejitcode' + graph[-1] + def calldescr_canraise(self, calldescr): + return True + def test_optimize_goto_if_not(): v1 = Variable() @@ -156,11 +171,13 @@ if with_r: ARGS += [rclass.OBJECTPTR, lltype.Ptr(rstr.STR)] if with_f: ARGS += [lltype.Float, lltype.Float] random.shuffle(ARGS) + if RESTYPE == lltype.Float: with_f = True if with_f: expectedkind = 'irf' # all kinds elif with_i: expectedkind = 'ir' # integers and references else: expectedkind = 'r' # only references yield residual_call_test, ARGS, RESTYPE, expectedkind yield direct_call_test, ARGS, RESTYPE, expectedkind + yield indirect_call_test, ARGS, RESTYPE, expectedkind def get_direct_call_op(argtypes, restype): FUNC = lltype.FuncType(argtypes, restype) @@ -207,6 +224,29 @@ kind = getkind(v.concretetype) assert kind == 'void' or kind[0] in expectedkind +def indirect_call_test(argtypes, restype, expectedkind): + op = get_direct_call_op(argtypes, restype) + op.opname = 'indirect_call' + op.args[0] = varoftype(op.args[0].concretetype) + op.args.append(Constant(['somegraph1', 'somegraph2'], lltype.Void)) + tr = Transformer(FakeCPU(), FakeIndirectCallControl()) + tr.graph = 'someinitialgraph' + op1 = tr.rewrite_operation(op) + reskind = getkind(restype)[0] + assert op1.opname == 'G_residual_call_%s_%s' % (expectedkind, reskind) + assert op1.result == op.result + assert op1.args[0] == op.args[0] + assert op1.args[1] == 'calldescr' + assert op1.args[2].lst == ['somejitcode1', 'somejitcode2'] + assert len(op1.args) == 3 + len(expectedkind) + for sublist, kind1 in zip(op1.args[3:], expectedkind): + assert sublist.kind.startswith(kind1) + assert list(sublist) == [v for v in op.args[1:] + if getkind(v.concretetype) == sublist.kind] + for v in op.args[1:]: + kind = getkind(v.concretetype) + assert kind == 'void' or kind[0] in expectedkind + def test_getfield(): # XXX a more compact encoding would be possible, something along # the lines of getfield_gc_r %r0, $offset, %r1 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Wed May 12 09:18:28 2010 @@ -18,7 +18,7 @@ def check_assembler(self, graph, expected, transform=False): # 'transform' can be False only for simple graphs. More complex - # graphs must first be transformed by jitter.py before they can be + # graphs must first be transformed by jtransform.py before they can be # subjected to register allocation and flattening. if transform: from pypy.jit.codewriter.jtransform import transform_graph Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Wed May 12 09:18:28 2010 @@ -218,7 +218,7 @@ # Get the bhimpl_xxx method. If we get an AttributeError here, # it means that either the implementation is missing, or that it # should not appear here at all but instead be transformed away - # by codewriter/jitter.py. + # by codewriter/jtransform.py. unboundmethod = getattr(BlackholeInterpreter, 'bhimpl_' + name).im_func verbose = self.verbose argtypes = unrolling_iterable(unboundmethod.argtypes) @@ -726,9 +726,6 @@ @arguments("cpu", "i", "d", "R", returns="r") def bhimpl_residual_call_r_r(cpu, func, calldescr, args_r): return cpu.bh_call_r(func, calldescr, None, args_r, None) - @arguments("cpu", "i", "d", "R", returns="f") - def bhimpl_residual_call_r_f(cpu, func, calldescr, args_r): - return cpu.bh_call_f(func, calldescr, None, args_r, None) @arguments("cpu", "i", "d", "R") def bhimpl_residual_call_r_v(cpu, func, calldescr, args_r): cpu.bh_call_v(func, calldescr, None, args_r, None) @@ -739,9 +736,6 @@ @arguments("cpu", "i", "d", "I", "R", returns="r") def bhimpl_residual_call_ir_r(cpu, func, calldescr, args_i, args_r): return cpu.bh_call_r(func, calldescr, args_i, args_r, None) - @arguments("cpu", "i", "d", "I", "R", returns="f") - def bhimpl_residual_call_ir_f(cpu, func, calldescr, args_i, args_r): - return cpu.bh_call_f(func, calldescr, args_i, args_r, None) @arguments("cpu", "i", "d", "I", "R") def bhimpl_residual_call_ir_v(cpu, func, calldescr, args_i, args_r): cpu.bh_call_v(func, calldescr, args_i, args_r, None) @@ -767,10 +761,6 @@ def bhimpl_inline_call_r_r(cpu, jitcode, args_r): return cpu.bh_call_r(jitcode.get_fnaddr_as_int(), jitcode.calldescr, None, args_r, None) - @arguments("cpu", "j", "R", returns="f") - def bhimpl_inline_call_r_f(cpu, jitcode, args_r): - return cpu.bh_call_f(jitcode.get_fnaddr_as_int(), jitcode.calldescr, - None, args_r, None) @arguments("cpu", "j", "R") def bhimpl_inline_call_r_v(cpu, jitcode, args_r): return cpu.bh_call_v(jitcode.get_fnaddr_as_int(), jitcode.calldescr, @@ -784,10 +774,6 @@ def bhimpl_inline_call_ir_r(cpu, jitcode, args_i, args_r): return cpu.bh_call_r(jitcode.get_fnaddr_as_int(), jitcode.calldescr, args_i, args_r, None) - @arguments("cpu", "j", "I", "R", returns="f") - def bhimpl_inline_call_ir_f(cpu, jitcode, args_i, args_r): - return cpu.bh_call_f(jitcode.get_fnaddr_as_int(), jitcode.calldescr, - args_i, args_r, None) @arguments("cpu", "j", "I", "R") def bhimpl_inline_call_ir_v(cpu, jitcode, args_i, args_r): return cpu.bh_call_v(jitcode.get_fnaddr_as_int(), jitcode.calldescr, @@ -880,3 +866,29 @@ @arguments("cpu", "r", returns="i") def bhimpl_cast_ptr_to_int(cpu, p): return cpu.bh_cast_ptr_to_int(p) + + @arguments("cpu", "i", returns="r") + def bhimpl_newstr(cpu, length): + return cpu.bh_newstr(length) + @arguments("cpu", "r", returns="i") + def bhimpl_strlen(cpu, string): + return cpu.bh_strlen(string) + @arguments("cpu", "r", "i", returns="i") + def bhimpl_strgetitem(cpu, string, index): + return cpu.bh_strgetitem(string, index) + @arguments("cpu", "r", "i", "i") + def bhimpl_strsetitem(cpu, string, index, newchr): + cpu.bh_strsetitem(string, index, newchr) + + @arguments("cpu", "i", returns="r") + def bhimpl_newunicode(cpu, length): + return cpu.bh_newunicode(length) + @arguments("cpu", "r", returns="i") + def bhimpl_unicodelen(cpu, unicode): + return cpu.bh_unicodelen(unicode) + @arguments("cpu", "r", "i", returns="i") + def bhimpl_unicodegetitem(cpu, unicode, index): + return cpu.bh_unicodegetitem(unicode, index) + @arguments("cpu", "r", "i", "i") + def bhimpl_unicodesetitem(cpu, unicode, index, newchr): + cpu.bh_unicodesetitem(unicode, index, newchr) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py Wed May 12 09:18:28 2010 @@ -360,7 +360,11 @@ def _get_str(self): # for debugging only from pypy.rpython.annlowlevel import hlstr from pypy.rpython.lltypesystem import rstr - return hlstr(lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), self.value)) + try: + return hlstr(lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), + self.value)) + except lltype.UninitializedMemoryAccess: + return '' class ConstObj(Const): type = REF Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Wed May 12 09:18:28 2010 @@ -22,6 +22,7 @@ from pypy.rlib.jit import DEBUG_OFF, DEBUG_PROFILE, DEBUG_STEPS, DEBUG_DETAILED from pypy.jit.metainterp.compile import GiveUp from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr +from pypy.jit.codewriter import effectinfo as ef # ____________________________________________________________ @@ -167,10 +168,10 @@ for _opimpl in ['int_add_ovf', 'int_sub_ovf', 'int_mul_ovf']: exec py.code.Source(''' - @arguments("orgpc", "box", "box") - def opimpl_%s(self, pc, b1, b2): + @arguments("box", "box") + def opimpl_%s(self, b1, b2): resbox = self.execute(rop.%s, b1, b2) - self.metainterp.handle_possible_overflow_error(pc) + self.metainterp.handle_possible_overflow_error() return resbox ''' % (_opimpl, _opimpl.upper())).compile() @@ -289,9 +290,9 @@ self.load_int() # past the 'box' argument self.ignore_varargs() # past the 'livelist' argument - @arguments("orgpc", "box", "descr") - def opimpl_switch(self, pc, valuebox, switchdict): - box = self.implement_guard_value(pc, valuebox) + @arguments("box", "descr") + def opimpl_switch(self, valuebox, switchdict): + box = self.implement_guard_value(valuebox) switchvalue = box.getint() assert isinstance(switchdict, SwitchDictDescr) try: @@ -632,11 +633,9 @@ opimpl_inline_call_r_i = _opimpl_inline_call1 opimpl_inline_call_r_r = _opimpl_inline_call1 - opimpl_inline_call_r_f = _opimpl_inline_call1 opimpl_inline_call_r_v = _opimpl_inline_call1 opimpl_inline_call_ir_i = _opimpl_inline_call2 opimpl_inline_call_ir_r = _opimpl_inline_call2 - opimpl_inline_call_ir_f = _opimpl_inline_call2 opimpl_inline_call_ir_v = _opimpl_inline_call2 opimpl_inline_call_irf_i = _opimpl_inline_call3 opimpl_inline_call_irf_r = _opimpl_inline_call3 @@ -645,21 +644,19 @@ @arguments("box", "descr", "boxes") def _opimpl_residual_call1(self, funcbox, calldescr, argboxes): - return self.do_residual_call(funcbox, calldescr, argboxes, exc=True) + return self.do_residual_or_indirect_call(funcbox, calldescr, argboxes) @arguments("box", "descr", "boxes2") def _opimpl_residual_call2(self, funcbox, calldescr, argboxes): - return self.do_residual_call(funcbox, calldescr, argboxes, exc=True) + return self.do_residual_or_indirect_call(funcbox, calldescr, argboxes) @arguments("box", "descr", "boxes3") def _opimpl_residual_call3(self, funcbox, calldescr, argboxes): - return self.do_residual_call(funcbox, calldescr, argboxes, exc=True) + return self.do_residual_or_indirect_call(funcbox, calldescr, argboxes) opimpl_residual_call_r_i = _opimpl_residual_call1 opimpl_residual_call_r_r = _opimpl_residual_call1 - opimpl_residual_call_r_f = _opimpl_residual_call1 opimpl_residual_call_r_v = _opimpl_residual_call1 opimpl_residual_call_ir_i = _opimpl_residual_call2 opimpl_residual_call_ir_r = _opimpl_residual_call2 - opimpl_residual_call_ir_f = _opimpl_residual_call2 opimpl_residual_call_ir_v = _opimpl_residual_call2 opimpl_residual_call_irf_i = _opimpl_residual_call3 opimpl_residual_call_irf_r = _opimpl_residual_call3 @@ -705,28 +702,6 @@ call_position) return res - @XXX #arguments("descr", "varargs") - def opimpl_residual_call_noexception(self, calldescr, varargs): - self.do_residual_call(varargs, descr=calldescr, exc=False) - - @XXX #arguments("descr", "varargs") - def opimpl_residual_call_pure(self, calldescr, varargs): - self.execute_varargs(rop.CALL_PURE, varargs, descr=calldescr, exc=False) - - @XXX #arguments("orgpc", "descr", "box", "varargs") - def opimpl_indirect_call(self, pc, calldescr, box, varargs): - box = self.implement_guard_value(pc, box) - cpu = self.metainterp.cpu - key = cpu.ts.getaddr_for_box(cpu, box) - jitcode = self.metainterp.staticdata.bytecode_for_address(key) - if jitcode is not None: - # we should follow calls to this graph - return self.perform_call(jitcode, varargs) - else: - # but we should not follow calls to that graph - return self.do_residual_call([box] + varargs, - descr=calldescr, exc=True) - @XXX #arguments("orgpc", "methdescr", "varargs") def opimpl_oosend(self, pc, methdescr, varargs): objbox = varargs[0] @@ -743,37 +718,37 @@ return self.execute_varargs(rop.OOSEND, varargs, descr=methdescr, exc=True) - @XXX #arguments("box") - def opimpl_strlen(self, str): - self.execute(rop.STRLEN, str) - - @XXX #arguments("box") - def opimpl_unicodelen(self, str): - self.execute(rop.UNICODELEN, str) - - @XXX #arguments("box", "box") - def opimpl_strgetitem(self, str, index): - self.execute(rop.STRGETITEM, str, index) + @arguments("box") + def opimpl_strlen(self, strbox): + return self.execute(rop.STRLEN, strbox) - @XXX #arguments("box", "box") - def opimpl_unicodegetitem(self, str, index): - self.execute(rop.UNICODEGETITEM, str, index) + @arguments("box") + def opimpl_unicodelen(self, unicodebox): + return self.execute(rop.UNICODELEN, unicodebox) - @XXX #arguments("box", "box", "box") - def opimpl_strsetitem(self, str, index, newchar): - self.execute(rop.STRSETITEM, str, index, newchar) - - @XXX #arguments("box", "box", "box") - def opimpl_unicodesetitem(self, str, index, newchar): - self.execute(rop.UNICODESETITEM, str, index, newchar) + @arguments("box", "box") + def opimpl_strgetitem(self, strbox, indexbox): + return self.execute(rop.STRGETITEM, strbox, indexbox) + + @arguments("box", "box") + def opimpl_unicodegetitem(self, unicodebox, indexbox): + return self.execute(rop.UNICODEGETITEM, unicodebox, indexbox) + + @arguments("box", "box", "box") + def opimpl_strsetitem(self, strbox, indexbox, newcharbox): + return self.execute(rop.STRSETITEM, strbox, indexbox, newcharbox) + + @arguments("box", "box", "box") + def opimpl_unicodesetitem(self, unicodebox, indexbox, newcharbox): + self.execute(rop.UNICODESETITEM, unicodebox, indexbox, newcharbox) - @XXX #arguments("box") - def opimpl_newstr(self, length): - self.execute(rop.NEWSTR, length) + @arguments("box") + def opimpl_newstr(self, lengthbox): + return self.execute(rop.NEWSTR, lengthbox) - @XXX #arguments("box") - def opimpl_newunicode(self, length): - self.execute(rop.NEWUNICODE, length) + @arguments("box") + def opimpl_newunicode(self, lengthbox): + return self.execute(rop.NEWUNICODE, lengthbox) @XXX #arguments("descr", "varargs") def opimpl_residual_oosend_canraise(self, methdescr, varargs): @@ -787,9 +762,9 @@ def opimpl_residual_oosend_pure(self, methdescr, boxes): self.execute_varargs(rop.OOSEND_PURE, boxes, descr=methdescr, exc=False) - @arguments("orgpc", "box") - def opimpl_int_guard_value(self, pc, box): - return self.implement_guard_value(pc, box) + @arguments("box") + def opimpl_int_guard_value(self, box): + return self.implement_guard_value(box) @XXX #arguments("orgpc", "int") def opimpl_guard_green(self, pc, boxindex): @@ -802,10 +777,10 @@ constbox = self.implement_guard_value(pc, box) self.env[boxindex] = constbox - @arguments("orgpc", "box") - def opimpl_guard_class(self, pc, box): + @arguments("box") + def opimpl_guard_class(self, box): clsbox = self.cls_of_box(box) - self.generate_guard(pc, rop.GUARD_CLASS, box, [clsbox]) + self.generate_guard(rop.GUARD_CLASS, box, [clsbox]) return clsbox ## @XXX #arguments("orgpc", "box", "builtin") @@ -1029,7 +1004,7 @@ metainterp.attach_debug_info(guard_op) return guard_op - def implement_guard_value(self, pc, box): + def implement_guard_value(self, box): """Promote the given Box into a Const. Note: be careful, it's a bit unclear what occurs if a single opcode needs to generate several ones and/or ones not near the beginning.""" @@ -1037,7 +1012,7 @@ return box # no promotion needed, already a Const else: promoted_box = box.constbox() - self.generate_guard(pc, rop.GUARD_VALUE, box, [promoted_box]) + self.generate_guard(rop.GUARD_VALUE, box, [promoted_box]) self.metainterp.replace_box(box, promoted_box) return promoted_box @@ -1057,15 +1032,16 @@ resbox = self.metainterp.execute_and_record_varargs(opnum, argboxes, descr=descr) if exc: - self.metainterp.handle_possible_exception(self.pc) + self.metainterp.handle_possible_exception() else: self.metainterp.assert_no_exception() return resbox - def do_residual_call(self, funcbox, descr, argboxes, exc): + def do_residual_call(self, funcbox, descr, argboxes): allboxes = [funcbox] + argboxes effectinfo = descr.get_extra_info() - if 0:# XXX effectinfo is None or effectinfo.forces_virtual_or_virtualizable: + if (effectinfo is None or + effectinfo.extraeffect == ef.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE): # residual calls require attention to keep virtualizables in-sync self.metainterp.vable_and_vrefs_before_residual_call() # xxx do something about code duplication @@ -1080,7 +1056,31 @@ else: return self.metainterp.assert_no_exception() else: - return self.execute_varargs(rop.CALL, allboxes, descr, exc) + effect = effectinfo.extraeffect + if effect == ef.EF_CANNOT_RAISE: + opnum, exc = rop.CALL, False + elif effect == ef.EF_PURE: + opnum, exc = rop.CALL_PURE, False + elif effect == ef.EF_LOOPINVARIANT: + opnum, exc = rop.CALL_LOOPINVARIANT, True + else: + opnum, exc = rop.CALL, True + return self.execute_varargs(opnum, allboxes, descr, exc) + + def do_residual_or_indirect_call(self, funcbox, calldescr, argboxes): + """The 'residual_call' operation is emitted in two cases: + when we have to generate a residual CALL operation, but also + to handle an indirect_call that may need to be inlined.""" + funcbox = self.implement_guard_value(funcbox) + sd = self.metainterp.staticdata + key = sd.cpu.ts.getaddr_for_box(funcbox) + jitcode = sd.bytecode_for_address(key) + if jitcode is not None: + # we should follow calls to this graph + return self.metainterp.perform_call(jitcode, argboxes) + else: + # but we should not follow calls to that graph + return self.do_residual_call(funcbox, calldescr, argboxes) # ____________________________________________________________ @@ -1100,14 +1100,12 @@ RESULT = codewriter.portal_graph.getreturnvar().concretetype self.result_type = history.getkind(RESULT) - self.setup_insns(codewriter.assembler.insns) - self.setup_descrs(codewriter.assembler.descrs) + asm = codewriter.assembler + self.setup_insns(asm.insns) + self.setup_descrs(asm.descrs) + self.setup_indirectcalltargets(asm.indirectcalltargets) self.profiler = ProfilerClass() - - self.indirectcall_keys = [] - self.indirectcall_values = [] - self.warmrunnerdesc = warmrunnerdesc backendmodule = self.cpu.__module__ @@ -1140,6 +1138,9 @@ def setup_descrs(self, descrs): self.opcode_descrs = descrs + def setup_indirectcalltargets(self, indirectcalltargets): + self.indirectcalltargets = list(indirectcalltargets) + def finish_setup(self, optimizer=None): warmrunnerdesc = self.warmrunnerdesc if warmrunnerdesc is not None: @@ -1192,24 +1193,16 @@ # because the keys are function addresses, so they # can change from run to run. d = {} - keys = self.indirectcall_keys - values = self.indirectcall_values - for i in range(len(keys)): - d[keys[i]] = values[i] + for jitcode in self.indirectcalltargets: + d[jitcode.fnaddr] = jitcode self.globaldata.indirectcall_dict = d return d.get(fnaddress, None) else: - for i in range(len(self.indirectcall_keys)): - if fnaddress == self.indirectcall_keys[i]: - return self.indirectcall_values[i] + for jitcode in self.indirectcalltargets: + if jitcode.fnaddr == fnaddress: + return jitcode return None - # ---------- construction-time interface ---------- - - def _register_indirect_call_target(self, fnaddress, jitcode): - self.indirectcall_keys.append(fnaddress) - self.indirectcall_values.append(jitcode) - # ---------------- logging ------------------------ def log(self, msg): @@ -1273,7 +1266,7 @@ return False # XXX get rid of this method def perform_call(self, jitcode, boxes, greenkey=None): - # when tracing, this bytecode causes the subfunction to be entered + # causes the metainterp to enter the given subfunction f = self.newframe(jitcode, greenkey) f.setup_call(boxes) raise ChangeFrame @@ -1865,26 +1858,26 @@ # mark by replacing it with ConstPtr(NULL) self.virtualref_boxes[i+1] = self.cpu.ts.CONST_NULL - def handle_possible_exception(self, pc): + def handle_possible_exception(self): frame = self.framestack[-1] if self.last_exc_value_box: exception_box = self.cpu.ts.cls_of_box(self.last_exc_value_box) - op = frame.generate_guard(pc, rop.GUARD_EXCEPTION, + op = frame.generate_guard(rop.GUARD_EXCEPTION, None, [exception_box]) if op: op.result = self.last_exc_value_box self.finishframe_exception() else: - frame.generate_guard(pc, rop.GUARD_NO_EXCEPTION, None, []) + frame.generate_guard(rop.GUARD_NO_EXCEPTION, None, []) - def handle_possible_overflow_error(self, pc): + def handle_possible_overflow_error(self): frame = self.framestack[-1] if self.last_exc_value_box: - frame.generate_guard(pc, rop.GUARD_OVERFLOW, None) + frame.generate_guard(rop.GUARD_OVERFLOW, None) assert isinstance(self.last_exc_value_box, Const) self.finishframe_exception() else: - frame.generate_guard(pc, rop.GUARD_NO_OVERFLOW, None) + frame.generate_guard(rop.GUARD_NO_OVERFLOW, None) def assert_no_exception(self): assert not self.last_exc_value_box @@ -2066,7 +2059,6 @@ args = () next_argcode = 0 code = self.bytecode - orgpc = position position += 1 for argtype in argtypes: if argtype == "box": # a box, of whatever type @@ -2129,8 +2121,6 @@ argcodes[next_argcode + 2]) next_argcode = next_argcode + 3 position = position3 + 1 + length3 - elif argtype == "orgpc": - value = orgpc else: raise AssertionError("bad argtype: %r" % (argtype,)) args += (value,) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/typesystem.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/typesystem.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/typesystem.py Wed May 12 09:18:28 2010 @@ -128,8 +128,8 @@ return lltype.cast_opaque_ptr(llmemory.GCREF, value) cast_to_ref._annspecialcase_ = 'specialize:ll' - def getaddr_for_box(self, cpu, box): - return box.getaddr(cpu) + def getaddr_for_box(self, box): + return box.getaddr() def rd_eq(ref1, ref2): return ref1 == ref2 @@ -230,7 +230,7 @@ return ootype.cast_to_object(value) cast_to_ref._annspecialcase_ = 'specialize:ll' - def getaddr_for_box(self, cpu, box): + def getaddr_for_box(self, box): return box.getref_base() llhelper = LLTypeHelper() From afa at codespeak.net Wed May 12 10:19:38 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 12 May 2010 10:19:38 +0200 (CEST) Subject: [pypy-svn] r74495 - pypy/trunk/pypy/module/cpyext/test Message-ID: <20100512081938.76B8D282BDE@codespeak.net> Author: afa Date: Wed May 12 10:19:35 2010 New Revision: 74495 Modified: pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py Log: Skip this test on non-windows platorms, ll2ctypes seems to change the errno value. Modified: pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py Wed May 12 10:19:35 2010 @@ -120,6 +120,9 @@ assert module.check_error() def test_SetFromErrno(self): + import sys + if sys.platform != 'win32': + skip("callbacks through ll2ctypes modify errno") import errno, os module = self.import_extension('foo', [ From arigo at codespeak.net Wed May 12 10:46:28 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 12 May 2010 10:46:28 +0200 (CEST) Subject: [pypy-svn] r74496 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100512084628.945BC282BDE@codespeak.net> Author: arigo Date: Wed May 12 10:46:26 2010 New Revision: 74496 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Log: Placeholder. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Wed May 12 10:46:26 2010 @@ -711,6 +711,12 @@ assert real_instance return lltype.cast_opaque_ptr(llmemory.GCREF, real_instance) + @arguments("self", "r") + def bhimpl_raise(self, excvalue): + import pdb; pdb.set_trace() + XXX + raise real_instance + @arguments("self") def bhimpl_reraise(self): real_instance = self.exception_last_value From arigo at codespeak.net Wed May 12 10:46:38 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 12 May 2010 10:46:38 +0200 (CEST) Subject: [pypy-svn] r74497 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter metainterp Message-ID: <20100512084638.13F3D282BF1@codespeak.net> Author: arigo Date: Wed May 12 10:46:36 2010 New Revision: 74497 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py Log: builtin calls. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Wed May 12 10:46:36 2010 @@ -256,6 +256,11 @@ called_from=self.graph) return self.rewrite_call(op, 'G_inline_call', [jitcode]) + def handle_builtin_call(self, op): + oopspec_name, args = support.decode_builtin_call(op) + op1 = self._prepare_builtin_call(op, oopspec_name, args) + return self.handle_residual_call(op1) + handle_residual_indirect_call = handle_residual_call def handle_regular_indirect_call(self, op): @@ -267,15 +272,18 @@ lst.append(jitcode) return self.handle_residual_call(op, [IndirectCallTargets(lst)]) - def _do_builtin_call(self, op, oopspec_name=None, args=None, extra=None): - if oopspec_name is None: oopspec_name = op.opname - if args is None: args = op.args + def _prepare_builtin_call(self, op, oopspec_name, args, extra=None): argtypes = [v.concretetype for v in args] resulttype = op.result.concretetype c_func, TP = support.builtin_func_for_spec(self.cpu.rtyper, oopspec_name, argtypes, resulttype, extra) - op1 = SpaceOperation('direct_call', [c_func] + args, op.result) + return SpaceOperation('direct_call', [c_func] + args, op.result) + + def _do_builtin_call(self, op, oopspec_name=None, args=None, extra=None): + if oopspec_name is None: oopspec_name = op.opname + if args is None: args = op.args + op1 = self._prepare_builtin_call(op, oopspec_name, args, extra) return self.rewrite_op_direct_call(op1) rewrite_op_int_floordiv_ovf_zer = _do_builtin_call Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Wed May 12 10:46:36 2010 @@ -247,8 +247,16 @@ if not switchcase: self.pc = target + for _opimpl in ['int_is_zero', 'ptr_iszero', 'ptr_nonzero']: + exec py.code.Source(''' + @arguments("label", "box") + def opimpl_goto_if_not_%s(self, target, box): + condbox = self.execute(rop.%s, box) + self.opimpl_goto_if_not(target, condbox) + ''' % (_opimpl, _opimpl.upper())).compile() + for _opimpl in ['int_lt', 'int_le', 'int_eq', 'int_ne', 'int_gt', 'int_ge', - ]: + 'ptr_eq', 'ptr_ne']: exec py.code.Source(''' @arguments("label", "box", "box") def opimpl_goto_if_not_%s(self, target, b1, b2): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py Wed May 12 10:46:36 2010 @@ -183,8 +183,10 @@ # 'SAME_AS/1', # gets a Const or a Box, turns it into another Box # - 'OOIS/2b', - 'OOISNOT/2b', + 'PTR_ISZERO/1b', + 'PTR_NONZERO/1b', + 'PTR_EQ/2b', + 'PTR_NE/2b', # 'ARRAYLEN_GC/1d', 'STRLEN/1', From afa at codespeak.net Wed May 12 13:29:20 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 12 May 2010 13:29:20 +0200 (CEST) Subject: [pypy-svn] r74498 - pypy/trunk/pypy/rpython/lltypesystem Message-ID: <20100512112920.97E1A282BDE@codespeak.net> Author: afa Date: Wed May 12 13:29:17 2010 New Revision: 74498 Modified: pypy/trunk/pypy/rpython/lltypesystem/rstr.py Log: The 'if else' syntax is not clearer here, let the module import even with python2.4 Modified: pypy/trunk/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rstr.py Wed May 12 13:29:17 2010 @@ -607,13 +607,19 @@ i += mlast continue - c = '\0' if i+m == len(s1.chars) else s1.chars[i+m] + if i + m < len(s1.chars): + c = s1.chars[i + m] + else: + c = '\0' if not bloom(mask, c): i += m else: i += skip else: - c = '\0' if i+m == len(s1.chars) else s1.chars[i+m] + if i + m < len(s1.chars): + c = s1.chars[i + m] + else: + c = '\0' if not bloom(mask, c): i += m else: From arigo at codespeak.net Wed May 12 17:21:45 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 12 May 2010 17:21:45 +0200 (CEST) Subject: [pypy-svn] r74499 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter codewriter/test metainterp metainterp/test Message-ID: <20100512152145.E058B282BDE@codespeak.net> Author: arigo Date: Wed May 12 17:21:43 2010 New Revision: 74499 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/virtualref.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py Log: Start reimplementing the tests that use jitdriver.jit_merge_point. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py Wed May 12 17:21:43 2010 @@ -21,6 +21,7 @@ self.portal_graph = portal_graph self.jitcodes = {} # map {graph: jitcode} self.unfinished_graphs = [] # list of graphs with pending jitcodes + self.jitdriver = None if cpu is not None: self.rtyper = cpu.rtyper translator = self.rtyper.annotator.translator @@ -29,6 +30,11 @@ self.virtualizable_analyzer = VirtualizableAnalyzer(translator) def find_all_graphs(self, policy): + try: + return self.candidate_graphs + except AttributeError: + pass + def is_candidate(graph): return policy.look_inside_graph(graph) @@ -223,3 +229,9 @@ def calldescr_canraise(self, calldescr): effectinfo = calldescr.get_extra_info() return effectinfo is None or effectinfo.extraeffect >= ef.EF_CAN_RAISE + + def found_jitdriver(self, jitdriver): + if self.jitdriver is None: + self.jitdriver = jitdriver + else: + assert self.jitdriver is jitdriver Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py Wed May 12 17:21:43 2010 @@ -14,9 +14,11 @@ class CodeWriter(object): callcontrol = None # for tests - def __init__(self, cpu=None): + def __init__(self, cpu=None, maingraph=None): self.cpu = cpu self.assembler = Assembler() + self.portal_graph = maingraph + self.callcontrol = CallControl(cpu, maingraph) def transform_func_to_jitcode(self, func, values, type_system='lltype'): """For testing.""" @@ -59,15 +61,15 @@ # Float). self.assembler.assemble(ssarepr, jitcode) - def make_jitcodes(self, maingraph, policy, verbose=False): - self.portal_graph = maingraph - self.callcontrol = CallControl(self.cpu, maingraph) - self.callcontrol.find_all_graphs(policy) - mainjitcode = self.callcontrol.get_jitcode(maingraph) + def make_jitcodes(self, verbose=False): + maingraph = self.portal_graph + self.mainjitcode = self.callcontrol.get_jitcode(maingraph) for graph, jitcode in self.callcontrol.enum_pending_graphs(): self.transform_graph_to_jitcode(graph, jitcode, graph is maingraph, verbose) - return mainjitcode + + def find_all_graphs(self, policy): + return self.callcontrol.find_all_graphs(policy) def print_ssa_repr(self, ssarepr, portal, verbose): if verbose: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Wed May 12 17:21:43 2010 @@ -608,6 +608,19 @@ return Constant(value, lltype.Bool) return op + def rewrite_op_jit_marker(self, op): + opname = op.args[0].value + jitdriver = op.args[1].value + self.callcontrol.found_jitdriver(jitdriver) + args_i = [] + args_r = [] + args_f = [] + for v in op.args[2:]: + self.add_in_correct_list(v, args_i, args_r, args_f) + return SpaceOperation(opname, [ListOfKind('int', args_i), + ListOfKind('ref', args_r), + ListOfKind('float', args_f)], op.result) + # ____________________________________________________________ def _with_prefix(prefix): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py Wed May 12 17:21:43 2010 @@ -57,15 +57,17 @@ return ggg(b) - ggg(a) rtyper = support.annotate(fff, [35, 42]) maingraph = rtyper.annotator.translator.graphs[0] - cw = CodeWriter(FakeCPU()) - jitcode = cw.make_jitcodes(maingraph, FakePolicy(), verbose=True) - print jitcode._dump + cw = CodeWriter(FakeCPU(), maingraph) + cw.find_all_graphs(FakePolicy()) + cw.make_jitcodes(verbose=True) + jitcode = cw.mainjitcode + print jitcode.dump() [jitcode2] = cw.assembler.descrs - print jitcode2._dump + print jitcode2.dump() assert jitcode is not jitcode2 assert jitcode.name == 'fff' assert jitcode2.name == 'ggg' - assert 'ggg' in jitcode._dump + assert 'ggg' in jitcode.dump() assert lltype.typeOf(jitcode2.fnaddr) == llmemory.Address assert jitcode2.calldescr[0] == 'calldescr' Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Wed May 12 17:21:43 2010 @@ -8,8 +8,7 @@ from pypy.objspace.flow.model import SpaceOperation, Variable, Constant from pypy.translator.unsimplify import varoftype from pypy.rlib.rarithmetic import ovfcheck -from pypy.rlib.jit import dont_look_inside -from pypy.rlib.jit import _we_are_jitted +from pypy.rlib.jit import dont_look_inside, _we_are_jitted, JitDriver class FakeRegAlloc: @@ -57,6 +56,8 @@ return FakeDescr() def calldescr_canraise(self, calldescr): return calldescr is not self._descr_cannot_raise + def found_jitdriver(self, jitdriver): + assert isinstance(jitdriver, JitDriver) def fake_regallocs(): return {'int': FakeRegAlloc(), @@ -449,3 +450,14 @@ self.encoding_test(f, [5], """ int_return $2 """, transform=True) + + def test_jitdriver(self): + myjitdriver = JitDriver(greens = ['x'], reds = ['y']) + def f(x, y): + myjitdriver.jit_merge_point(x=x, y=y) + myjitdriver.can_enter_jit(x=y, y=x) + self.encoding_test(f, [4, 5], """ + jit_merge_point I[%i0, %i1], R[], F[] + can_enter_jit I[%i1, %i0], R[], F[] + void_return + """, transform=True) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Wed May 12 17:21:43 2010 @@ -831,8 +831,9 @@ else: raise self.metainterp.staticdata.ContinueRunningNormally(varargs) - @XXX #arguments() - def opimpl_can_enter_jit(self): + @arguments("boxes3") + def opimpl_can_enter_jit(self, boxes): + xxx # Note: when running with a BlackHole history, this 'can_enter_jit' # may be completely skipped by the logic that replaces perform_call # with rop.CALL. But in that case, no-one will check the flag anyway, @@ -842,8 +843,9 @@ raise CannotInlineCanEnterJit() self.metainterp.seen_can_enter_jit = True - @XXX #arguments() - def opimpl_jit_merge_point(self): + @arguments("boxes3") + def opimpl_jit_merge_point(self, boxes): + xxx if not self.metainterp.is_blackholing(): self.verify_green_args(self.env) # xxx we may disable the following line in some context later @@ -1097,22 +1099,14 @@ logger_noopt = None logger_ops = None - def __init__(self, codewriter, options, + def __init__(self, cpu, options, ProfilerClass=EmptyProfiler, warmrunnerdesc=None): - self.cpu = codewriter.cpu + self.cpu = cpu self.stats = self.cpu.stats self.options = options self.logger_noopt = Logger(self) self.logger_ops = Logger(self, guard_number=True) - RESULT = codewriter.portal_graph.getreturnvar().concretetype - self.result_type = history.getkind(RESULT) - - asm = codewriter.assembler - self.setup_insns(asm.insns) - self.setup_descrs(asm.descrs) - self.setup_indirectcalltargets(asm.indirectcalltargets) - self.profiler = ProfilerClass() self.warmrunnerdesc = warmrunnerdesc @@ -1120,7 +1114,6 @@ backendmodule = backendmodule.split('.')[-2] self.jit_starting_line = 'JIT starting (%s)' % backendmodule - self.portal_code = None self._addr2name_keys = [] self._addr2name_values = [] @@ -1149,7 +1142,16 @@ def setup_indirectcalltargets(self, indirectcalltargets): self.indirectcalltargets = list(indirectcalltargets) - def finish_setup(self, optimizer=None): + def finish_setup(self, codewriter, optimizer=None): + asm = codewriter.assembler + self.setup_insns(asm.insns) + self.setup_descrs(asm.descrs) + self.setup_indirectcalltargets(asm.indirectcalltargets) + # + self.portal_code = codewriter.mainjitcode + RESULT = codewriter.portal_graph.getreturnvar().concretetype + self.result_type = history.getkind(RESULT) + # warmrunnerdesc = self.warmrunnerdesc if warmrunnerdesc is not None: self.num_green_args = warmrunnerdesc.num_green_args Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Wed May 12 17:21:43 2010 @@ -20,11 +20,12 @@ graphs = rtyper.annotator.translator.graphs stats = history.Stats() cpu = CPUClass(rtyper, stats, None, False) - cw = codewriter.CodeWriter(cpu) - mainjitcode = cw.make_jitcodes(graphs[0], JitPolicy(), verbose=True) - return cw, mainjitcode + cw = codewriter.CodeWriter(cpu, graphs[0]) + cw.find_all_graphs(JitPolicy()) + cw.make_jitcodes(verbose=True) + return cw -def _run_with_blackhole(cw, mainjitcode, args): +def _run_with_blackhole(cw, args): from pypy.jit.metainterp.blackhole import BlackholeInterpBuilder blackholeinterpbuilder = BlackholeInterpBuilder(cw) blackholeinterp = blackholeinterpbuilder.acquire_interp() @@ -42,10 +43,10 @@ count_f += 1 else: raise TypeError(T) - blackholeinterp.run(mainjitcode, 0) + blackholeinterp.run(cw.mainjitcode, 0) return blackholeinterp._get_result_anytype() -def _run_with_pyjitpl(cw, mainjitcode, args, testself): +def _run_with_pyjitpl(cw, args, testself): from pypy.jit.metainterp import simple_optimize class DoneWithThisFrame(Exception): @@ -67,16 +68,14 @@ debug_level = 2 opt = history.Options(listops=True) - cpu = cw.cpu - metainterp_sd = pyjitpl.MetaInterpStaticData(cw, opt) - metainterp_sd.finish_setup(optimizer="bogus") + metainterp_sd = pyjitpl.MetaInterpStaticData(cw.cpu, opt) + metainterp_sd.finish_setup(cw, optimizer="bogus") metainterp_sd.state = FakeWarmRunnerState() metainterp_sd.state.cpu = metainterp_sd.cpu metainterp = pyjitpl.MetaInterp(metainterp_sd) if hasattr(testself, 'finish_metainterp_for_interp_operations'): testself.finish_metainterp_for_interp_operations(metainterp) - metainterp_sd.portal_code = mainjitcode metainterp_sd.DoneWithThisFrameInt = DoneWithThisFrame metainterp_sd.DoneWithThisFrameRef = DoneWithThisFrameRef metainterp_sd.DoneWithThisFrameFloat = DoneWithThisFrame @@ -126,12 +125,11 @@ def interp_operations(self, f, args, **kwds): # get the JitCodes for the function f - cw, mainjitcode = _get_jitcodes(self.CPUClass, f, args, - self.type_system) + cw = _get_jitcodes(self.CPUClass, f, args, self.type_system) # try to run it with blackhole.py - result1 = _run_with_blackhole(cw, mainjitcode, args) + result1 = _run_with_blackhole(cw, args) # try to run it with pyjitpl.py - result2 = _run_with_pyjitpl(cw, mainjitcode, args, self) + result2 = _run_with_pyjitpl(cw, args, self) assert result1 == result2 return result1 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/virtualref.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/virtualref.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/virtualref.py Wed May 12 17:21:43 2010 @@ -20,7 +20,8 @@ 'jit_virtual_ref') # build some constants adr = llmemory.cast_ptr_to_adr(self.jit_virtual_ref_vtable) - self.jit_virtual_ref_const_class = history.ConstAddr(adr, self.cpu) + adr = llmemory.cast_adr_to_int(adr) + self.jit_virtual_ref_const_class = history.ConstInt(adr) fielddescrof = self.cpu.fielddescrof self.descr_virtual_token = fielddescrof(self.JIT_VIRTUAL_REF, 'virtual_token') Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py Wed May 12 17:21:43 2010 @@ -20,7 +20,7 @@ from pypy.jit.metainterp.pyjitpl import MetaInterpStaticData, MetaInterp from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper from pypy.jit.metainterp.jitprof import Profiler, EmptyProfiler -from pypy.jit.codewriter import support +from pypy.jit.codewriter import support, codewriter from pypy.jit.codewriter.policy import JitPolicy from pypy.rlib.jit import DEBUG_STEPS, DEBUG_DETAILED, DEBUG_OFF, DEBUG_PROFILE @@ -144,22 +144,22 @@ class WarmRunnerDesc(object): def __init__(self, translator, policy=None, backendopt=True, CPUClass=None, - optimizer=None, **kwds): - import py; py.test.skip("XXX") + optimizer=None, ProfilerClass=EmptyProfiler, **kwds): pyjitpl._warmrunnerdesc = self # this is a global for debugging only! - if policy is None: - policy = JitPolicy() self.set_translator(translator) + self.build_cpu(CPUClass, **kwds) self.find_portal() - self.codewriter = codewriter.CodeWriter(self.rtyper) - policy.set_supports_floats(CPUClass.supports_floats) - graphs = self.codewriter.find_all_graphs(self.portal_graph, policy) + self.codewriter = codewriter.CodeWriter(self.cpu, self.portal_graph) + if policy is None: + policy = JitPolicy() + policy.set_supports_floats(self.cpu.supports_floats) + graphs = self.codewriter.find_all_graphs(policy) policy.dump_unsafe_loops() self.check_access_directly_sanity(graphs) if backendopt: self.prejit_optimizations(policy, graphs) - self.build_meta_interp(CPUClass, **kwds) + self.build_meta_interp(ProfilerClass) self.make_args_specification() # from pypy.jit.metainterp.virtualref import VirtualRefInfo @@ -172,16 +172,14 @@ self.make_driverhook_graphs() self.make_enter_function() self.rewrite_jit_merge_point(policy) - - self.codewriter.generate_bytecode(self.metainterp_sd, - self.portal_graph, - self.portal_runner_ptr - ) + + verbose = not self.cpu.translate_support_code + self.codewriter.make_jitcodes(verbose=verbose) self.rewrite_can_enter_jit() self.rewrite_set_param() self.rewrite_force_virtual() self.add_finish() - self.metainterp_sd.finish_setup(optimizer=optimizer) + self.metainterp_sd.finish_setup(self.codewriter, optimizer=optimizer) def finish(self): vinfo = self.metainterp_sd.virtualizable_info @@ -244,11 +242,10 @@ remove_asserts=True, really_remove_asserts=True) - def build_meta_interp(self, CPUClass, translate_support_code=False, - view="auto", no_stats=False, - ProfilerClass=EmptyProfiler, **kwds): + def build_cpu(self, CPUClass, translate_support_code=False, + no_stats=False, **kwds): assert CPUClass is not None - opt = history.Options(**kwds) + self.opt = history.Options(**kwds) if no_stats: stats = history.NoStats() else: @@ -259,12 +256,13 @@ annhelper = self.annhelper else: annhelper = None - cpu = CPUClass(self.translator.rtyper, self.stats, opt, + cpu = CPUClass(self.translator.rtyper, self.stats, self.opt, translate_support_code, gcdescr=self.gcdescr) self.cpu = cpu - self.metainterp_sd = MetaInterpStaticData(self.portal_graph, # xxx - cpu, - self.stats, opt, + + def build_meta_interp(self, ProfilerClass): + self.metainterp_sd = MetaInterpStaticData(self.cpu, + self.opt, ProfilerClass=ProfilerClass, warmrunnerdesc=self) From fijal at codespeak.net Wed May 12 18:59:30 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 12 May 2010 18:59:30 +0200 (CEST) Subject: [pypy-svn] r74505 - pypy/branch/chunked-list/pypy/rpython/lltypesystem/test Message-ID: <20100512165930.410C0282BDE@codespeak.net> Author: fijal Date: Wed May 12 18:59:28 2010 New Revision: 74505 Added: pypy/branch/chunked-list/pypy/rpython/lltypesystem/test/test_rlist.py (contents, props changed) Log: Start writing direct tests for rlist. Implementation is quickly getting more complex, so it might be a good idea Added: pypy/branch/chunked-list/pypy/rpython/lltypesystem/test/test_rlist.py ============================================================================== --- (empty file) +++ pypy/branch/chunked-list/pypy/rpython/lltypesystem/test/test_rlist.py Wed May 12 18:59:28 2010 @@ -0,0 +1,38 @@ + +from pypy.rpython.lltypesystem import rlist as ll_rlist +from pypy.rpython.rlist import ADTIList +from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem.lltype import GcStruct, Signed, Ptr + +class TestRListDirect(object): + def setup_class(cls): + cls.MIN_CHUNKED_SIZE = ll_rlist.MIN_CHUNKED_SIZE + cls.CHUNK_SIZE = ll_rlist.CHUNK_SIZE + ll_rlist.MIN_CHUNKED_SIZE = 100 + ll_rlist.CHUNK_SIZE = 100 + + ITEM = lltype.Signed + ITEMARRAY = lltype.GcArray(ITEM) + + cls.LISTTP = GcStruct("list", ("length", Signed), + ("items", Ptr(ITEMARRAY)), + adtmeths = ADTIList({ + "ll_newlist": ll_rlist.ll_newlist, + "ll_newemptylist": ll_rlist.ll_newemptylist, + "ll_length": ll_rlist.ll_length, + "ll_items": ll_rlist.ll_items, + "ITEM": ITEM, + "ll_getitem_fast": ll_rlist.ll_getitem_fast, + "ll_setitem_fast": ll_rlist.ll_setitem_fast, + "_ll_resize_ge": ll_rlist._ll_list_resize_ge, + "_ll_resize_le": ll_rlist._ll_list_resize_le, + "_ll_resize": ll_rlist._ll_list_resize, + }), + hints = {'list': True}) + + def teardown_class(cls): + ll_rlist.MIN_CHUNKED_SIZE = cls.MIN_CHUNKED_SIZE + ll_rlist.CHUNK_SIZE = cls.CHUNK_SIZE + + def test_resize_unchunked_chunked(self): + l = self.LISTTP.ll_newlist(self.CHUNK_SIZE - 3) From fijal at codespeak.net Wed May 12 19:23:26 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 12 May 2010 19:23:26 +0200 (CEST) Subject: [pypy-svn] r74506 - in pypy/branch/chunked-list/pypy/rpython: lltypesystem lltypesystem/test test Message-ID: <20100512172326.205D2282BDE@codespeak.net> Author: fijal Date: Wed May 12 19:23:24 2010 New Revision: 74506 Modified: pypy/branch/chunked-list/pypy/rpython/lltypesystem/lltype.py pypy/branch/chunked-list/pypy/rpython/lltypesystem/rlist.py pypy/branch/chunked-list/pypy/rpython/lltypesystem/test/test_rlist.py pypy/branch/chunked-list/pypy/rpython/test/test_rlist.py Log: Pass the first direct test. Right now both ll_getitem_fast and ll_setitem_fast would check for boundaries. To be changed in future I believe Modified: pypy/branch/chunked-list/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/chunked-list/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/chunked-list/pypy/rpython/lltypesystem/lltype.py Wed May 12 19:23:24 2010 @@ -1,7 +1,7 @@ from pypy.rlib.rarithmetic import (r_int, r_uint, intmask, r_singlefloat, r_ulonglong, r_longlong, base_int, normalizedinttype) -from pypy.rlib.objectmodel import Symbolic +from pypy.rlib.objectmodel import Symbolic, specialize from pypy.tool.uid import Hashable from pypy.tool.tls import tlsobject from pypy.lib.identity_dict import identity_dict @@ -316,7 +316,7 @@ _gckind = 'raw' __name__ = 'array' _anonym_struct = False - + def __init__(self, *fields, **kwds): if len(fields) == 1 and isinstance(fields[0], LowLevelType): self.OF = fields[0] @@ -370,6 +370,9 @@ def _inline_is_varsize(self, last): raise TypeError("cannot inline a GC array inside a structure") + at specialize.memo() +def new_gc_array(arg): + return GcArray(arg) class FixedSizeArray(Struct): # behaves more or less like a Struct with fields item0, item1, ... Modified: pypy/branch/chunked-list/pypy/rpython/lltypesystem/rlist.py ============================================================================== --- pypy/branch/chunked-list/pypy/rpython/lltypesystem/rlist.py (original) +++ pypy/branch/chunked-list/pypy/rpython/lltypesystem/rlist.py Wed May 12 19:23:24 2010 @@ -11,7 +11,7 @@ GcForwardReference, Ptr, GcArray, GcStruct, \ Void, Signed, malloc, typeOf, Primitive, \ Bool, nullptr, typeMethod -from pypy.rpython.lltypesystem import rstr +from pypy.rpython.lltypesystem import rstr, lltype from pypy.rpython import robject from pypy.rlib.debug import ll_assert from pypy.rlib.rarithmetic import ovfcheck @@ -20,6 +20,10 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib import rgc +MIN_CHUNKED_SIZE = 32*1024 # should be somehow correlated with size of GC +# structure that gets allocated using raw_malloc in the hybrid gc +CHUNK_SIZE = MIN_CHUNKED_SIZE + # ____________________________________________________________ # # Concrete implementation of RPython lists: @@ -105,6 +109,7 @@ "ll_newemptylist": ll_newemptylist, "ll_length": ll_length, "ll_items": ll_items, + 'll_chunks': ll_chunks, "ITEM": ITEM, "ll_getitem_fast": ll_getitem_fast, "ll_setitem_fast": ll_setitem_fast, @@ -206,20 +211,48 @@ new_allocated = ovfcheck(newsize + some) except OverflowError: raise MemoryError - # XXX consider to have a real realloc # new_allocated is a bit more than newsize, enough to ensure an amortized # linear complexity for e.g. repeated usage of l.append(). items = l.items - newitems = malloc(typeOf(l).TO.items.TO, new_allocated) + ARRAYTP = typeOf(l).TO.items.TO before_len = l.length if before_len: # avoids copying GC flags from the prebuilt_empty_array if before_len < newsize: p = before_len else: p = newsize - rgc.ll_arraycopy(items, newitems, 0, 0, p) + else: + p = 0 + if new_allocated > MIN_CHUNKED_SIZE: + newarr = malloc(lltype.new_gc_array(lltype.Ptr(ARRAYTP)), + (new_allocated / CHUNK_SIZE) + 1) + if before_len <= MIN_CHUNKED_SIZE: + # copy unchunked list -> chunked list, always grow + i = 0 + start = 0 + while i < (new_allocated / CHUNK_SIZE) + 1: + newarr[i] = malloc(ARRAYTP, CHUNK_SIZE) + if start < before_len: + size = CHUNK_SIZE + if start + size > before_len: + size = before_len - start + rgc.ll_arraycopy(items, newarr[i], start, 0, size) + start += CHUNK_SIZE + i += 1 + else: + # copy chunked list -> larger chunked list + raise NotImplementedError + l.length = newsize + l.items = rffi.cast(lltype.Ptr(ARRAYTP), newarr) + else: + if before_len > MIN_CHUNKED_SIZE: + # copy chunked list -> unchunked list + raise NotImplementedError + newitems = malloc(ARRAYTP, new_allocated) + if p: + rgc.ll_arraycopy(items, newitems, 0, 0, p) + l.items = newitems l.length = newsize - l.items = newitems _ll_list_resize_really._annenforceargs_ = (None, int) # this common case was factored out of _ll_list_resize @@ -302,16 +335,23 @@ def ll_items(l): return l.items +def ll_chunks(l): + ARRAYTP = lltype.typeOf(l).TO.items + return rffi.cast(Ptr(lltype.new_gc_array(ARRAYTP)), l.items) + def ll_getitem_fast(l, index): - if l.length == 0: - raise IndexError ll_assert(index < l.length, "getitem out of bounds") + if l.length > MIN_CHUNKED_SIZE: + return l.ll_chunks()[index / CHUNK_SIZE][index % CHUNK_SIZE] return l.ll_items()[index] ll_getitem_fast.oopspec = 'list.getitem(l, index)' def ll_setitem_fast(l, index, item): ll_assert(index < l.length, "setitem out of bounds") - l.ll_items()[index] = item + if l.length > MIN_CHUNKED_SIZE: + l.ll_chunks()[index / CHUNK_SIZE][index % CHUNK_SIZE] = item + else: + l.ll_items()[index] = item ll_setitem_fast.oopspec = 'list.setitem(l, index, item)' # fixed size versions Modified: pypy/branch/chunked-list/pypy/rpython/lltypesystem/test/test_rlist.py ============================================================================== --- pypy/branch/chunked-list/pypy/rpython/lltypesystem/test/test_rlist.py (original) +++ pypy/branch/chunked-list/pypy/rpython/lltypesystem/test/test_rlist.py Wed May 12 19:23:24 2010 @@ -1,8 +1,8 @@ from pypy.rpython.lltypesystem import rlist as ll_rlist from pypy.rpython.rlist import ADTIList -from pypy.rpython.lltypesystem import lltype -from pypy.rpython.lltypesystem.lltype import GcStruct, Signed, Ptr +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rpython.lltypesystem.lltype import GcStruct, Signed, Ptr, GcArray class TestRListDirect(object): def setup_class(cls): @@ -21,6 +21,7 @@ "ll_newemptylist": ll_rlist.ll_newemptylist, "ll_length": ll_rlist.ll_length, "ll_items": ll_rlist.ll_items, + "ll_chunks": ll_rlist.ll_chunks, "ITEM": ITEM, "ll_getitem_fast": ll_rlist.ll_getitem_fast, "ll_setitem_fast": ll_rlist.ll_setitem_fast, @@ -35,4 +36,18 @@ ll_rlist.CHUNK_SIZE = cls.CHUNK_SIZE def test_resize_unchunked_chunked(self): - l = self.LISTTP.ll_newlist(self.CHUNK_SIZE - 3) + l = self.LISTTP.ll_newlist(ll_rlist.MIN_CHUNKED_SIZE - 3) + for i in range(l.length): + l.ll_setitem_fast(i, i) + l._ll_resize(ll_rlist.MIN_CHUNKED_SIZE + 10) + for i in range(ll_rlist.MIN_CHUNKED_SIZE - 3, + ll_rlist.MIN_CHUNKED_SIZE + 10): + l.ll_setitem_fast(i, 10*i) + # this should resize above the chunked threshold + CHUNK_TP = Ptr(GcArray(Ptr(GcArray(Signed)))) + chunked_items = rffi.cast(CHUNK_TP, l.items) + CHUNK_SIZE = ll_rlist.CHUNK_SIZE + assert chunked_items[0][10] == 10 + assert chunked_items[0][CHUNK_SIZE - 1] == (CHUNK_SIZE - 1) * 10 + assert chunked_items[1][0] == CHUNK_SIZE * 10 + assert chunked_items[1][9] == (CHUNK_SIZE + 9) * 10 Modified: pypy/branch/chunked-list/pypy/rpython/test/test_rlist.py ============================================================================== --- pypy/branch/chunked-list/pypy/rpython/test/test_rlist.py (original) +++ pypy/branch/chunked-list/pypy/rpython/test/test_rlist.py Wed May 12 19:23:24 2010 @@ -1430,5 +1430,26 @@ assert func1.oopspec == 'list.getitem_foldable(l, index)' assert func2.oopspec == 'list.getitem(l, index)' +class TestLLtypeChunkedList(BaseRtypingTest, LLRtypeMixin): + def setup_class(cls): + cls.MIN_CHUNKED_SIZE = ll_rlist.MIN_CHUNKED_SIZE + cls.CHUNK_SIZE = ll_rlist.CHUNK_SIZE + ll_rlist.MIN_CHUNKED_SIZE = 100 + ll_rlist.CHUNK_SIZE = 100 + + def teardown_class(cls): + ll_rlist.MIN_CHUNKED_SIZE = cls.MIN_CHUNKED_SIZE + ll_rlist.CHUNK_SIZE = cls.CHUNK_SIZE + + def test_big_getitem(self): + def f(i): + l = [0] * (ll_rlist.MIN_CHUNKED_SIZE - 3) + for k in range(10): + l.append(i) + return l[-1] + + res = self.interpret(f, [38]) + assert res == 38 + class TestOOtype(BaseTestRlist, OORtypeMixin): rlist = oo_rlist From arigo at codespeak.net Thu May 13 10:59:05 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 May 2010 10:59:05 +0200 (CEST) Subject: [pypy-svn] r74507 - pypy/trunk/pypy/module/pypyjit/test Message-ID: <20100513085905.68A4436C22B@codespeak.net> Author: arigo Date: Thu May 13 10:59:03 2010 New Revision: 74507 Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Log: Fix test. Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Thu May 13 10:59:03 2010 @@ -613,7 +613,7 @@ for i in range(100000): total += f(i, 5) return total - ''', 100, ([], 4999450000L)) + ''', 170, ([], 4999450000L)) class AppTestJIT(PyPyCJITTests): def setup_class(cls): From arigo at codespeak.net Thu May 13 16:28:37 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 May 2010 16:28:37 +0200 (CEST) Subject: [pypy-svn] r74508 - in pypy/branch/blackhole-improvement/pypy/jit: backend/llgraph codewriter codewriter/test metainterp Message-ID: <20100513142837.7646F282BFE@codespeak.net> Author: arigo Date: Thu May 13 16:28:35 2010 New Revision: 74508 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: Progress. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py Thu May 13 16:28:35 2010 @@ -399,10 +399,9 @@ class Frame(object): OPHANDLERS = [None] * (rop._LAST+1) - def __init__(self, memocast, cpu): + def __init__(self, cpu): self.verbose = False self.cpu = cpu - self.memocast = memocast self.opindex = 1 self._forced = False self._may_force = -1 @@ -529,7 +528,7 @@ return res def as_int(self, x): - return cast_to_int(x, self.memocast) + return cast_to_int(x) def as_ptr(self, x): return cast_to_ptr(x) @@ -553,34 +552,32 @@ try: op = getattr(cls, 'op_' + opname.lower()) # op_guard_true etc. except AttributeError: - name = 'do_' + opname.lower() try: - impl = globals()[name] # do_arraylen_gc etc. - def op(self, descr, *args): - return impl(descr, *args) - # + impl = globals()['do_' + opname.lower()] # do_arraylen_gc etc. + op = staticmethod(impl) except KeyError: - from pypy.jit.backend.llgraph import llimpl - impl = getattr(executor, name) # do_int_add etc. - def _op_default_implementation(self, descr, *args): - # for all operations implemented in execute.py - boxedargs = [] - for x in args: - if type(x) is int: - boxedargs.append(BoxInt(x)) - elif type(x) is float: - boxedargs.append(BoxFloat(x)) - elif isinstance(ootype.typeOf(x), ootype.OOType): - boxedargs.append(BoxObj(ootype.cast_to_object(x))) - else: - boxedargs.append(BoxPtr(x)) - # xxx this passes the 'llimpl' module as the CPU argument - resbox = impl(llimpl, *boxedargs) - return getattr(resbox, 'value', None) - op = _op_default_implementation - # + op = cls._make_impl_from_blackhole_interp(opname) cls.OPHANDLERS[opnum] = op + @classmethod + def _make_impl_from_blackhole_interp(cls, opname): + from pypy.jit.metainterp.blackhole import BlackholeInterpreter + name = 'bhimpl_' + opname.lower() + func = BlackholeInterpreter.__dict__[name] + for argtype in func.argtypes: + assert argtype in ('i', 'r', 'f') + # + def _op_default_implementation(self, descr, *args): + # for all operations implemented in the blackhole interpreter + return func(*args) + # + return _op_default_implementation + + def op_debug_merge_point(self, _, value): + from pypy.jit.metainterp.warmspot import get_stats + loc = ConstPtr(value)._get_str() + get_stats().add_merge_point_location(loc) + def op_guard_true(self, _, value): if not value: raise GuardFailed @@ -800,7 +797,7 @@ return do_new_array(arraydescr.ofs, count) def op_cast_ptr_to_int(self, descr, ptr): - return cast_to_int(ptr, self.memocast) + return cast_to_int(ptr) def op_uint_xor(self, descr, arg1, arg2): return arg1 ^ arg2 @@ -1012,11 +1009,11 @@ return x -def new_frame(memocast, is_oo, cpu): +def new_frame(is_oo, cpu): if is_oo: - frame = OOFrame(memocast, cpu) + frame = OOFrame(cpu) else: - frame = Frame(memocast, cpu) + frame = Frame(cpu) return _to_opaque(frame) _future_values = [] Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py Thu May 13 16:28:35 2010 @@ -209,7 +209,7 @@ def _execute_token(self, loop_token): compiled_version = loop_token._llgraph_compiled_version - frame = llimpl.new_frame(self.memo_cast, self.is_oo, self) + frame = llimpl.new_frame(self.is_oo, self) # setup the frame llimpl.frame_clear(frame, compiled_version) # run the loop @@ -869,10 +869,3 @@ def __repr__(self): return '' % self.fieldname - - -# ____________________________________________________________ - -import pypy.jit.metainterp.executor -pypy.jit.metainterp.executor.make_execute_list(LLtypeCPU) -##pypy.jit.metainterp.executor.make_execute_list(OOtypeCPU) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Thu May 13 16:28:35 2010 @@ -43,6 +43,8 @@ return '%s%s' % (self.kind[0], self.content) def __iter__(self): return iter(self.content) + def __nonzero__(self): + return bool(self.content) class IndirectCallTargets(object): def __init__(self, lst): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Thu May 13 16:28:35 2010 @@ -17,10 +17,6 @@ t.transform(graph, portal) -class NoOp(Exception): - pass - - class Transformer(object): def __init__(self, cpu=None, callcontrol=None): @@ -36,36 +32,40 @@ def optimize_block(self, block): if block.operations == (): return - rename = {} + renamings = {} newoperations = [] - if block.exitswitch == c_last_exception: - op_raising_exception = block.operations[-1] - else: - op_raising_exception = Ellipsis + # for op in block.operations: - try: - op1 = self.rewrite_operation(op) - except NoOp: - if op.result is not None: - rename[op.result] = rename.get(op.args[0], op.args[0]) - if op is op_raising_exception: - self.killed_exception_raising_operation(block) - else: + oplist = self.rewrite_operation(op) + # + count_before_last_operation = len(newoperations) + if not isinstance(oplist, list): + oplist = [oplist] + for op1 in oplist: if isinstance(op1, SpaceOperation): - op2 = self.do_renaming(rename, op1) - newoperations.append(op2) + newoperations.append(self._do_renaming(renamings, op1)) + elif op1 is None: + # rewrite_operation() returns None to mean "has no real + # effect, the result should just be renamed to args[0]" + if op.result is not None: + renamings[op.result] = renamings.get(op.args[0], + op.args[0]) elif isinstance(op1, Constant): - rename[op.result] = op1 + renamings[op.result] = op1 else: raise TypeError(repr(op1)) + # + if block.exitswitch == c_last_exception: + if len(newoperations) == count_before_last_operation: + self._killed_exception_raising_operation(block) block.operations = newoperations - block.exitswitch = rename.get(block.exitswitch, block.exitswitch) + block.exitswitch = renamings.get(block.exitswitch, block.exitswitch) self.follow_constant_exit(block) self.optimize_goto_if_not(block) for link in block.exits: - self.do_renaming_on_link(rename, link) + self._do_renaming_on_link(renamings, link) - def do_renaming(self, rename, op): + def _do_renaming(self, rename, op): op = SpaceOperation(op.opname, op.args[:], op.result) for i, v in enumerate(op.args): if isinstance(v, Variable): @@ -80,13 +80,13 @@ op.args[i] = ListOfKind(v.kind, newlst) return op - def do_renaming_on_link(self, rename, link): + def _do_renaming_on_link(self, rename, link): for i, v in enumerate(link.args): if isinstance(v, Variable): if v in rename: link.args[i] = rename[v] - def killed_exception_raising_operation(self, block): + def _killed_exception_raising_operation(self, block): assert block.exits[0].exitcase is None del block.exits[1:] block.exitswitch = None @@ -140,21 +140,21 @@ try: rewrite = _rewrite_ops[op.opname] except KeyError: - return op + return op # default: keep the operation unchanged else: return rewrite(self, op) - def rewrite_op_same_as(self, op): raise NoOp - def rewrite_op_cast_pointer(self, op): raise NoOp - def rewrite_op_cast_primitive(self, op): raise NoOp - def rewrite_op_cast_bool_to_int(self, op): raise NoOp - def rewrite_op_cast_bool_to_uint(self, op): raise NoOp - def rewrite_op_cast_char_to_int(self, op): raise NoOp - def rewrite_op_cast_unichar_to_int(self, op): raise NoOp - def rewrite_op_cast_int_to_char(self, op): raise NoOp - def rewrite_op_cast_int_to_unichar(self, op): raise NoOp - def rewrite_op_cast_int_to_uint(self, op): raise NoOp - def rewrite_op_cast_uint_to_int(self, op): raise NoOp + def rewrite_op_same_as(self, op): pass + def rewrite_op_cast_pointer(self, op): pass + def rewrite_op_cast_primitive(self, op): pass + def rewrite_op_cast_bool_to_int(self, op): pass + def rewrite_op_cast_bool_to_uint(self, op): pass + def rewrite_op_cast_char_to_int(self, op): pass + def rewrite_op_cast_unichar_to_int(self, op): pass + def rewrite_op_cast_int_to_char(self, op): pass + def rewrite_op_cast_int_to_unichar(self, op): pass + def rewrite_op_cast_int_to_uint(self, op): pass + def rewrite_op_cast_uint_to_int(self, op): pass def _rewrite_symmetric(self, op): """Rewrite 'c1+v2' into 'v2+c1' in an attempt to avoid generating @@ -211,22 +211,28 @@ """Turn 'i0 = direct_call(fn, i1, i2, ref1, ref2)' into 'i0 = xxx_call_ir_i(fn, descr, [i1,i2], [ref1,ref2])'. The name is one of '{residual,direct}_call_{r,ir,irf}_{i,r,f,v}'.""" - args_i = [] - args_r = [] - args_f = [] - for v in op.args[1:]: - self.add_in_correct_list(v, args_i, args_r, args_f) + lst_i, lst_r, lst_f = self.make_three_lists(op.args[1:]) reskind = getkind(op.result.concretetype)[0] - if args_f or reskind == 'f': kinds = 'irf' - elif args_i: kinds = 'ir' - else: kinds = 'r' + if lst_f or reskind == 'f': kinds = 'irf' + elif lst_i: kinds = 'ir' + else: kinds = 'r' sublists = [] - if 'i' in kinds: sublists.append(ListOfKind('int', args_i)) - if 'r' in kinds: sublists.append(ListOfKind('ref', args_r)) - if 'f' in kinds: sublists.append(ListOfKind('float', args_f)) + if 'i' in kinds: sublists.append(lst_i) + if 'r' in kinds: sublists.append(lst_r) + if 'f' in kinds: sublists.append(lst_f) return SpaceOperation('%s_%s_%s' % (namebase, kinds, reskind), initialargs + sublists, op.result) + def make_three_lists(self, vars): + args_i = [] + args_r = [] + args_f = [] + for v in vars: + self.add_in_correct_list(v, args_i, args_r, args_f) + return [ListOfKind('int', args_i), + ListOfKind('ref', args_r), + ListOfKind('float', args_f)] + def add_in_correct_list(self, v, lst_i, lst_r, lst_f): kind = getkind(v.concretetype) if kind == 'void': return @@ -305,11 +311,13 @@ kind = getkind(op.args[0].concretetype) # note: the 'G_' prefix tells that the operation might generate # a guard in pyjitpl (see liveness.py) - return SpaceOperation('G_%s_guard_value' % kind, - [op.args[0]], op.result) + op = SpaceOperation('G_%s_guard_value' % kind, + [op.args[0]], None) + # this special return value forces op.result to be considered + # equal to op.args[0] + return [op, None] else: log.WARNING('ignoring hint %r at %r' % (hints, self.graph)) - raise NoOp def rewrite_op_malloc_varsize(self, op): assert op.args[1].value == {'flavor': 'gc'} @@ -355,7 +363,7 @@ [v_inst, c_fieldname] = op.args RESULT = op.result.concretetype if RESULT is lltype.Void: - raise NoOp + return # check for virtualizable #try: # if self.is_virtualizable_getset(op): @@ -393,12 +401,12 @@ def rewrite_op_setfield(self, op): if self.is_typeptr_getset(op): # ignore the operation completely -- instead, it's done by 'new' - raise NoOp + return # turn the flow graph 'setfield' operation into our own version [v_inst, c_fieldname, v_value] = op.args RESULT = v_value.concretetype if RESULT is lltype.Void: - raise NoOp + return # check for virtualizable #if self.is_virtualizable_getset(op): # vinfo = self.codewriter.metainterp_sd.virtualizable_info @@ -530,8 +538,6 @@ def rewrite_op_cast_ptr_to_int(self, op): if self._is_gc(op.args[0]): return op - else: - raise NoOp # ---------- # Renames, from the _old opname to the _new one. @@ -609,17 +615,27 @@ return op def rewrite_op_jit_marker(self, op): - opname = op.args[0].value jitdriver = op.args[1].value self.callcontrol.found_jitdriver(jitdriver) - args_i = [] - args_r = [] - args_f = [] - for v in op.args[2:]: - self.add_in_correct_list(v, args_i, args_r, args_f) - return SpaceOperation(opname, [ListOfKind('int', args_i), - ListOfKind('ref', args_r), - ListOfKind('float', args_f)], op.result) + key = op.args[0].value + return getattr(self, 'handle_jit_marker__%s' % key)(op, jitdriver) + + def handle_jit_marker__jit_merge_point(self, op, jitdriver): + assert self.portal, "jit_merge_point in non-main graph!" + ops = [] + num_green_args = len(jitdriver.greens) + for v in op.args[2:2+num_green_args]: + if isinstance(v, Variable) and v.concretetype is not lltype.Void: + kind = getkind(v.concretetype) + ops.append(SpaceOperation('G_%s_guard_value' % kind, + [v], None)) + args = (self.make_three_lists(op.args[2:2+num_green_args]) + + self.make_three_lists(op.args[2+num_green_args:])) + ops.append(SpaceOperation('jit_merge_point', args, None)) + return ops + + def handle_jit_marker__can_enter_jit(self, op, jitdriver): + return SpaceOperation('can_enter_jit', [], None) # ____________________________________________________________ Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Thu May 13 16:28:35 2010 @@ -457,7 +457,8 @@ myjitdriver.jit_merge_point(x=x, y=y) myjitdriver.can_enter_jit(x=y, y=x) self.encoding_test(f, [4, 5], """ - jit_merge_point I[%i0, %i1], R[], F[] - can_enter_jit I[%i1, %i0], R[], F[] + G_int_guard_value %i0 + jit_merge_point I[%i0], R[], F[], I[%i1], R[], F[] + can_enter_jit void_return """, transform=True) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Thu May 13 16:28:35 2010 @@ -516,3 +516,34 @@ assert op1.opname == 'unicodesetitem' assert op1.args == [v, v_index, v_newchr] assert op1.result == v_void + +def test_promote_1(): + v1 = varoftype(lltype.Signed) + v2 = varoftype(lltype.Signed) + op = SpaceOperation('hint', + [v1, Constant({'promote': True}, lltype.Void)], + v2) + oplist = Transformer().rewrite_operation(op) + assert len(oplist) == 2 + assert oplist[1] is None + assert oplist[0].opname == 'G_int_guard_value' + assert oplist[0].args == [v1] + assert oplist[0].result is None + +def test_promote_2(): + v1 = varoftype(lltype.Signed) + v2 = varoftype(lltype.Signed) + op = SpaceOperation('hint', + [v1, Constant({'promote': True}, lltype.Void)], + v2) + returnblock = Block([varoftype(lltype.Signed)]) + returnblock.operations = () + block = Block([v1]) + block.operations = [op] + block.closeblock(Link([v2], returnblock)) + Transformer().optimize_block(block) + assert len(block.operations) == 1 + assert block.operations[0].opname == 'G_int_guard_value' + assert block.operations[0].args == [v1] + assert block.operations[0].result is None + assert block.exits[0].args == [v1] Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Thu May 13 16:28:35 2010 @@ -475,9 +475,15 @@ def bhimpl_float_copy(a): return a - bhimpl_int_guard_value = bhimpl_int_copy - bhimpl_ref_guard_value = bhimpl_ref_copy - bhimpl_float_guard_value = bhimpl_float_copy + @arguments("i") + def bhimpl_int_guard_value(a): + pass + @arguments("r") + def bhimpl_ref_guard_value(a): + pass + @arguments("f") + def bhimpl_float_guard_value(a): + pass @arguments("self", "i") def bhimpl_int_push(self, a): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py Thu May 13 16:28:35 2010 @@ -203,8 +203,7 @@ # ____________________________________________________________ -def make_execute_list(cpuclass): - from pypy.jit.backend.model import AbstractCPU +def _make_execute_list(): if 0: # enable this to trace calls to do_xxx def wrap(fn): def myfn(*args): @@ -259,7 +258,7 @@ execute[value] = func continue print "XXX warning: missing", key - cpuclass._execute_by_num_args = execute_by_num_args + return execute_by_num_args def make_execute_function_with_boxes(name, func): # Make a wrapper for 'func'. The func is a simple bhimpl_xxx function @@ -300,17 +299,19 @@ do.func_name = 'do_' + name return do -def get_execute_funclist(cpu, num_args): +def get_execute_funclist(num_args): # workaround, similar to the next one - return cpu._execute_by_num_args[num_args] + return EXECUTE_BY_NUM_ARGS[num_args] get_execute_funclist._annspecialcase_ = 'specialize:memo' -def get_execute_function(cpu, opnum, num_args): +def get_execute_function(opnum, num_args): # workaround for an annotation limitation: putting this code in # a specialize:memo function makes sure the following line is # constant-folded away. Only works if opnum and num_args are # constants, of course. - return cpu._execute_by_num_args[num_args][opnum] + func = EXECUTE_BY_NUM_ARGS[num_args][opnum] + assert func is not None, "EXECUTE_BY_NUM_ARGS[%s][%s]" % (num_args, opnum) + return func get_execute_function._annspecialcase_ = 'specialize:memo' def has_descr(opnum): @@ -326,8 +327,7 @@ argboxes = argboxes + (descr,) else: assert descr is None - func = get_execute_function(metainterp.cpu, opnum, len(argboxes)) - assert func is not None + func = get_execute_function(opnum, len(argboxes)) return func(metainterp, *argboxes) # note that the 'argboxes' tuple # optionally ends with the descr execute._annspecialcase_ = 'specialize:arg(1)' @@ -335,41 +335,45 @@ def execute_varargs(metainterp, opnum, argboxes, descr): # only for opnums with a variable arity (calls, typically) check_descr(descr) - func = get_execute_function(metainterp.cpu, opnum, -1) - assert func is not None + func = get_execute_function(opnum, -1) return func(metainterp, argboxes, descr) execute_varargs._annspecialcase_ = 'specialize:arg(1)' -def execute_nonspec(cpu, opnum, argboxes, descr=None): +def execute_nonspec(metainterp, opnum, argboxes, descr=None): + assert metainterp.__class__.__name__ == 'MetaInterp' # XXX kill me arity = resoperation.oparity[opnum] assert arity == -1 or len(argboxes) == arity if resoperation.opwithdescr[opnum]: check_descr(descr) if arity == -1: - func = get_execute_funclist(cpu, -1)[opnum] - return func(cpu, argboxes, descr) + func = get_execute_funclist(-1)[opnum] + return func(metainterp, argboxes, descr) if arity == 0: - func = get_execute_funclist(cpu, 1)[opnum] - return func(cpu, descr) + func = get_execute_funclist(1)[opnum] + return func(metainterp, descr) if arity == 1: - func = get_execute_funclist(cpu, 2)[opnum] - return func(cpu, argboxes[0], descr) + func = get_execute_funclist(2)[opnum] + return func(metainterp, argboxes[0], descr) if arity == 2: - func = get_execute_funclist(cpu, 3)[opnum] - return func(cpu, argboxes[0], argboxes[1], descr) + func = get_execute_funclist(3)[opnum] + return func(metainterp, argboxes[0], argboxes[1], descr) if arity == 3: - func = get_execute_funclist(cpu, 4)[opnum] - return func(cpu, argboxes[0], argboxes[1], argboxes[2], descr) + func = get_execute_funclist(4)[opnum] + return func(metainterp, argboxes[0], argboxes[1], argboxes[2], + descr) else: assert descr is None if arity == 1: - func = get_execute_funclist(cpu, 1)[opnum] - return func(cpu, argboxes[0]) + func = get_execute_funclist(1)[opnum] + return func(metainterp, argboxes[0]) if arity == 2: - func = get_execute_funclist(cpu, 2)[opnum] - return func(cpu, argboxes[0], argboxes[1]) + func = get_execute_funclist(2)[opnum] + return func(metainterp, argboxes[0], argboxes[1]) if arity == 3: - func = get_execute_funclist(cpu, 3)[opnum] - return func(cpu, argboxes[0], argboxes[1], argboxes[2]) + func = get_execute_funclist(3)[opnum] + return func(metainterp, argboxes[0], argboxes[1], argboxes[2]) raise NotImplementedError + + +EXECUTE_BY_NUM_ARGS = _make_execute_list() Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Thu May 13 16:28:35 2010 @@ -51,7 +51,7 @@ parent_resumedata_snapshot = None parent_resumedata_frame_info_list = None - env = property(lambda: xxx, lambda x: xxx) # no read/write! + env = property(lambda: xxx, lambda: xxx) # temporary: no read/write! def __init__(self, metainterp): self.metainterp = metainterp @@ -771,19 +771,12 @@ self.execute_varargs(rop.OOSEND_PURE, boxes, descr=methdescr, exc=False) @arguments("box") - def opimpl_int_guard_value(self, box): - return self.implement_guard_value(box) + def _opimpl_guard_value(self, box): + self.implement_guard_value(box) - @XXX #arguments("orgpc", "int") - def opimpl_guard_green(self, pc, boxindex): - """Like guard_value, but overwrites the original box with the const. - Used to prevent Boxes from showing up in the greenkey of some - operations, like jit_merge_point. The in-place overwriting is - convenient for jit_merge_point, which expects self.env to contain - not more than the greens+reds described in the jitdriver.""" - box = self.env[boxindex] - constbox = self.implement_guard_value(pc, box) - self.env[boxindex] = constbox + opimpl_int_guard_value = _opimpl_guard_value + opimpl_ref_guard_value = _opimpl_guard_value + opimpl_float_guard_value = _opimpl_guard_value @arguments("box") def opimpl_guard_class(self, box): @@ -831,39 +824,28 @@ else: raise self.metainterp.staticdata.ContinueRunningNormally(varargs) - @arguments("boxes3") - def opimpl_can_enter_jit(self, boxes): - xxx - # Note: when running with a BlackHole history, this 'can_enter_jit' - # may be completely skipped by the logic that replaces perform_call - # with rop.CALL. But in that case, no-one will check the flag anyway, - # so it's fine. + @arguments() + def opimpl_can_enter_jit(self): if self.metainterp.in_recursion: from pypy.jit.metainterp.warmspot import CannotInlineCanEnterJit raise CannotInlineCanEnterJit() self.metainterp.seen_can_enter_jit = True - @arguments("boxes3") - def opimpl_jit_merge_point(self, boxes): - xxx - if not self.metainterp.is_blackholing(): - self.verify_green_args(self.env) - # xxx we may disable the following line in some context later - self.debug_merge_point() - if self.metainterp.seen_can_enter_jit: - self.metainterp.seen_can_enter_jit = False - try: - self.metainterp.reached_can_enter_jit(self.env) - except GiveUp: - self.metainterp.switch_to_blackhole(ABORT_BRIDGE) - if self.metainterp.is_blackholing(): - self.blackhole_reached_merge_point(self.env) - return True + @arguments("boxes3", "boxes3") + def opimpl_jit_merge_point(self, greenboxes, redboxes): + self.verify_green_args(greenboxes) + # xxx we may disable the following line in some context later + self.debug_merge_point(greenboxes) + if self.metainterp.seen_can_enter_jit: + self.metainterp.seen_can_enter_jit = False + try: + self.metainterp.reached_can_enter_jit(greenboxes, redboxes) + except GiveUp: + XXX + self.metainterp.switch_to_blackhole(ABORT_BRIDGE) - def debug_merge_point(self): + def debug_merge_point(self, greenkey): # debugging: produce a DEBUG_MERGE_POINT operation - num_green_args = self.metainterp.staticdata.num_green_args - greenkey = self.env[:num_green_args] sd = self.metainterp.staticdata loc = sd.state.get_location_str(greenkey) debug_print(loc) @@ -1583,9 +1565,8 @@ key.reset_counter_from_failure(self) raise - def remove_consts_and_duplicates(self, boxes, startindex, endindex, - duplicates): - for i in range(startindex, endindex): + def remove_consts_and_duplicates(self, boxes, endindex, duplicates): + for i in range(endindex): box = boxes[i] if isinstance(box, Const) or box in duplicates: oldbox = box @@ -1595,19 +1576,15 @@ else: duplicates[box] = None - def reached_can_enter_jit(self, live_arg_boxes): - num_green_args = self.staticdata.num_green_args + def reached_can_enter_jit(self, greenboxes, redboxes): duplicates = {} - self.remove_consts_and_duplicates(live_arg_boxes, - num_green_args, - len(live_arg_boxes), + self.remove_consts_and_duplicates(redboxes, len(redboxes), duplicates) - live_arg_boxes = live_arg_boxes[:] + live_arg_boxes = greenboxes + redboxes if self.staticdata.virtualizable_info is not None: # we use ':-1' to remove the last item, which is the virtualizable # itself self.remove_consts_and_duplicates(self.virtualizable_boxes, - 0, len(self.virtualizable_boxes)-1, duplicates) live_arg_boxes += self.virtualizable_boxes[:-1] @@ -2140,7 +2117,7 @@ self.pc = position + num_return_args # if not we_are_translated(): - print '\tjitcode:', name, list(args), + print '\tjitcode: %s(%s)' % (name, ', '.join(map(repr, args))), try: resultbox = unboundmethod(self, *args) except Exception, e: @@ -2150,7 +2127,7 @@ print assert resultbox is None else: - print '->', resultbox + print '-> %r' % (resultbox,) result_argcode = argcodes[next_argcode] assert resultbox.type == {'i': history.INT, 'r': history.REF, From agaynor at codespeak.net Thu May 13 17:46:31 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Thu, 13 May 2010 17:46:31 +0200 (CEST) Subject: [pypy-svn] r74509 - pypy/trunk/pypy/module/cpyext/patches Message-ID: <20100513154631.41E27282BFE@codespeak.net> Author: agaynor Date: Thu May 13 17:46:29 2010 New Revision: 74509 Added: pypy/trunk/pypy/module/cpyext/patches/mysqldb.patch Log: Added current patch for MySQLDB, it passes tests but it blows up under Django Added: pypy/trunk/pypy/module/cpyext/patches/mysqldb.patch ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/cpyext/patches/mysqldb.patch Thu May 13 17:46:29 2010 @@ -0,0 +1,45 @@ +*** _mysql-orig.c 2009-03-30 18:59:28.000000000 -0400 +--- _mysql.c 2010-05-13 11:43:59.000000000 -0400 +*************** +*** 413,418 **** +--- 413,419 ---- + PyObject *pmask=NULL; + pmask = PyTuple_GET_ITEM(t, 0); + fun2 = PyTuple_GET_ITEM(t, 1); ++ Py_XINCREF(fun2); + if (PyInt_Check(pmask)) { + mask = PyInt_AS_LONG(pmask); + if (mask & fields[i].flags) { +*************** +*** 430,437 **** + cleanup: + Py_DECREF(t); + } +! if (!fun2) fun2 = Py_None; +! Py_INCREF(fun2); + Py_DECREF(fun); + fun = fun2; + } +--- 431,440 ---- + cleanup: + Py_DECREF(t); + } +! if (!fun2) { +! fun2 = Py_None; +! Py_INCREF(fun2); +! } + Py_DECREF(fun); + fun = fun2; + } +*************** +*** 2785,2790 **** +--- 2788,2796 ---- + if (!(e = PyDict_GetItemString(edict, name))) + return NULL; + if (PyDict_SetItemString(dict, name, e)) return NULL; ++ #ifdef PYPY_VERSION ++ Py_INCREF(e); ++ #endif + return e; + } + From fijal at codespeak.net Thu May 13 22:41:18 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 13 May 2010 22:41:18 +0200 (CEST) Subject: [pypy-svn] r74510 - pypy/build/bot2/pypybuildbot Message-ID: <20100513204118.7169A282BFE@codespeak.net> Author: fijal Date: Thu May 13 22:41:15 2010 New Revision: 74510 Modified: pypy/build/bot2/pypybuildbot/master.py Log: Move benchmarks to tannit Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Thu May 13 22:41:15 2010 @@ -226,7 +226,7 @@ "category": 'own' }, {"name": JITBENCH, - "slavenames": ["bigdogvm2"], + "slavenames": ["tannit32"], "builddir": JITBENCH, "factory": pypyJITBenchmarkFactory, "category": 'benchmark-run', From agaynor at codespeak.net Fri May 14 08:19:48 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Fri, 14 May 2010 08:19:48 +0200 (CEST) Subject: [pypy-svn] r74511 - pypy/trunk/pypy/module/cpyext/patches Message-ID: <20100514061948.85792282BFE@codespeak.net> Author: agaynor Date: Fri May 14 08:19:46 2010 New Revision: 74511 Modified: pypy/trunk/pypy/module/cpyext/patches/mysqldb.patch Log: Updated MySQLdb patch, Django now runs on it, however this fix may be in the wrong place. Modified: pypy/trunk/pypy/module/cpyext/patches/mysqldb.patch ============================================================================== --- pypy/trunk/pypy/module/cpyext/patches/mysqldb.patch (original) +++ pypy/trunk/pypy/module/cpyext/patches/mysqldb.patch Fri May 14 08:19:46 2010 @@ -1,5 +1,5 @@ *** _mysql-orig.c 2009-03-30 18:59:28.000000000 -0400 ---- _mysql.c 2010-05-13 11:43:59.000000000 -0400 +--- _mysql.c 2010-05-13 20:10:19.000000000 -0400 *************** *** 413,418 **** --- 413,419 ---- @@ -32,8 +32,20 @@ fun = fun2; } *************** +*** 1053,1059 **** +--- 1056,1064 ---- + "no default type converter defined"); + goto error; + } ++ Py_INCREF(d); + quoted = PyObject_CallFunction(itemconv, "OO", item, d); ++ Py_DECREF(d); + Py_DECREF(itemconv); + error: + return quoted; +*************** *** 2785,2790 **** ---- 2788,2796 ---- +--- 2790,2798 ---- if (!(e = PyDict_GetItemString(edict, name))) return NULL; if (PyDict_SetItemString(dict, name, e)) return NULL; From afa at codespeak.net Fri May 14 11:04:29 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 14 May 2010 11:04:29 +0200 (CEST) Subject: [pypy-svn] r74512 - pypy/trunk/pypy/module/_socket/test Message-ID: <20100514090429.C083B282BFE@codespeak.net> Author: afa Date: Fri May 14 11:04:27 2010 New Revision: 74512 Modified: pypy/trunk/pypy/module/_socket/test/test_sock_app.py Log: Now that teardown_method() is called even when setup_method() failed, be more robust against errors when creating the socket. Modified: pypy/trunk/pypy/module/_socket/test/test_sock_app.py ============================================================================== --- pypy/trunk/pypy/module/_socket/test/test_sock_app.py (original) +++ pypy/trunk/pypy/module/_socket/test/test_sock_app.py Fri May 14 11:04:27 2010 @@ -501,9 +501,9 @@ return serv ''') def teardown_method(self, method): - space.appexec([self.w_serv], '(serv): serv.close()') - self.w_serv = None - + if hasattr(self, 'w_serv'): + space.appexec([self.w_serv], '(serv): serv.close()') + self.w_serv = None def test_timeout(self): from _socket import timeout From tobami at codespeak.net Fri May 14 18:14:47 2010 From: tobami at codespeak.net (tobami at codespeak.net) Date: Fri, 14 May 2010 18:14:47 +0200 (CEST) Subject: [pypy-svn] r74513 - pypy/benchmarks Message-ID: <20100514161447.8BDDD282BFE@codespeak.net> Author: tobami Date: Fri May 14 18:14:44 2010 New Revision: 74513 Modified: pypy/benchmarks/runner.py pypy/benchmarks/saveresults.py Log: adjusted saving options for new speed version Modified: pypy/benchmarks/runner.py ============================================================================== --- pypy/benchmarks/runner.py (original) +++ pypy/benchmarks/runner.py Fri May 14 18:14:44 2010 @@ -39,7 +39,7 @@ name = "pypy-c" else: name = "pypy-c-jit" - optionsname = "gc=hybrid" + optionsname = "" if "psyco.sh" in pypy_c_path: name = "cpython" optionsname = "psyco-profile" Modified: pypy/benchmarks/saveresults.py ============================================================================== --- pypy/benchmarks/saveresults.py (original) +++ pypy/benchmarks/saveresults.py Fri May 14 18:14:44 2010 @@ -14,7 +14,6 @@ #Parse data data = {} current_date = datetime.today() - if branch == "": branch = 'trunk' for b in results: bench_name = b[0] @@ -31,7 +30,6 @@ data = { 'commitid': revision, 'project': project, - 'branch': branch, 'executable_name': interpreter, 'executable_coptions': int_options, 'benchmark': bench_name, From fijal at codespeak.net Fri May 14 19:09:22 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 14 May 2010 19:09:22 +0200 (CEST) Subject: [pypy-svn] r74514 - in pypy/trunk/pypy/jit/backend/x86: . test Message-ID: <20100514170922.5934D282BFE@codespeak.net> Author: fijal Date: Fri May 14 19:09:20 2010 New Revision: 74514 Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_platform.py Log: issue533 fixed newer gcc require 16 byte stack alignment. Adapt. http://gcc.gnu.org/bugzilla/show_bug.cgi?id=40838 Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Fri May 14 19:09:20 2010 @@ -21,11 +21,9 @@ # our calling convention - we pass first 6 args in registers # and the rest stays on the stack -if sys.platform == 'darwin': - # darwin requires the stack to be 16 bytes aligned on calls - CALL_ALIGN = 4 -else: - CALL_ALIGN = 1 +# darwin requires the stack to be 16 bytes aligned on calls. Same for gcc 4.5.0, +# better safe than sorry +CALL_ALIGN = 4 def align_stack_words(words): return (words + CALL_ALIGN - 1) & ~(CALL_ALIGN-1) Modified: pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_platform.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_platform.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_platform.py Fri May 14 19:09:20 2010 @@ -10,6 +10,7 @@ def test_stack_alignment(): + # we can detect gcc 4.5.0 to test those as well if sys.platform != 'darwin': py.test.skip("tests darwin only stack alignment requirements") From afa at codespeak.net Sat May 15 10:12:13 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 15 May 2010 10:12:13 +0200 (CEST) Subject: [pypy-svn] r74515 - pypy/trunk/pypy/module/cpyext Message-ID: <20100515081213.1DF80282B90@codespeak.net> Author: afa Date: Sat May 15 10:12:11 2010 New Revision: 74515 Modified: pypy/trunk/pypy/module/cpyext/stubs.py Log: These functions are already implemented Modified: pypy/trunk/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/stubs.py (original) +++ pypy/trunk/pypy/module/cpyext/stubs.py Sat May 15 10:12:11 2010 @@ -1164,12 +1164,6 @@ Use PyObject_Repr() or PyOS_double_to_string() instead.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyFunction_Check(space, o): - """Return true if o is a function object (has type PyFunction_Type). - The parameter must not be NULL.""" - raise NotImplementedError - @cpython_api([PyObject, PyObject], PyObject) def PyFunction_New(space, code, globals): """Return a new function object associated with the code object code. globals @@ -3128,13 +3122,6 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject], PyObject) -def PyUnicode_AsUTF8String(space, unicode): - """Encode a Unicode object using UTF-8 and return the result as Python string - object. Error handling is "strict". Return NULL if an exception was raised - by the codec.""" - raise NotImplementedError - @cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.INTP], PyObject) def PyUnicode_DecodeUTF32(space, s, size, errors, byteorder): """Decode length bytes from a UTF-32 encoded buffer string and return the From afa at codespeak.net Sun May 16 20:15:46 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sun, 16 May 2010 20:15:46 +0200 (CEST) Subject: [pypy-svn] r74516 - pypy/trunk/pypy/module/cpyext Message-ID: <20100516181546.E7BD4282BAD@codespeak.net> Author: afa Date: Sun May 16 20:15:43 2010 New Revision: 74516 Modified: pypy/trunk/pypy/module/cpyext/api.py Log: Remove the 'name' parameter of cpython_api. More docstrings. Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Sun May 16 20:15:43 2010 @@ -139,8 +139,17 @@ wrapper.relax_sig_check = True return wrapper -def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, - external=True, name=None): +def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, external=True): + """ + Declares a function to be exported. + - `argtypes`, `restype` are lltypes and describe the function signature. + - `error` is the value returned when an applevel exception is raised. The + special value 'CANNOT_FAIL' (also when restype is Void) turns an eventual + exception into a wrapped SystemError. Unwrapped exceptions also cause a + SytemError. + - set `external` to False to get a C function pointer, but not exported by + the API headers. + """ if error is _NOT_SPECIFIED: if restype is PyObject: error = lltype.nullptr(PyObject.TO) @@ -150,15 +159,11 @@ error = rffi.cast(restype, error) def decorate(func): - if name is None: - func_name = func.func_name - else: - func_name = name - func = func_with_new_name(func, name) + func_name = func.func_name api_function = ApiFunction(argtypes, restype, func, error) func.api_func = api_function - assert func_name not in FUNCTIONS + assert func_name not in FUNCTIONS, "%s already registered" % func_name assert func_name not in FUNCTIONS_STATIC if error is _NOT_SPECIFIED: @@ -363,6 +368,12 @@ TYPES[name].become(TYPE) def build_type_checkers(type_name, cls=None): + """ + Builds two api functions: Py_XxxCheck() and Py_XxxCheckExact(). + - if `cls` is None, the type is space.w_[type]. + - if `cls` is a string, it is the name of a space attribute, e.g. 'w_str'. + - else `cls` must be a W_Class with a typedef. + """ if cls is None: attrname = "w_" + type_name.lower() def get_w_type(space): @@ -374,18 +385,23 @@ def get_w_type(space): return space.gettypeobject(cls.typedef) check_name = "Py" + type_name + "_Check" - @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL, name=check_name) + def check(space, w_obj): + "Implements the Py_Xxx_Check function" w_obj_type = space.type(w_obj) w_type = get_w_type(space) - return int(space.is_w(w_obj_type, w_type) or - space.is_true(space.issubtype(w_obj_type, w_type))) - @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL, - name=check_name + "Exact") + return (space.is_w(w_obj_type, w_type) or + space.is_true(space.issubtype(w_obj_type, w_type))) def check_exact(space, w_obj): + "Implements the Py_Xxx_CheckExact function" w_obj_type = space.type(w_obj) w_type = get_w_type(space) - return int(space.is_w(w_obj_type, w_type)) + return space.is_w(w_obj_type, w_type) + + check = cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)( + func_with_new_name(check, check_name)) + check_exact = cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)( + func_with_new_name(check_exact, check_name + "Exact")) return check, check_exact pypy_debug_catch_fatal_exception = rffi.llexternal('pypy_debug_catch_fatal_exception', [], lltype.Void) From afa at codespeak.net Sun May 16 21:40:13 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sun, 16 May 2010 21:40:13 +0200 (CEST) Subject: [pypy-svn] r74517 - pypy/trunk/pypy/module/cpyext Message-ID: <20100516194013.36047282BAD@codespeak.net> Author: afa Date: Sun May 16 21:40:10 2010 New Revision: 74517 Modified: pypy/trunk/pypy/module/cpyext/api.py pypy/trunk/pypy/module/cpyext/funcobject.py pypy/trunk/pypy/module/cpyext/methodobject.py pypy/trunk/pypy/module/cpyext/pyobject.py pypy/trunk/pypy/module/cpyext/slotdefs.py pypy/trunk/pypy/module/cpyext/stringobject.py pypy/trunk/pypy/module/cpyext/typeobject.py pypy/trunk/pypy/module/cpyext/unicodeobject.py Log: Remove the "external" parameter of cpython_api, provide make_static_function() to build function pointers. also simplify 'dealloc' passed to make_typedescr(). Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Sun May 16 21:40:10 2010 @@ -139,7 +139,27 @@ wrapper.relax_sig_check = True return wrapper -def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, external=True): +def make_static_function(argtypes, restype, error=_NOT_SPECIFIED): + """ + Helper to build a static function pointer. + """ + if error is _NOT_SPECIFIED: + if restype is PyObject: + error = lltype.nullptr(PyObject.TO) + elif restype is lltype.Void: + error = CANNOT_FAIL + if type(error) is int: + error = rffi.cast(restype, error) + + def decorate(func): + func_name = func.func_name + api_function = ApiFunction(argtypes, restype, func, error) + func.api_func = api_function + FUNCTIONS_STATIC[func_name] = api_function + return func + return decorate + +def cpython_api(argtypes, restype, error=_NOT_SPECIFIED): """ Declares a function to be exported. - `argtypes`, `restype` are lltypes and describe the function signature. @@ -229,6 +249,8 @@ finally: for arg in to_decref: Py_DecRef(space, arg) + unwrapper = func_with_new_name(unwrapper, + "unwrapper_%s" % (func_name,)) unwrapper.func = func unwrapper.api_func = api_function unwrapper._always_inline_ = True @@ -236,10 +258,7 @@ unwrapper_catch = make_unwrapper(True) unwrapper_raise = make_unwrapper(False) - if external: - FUNCTIONS[func_name] = api_function - else: - FUNCTIONS_STATIC[func_name] = api_function + FUNCTIONS[func_name] = api_function INTERPLEVEL_API[func_name] = unwrapper_catch # used in tests return unwrapper_raise # used in 'normal' RPython code. return decorate Modified: pypy/trunk/pypy/module/cpyext/funcobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/funcobject.py (original) +++ pypy/trunk/pypy/module/cpyext/funcobject.py Sun May 16 21:40:10 2010 @@ -27,7 +27,6 @@ assert isinstance(w_obj, Function) py_func.c_func_name = make_ref(space, space.wrap(w_obj.name)) - at cpython_api([PyObject], lltype.Void, external=False) def function_dealloc(space, py_obj): py_func = rffi.cast(PyFunctionObject, py_obj) Py_DecRef(space, py_func.c_func_name) Modified: pypy/trunk/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/methodobject.py (original) +++ pypy/trunk/pypy/module/cpyext/methodobject.py Sun May 16 21:40:10 2010 @@ -49,7 +49,6 @@ py_func.c_m_ml = w_obj.ml py_func.c_m_self = make_ref(space, w_obj.w_self) - at cpython_api([PyObject], lltype.Void, external=False) def cfunction_dealloc(space, py_obj): py_func = rffi.cast(PyCFunctionObject, py_obj) Py_DecRef(space, py_func.c_m_self) Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Sun May 16 21:40:10 2010 @@ -2,9 +2,9 @@ from pypy.interpreter.baseobjspace import W_Root, SpaceCache from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import cpython_api, bootstrap_function, \ - PyObject, PyObjectP, ADDR, CANNOT_FAIL, \ - Py_TPFLAGS_HEAPTYPE, PyTypeObjectPtr +from pypy.module.cpyext.api import ( + cpython_api, make_static_function, bootstrap_function, PyObject, PyObjectP, + ADDR, CANNOT_FAIL, Py_TPFLAGS_HEAPTYPE, PyTypeObjectPtr) from pypy.module.cpyext.state import State from pypy.objspace.std.typeobject import W_TypeObject from pypy.rlib.objectmodel import specialize, we_are_translated @@ -36,7 +36,7 @@ alloc : allocate and basic initialization of a raw PyObject attach : Function called to tie a raw structure to a pypy object realize : Function called to create a pypy object from a raw struct - dealloc : a cpython_api(external=False), similar to PyObject_dealloc + dealloc : Function called to free a structure """ tp_basestruct = kw.pop('basestruct', PyObject.TO) @@ -46,22 +46,19 @@ tp_dealloc = kw.pop('dealloc', None) assert not kw, "Extra arguments to make_typedescr" - null_dealloc = lltype.nullptr(lltype.FuncType([PyObject], lltype.Void)) + if not tp_dealloc: + from pypy.module.cpyext.typeobject import subtype_dealloc + tp_dealloc = subtype_dealloc + tp_dealloc = make_static_function([PyObject], lltype.Void)(tp_dealloc) class CpyTypedescr(BaseCpyTypedescr): basestruct = tp_basestruct realize = tp_realize def get_dealloc(self, space): - if tp_dealloc: - return llhelper( - tp_dealloc.api_func.functype, - tp_dealloc.api_func.get_wrapper(space)) - else: - from pypy.module.cpyext.typeobject import subtype_dealloc - return llhelper( - subtype_dealloc.api_func.functype, - subtype_dealloc.api_func.get_wrapper(space)) + return llhelper( + tp_dealloc.api_func.functype, + tp_dealloc.api_func.get_wrapper(space)) def allocate(self, space, w_type, itemcount=0): # similar to PyType_GenericAlloc? @@ -124,7 +121,7 @@ from pypy.module.cpyext.object import PyObject_dealloc # typedescr for the 'object' type make_typedescr(space.w_object.instancetypedef, - dealloc=PyObject_dealloc) + dealloc=PyObject_dealloc.func) # almost all types, which should better inherit from object. make_typedescr(None) Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/slotdefs.py (original) +++ pypy/trunk/pypy/module/cpyext/slotdefs.py Sun May 16 21:40:10 2010 @@ -106,7 +106,8 @@ raise OperationError(space.w_StopIteration, space.w_None) return w_res - at cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject, external=True) +# XXX why is it a "cpython_api"? + at cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject) def slot_tp_new(space, type, w_args, w_kwds): from pypy.module.cpyext.tupleobject import PyTuple_Check pyo = rffi.cast(PyObject, type) Modified: pypy/trunk/pypy/module/cpyext/stringobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/stringobject.py (original) +++ pypy/trunk/pypy/module/cpyext/stringobject.py Sun May 16 21:40:10 2010 @@ -102,7 +102,6 @@ track_reference(space, py_obj, w_obj) return w_obj - at cpython_api([PyObject], lltype.Void, external=False) def string_dealloc(space, py_obj): """Frees allocated PyStringObject resources. """ Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Sun May 16 21:40:10 2010 @@ -11,8 +11,8 @@ from pypy.objspace.std.typetype import _precheck_for_new from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.module.cpyext.api import ( - cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, - generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING, + cpython_api, make_static_function, cpython_struct, bootstrap_function, + Py_ssize_t, generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING, Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL, PyBufferProcs, build_type_checkers) from pypy.module.cpyext.pyobject import ( @@ -155,7 +155,7 @@ if pto.c_tp_new: add_tp_new_wrapper(space, dict_w, pto) - at cpython_api([PyObject, PyObject, PyObject], PyObject, external=False) + at make_static_function([PyObject, PyObject, PyObject], PyObject) def tp_new_wrapper(space, self, w_args, w_kwds): tp_new = rffi.cast(PyTypeObjectPtr, self).c_tp_new @@ -405,7 +405,6 @@ track_reference(space, py_tuple, space.w_tuple, replace=True) - at cpython_api([PyObject], lltype.Void, external=False) def subtype_dealloc(space, obj): pto = obj.c_ob_type base = pto @@ -433,15 +432,14 @@ lifeline_dict.set(w_obj, PyOLifeline(space, py_obj)) return py_obj - at cpython_api([PyObject, rffi.INTP], lltype.Signed, external=False, - error=CANNOT_FAIL) + at make_static_function([PyObject, rffi.INTP], lltype.Signed, error=CANNOT_FAIL) def str_segcount(space, w_obj, ref): if ref: ref[0] = rffi.cast(rffi.INT, space.int_w(space.len(w_obj))) return 1 - at cpython_api([PyObject, lltype.Signed, rffi.VOIDPP], lltype.Signed, - external=False, error=-1) + at make_static_function([PyObject, lltype.Signed, rffi.VOIDPP], lltype.Signed, + error=-1) def str_getreadbuffer(space, w_str, segment, ref): from pypy.module.cpyext.stringobject import PyString_AsString if segment != 0: @@ -461,7 +459,6 @@ str_getreadbuffer.api_func.get_wrapper(space)) pto.c_tp_as_buffer = c_buf - at cpython_api([PyObject], lltype.Void, external=False) def type_dealloc(space, obj): obj_pto = rffi.cast(PyTypeObjectPtr, obj) type_pto = obj.c_ob_type Modified: pypy/trunk/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/unicodeobject.py Sun May 16 21:40:10 2010 @@ -42,7 +42,6 @@ py_unicode.c_size = len(space.unicode_w(w_obj)) py_unicode.c_buffer = lltype.nullptr(rffi.CWCHARP.TO) - at cpython_api([PyObject], lltype.Void, external=False) def unicode_dealloc(space, py_obj): py_unicode = rffi.cast(PyUnicodeObject, py_obj) if py_unicode.c_buffer: From agaynor at codespeak.net Sun May 16 21:47:31 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sun, 16 May 2010 21:47:31 +0200 (CEST) Subject: [pypy-svn] r74518 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100516194731.54EF5282BAD@codespeak.net> Author: agaynor Date: Sun May 16 21:47:29 2010 New Revision: 74518 Added: pypy/trunk/pypy/module/cpyext/test/test_cell.py Modified: pypy/trunk/pypy/module/cpyext/api.py Log: Added PyCell_Type to cpyext, with tests. Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Sun May 16 21:47:29 2010 @@ -17,6 +17,7 @@ from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.gateway import ObjSpace, unwrap_spec +from pypy.interpreter.nestedscope import Cell from pypy.rlib.entrypoint import entrypoint from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.objectmodel import specialize @@ -329,6 +330,7 @@ "BaseObject": "space.w_object", 'None': 'space.type(space.w_None)', 'NotImplemented': 'space.type(space.w_NotImplemented)', + 'Cell': 'space.gettypeobject(Cell.typedef)', }.items(): GLOBALS['Py%s_Type#' % (cpyname, )] = ('PyTypeObject*', pypyexpr) @@ -502,6 +504,7 @@ print str(e) # we can't do much here, since we're in ctypes, swallow else: + print str(e) pypy_debug_catch_fatal_exception() rffi.stackcounter.stacks_counter -= 1 return retval Added: pypy/trunk/pypy/module/cpyext/test/test_cell.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/cpyext/test/test_cell.py Sun May 16 21:47:29 2010 @@ -0,0 +1,20 @@ +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase + + +class AppTestCell(AppTestCpythonExtensionBase): + def test_cell_type(self): + module = self.import_extension('foo', [ + ("cell_type", "METH_O", + """ + PyDict_SetItemString(args, "cell", &PyCell_Type); + Py_RETURN_NONE; + """)]) + d = {} + module.cell_type(d) + def f(o): + def g(): + return o + return g + + cell_type = type(f(0).func_closure[0]) + assert d["cell"] is cell_type From afa at codespeak.net Sun May 16 22:57:19 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sun, 16 May 2010 22:57:19 +0200 (CEST) Subject: [pypy-svn] r74519 - pypy/trunk/pypy/module/cpyext Message-ID: <20100516205719.DCB7B282BAD@codespeak.net> Author: afa Date: Sun May 16 22:57:18 2010 New Revision: 74519 Modified: pypy/trunk/pypy/module/cpyext/pyobject.py pypy/trunk/pypy/module/cpyext/typeobject.py Log: use get_llhelper() whenever possible, this may fix translation because this function uses a memo (got duplicate definitions for PyObject_dealloc) Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Sun May 16 22:57:18 2010 @@ -56,9 +56,7 @@ realize = tp_realize def get_dealloc(self, space): - return llhelper( - tp_dealloc.api_func.functype, - tp_dealloc.api_func.get_wrapper(space)) + return tp_dealloc.api_func.get_llhelper(space) def allocate(self, space, w_type, itemcount=0): # similar to PyType_GenericAlloc? Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Sun May 16 22:57:18 2010 @@ -115,8 +115,7 @@ if WARN_ABOUT_MISSING_SLOT_FUNCTIONS: os.write(2, method_name + " defined by the type but no slot function defined!\n") continue - slot_func_helper = llhelper(slot_func.api_func.functype, - slot_func.api_func.get_wrapper(space)) + slot_func_helper = slot_func.api_func.get_llhelper(space) # XXX special case wrapper-functions and use a "specific" slot func # the special case of __new__ in CPython works a bit differently, hopefully @@ -191,8 +190,7 @@ def setup_new_method_def(space): ptr = get_new_method_def(space) ptr.c_ml_meth = rffi.cast(PyCFunction, - llhelper(tp_new_wrapper.api_func.functype, - tp_new_wrapper.api_func.get_wrapper(space))) + tp_new_wrapper.api_func.get_llhelper(space)) def add_tp_new_wrapper(space, dict_w, pto): if "__new__" in dict_w: @@ -408,8 +406,7 @@ def subtype_dealloc(space, obj): pto = obj.c_ob_type base = pto - this_func_ptr = llhelper(subtype_dealloc.api_func.functype, - subtype_dealloc.api_func.get_wrapper(space)) + this_func_ptr = subtype_dealloc.api_func.get_llhelper(space) while base.c_tp_dealloc == this_func_ptr: base = base.c_tp_base assert base @@ -453,10 +450,8 @@ def setup_string_buffer_procs(space, pto): c_buf = lltype.malloc(PyBufferProcs, flavor='raw', zero=True) - c_buf.c_bf_getsegcount = llhelper(str_segcount.api_func.functype, - str_segcount.api_func.get_wrapper(space)) - c_buf.c_bf_getreadbuffer = llhelper(str_getreadbuffer.api_func.functype, - str_getreadbuffer.api_func.get_wrapper(space)) + c_buf.c_bf_getsegcount = str_segcount.api_func.get_llhelper(space) + c_buf.c_bf_getreadbuffer = str_getreadbuffer.api_func.get_llhelper(space) pto.c_tp_as_buffer = c_buf def type_dealloc(space, obj): @@ -496,10 +491,8 @@ setup_string_buffer_procs(space, pto) pto.c_tp_flags = Py_TPFLAGS_HEAPTYPE - pto.c_tp_free = llhelper(PyObject_Del.api_func.functype, - PyObject_Del.api_func.get_wrapper(space)) - pto.c_tp_alloc = llhelper(PyType_GenericAlloc.api_func.functype, - PyType_GenericAlloc.api_func.get_wrapper(space)) + pto.c_tp_free = PyObject_Del.api_func.get_llhelper(space) + pto.c_tp_alloc = PyType_GenericAlloc.api_func.get_llhelper(space) pto.c_tp_name = rffi.str2charp(w_type.getname(space, "?")) pto.c_tp_basicsize = -1 # hopefully this makes malloc bail out pto.c_tp_itemsize = 0 From afa at codespeak.net Sun May 16 23:06:32 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sun, 16 May 2010 23:06:32 +0200 (CEST) Subject: [pypy-svn] r74520 - pypy/trunk/pypy/module/cpyext Message-ID: <20100516210632.AD2B3282BAD@codespeak.net> Author: afa Date: Sun May 16 23:06:31 2010 New Revision: 74520 Modified: pypy/trunk/pypy/module/cpyext/api.py Log: More translation fixes Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Sun May 16 23:06:31 2010 @@ -124,6 +124,7 @@ def _freeze_(self): return True + @specialize.memo() def get_llhelper(self, space): llh = getattr(self, '_llhelper', None) if llh is None: From afa at codespeak.net Sun May 16 23:06:52 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sun, 16 May 2010 23:06:52 +0200 (CEST) Subject: [pypy-svn] r74521 - pypy/trunk/pypy/module/cpyext Message-ID: <20100516210652.E502C282BAD@codespeak.net> Author: afa Date: Sun May 16 23:06:51 2010 New Revision: 74521 Modified: pypy/trunk/pypy/module/cpyext/pyobject.py pypy/trunk/pypy/module/cpyext/typeobject.py Log: remove unused imports Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Sun May 16 23:06:51 2010 @@ -8,7 +8,6 @@ from pypy.module.cpyext.state import State from pypy.objspace.std.typeobject import W_TypeObject from pypy.rlib.objectmodel import specialize, we_are_translated -from pypy.rpython.annlowlevel import llhelper #________________________________________________________ # type description Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Sun May 16 23:06:51 2010 @@ -2,7 +2,6 @@ import sys from pypy.rpython.lltypesystem import rffi, lltype -from pypy.rpython.annlowlevel import llhelper from pypy.rlib.rweakref import RWeakKeyDictionary from pypy.interpreter.gateway import ObjSpace, W_Root, Arguments from pypy.interpreter.gateway import interp2app, unwrap_spec From afa at codespeak.net Sun May 16 23:41:50 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sun, 16 May 2010 23:41:50 +0200 (CEST) Subject: [pypy-svn] r74522 - pypy/trunk/pypy/module/cpyext Message-ID: <20100516214150.3B1A4282BAD@codespeak.net> Author: afa Date: Sun May 16 23:41:48 2010 New Revision: 74522 Modified: pypy/trunk/pypy/module/cpyext/api.py Log: Use the dedicated function Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Sun May 16 23:41:48 2010 @@ -33,7 +33,7 @@ # update these for other platforms Py_ssize_t = lltype.Signed -Py_ssize_tP = lltype.Ptr(lltype.Array(Py_ssize_t, hints={'nolength': True})) +Py_ssize_tP = rffi.CArrayPtr(Py_ssize_t) size_t = rffi.ULONG ADDR = lltype.Signed From afa at codespeak.net Mon May 17 00:10:52 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 17 May 2010 00:10:52 +0200 (CEST) Subject: [pypy-svn] r74523 - pypy/trunk/pypy/module/cpyext Message-ID: <20100516221052.F131D282BAD@codespeak.net> Author: afa Date: Mon May 17 00:10:50 2010 New Revision: 74523 Modified: pypy/trunk/pypy/module/cpyext/api.py pypy/trunk/pypy/module/cpyext/funcobject.py pypy/trunk/pypy/module/cpyext/methodobject.py pypy/trunk/pypy/module/cpyext/pyobject.py pypy/trunk/pypy/module/cpyext/slotdefs.py pypy/trunk/pypy/module/cpyext/stringobject.py pypy/trunk/pypy/module/cpyext/typeobject.py pypy/trunk/pypy/module/cpyext/unicodeobject.py Log: Revert my changes since r74517: they break translation. I'll think harder about it. Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Mon May 17 00:10:50 2010 @@ -124,7 +124,6 @@ def _freeze_(self): return True - @specialize.memo() def get_llhelper(self, space): llh = getattr(self, '_llhelper', None) if llh is None: @@ -141,27 +140,7 @@ wrapper.relax_sig_check = True return wrapper -def make_static_function(argtypes, restype, error=_NOT_SPECIFIED): - """ - Helper to build a static function pointer. - """ - if error is _NOT_SPECIFIED: - if restype is PyObject: - error = lltype.nullptr(PyObject.TO) - elif restype is lltype.Void: - error = CANNOT_FAIL - if type(error) is int: - error = rffi.cast(restype, error) - - def decorate(func): - func_name = func.func_name - api_function = ApiFunction(argtypes, restype, func, error) - func.api_func = api_function - FUNCTIONS_STATIC[func_name] = api_function - return func - return decorate - -def cpython_api(argtypes, restype, error=_NOT_SPECIFIED): +def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, external=True): """ Declares a function to be exported. - `argtypes`, `restype` are lltypes and describe the function signature. @@ -251,8 +230,6 @@ finally: for arg in to_decref: Py_DecRef(space, arg) - unwrapper = func_with_new_name(unwrapper, - "unwrapper_%s" % (func_name,)) unwrapper.func = func unwrapper.api_func = api_function unwrapper._always_inline_ = True @@ -260,7 +237,10 @@ unwrapper_catch = make_unwrapper(True) unwrapper_raise = make_unwrapper(False) - FUNCTIONS[func_name] = api_function + if external: + FUNCTIONS[func_name] = api_function + else: + FUNCTIONS_STATIC[func_name] = api_function INTERPLEVEL_API[func_name] = unwrapper_catch # used in tests return unwrapper_raise # used in 'normal' RPython code. return decorate Modified: pypy/trunk/pypy/module/cpyext/funcobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/funcobject.py (original) +++ pypy/trunk/pypy/module/cpyext/funcobject.py Mon May 17 00:10:50 2010 @@ -27,6 +27,7 @@ assert isinstance(w_obj, Function) py_func.c_func_name = make_ref(space, space.wrap(w_obj.name)) + at cpython_api([PyObject], lltype.Void, external=False) def function_dealloc(space, py_obj): py_func = rffi.cast(PyFunctionObject, py_obj) Py_DecRef(space, py_func.c_func_name) Modified: pypy/trunk/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/methodobject.py (original) +++ pypy/trunk/pypy/module/cpyext/methodobject.py Mon May 17 00:10:50 2010 @@ -49,6 +49,7 @@ py_func.c_m_ml = w_obj.ml py_func.c_m_self = make_ref(space, w_obj.w_self) + at cpython_api([PyObject], lltype.Void, external=False) def cfunction_dealloc(space, py_obj): py_func = rffi.cast(PyCFunctionObject, py_obj) Py_DecRef(space, py_func.c_m_self) Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Mon May 17 00:10:50 2010 @@ -2,12 +2,13 @@ from pypy.interpreter.baseobjspace import W_Root, SpaceCache from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import ( - cpython_api, make_static_function, bootstrap_function, PyObject, PyObjectP, - ADDR, CANNOT_FAIL, Py_TPFLAGS_HEAPTYPE, PyTypeObjectPtr) +from pypy.module.cpyext.api import cpython_api, bootstrap_function, \ + PyObject, PyObjectP, ADDR, CANNOT_FAIL, \ + Py_TPFLAGS_HEAPTYPE, PyTypeObjectPtr from pypy.module.cpyext.state import State from pypy.objspace.std.typeobject import W_TypeObject from pypy.rlib.objectmodel import specialize, we_are_translated +from pypy.rpython.annlowlevel import llhelper #________________________________________________________ # type description @@ -35,7 +36,7 @@ alloc : allocate and basic initialization of a raw PyObject attach : Function called to tie a raw structure to a pypy object realize : Function called to create a pypy object from a raw struct - dealloc : Function called to free a structure + dealloc : a cpython_api(external=False), similar to PyObject_dealloc """ tp_basestruct = kw.pop('basestruct', PyObject.TO) @@ -45,17 +46,22 @@ tp_dealloc = kw.pop('dealloc', None) assert not kw, "Extra arguments to make_typedescr" - if not tp_dealloc: - from pypy.module.cpyext.typeobject import subtype_dealloc - tp_dealloc = subtype_dealloc - tp_dealloc = make_static_function([PyObject], lltype.Void)(tp_dealloc) + null_dealloc = lltype.nullptr(lltype.FuncType([PyObject], lltype.Void)) class CpyTypedescr(BaseCpyTypedescr): basestruct = tp_basestruct realize = tp_realize def get_dealloc(self, space): - return tp_dealloc.api_func.get_llhelper(space) + if tp_dealloc: + return llhelper( + tp_dealloc.api_func.functype, + tp_dealloc.api_func.get_wrapper(space)) + else: + from pypy.module.cpyext.typeobject import subtype_dealloc + return llhelper( + subtype_dealloc.api_func.functype, + subtype_dealloc.api_func.get_wrapper(space)) def allocate(self, space, w_type, itemcount=0): # similar to PyType_GenericAlloc? @@ -118,7 +124,7 @@ from pypy.module.cpyext.object import PyObject_dealloc # typedescr for the 'object' type make_typedescr(space.w_object.instancetypedef, - dealloc=PyObject_dealloc.func) + dealloc=PyObject_dealloc) # almost all types, which should better inherit from object. make_typedescr(None) Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/slotdefs.py (original) +++ pypy/trunk/pypy/module/cpyext/slotdefs.py Mon May 17 00:10:50 2010 @@ -106,8 +106,7 @@ raise OperationError(space.w_StopIteration, space.w_None) return w_res -# XXX why is it a "cpython_api"? - at cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject) + at cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject, external=True) def slot_tp_new(space, type, w_args, w_kwds): from pypy.module.cpyext.tupleobject import PyTuple_Check pyo = rffi.cast(PyObject, type) Modified: pypy/trunk/pypy/module/cpyext/stringobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/stringobject.py (original) +++ pypy/trunk/pypy/module/cpyext/stringobject.py Mon May 17 00:10:50 2010 @@ -102,6 +102,7 @@ track_reference(space, py_obj, w_obj) return w_obj + at cpython_api([PyObject], lltype.Void, external=False) def string_dealloc(space, py_obj): """Frees allocated PyStringObject resources. """ Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Mon May 17 00:10:50 2010 @@ -2,6 +2,7 @@ import sys from pypy.rpython.lltypesystem import rffi, lltype +from pypy.rpython.annlowlevel import llhelper from pypy.rlib.rweakref import RWeakKeyDictionary from pypy.interpreter.gateway import ObjSpace, W_Root, Arguments from pypy.interpreter.gateway import interp2app, unwrap_spec @@ -10,8 +11,8 @@ from pypy.objspace.std.typetype import _precheck_for_new from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.module.cpyext.api import ( - cpython_api, make_static_function, cpython_struct, bootstrap_function, - Py_ssize_t, generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING, + cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, + generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING, Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL, PyBufferProcs, build_type_checkers) from pypy.module.cpyext.pyobject import ( @@ -114,7 +115,8 @@ if WARN_ABOUT_MISSING_SLOT_FUNCTIONS: os.write(2, method_name + " defined by the type but no slot function defined!\n") continue - slot_func_helper = slot_func.api_func.get_llhelper(space) + slot_func_helper = llhelper(slot_func.api_func.functype, + slot_func.api_func.get_wrapper(space)) # XXX special case wrapper-functions and use a "specific" slot func # the special case of __new__ in CPython works a bit differently, hopefully @@ -153,7 +155,7 @@ if pto.c_tp_new: add_tp_new_wrapper(space, dict_w, pto) - at make_static_function([PyObject, PyObject, PyObject], PyObject) + at cpython_api([PyObject, PyObject, PyObject], PyObject, external=False) def tp_new_wrapper(space, self, w_args, w_kwds): tp_new = rffi.cast(PyTypeObjectPtr, self).c_tp_new @@ -189,7 +191,8 @@ def setup_new_method_def(space): ptr = get_new_method_def(space) ptr.c_ml_meth = rffi.cast(PyCFunction, - tp_new_wrapper.api_func.get_llhelper(space)) + llhelper(tp_new_wrapper.api_func.functype, + tp_new_wrapper.api_func.get_wrapper(space))) def add_tp_new_wrapper(space, dict_w, pto): if "__new__" in dict_w: @@ -402,10 +405,12 @@ track_reference(space, py_tuple, space.w_tuple, replace=True) + at cpython_api([PyObject], lltype.Void, external=False) def subtype_dealloc(space, obj): pto = obj.c_ob_type base = pto - this_func_ptr = subtype_dealloc.api_func.get_llhelper(space) + this_func_ptr = llhelper(subtype_dealloc.api_func.functype, + subtype_dealloc.api_func.get_wrapper(space)) while base.c_tp_dealloc == this_func_ptr: base = base.c_tp_base assert base @@ -428,14 +433,15 @@ lifeline_dict.set(w_obj, PyOLifeline(space, py_obj)) return py_obj - at make_static_function([PyObject, rffi.INTP], lltype.Signed, error=CANNOT_FAIL) + at cpython_api([PyObject, rffi.INTP], lltype.Signed, external=False, + error=CANNOT_FAIL) def str_segcount(space, w_obj, ref): if ref: ref[0] = rffi.cast(rffi.INT, space.int_w(space.len(w_obj))) return 1 - at make_static_function([PyObject, lltype.Signed, rffi.VOIDPP], lltype.Signed, - error=-1) + at cpython_api([PyObject, lltype.Signed, rffi.VOIDPP], lltype.Signed, + external=False, error=-1) def str_getreadbuffer(space, w_str, segment, ref): from pypy.module.cpyext.stringobject import PyString_AsString if segment != 0: @@ -449,10 +455,13 @@ def setup_string_buffer_procs(space, pto): c_buf = lltype.malloc(PyBufferProcs, flavor='raw', zero=True) - c_buf.c_bf_getsegcount = str_segcount.api_func.get_llhelper(space) - c_buf.c_bf_getreadbuffer = str_getreadbuffer.api_func.get_llhelper(space) + c_buf.c_bf_getsegcount = llhelper(str_segcount.api_func.functype, + str_segcount.api_func.get_wrapper(space)) + c_buf.c_bf_getreadbuffer = llhelper(str_getreadbuffer.api_func.functype, + str_getreadbuffer.api_func.get_wrapper(space)) pto.c_tp_as_buffer = c_buf + at cpython_api([PyObject], lltype.Void, external=False) def type_dealloc(space, obj): obj_pto = rffi.cast(PyTypeObjectPtr, obj) type_pto = obj.c_ob_type @@ -490,8 +499,10 @@ setup_string_buffer_procs(space, pto) pto.c_tp_flags = Py_TPFLAGS_HEAPTYPE - pto.c_tp_free = PyObject_Del.api_func.get_llhelper(space) - pto.c_tp_alloc = PyType_GenericAlloc.api_func.get_llhelper(space) + pto.c_tp_free = llhelper(PyObject_Del.api_func.functype, + PyObject_Del.api_func.get_wrapper(space)) + pto.c_tp_alloc = llhelper(PyType_GenericAlloc.api_func.functype, + PyType_GenericAlloc.api_func.get_wrapper(space)) pto.c_tp_name = rffi.str2charp(w_type.getname(space, "?")) pto.c_tp_basicsize = -1 # hopefully this makes malloc bail out pto.c_tp_itemsize = 0 Modified: pypy/trunk/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/unicodeobject.py Mon May 17 00:10:50 2010 @@ -42,6 +42,7 @@ py_unicode.c_size = len(space.unicode_w(w_obj)) py_unicode.c_buffer = lltype.nullptr(rffi.CWCHARP.TO) + at cpython_api([PyObject], lltype.Void, external=False) def unicode_dealloc(space, py_obj): py_unicode = rffi.cast(PyUnicodeObject, py_obj) if py_unicode.c_buffer: From agaynor at codespeak.net Mon May 17 15:33:20 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Mon, 17 May 2010 15:33:20 +0200 (CEST) Subject: [pypy-svn] r74526 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100517133320.C771B282BE0@codespeak.net> Author: agaynor Date: Mon May 17 15:33:18 2010 New Revision: 74526 Modified: pypy/trunk/pypy/module/cpyext/stubs.py pypy/trunk/pypy/module/cpyext/test/test_unicodeobject.py pypy/trunk/pypy/module/cpyext/unicodeobject.py Log: Implemented PyUnicode_DecodeUTF8 in cpyext Modified: pypy/trunk/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/stubs.py (original) +++ pypy/trunk/pypy/module/cpyext/stubs.py Mon May 17 15:33:18 2010 @@ -3091,15 +3091,6 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP], PyObject) -def PyUnicode_DecodeUTF8(space, s, size, errors): - """Create a Unicode object by decoding size bytes of the UTF-8 encoded string - s. Return NULL if an exception was raised by the codec. - - This function used an int type for size. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, Py_ssize_t], PyObject) def PyUnicode_DecodeUTF8Stateful(space, s, size, errors, consumed): """If consumed is NULL, behave like PyUnicode_DecodeUTF8(). If Modified: pypy/trunk/pypy/module/cpyext/test/test_unicodeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_unicodeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_unicodeobject.py Mon May 17 15:33:18 2010 @@ -63,6 +63,17 @@ w_res = api.PyUnicode_AsUTF8String(w_u) assert space.type(w_res) is space.w_str assert space.unwrap(w_res) == 'sp\xc3\xa4m' + + def test_decode_utf8(self, space, api): + u = rffi.str2charp(u'sp?m'.encode("utf-8")) + w_u = api.PyUnicode_DecodeUTF8(u, 5, None) + assert space.type(w_u) is space.w_unicode + assert space.unwrap(w_u) == u'sp?m' + + w_u = api.PyUnicode_DecodeUTF8(u, 2, None) + assert space.type(w_u) is space.w_unicode + assert space.unwrap(w_u) == 'sp' + rffi.free_charp(u) def test_IS(self, space, api): for char in [0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x1c, 0x1d, 0x1e, 0x1f, Modified: pypy/trunk/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/unicodeobject.py Mon May 17 15:33:18 2010 @@ -288,6 +288,18 @@ PyErr_BadArgument(space) return unicodetype.encode_object(space, w_unicode, "utf-8", "strict") + at cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP], PyObject) +def PyUnicode_DecodeUTF8(space, s, size, errors): + """Create a Unicode object by decoding size bytes of the UTF-8 encoded string + s. Return NULL if an exception was raised by the codec. + """ + w_str = space.wrap(rffi.charpsize2str(s, size)) + if errors: + w_errors = space.wrap(rffi.charp2str(errors)) + else: + w_errors = space.w_None + return space.call_method(w_str, 'decode', space.wrap("utf-8"), w_errors) + if sys.platform == 'win32': @cpython_api([CONST_WSTRING, Py_ssize_t, CONST_STRING], PyObject) def PyUnicode_EncodeMBCS(space, wchar_p, length, errors): From afa at codespeak.net Mon May 17 16:29:55 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 17 May 2010 16:29:55 +0200 (CEST) Subject: [pypy-svn] r74527 - pypy/trunk/pypy/module/cpyext Message-ID: <20100517142955.3C36C282BE0@codespeak.net> Author: afa Date: Mon May 17 16:29:52 2010 New Revision: 74527 Modified: pypy/trunk/pypy/module/cpyext/unicodeobject.py Log: This function can use CONST_STRING => 'const char *' Modified: pypy/trunk/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/unicodeobject.py Mon May 17 16:29:52 2010 @@ -288,7 +288,7 @@ PyErr_BadArgument(space) return unicodetype.encode_object(space, w_unicode, "utf-8", "strict") - at cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP], PyObject) + at cpython_api([CONST_STRING, Py_ssize_t, CONST_STRING], PyObject) def PyUnicode_DecodeUTF8(space, s, size, errors): """Create a Unicode object by decoding size bytes of the UTF-8 encoded string s. Return NULL if an exception was raised by the codec. @@ -299,7 +299,7 @@ else: w_errors = space.w_None return space.call_method(w_str, 'decode', space.wrap("utf-8"), w_errors) - + if sys.platform == 'win32': @cpython_api([CONST_WSTRING, Py_ssize_t, CONST_STRING], PyObject) def PyUnicode_EncodeMBCS(space, wchar_p, length, errors): From jcreigh at codespeak.net Mon May 17 16:33:51 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Mon, 17 May 2010 16:33:51 +0200 (CEST) Subject: [pypy-svn] r74528 - pypy/branch/x86-64-jit-backend Message-ID: <20100517143351.0133A282BE0@codespeak.net> Author: jcreigh Date: Mon May 17 16:33:50 2010 New Revision: 74528 Added: pypy/branch/x86-64-jit-backend/ (props changed) - copied from r74527, pypy/trunk/ Log: Add branch for developing x86-64 JIT backend From afa at codespeak.net Mon May 17 16:35:34 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 17 May 2010 16:35:34 +0200 (CEST) Subject: [pypy-svn] r74529 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100517143534.CC8AC282BE0@codespeak.net> Author: afa Date: Mon May 17 16:35:32 2010 New Revision: 74529 Modified: pypy/trunk/pypy/module/cpyext/object.py pypy/trunk/pypy/module/cpyext/stubs.py pypy/trunk/pypy/module/cpyext/test/test_bufferobject.py pypy/trunk/pypy/module/cpyext/test/test_stringobject.py Log: PyObject_AsCharBuffer, implemented only for strings and PyBuffer objects Modified: pypy/trunk/pypy/module/cpyext/object.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/object.py (original) +++ pypy/trunk/pypy/module/cpyext/object.py Mon May 17 16:35:32 2010 @@ -1,7 +1,8 @@ from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import cpython_api, generic_cpy_call, CANNOT_FAIL,\ - Py_ssize_t, PyVarObject, Py_TPFLAGS_HEAPTYPE,\ - Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT, Py_GE, CONST_STRING +from pypy.module.cpyext.api import ( + cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP, + PyVarObject, Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT, + Py_GE, CONST_STRING) from pypy.module.cpyext.pyobject import ( PyObject, PyObjectP, create_ref, from_ref, Py_IncRef, Py_DecRef, track_reference) @@ -341,3 +342,29 @@ Compute and return the hash value of an object o. On failure, return -1. This is the equivalent of the Python expression hash(o).""" return space.int_w(space.hash(w_obj)) + + at cpython_api([PyObject, rffi.CCHARPP, Py_ssize_tP], rffi.INT_real, error=-1) +def PyObject_AsCharBuffer(space, obj, bufferp, sizep): + """Returns a pointer to a read-only memory location usable as + character-based input. The obj argument must support the single-segment + character buffer interface. On success, returns 0, sets buffer to the + memory location and size to the buffer length. Returns -1 and sets a + TypeError on error. + """ + pto = obj.c_ob_type + + pb = pto.c_tp_as_buffer + if not (pb and pb.c_bf_getreadbuffer and pb.c_bf_getsegcount): + raise OperationError(space.w_TypeError, space.wrap( + "expected a character buffer object")) + if generic_cpy_call(space, pb.c_bf_getsegcount, + obj, lltype.nullptr(rffi.INTP.TO)) != 1: + raise OperationError(space.w_TypeError, space.wrap( + "expected a single-segment buffer object")) + size = generic_cpy_call(space, pb.c_bf_getreadbuffer, + obj, 0, bufferp) + if size < 0: + return -1 + sizep[0] = size + return 0 + Modified: pypy/trunk/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/stubs.py (original) +++ pypy/trunk/pypy/module/cpyext/stubs.py Mon May 17 16:35:32 2010 @@ -2373,20 +2373,6 @@ """ raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARPP, Py_ssize_t], rffi.INT_real, error=-1) -def PyObject_AsCharBuffer(space, obj, buffer, buffer_len): - """Returns a pointer to a read-only memory location usable as character-based - input. The obj argument must support the single-segment character buffer - interface. On success, returns 0, sets buffer to the memory location - and buffer_len to the buffer length. Returns -1 and sets a - TypeError on error. - - - - This function used an int * type for buffer_len. This might - require changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([PyObject, rffi.CCHARP], rffi.INT_real, error=-1) def PyObject_DelAttrString(space, o, attr_name): """Delete attribute named attr_name, for object o. Returns -1 on failure. Modified: pypy/trunk/pypy/module/cpyext/test/test_bufferobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_bufferobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_bufferobject.py Mon May 17 16:35:32 2010 @@ -20,10 +20,21 @@ """ free(cbuf); Py_RETURN_NONE; + """), + ("check_ascharbuffer", "METH_O", + """ + char *ptr; + Py_ssize_t size; + if (PyObject_AsCharBuffer(args, &ptr, &size) < 0) + return NULL; + return PyString_FromStringAndSize(ptr, size); """) ], prologue = """ static char* cbuf = NULL; """) - buffer = module.get_FromMemory() - assert str(buffer) == 'abc\0' + buf = module.get_FromMemory() + assert str(buf) == 'abc\0' + + assert module.check_ascharbuffer(buf) == 'abc\0' + module.free_buffer() Modified: pypy/trunk/pypy/module/cpyext/test/test_stringobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_stringobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_stringobject.py Mon May 17 16:35:32 2010 @@ -2,7 +2,7 @@ from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.stringobject import new_empty_str, PyStringObject -from pypy.module.cpyext.api import PyObjectP, PyObject +from pypy.module.cpyext.api import PyObjectP, PyObject, Py_ssize_tP from pypy.module.cpyext.pyobject import Py_DecRef, from_ref, make_ref import py @@ -234,3 +234,15 @@ def test_format(self, space, api): assert "1 2" == space.unwrap( api.PyString_Format(space.wrap('%s %d'), space.wrap((1, 2)))) + + def test_asbuffer(self, space, api): + bufp = lltype.malloc(rffi.VOIDPP.TO, 1, flavor='raw') + lenp = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw') + + w_text = space.wrap("text") + assert api.PyObject_AsCharBuffer(w_text, bufp, lenp) == 0 + assert lenp[0] == 4 + assert rffi.charp2str(bufp[0]) == 'text' + + lltype.free(bufp, flavor='raw') + lltype.free(lenp, flavor='raw') From cfbolz at codespeak.net Mon May 17 16:41:47 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 17 May 2010 16:41:47 +0200 (CEST) Subject: [pypy-svn] r74530 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20100517144147.779C2282BE0@codespeak.net> Author: cfbolz Date: Mon May 17 16:41:46 2010 New Revision: 74530 Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Log: an optimization for int_and and int_or: a & 0 = 0, a | 0 = a Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Mon May 17 16:41:46 2010 @@ -954,7 +954,25 @@ self.optimize_default(op) resvalue = self.getvalue(op.result) self.loop_invariant_results[op.args[0].getint()] = resvalue - + + def optimize_INT_AND(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + if v1.is_null() or v2.is_null(): + self.make_constant_int(op.result, 0) + else: + self.optimize_default(op) + + def optimize_INT_OR(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + if v1.is_null(): + self.make_equal_to(op.result, v2) + elif v2.is_null(): + self.make_equal_to(op.result, v1) + else: + self.optimize_default(op) + optimize_ops = _findall(Optimizer, 'optimize_') Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Mon May 17 16:41:46 2010 @@ -1999,6 +1999,22 @@ """ self.optimize_loop(ops, 'Not', expected) + def test_int_and_or_with_zero(self): + ops = """ + [i0, i1] + i2 = int_and(i0, 0) + i3 = int_and(0, i2) + i4 = int_or(i2, i1) + i5 = int_or(i0, i3) + jump(i4, i5) + """ + expected = """ + [i0, i1] + jump(i1, i0) + """ + self.optimize_loop(ops, 'Not, Not', expected) + + # ---------- def make_fail_descr(self): From agaynor at codespeak.net Mon May 17 17:35:40 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Mon, 17 May 2010 17:35:40 +0200 (CEST) Subject: [pypy-svn] r74531 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100517153540.B99E0282BE0@codespeak.net> Author: agaynor Date: Mon May 17 17:35:39 2010 New Revision: 74531 Added: pypy/trunk/pypy/module/cpyext/test/test_iterator.py Modified: pypy/trunk/pypy/module/cpyext/iterator.py pypy/trunk/pypy/module/cpyext/stubs.py Log: Implemented PyIter_Check for cpyext. Modified: pypy/trunk/pypy/module/cpyext/iterator.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/iterator.py (original) +++ pypy/trunk/pypy/module/cpyext/iterator.py Mon May 17 17:35:39 2010 @@ -1,6 +1,8 @@ -from pypy.module.cpyext.api import generic_cpy_call, cpython_api, PyObject from pypy.interpreter.error import OperationError +from pypy.module.cpyext.api import (generic_cpy_call, cpython_api, PyObject, + CANNOT_FAIL) import pypy.module.__builtin__.operation as operation +from pypy.rpython.lltypesystem import rffi @cpython_api([PyObject, PyObject], PyObject) @@ -12,3 +14,12 @@ """ return operation.iter_sentinel(space, w_callable, w_sentinel) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) +def PyIter_Check(space, w_obj): + """Return true if the object o supports the iterator protocol.""" + try: + w_attr = space.getattr(space.type(w_obj), space.wrap("next")) + except: + return False + else: + return space.is_true(space.callable(w_attr)) Modified: pypy/trunk/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/stubs.py (original) +++ pypy/trunk/pypy/module/cpyext/stubs.py Mon May 17 17:35:39 2010 @@ -2084,11 +2084,6 @@ raise NotImplementedError @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyIter_Check(space, o): - """Return true if the object o supports the iterator protocol.""" - raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PySeqIter_Check(space, seq): """Return true if the type of op is PySeqIter_Type. """ Added: pypy/trunk/pypy/module/cpyext/test/test_iterator.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/cpyext/test/test_iterator.py Mon May 17 17:35:39 2010 @@ -0,0 +1,9 @@ +from pypy.module.cpyext.test.test_api import BaseApiTest + + +class TestIterator(BaseApiTest): + def test_check_iter(self, space, api): + assert api.PyIter_Check(space.wrap(iter("a"))) + assert api.PyIter_Check(space.wrap(iter([]))) + assert not api.PyIter_Check(space.wrap(type)) + assert not api.PyIter_Check(space.wrap(2)) From afa at codespeak.net Mon May 17 17:47:22 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 17 May 2010 17:47:22 +0200 (CEST) Subject: [pypy-svn] r74532 - in pypy/trunk/pypy/rpython: . module test Message-ID: <20100517154722.5BEB3282BE0@codespeak.net> Author: afa Date: Mon May 17 17:47:20 2010 New Revision: 74532 Modified: pypy/trunk/pypy/rpython/extfunc.py pypy/trunk/pypy/rpython/module/ll_os.py pypy/trunk/pypy/rpython/test/test_extfuncregister.py Log: hasattr(os, 'kill') is not enough to disable the function with CPython2.7 on Windows, which provides os.kill, with a special implementation of course. Add an additional "condition" parameter. Modified: pypy/trunk/pypy/rpython/extfunc.py ============================================================================== --- pypy/trunk/pypy/rpython/extfunc.py (original) +++ pypy/trunk/pypy/rpython/extfunc.py Mon May 17 17:47:20 2010 @@ -58,13 +58,16 @@ return method return decorator -def registering_if(ns, name): +def registering_if(ns, name, condition=True): try: func = getattr(ns, name) except AttributeError: - return lambda method: None - else: + condition = False + + if condition: return registering(func) + else: + return lambda method: None class LazyRegisteringMeta(type): def __new__(self, _name, _type, _vars): Modified: pypy/trunk/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/trunk/pypy/rpython/module/ll_os.py (original) +++ pypy/trunk/pypy/rpython/module/ll_os.py Mon May 17 17:47:20 2010 @@ -1329,7 +1329,7 @@ return extdef([int], int, llimpl=umask_llimpl, export_name="ll_os.ll_os_umask") - @registering_if(os, 'kill') + @registering_if(os, 'kill', sys.platform != 'win32') def register_os_kill(self): os_kill = self.llexternal('kill', [rffi.PID_T, rffi.INT], rffi.INT) Modified: pypy/trunk/pypy/rpython/test/test_extfuncregister.py ============================================================================== --- pypy/trunk/pypy/rpython/test/test_extfuncregister.py (original) +++ pypy/trunk/pypy/rpython/test/test_extfuncregister.py Mon May 17 17:47:20 2010 @@ -105,3 +105,9 @@ assert bar is None + @registering_if(A, 'f', False) + def baz(): + pass + + assert baz is None + From afa at codespeak.net Mon May 17 19:07:58 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 17 May 2010 19:07:58 +0200 (CEST) Subject: [pypy-svn] r74533 - pypy/trunk/pypy/module/posix Message-ID: <20100517170758.EA972282BE0@codespeak.net> Author: afa Date: Mon May 17 19:07:57 2010 New Revision: 74533 Modified: pypy/trunk/pypy/module/posix/__init__.py Log: Fix translation on windows with CPython2.7 Modified: pypy/trunk/pypy/module/posix/__init__.py ============================================================================== --- pypy/trunk/pypy/module/posix/__init__.py (original) +++ pypy/trunk/pypy/module/posix/__init__.py Mon May 17 19:07:57 2010 @@ -2,7 +2,7 @@ from pypy.interpreter.mixedmodule import MixedModule from pypy.rpython.module.ll_os import RegisterOs -import os +import os, sys exec 'import %s as posix' % os.name class Module(MixedModule): @@ -81,7 +81,7 @@ interpleveldefs['putenv'] = 'interp_posix.putenv' if hasattr(posix, 'unsetenv'): # note: emulated in os interpleveldefs['unsetenv'] = 'interp_posix.unsetenv' - if hasattr(os, 'kill'): + if hasattr(os, 'kill') and sys.platform != 'win32': interpleveldefs['kill'] = 'interp_posix.kill' interpleveldefs['abort'] = 'interp_posix.abort' if hasattr(os, 'getpid'): From afa at codespeak.net Mon May 17 19:13:44 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 17 May 2010 19:13:44 +0200 (CEST) Subject: [pypy-svn] r74534 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100517171344.E5356282BE0@codespeak.net> Author: afa Date: Mon May 17 19:13:43 2010 New Revision: 74534 Modified: pypy/trunk/pypy/module/cpyext/iterator.py pypy/trunk/pypy/module/cpyext/stubsactive.py pypy/trunk/pypy/module/cpyext/test/test_iterator.py Log: Provide real implementations of PyObject_GetIter() and PyIter_Next() Modified: pypy/trunk/pypy/module/cpyext/iterator.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/iterator.py (original) +++ pypy/trunk/pypy/module/cpyext/iterator.py Mon May 17 19:13:43 2010 @@ -14,6 +14,28 @@ """ return operation.iter_sentinel(space, w_callable, w_sentinel) + at cpython_api([PyObject], PyObject) +def PyObject_GetIter(space, w_obj): + """This is equivalent to the Python expression iter(o). It returns a new + iterator for the object argument, or the object itself if the object is + already an iterator. Raises TypeError and returns NULL if the object + cannot be iterated.""" + return space.iter(w_obj) + + at cpython_api([PyObject], PyObject, error=CANNOT_FAIL) +def PyIter_Next(space, w_obj): + """Return the next value from the iteration o. If the object is an + iterator, this retrieves the next value from the iteration, and returns + NULL with no exception set if there are no remaining items. If the object + is not an iterator, TypeError is raised, or if there is an error in + retrieving the item, returns NULL and passes along the exception.""" + try: + return space.next(w_obj) + except OperationError, e: + if not e.match(space, space.w_StopIteration): + raise + return None + @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyIter_Check(space, w_obj): """Return true if the object o supports the iterator protocol.""" Modified: pypy/trunk/pypy/module/cpyext/stubsactive.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/stubsactive.py (original) +++ pypy/trunk/pypy/module/cpyext/stubsactive.py Mon May 17 19:13:43 2010 @@ -13,23 +13,6 @@ PyFile_DecUseCount() functions described below as appropriate.""" raise NotImplementedError - at cpython_api([PyObject], PyObject) -def PyObject_GetIter(space, o): - """This is equivalent to the Python expression iter(o). It returns a new - iterator for the object argument, or the object itself if the object is already - an iterator. Raises TypeError and returns NULL if the object cannot be - iterated.""" - raise NotImplementedError - - at cpython_api([PyObject], PyObject) -def PyIter_Next(space, o): - """Return the next value from the iteration o. If the object is an iterator, - this retrieves the next value from the iteration, and returns NULL with no - exception set if there are no remaining items. If the object is not an - iterator, TypeError is raised, or if there is an error in retrieving the - item, returns NULL and passes along the exception.""" - raise NotImplementedError - @cpython_api([rffi.ULONG], PyObject) def PyLong_FromUnsignedLong(space, v): """Return a new PyLongObject object from a C unsigned long, or Modified: pypy/trunk/pypy/module/cpyext/test/test_iterator.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_iterator.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_iterator.py Mon May 17 19:13:43 2010 @@ -7,3 +7,11 @@ assert api.PyIter_Check(space.wrap(iter([]))) assert not api.PyIter_Check(space.wrap(type)) assert not api.PyIter_Check(space.wrap(2)) + + def test_getIter(self, space, api): + w_iter = api.PyObject_GetIter(space.wrap([1, 2, 3])) + assert space.unwrap(api.PyIter_Next(w_iter)) == 1 + assert space.unwrap(api.PyIter_Next(w_iter)) == 2 + assert space.unwrap(api.PyIter_Next(w_iter)) == 3 + assert api.PyIter_Next(w_iter) is None + assert not api.PyErr_Occurred() From afa at codespeak.net Mon May 17 19:26:11 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 17 May 2010 19:26:11 +0200 (CEST) Subject: [pypy-svn] r74535 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100517172611.A9A67282BE0@codespeak.net> Author: afa Date: Mon May 17 19:26:10 2010 New Revision: 74535 Modified: pypy/trunk/pypy/module/cpyext/longobject.py pypy/trunk/pypy/module/cpyext/stubsactive.py pypy/trunk/pypy/module/cpyext/test/test_longobject.py Log: Implement PyLong_FromUnsignedLong, don't know why it was put into "stubsactive" Modified: pypy/trunk/pypy/module/cpyext/longobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/longobject.py (original) +++ pypy/trunk/pypy/module/cpyext/longobject.py Mon May 17 19:26:10 2010 @@ -18,6 +18,12 @@ on failure.""" return space.wrap(val) + at cpython_api([rffi.ULONG], PyObject) +def PyLong_FromUnsignedLong(space, val): + """Return a new PyLongObject object from a C unsigned long, or + NULL on failure.""" + return space.wrap(val) + @cpython_api([rffi.ULONGLONG], PyObject) def PyLong_FromUnsignedLongLong(space, val): """Return a new PyLongObject object from a C unsigned long long, Modified: pypy/trunk/pypy/module/cpyext/stubsactive.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/stubsactive.py (original) +++ pypy/trunk/pypy/module/cpyext/stubsactive.py Mon May 17 19:26:10 2010 @@ -13,12 +13,6 @@ PyFile_DecUseCount() functions described below as appropriate.""" raise NotImplementedError - at cpython_api([rffi.ULONG], PyObject) -def PyLong_FromUnsignedLong(space, v): - """Return a new PyLongObject object from a C unsigned long, or - NULL on failure.""" - raise NotImplementedError - FILE = rffi.VOIDP_real.TO FILEP = lltype.Ptr(FILE) @cpython_api([PyObject, FILEP, rffi.INT_real], rffi.INT_real, error=-1) Modified: pypy/trunk/pypy/module/cpyext/test/test_longobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_longobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_longobject.py Mon May 17 19:26:10 2010 @@ -70,6 +70,14 @@ assert api.PyLong_AsVoidPtr(w_l) == lltype.nullptr(rffi.VOIDP_real.TO) class AppTestLongObject(AppTestCpythonExtensionBase): + def test_fromunsignedlong(self): + module = self.import_extension('foo', [ + ("from_unsignedlong", "METH_NOARGS", + """ + return PyLong_FromUnsignedLong((unsigned long)-1); + """)]) + assert module.from_unsignedlong() == (1<<32) - 1 + def test_fromlonglong(self): module = self.import_extension('foo', [ ("from_longlong", "METH_NOARGS", From fijal at codespeak.net Mon May 17 23:28:11 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 17 May 2010 23:28:11 +0200 (CEST) Subject: [pypy-svn] r74536 - pypy/trunk/pypy/rlib Message-ID: <20100517212811.60DC8282BE0@codespeak.net> Author: fijal Date: Mon May 17 23:28:07 2010 New Revision: 74536 Modified: pypy/trunk/pypy/rlib/rlocale.py Log: (WildChild) A patch to look for locale dev installed by macports as well Modified: pypy/trunk/pypy/rlib/rlocale.py ============================================================================== --- pypy/trunk/pypy/rlib/rlocale.py (original) +++ pypy/trunk/pypy/rlib/rlocale.py Mon May 17 23:28:07 2010 @@ -17,14 +17,19 @@ class CConfig: includes = ['locale.h', 'limits.h'] + include_dirs = [] + if HAVE_LANGINFO: includes += ['langinfo.h'] if HAVE_LIBINTL: includes += ['libintl.h'] if sys.platform == 'win32': includes += ['windows.h'] + if sys.platform == 'darwin': + include_dirs += ['/opt/local/include'] _compilation_info_ = ExternalCompilationInfo( includes=includes, + include_dirs=include_dirs ) HAVE_BIND_TEXTDOMAIN_CODESET = platform.Has('bind_textdomain_codeset') lconv = platform.Struct("struct lconv", [ From afa at codespeak.net Tue May 18 00:34:28 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 18 May 2010 00:34:28 +0200 (CEST) Subject: [pypy-svn] r74537 - pypy/trunk/pypy/module/cpyext Message-ID: <20100517223428.45B4F282BE0@codespeak.net> Author: afa Date: Tue May 18 00:34:26 2010 New Revision: 74537 Modified: pypy/trunk/pypy/module/cpyext/api.py Log: Remove tab Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Tue May 18 00:34:26 2010 @@ -17,7 +17,7 @@ from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.gateway import ObjSpace, unwrap_spec -from pypy.interpreter.nestedscope import Cell +from pypy.interpreter.nestedscope import Cell from pypy.rlib.entrypoint import entrypoint from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.objectmodel import specialize From afa at codespeak.net Tue May 18 10:57:28 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 18 May 2010 10:57:28 +0200 (CEST) Subject: [pypy-svn] r74539 - pypy/trunk/pypy/module/cpyext/test Message-ID: <20100518085728.03390282BE0@codespeak.net> Author: afa Date: Tue May 18 10:57:26 2010 New Revision: 74539 Modified: pypy/trunk/pypy/module/cpyext/test/test_api.py Log: Fix refcount checks when the test contains an applevel print: we force sys.stdout to allocate its file lock before tests are run. Modified: pypy/trunk/pypy/module/cpyext/test/test_api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_api.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_api.py Tue May 18 10:57:26 2010 @@ -22,12 +22,15 @@ # warm up reference counts: # - the posix module allocates a HCRYPTPROV on Windows - # - writing to stderr allocates a file lock + # - writing to stdout and stderr allocates a file lock space.getbuiltinmodule("cpyext") space.getbuiltinmodule(os.name) space.call_function(space.getattr(space.sys.get("stderr"), space.wrap("write")), space.wrap("")) + space.call_function(space.getattr(space.sys.get("stdout"), + space.wrap("write")), + space.wrap("")) class CAPI: def __getattr__(self, name): From jcreigh at codespeak.net Tue May 18 16:44:42 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Tue, 18 May 2010 16:44:42 +0200 (CEST) Subject: [pypy-svn] r74542 - in pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86: . test Message-ID: <20100518144442.C99E8282BE0@codespeak.net> Author: jcreigh Date: Tue May 18 16:44:38 2010 New Revision: 74542 Added: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py - copied unchanged from r74541, pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/rx86.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py - copied unchanged from r74541, pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py - copied unchanged from r74541, pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py - copied unchanged from r74541, pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/conftest.py Log: import rx86.py from remove-ri386-multimethod-2 branch Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/conftest.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/conftest.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/conftest.py Tue May 18 16:44:38 2010 @@ -3,6 +3,9 @@ cpu = detect_cpu.autodetect() def pytest_runtest_setup(item): - if cpu != 'x86': - py.test.skip("x86 directory skipped: cpu is %r" % (cpu,)) - + # XXX + if "test_rx86" in str(item.fspath): + # Don't skip + pass + elif cpu != 'x86': + py.test.skip("x86 tests skipped: cpu is %r" % (cpu,)) From arigo at codespeak.net Tue May 18 17:04:46 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 18 May 2010 17:04:46 +0200 (CEST) Subject: [pypy-svn] r74543 - in pypy/branch/blackhole-improvement/pypy/jit: backend backend/llgraph backend/x86 codewriter codewriter/test metainterp metainterp/test Message-ID: <20100518150446.EB5C3282BE0@codespeak.net> Author: arigo Date: Tue May 18 17:04:44 2010 New Revision: 74543 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py pypy/branch/blackhole-improvement/pypy/jit/backend/model.py pypy/branch/blackhole-improvement/pypy/jit/backend/x86/assembler.py pypy/branch/blackhole-improvement/pypy/jit/backend/x86/support.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitcode.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/typesystem.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmstate.py Log: Start hooking the blackhole interpreter to the rest of the metainterp. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py Tue May 18 17:04:44 2010 @@ -1063,17 +1063,38 @@ def frame_int_getvalue(frame, num): frame = _from_opaque(frame) + assert num >= 0 return frame.fail_args[num] def frame_float_getvalue(frame, num): frame = _from_opaque(frame) + assert num >= 0 return frame.fail_args[num] def frame_ptr_getvalue(frame, num): frame = _from_opaque(frame) - result = frame.fail_args[num] - frame.fail_args[num] = None - return result + assert num >= 0 + return frame.fail_args[num] + +def frame_get_value_kind(frame, num): + frame = _from_opaque(frame) + assert num >= 0 + TYPE = lltype.typeOf(frame.fail_args[num]) + if TYPE is lltype.Signed: + return INT + if TYPE == llmemory.GCREF: + return REF + if TYPE is lltype.Float: + return FLOAT + raise TypeError("frame.fail_args[%d] is of type %r" % (num, TYPE)) + +def get_latest_value_count(frame): + frame = _from_opaque(frame) + return len(frame.fail_args) + +def frame_clear_latest_values(frame): + frame = _from_opaque(frame) + del frame.fail_args _last_exception = None Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py Tue May 18 17:04:44 2010 @@ -243,26 +243,18 @@ def get_latest_value_float(self, index): return llimpl.frame_float_getvalue(self.latest_frame, index) + def get_latest_value_kind(self, index): + return llimpl.frame_get_value_kind(self.latest_frame, index) + + def get_latest_value_count(self): + return llimpl.frame_get_value_count(self.latest_frame) + def get_latest_force_token(self): token = llimpl.get_frame_forced_token(self.latest_frame) return self.cast_adr_to_int(token) - def make_boxes_from_latest_values(self, faildescr): - inputargs_and_holes = [] - for i in range(len(faildescr._fail_args_types)): - boxtype = faildescr._fail_args_types[i] - if boxtype == history.INT: - box = history.BoxInt(self.get_latest_value_int(i)) - elif boxtype == history.REF: - box = self.ts.BoxRef(self.get_latest_value_ref(i)) - elif boxtype == history.FLOAT: - box = history.BoxFloat(self.get_latest_value_float(i)) - elif boxtype == history.HOLE: - box = None - else: - assert False, "bad box type: num=%d" % ord(boxtype) - inputargs_and_holes.append(box) - return inputargs_and_holes + def clear_latest_values(self): + llimpl.frame_clear_latest_values(self.latest_frame) # ---------- Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/model.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/model.py Tue May 18 17:04:44 2010 @@ -85,15 +85,24 @@ or from 'args' if it was a FINISH). Returns a ptr or an obj.""" raise NotImplementedError + def get_latest_value_kind(self, index): + """Return the kind (history.INT, REF or FLOAT) of the index'th + argument to the last executed operation.""" + raise NotImplementedError + + def get_latest_value_count(self): + """Return how many values are ready to be returned by + get_latest_value_xxx().""" + raise NotImplementedError + def get_latest_force_token(self): """After a GUARD_NOT_FORCED fails, this function returns the same FORCE_TOKEN result as the one in the just-failed loop.""" raise NotImplementedError - def make_boxes_from_latest_value(self, faildescr): - """Build a list of Boxes (and None for holes) that contains - the current values, as would be returned by calls to - get_latest_value_xxx().""" + def clear_latest_values(self): + """Clear the latest values (at least the ref ones, so that + they no longer keep objects alive).""" raise NotImplementedError def get_exception(self): Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/x86/assembler.py Tue May 18 17:04:44 2010 @@ -1081,13 +1081,8 @@ n = self.CODE_HOLE mc.writechr(n) mc.writechr(self.CODE_STOP) - # preallocate the fail_boxes - i = len(failargs) - 1 - if i >= 0: - self.fail_boxes_int.get_addr_for_num(i) - self.fail_boxes_ptr.get_addr_for_num(i) - if self.cpu.supports_floats: - self.fail_boxes_float.get_addr_for_num(i) + # assert that the fail_boxes lists are big enough + assert len(failargs) <= self.fail_boxes_int.SIZE def rebuild_faillocs_from_descr(self, bytecode): from pypy.jit.backend.x86.regalloc import X86FrameManager Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/support.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/x86/support.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/x86/support.py Tue May 18 17:04:44 2010 @@ -4,6 +4,8 @@ ATP = lltype.GcArray(TP) class ValuesArray(object): + SIZE = size + def __init__(self): self.ar = lltype.malloc(ATP, size, zero=True, immortal=True) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Tue May 18 17:04:44 2010 @@ -47,6 +47,7 @@ self.liveness = {} self.nextlive = None self.startpoints = set() + self.alllabels = set() def emit_reg(self, reg): if reg.index >= self.count_regs[reg.kind]: @@ -113,6 +114,7 @@ else: argcodes.append(kind[0]) elif isinstance(x, TLabel): + self.alllabels.add(len(self.code)) self.tlabel_positions.append((x.name, len(self.code))) self.code.append("temp 1") self.code.append("temp 2") @@ -194,4 +196,5 @@ self.count_regs['ref'], self.count_regs['float'], liveness=self.liveness, - startpoints=self.startpoints) + startpoints=self.startpoints, + alllabels=self.alllabels) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Tue May 18 17:04:44 2010 @@ -210,8 +210,8 @@ assert block.exitswitch.concretetype == lltype.Bool opargs = [block.exitswitch] # - self.emitline(opname, TLabel(linkfalse), - *self.flatten_list(opargs)) + lst = self.flatten_list(opargs) + [TLabel(linkfalse)] + self.emitline(opname, *lst) # true path: self.make_link(linktrue) # false path: @@ -258,10 +258,10 @@ for switch in switches: # make the case described by 'switch' self.emitline('goto_if_not_int_eq', - TLabel(switch), color, Constant(switch.llexitcase, - block.exitswitch.concretetype)) + block.exitswitch.concretetype), + TLabel(switch)) # emit code for the "taken" path self.make_link(switch) # finally, emit the label for the "non-taken" path Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py Tue May 18 17:04:44 2010 @@ -16,31 +16,33 @@ self._ssarepr = None # debugging def setup(self, code='', constants_i=[], constants_r=[], constants_f=[], - num_regs_i=256, num_regs_r=256, num_regs_f=256, - liveness=None, startpoints=None): + num_regs_i=255, num_regs_r=255, num_regs_f=255, + liveness=None, startpoints=None, alllabels=None): self.code = code # if the following lists are empty, use a single shared empty list self.constants_i = constants_i or self._empty_i self.constants_r = constants_r or self._empty_r self.constants_f = constants_f or self._empty_f # encode the three num_regs into a single integer - self.num_regs_encoded = ((num_regs_i << 18) | - (num_regs_f << 9) | + assert num_regs_i < 256 and num_regs_r < 256 and num_regs_f < 256 + self.num_regs_encoded = ((num_regs_i << 16) | + (num_regs_f << 8) | (num_regs_r << 0)) self.liveness = liveness self._startpoints = startpoints # debugging + self._alllabels = alllabels # debugging def get_fnaddr_as_int(self): return llmemory.cast_adr_to_int(self.fnaddr) def num_regs_i(self): - return self.num_regs_encoded >> 18 + return self.num_regs_encoded >> 16 def num_regs_f(self): - return (self.num_regs_encoded >> 9) & 0x1FF + return (self.num_regs_encoded >> 8) & 0xFF def num_regs_r(self): - return self.num_regs_encoded & 0x1FF + return self.num_regs_encoded & 0xFF def has_liveness_info(self, pc): return pc in self.liveness Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py Tue May 18 17:04:44 2010 @@ -93,7 +93,7 @@ i0, i1 = Register('int', 0x16), Register('int', 0x17) ssarepr.insns = [ (Label('L1'),), - ('goto_if_not_int_gt', TLabel('L2'), i0, Constant(4, lltype.Signed)), + ('goto_if_not_int_gt', i0, Constant(4, lltype.Signed), TLabel('L2')), ('int_add', i1, i0, i1), ('int_sub', i0, Constant(1, lltype.Signed), i0), ('goto', TLabel('L1')), @@ -102,12 +102,12 @@ ] assembler = Assembler() jitcode = assembler.assemble(ssarepr) - assert jitcode.code == ("\x00\x10\x00\x16\x04" + assert jitcode.code == ("\x00\x16\x04\x10\x00" "\x01\x17\x16\x17" "\x02\x16\x01\x16" "\x03\x00\x00" "\x04\x17") - assert assembler.insns == {'goto_if_not_int_gt/Lic': 0, + assert assembler.insns == {'goto_if_not_int_gt/icL': 0, 'int_add/iii': 1, 'int_sub/ici': 2, 'goto/L': 3, Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py Tue May 18 17:04:44 2010 @@ -30,12 +30,12 @@ return b cw = CodeWriter() jitcode = cw.transform_func_to_jitcode(f, [5, 6]) - assert jitcode.code == ("\x00\x10\x00\x00\x00" # ends at 5 + assert jitcode.code == ("\x00\x00\x00\x10\x00" # ends at 5 "\x01\x01\x00\x01" "\x02\x00\x01\x00" "\x03\x00\x00" "\x04\x01") - assert cw.assembler.insns == {'goto_if_not_int_gt/Lic': 0, + assert cw.assembler.insns == {'goto_if_not_int_gt/icL': 0, 'int_add/iii': 1, 'int_sub/ici': 2, 'goto/L': 3, @@ -82,7 +82,8 @@ jitcode = cw.transform_func_to_jitcode(f, [5, 6]) blackholeinterpbuilder = BlackholeInterpBuilder(cw) blackholeinterp = blackholeinterpbuilder.acquire_interp() + blackholeinterp.setposition(jitcode, 0) blackholeinterp.setarg_i(0, 6) blackholeinterp.setarg_i(1, 100) - blackholeinterp.run(jitcode, 0) + blackholeinterp.run() assert blackholeinterp.get_result_i() == 100+6+5+4+3 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Tue May 18 17:04:44 2010 @@ -120,7 +120,7 @@ int_copy %i1, %i3 L1: int_gt %i2, $0, %i4 - goto_if_not L2, %i4 + goto_if_not %i4, L2 int_copy %i2, %i5 int_copy %i3, %i6 int_add %i6, %i5, %i7 @@ -142,7 +142,7 @@ int_copy %i0, %i2 int_copy %i1, %i3 L1: - goto_if_not_int_gt L2, %i2, $0 + goto_if_not_int_gt %i2, $0, L2 int_copy %i2, %i4 int_copy %i3, %i5 int_add %i5, %i4, %i6 @@ -210,13 +210,13 @@ else: return 42 self.encoding_test(f, [65], """ int_guard_value %i0, %i0 - goto_if_not_int_eq L1, %i0, $-5 + goto_if_not_int_eq %i0, $-5, L1 int_return $12 L1: - goto_if_not_int_eq L2, %i0, $2 + goto_if_not_int_eq %i0, $2, L2 int_return $51 L2: - goto_if_not_int_eq L3, %i0, $7 + goto_if_not_int_eq %i0, $7, L3 int_return $1212 L3: int_return $42 @@ -342,7 +342,7 @@ # note that 'goto_if_not_int_is_true' is actually the same thing # as just 'goto_if_not'. self.encoding_test(f, [7], """ - goto_if_not L1, %i0 + goto_if_not %i0, L1 int_return $False L1: int_return $True Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py Tue May 18 17:04:44 2010 @@ -52,7 +52,7 @@ i0, i1 = Register('int', 0), Register('int', 1) ssarepr.insns = [ (Label('L1'),), - ('goto_if_not_int_gt', TLabel('L2'), i0, Constant(0, lltype.Signed)), + ('goto_if_not_int_gt', i0, Constant(0, lltype.Signed), TLabel('L2')), ('int_add', i1, i0, i1), ('int_sub', i0, Constant(1, lltype.Signed), i0), ('goto', TLabel('L1')), @@ -62,7 +62,7 @@ asm = format_assembler(ssarepr) expected = """ L1: - goto_if_not_int_gt L2, %i0, $0 + goto_if_not_int_gt %i0, $0, L2 int_add %i1, %i0, %i1 int_sub %i0, $1, %i0 goto L1 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitcode.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitcode.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitcode.py Tue May 18 17:04:44 2010 @@ -11,10 +11,10 @@ assert j.num_regs_i() == 0 assert j.num_regs_r() == 0 assert j.num_regs_f() == 0 - j.setup(num_regs_i=256, num_regs_r=256, num_regs_f=256) - assert j.num_regs_i() == 256 - assert j.num_regs_r() == 256 - assert j.num_regs_f() == 256 + j.setup(num_regs_i=255, num_regs_r=255, num_regs_f=255) + assert j.num_regs_i() == 255 + assert j.num_regs_r() == 255 + assert j.num_regs_f() == 255 def test_liveness(): j = JitCode("test") Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py Tue May 18 17:04:44 2010 @@ -66,7 +66,7 @@ -live- %i0, %i1 G_int_add %i0, $5, %i2 int_is_true %i2, %i3 - goto_if_not L1, %i3 + goto_if_not %i3, L1 int_copy %i0, %i4 -live- G_int_add %i4, $1, %i5 @@ -87,7 +87,7 @@ -live- %i0, %i1 G_int_add %i0, $5, %i2 int_is_true %i2, %i3 - goto_if_not L1, %i3 + goto_if_not %i3, L1 int_copy %i0, %i4 int_copy %i1, %i5 -live- @@ -109,7 +109,7 @@ -live- %i0 G_int_add %i0, %i1, %i2 int_is_true %i2, %i3 - goto_if_not L1, %i3 + goto_if_not %i3, L1 int_copy %i0, %i4 -live- G_int_add %i4, $5, %i5 @@ -129,7 +129,7 @@ self.encoding_test(f, [5, 6], """ int_is_true %i0, %i2 -live- %i0, %i1 - goto_if_not L1, %i2 + goto_if_not %i2, L1 int_return %i0 L1: int_return %i1 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Tue May 18 17:04:44 2010 @@ -59,7 +59,7 @@ self.check_assembler(graph, """ L1: int_gt %i0, $0, %i2 - goto_if_not L2, %i2 + goto_if_not %i2, L2 int_add %i1, %i0, %i1 int_sub %i0, $1, %i0 goto L1 @@ -76,7 +76,7 @@ self.check_assembler(graph, """ L1: int_gt %i0, $0, %i2 - goto_if_not L2, %i2 + goto_if_not %i2, L2 int_push %i1 int_copy %i0, %i1 int_pop %i0 @@ -94,7 +94,7 @@ self.check_assembler(graph, """ L1: int_gt %i0, $0, %i0 - goto_if_not L2, %i0 + goto_if_not %i0, L2 int_copy %i1, %i0 int_copy $2, %i1 goto L1 @@ -111,7 +111,7 @@ self.check_assembler(graph, """ L1: int_gt %i0, $0, %i3 - goto_if_not L2, %i3 + goto_if_not %i3, L2 int_push %i1 int_copy %i2, %i1 int_copy %i0, %i2 @@ -130,7 +130,7 @@ self.check_assembler(graph, """ L1: int_gt %i0, $0, %i3 - goto_if_not L2, %i3 + goto_if_not %i3, L2 int_copy %i2, %i1 goto L1 L2: Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Tue May 18 17:04:44 2010 @@ -1,6 +1,7 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.debug import debug_start, debug_stop from pypy.rlib.debug import make_sure_not_resized from pypy.rpython.lltypesystem import lltype, llmemory, rclass from pypy.rpython.lltypesystem.lloperation import llop @@ -53,10 +54,11 @@ class BlackholeInterpBuilder(object): verbose = True - def __init__(self, codewriter): + def __init__(self, codewriter, metainterp_sd=None): self.cpu = codewriter.cpu self.setup_insns(codewriter.assembler.insns) self.setup_descrs(codewriter.assembler.descrs) + self.metainterp_sd = metainterp_sd self._freeze_() def _freeze_(self): @@ -81,7 +83,7 @@ def dispatch_loop(self, code, position): while True: if not we_are_translated(): - assert position in self._current_jitcode._startpoints, ( + assert position in self.jitcode._startpoints, ( "the current position %d is in the middle of " "an instruction!" % position) opcode = ord(code[position]) @@ -233,18 +235,18 @@ return BlackholeInterpreter(self) def release_interp(self, interp): - interp.cleanup_registers_r() + interp.cleanup_registers() self.blackholeinterps.append(interp) class BlackholeInterpreter(object): def __init__(self, builder): + self.builder = builder self.cpu = builder.cpu self.dispatch_loop = builder.dispatch_loop self.descrs = builder.descrs self.op_catch_exception = builder.op_catch_exception - self.cleanup_required_in_registers_r = 0 # if we_are_translated(): default_i = 0 @@ -257,6 +259,19 @@ self.registers_i = [default_i] * 256 self.registers_r = [default_r] * 256 self.registers_f = [default_f] * 256 + self.jitcode = None + + def setposition(self, jitcode, position): + if jitcode is not self.jitcode: + # the real performance impact of the following code is unclear, + # but it should be minimized by the fact that a given + # BlackholeInterpreter instance is likely to be reused with + # exactly the same jitcode, so we don't do the copy again. + self.copy_constants(self.registers_i, jitcode.constants_i) + self.copy_constants(self.registers_r, jitcode.constants_r) + self.copy_constants(self.registers_f, jitcode.constants_f) + self.jitcode = jitcode + self.position = position def setarg_i(self, index, value): self.registers_i[index] = value @@ -267,21 +282,14 @@ def setarg_f(self, index, value): self.registers_f[index] = value - def run(self, jitcode, position): - if not we_are_translated(): - self._current_jitcode = jitcode - self.copy_constants(self.registers_i, jitcode.constants_i) - self.copy_constants(self.registers_r, jitcode.constants_r) - self.copy_constants(self.registers_f, jitcode.constants_f) - code = jitcode.code - self.cleanup_required_in_registers_r = max( - self.cleanup_required_in_registers_r, - jitcode.num_regs_r()) + def run(self): + code = self.jitcode.code + position = self.position while True: try: self.dispatch_loop(self, code, position) except LeaveFrame: - return + break except Exception, e: e = get_llexception(self.cpu, e) position = self.handle_exception_in_frame(e, code) @@ -290,7 +298,12 @@ return self.tmpreg_i def get_result_r(self): - return self.tmpreg_r + result = self.tmpreg_r + if we_are_translated(): + self.tmpreg_r = NULL + else: + del self.tmpreg_r + return result def get_result_f(self): return self.tmpreg_f @@ -303,14 +316,12 @@ if self._return_type == 'void': return None raise ValueError(self._return_type) - def cleanup_registers_r(self): + def cleanup_registers(self): # To avoid keeping references alive, this cleans up the registers_r. # It does not clear the references set by copy_constants(), but # these are all prebuilt constants anyway. - for i in range(self.cleanup_required_in_registers_r): + for i in range(self.jitcode.num_regs_r()): self.registers_r[i] = NULL - self.cleanup_required_in_registers_r = 0 - self.tmpreg_r = NULL self.exception_last_value = None def handle_exception_in_frame(self, e, code): @@ -330,11 +341,12 @@ return target # XXX must be specialized - # XXX the real performance impact of the following loop is unclear def copy_constants(self, registers, constants): """Copy jitcode.constants[0] to registers[255], jitcode.constants[1] to registers[254], jitcode.constants[2] to registers[253], etc.""" + make_sure_not_resized(registers) + make_sure_not_resized(constants) i = len(constants) - 1 while i >= 0: j = 255 - i @@ -342,6 +354,18 @@ registers[j] = constants[i] i -= 1 + def follow_jump(self): + """Assuming that self.position points just after a bytecode + instruction that ends with a label, follow that label.""" + code = self.jitcode.code + position = self.position - 2 + assert position >= 0 + if not we_are_translated(): + assert position in self.jitcode._alllabels + labelvalue = ord(code[position]) | (ord(code[position+1])<<8) + assert labelvalue < len(code) + self.position = labelvalue + # ---------- @arguments("i", "i", returns="i") @@ -497,13 +521,13 @@ @arguments("self", returns="i") def bhimpl_int_pop(self): - return self.tmpreg_i + return self.get_result_i() @arguments("self", returns="r") def bhimpl_ref_pop(self): - return self.tmpreg_r + return self.get_result_r() @arguments("self", returns="f") def bhimpl_float_pop(self): - return self.tmpreg_f + return self.get_result_f() # ---------- # float operations @@ -590,85 +614,85 @@ self._return_type = "void" raise LeaveFrame - @arguments("L", "i", "pc", returns="L") - def bhimpl_goto_if_not(target, a, pc): + @arguments("i", "L", "pc", returns="L") + def bhimpl_goto_if_not(a, target, pc): if a: return pc else: return target - @arguments("L", "i", "i", "pc", returns="L") - def bhimpl_goto_if_not_int_lt(target, a, b, pc): + @arguments("i", "i", "L", "pc", returns="L") + def bhimpl_goto_if_not_int_lt(a, b, target, pc): if a < b: return pc else: return target - @arguments("L", "i", "i", "pc", returns="L") - def bhimpl_goto_if_not_int_le(target, a, b, pc): + @arguments("i", "i", "L", "pc", returns="L") + def bhimpl_goto_if_not_int_le(a, b, target, pc): if a <= b: return pc else: return target - @arguments("L", "i", "i", "pc", returns="L") - def bhimpl_goto_if_not_int_eq(target, a, b, pc): + @arguments("i", "i", "L", "pc", returns="L") + def bhimpl_goto_if_not_int_eq(a, b, target, pc): if a == b: return pc else: return target - @arguments("L", "i", "i", "pc", returns="L") - def bhimpl_goto_if_not_int_ne(target, a, b, pc): + @arguments("i", "i", "L", "pc", returns="L") + def bhimpl_goto_if_not_int_ne(a, b, target, pc): if a != b: return pc else: return target - @arguments("L", "i", "i", "pc", returns="L") - def bhimpl_goto_if_not_int_gt(target, a, b, pc): + @arguments("i", "i", "L", "pc", returns="L") + def bhimpl_goto_if_not_int_gt(a, b, target, pc): if a > b: return pc else: return target - @arguments("L", "i", "i", "pc", returns="L") - def bhimpl_goto_if_not_int_ge(target, a, b, pc): + @arguments("i", "i", "L", "pc", returns="L") + def bhimpl_goto_if_not_int_ge(a, b, target, pc): if a >= b: return pc else: return target - @arguments("L", "i", "pc", returns="L") - def bhimpl_goto_if_not_int_is_zero(target, a, pc): + @arguments("i", "L", "pc", returns="L") + def bhimpl_goto_if_not_int_is_zero(a, target, pc): if not a: return pc else: return target - @arguments("L", "r", "r", "pc", returns="L") - def bhimpl_goto_if_not_ptr_eq(target, a, b, pc): + @arguments("r", "r", "L", "pc", returns="L") + def bhimpl_goto_if_not_ptr_eq(a, b, target, pc): if a == b: return pc else: return target - @arguments("L", "r", "r", "pc", returns="L") - def bhimpl_goto_if_not_ptr_ne(target, a, b, pc): + @arguments("r", "r", "L", "pc", returns="L") + def bhimpl_goto_if_not_ptr_ne(a, b, target, pc): if a != b: return pc else: return target - @arguments("L", "r", "pc", returns="L") - def bhimpl_goto_if_not_ptr_iszero(target, a, pc): + @arguments("r", "L", "pc", returns="L") + def bhimpl_goto_if_not_ptr_iszero(a, target, pc): if not a: return pc else: return target - @arguments("L", "r", "pc", returns="L") - def bhimpl_goto_if_not_ptr_nonzero(target, a, pc): + @arguments("r", "L", "pc", returns="L") + def bhimpl_goto_if_not_ptr_nonzero(a, target, pc): if a: return pc else: @@ -729,6 +753,15 @@ assert real_instance raise real_instance + @arguments() + def bhimpl_can_enter_jit(): + pass + + @arguments("self", "I", "R", "F", "I", "R", "F") + def bhimpl_jit_merge_point(self, *results): + CRN = self.builder.metainterp_sd.ContinueRunningNormally + raise CRN(*results) + # ---------- # the following operations are directly implemented by the backend @@ -904,3 +937,60 @@ @arguments("cpu", "r", "i", "i") def bhimpl_unicodesetitem(cpu, unicode, index, newchr): cpu.bh_unicodesetitem(unicode, index, newchr) + +# ____________________________________________________________ + +def resume_in_blackhole(metainterp_sd, resumedescr): + from pypy.jit.metainterp.resume import blackhole_from_resumedata + debug_start('jit-blackhole') + metainterp_sd.profiler.start_blackhole() + blackholeinterp = blackhole_from_resumedata( + metainterp_sd.blackholeinterpbuilder, + resumedescr, + False) # XXX + # XXX virtualrefs + # XXX virtualizable + _prepare_resume_from_failure(blackholeinterp, resumedescr.guard_opnum) + try: + blackholeinterp = _resume_mainloop( + metainterp_sd.blackholeinterpbuilder, blackholeinterp) + finally: + metainterp_sd.profiler.end_blackhole() + debug_stop('jit-blackhole') + # rare case: we only get there if the blackhole interps all returned + # normally (in general we get a ContinueRunningNormally exception). + _done_with_this_frame(blackholeinterp) + +def _resume_mainloop(blackholeinterpbuilder, blackholeinterp): + while True: + try: + blackholeinterp.run() + finally: + blackholeinterpbuilder.release_interp(blackholeinterp) + #...x.x.x... + assert blackholeinterp.nextblackholeinterp is None # XXX + break + xxx + return blackholeinterp + +def _prepare_resume_from_failure(blackholeinterp, opnum): + from pypy.jit.metainterp.resoperation import rop + if opnum == rop.GUARD_TRUE: # a goto_if_not_xxx that jumps only now + blackholeinterp.follow_jump() + elif opnum == rop.GUARD_FALSE: # a goto_if_not that stops jumping + pass + else: + raise NotImplementedError(opnum) + +def _done_with_this_frame(blackholeinterp): + sd = blackholeinterp.builder.metainterp_sd + if sd.result_type == 'void': + raise sd.DoneWithThisFrameVoid() + elif sd.result_type == 'int': + raise sd.DoneWithThisFrameInt(blackholeinterp.get_result_i()) + elif sd.result_type == 'ref': + raise sd.DoneWithThisFrameRef(blackholeinterp.get_result_r()) + elif sd.result_type == 'float': + raise sd.DoneWithThisFrameFloat(blackholeinterp.get_result_f()) + else: + assert False Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py Tue May 18 17:04:44 2010 @@ -239,16 +239,20 @@ self._counter = ~i # use ~(index_of_guarded_box_in_fail_args) def handle_fail(self, metainterp_sd): + from pypy.jit.metainterp.blackhole import resume_in_blackhole + return resume_in_blackhole(metainterp_sd, self) + XXX from pypy.jit.metainterp.pyjitpl import MetaInterp metainterp = MetaInterp(metainterp_sd) return metainterp.handle_guard_failure(self) - def must_compile(self, metainterp_sd, inputargs_and_holes): + def must_compile(self, metainterp_sd): trace_eagerness = metainterp_sd.state.trace_eagerness if self._counter >= 0: self._counter += 1 return self._counter >= trace_eagerness else: + XXX box = inputargs_and_holes[~self._counter] if self._counters is None: self._counters = ResumeGuardCounters() Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Tue May 18 17:04:44 2010 @@ -36,11 +36,11 @@ return func return decorate -class XXX: # XXX temporary hack +class FixME: # XXX temporary hack def __init__(self, func): self.func = func def __getattr__(self, _): - raise Exception("@XXX: " + self.func.__name__) + raise Exception("@FixME: " + self.func.__name__) # ____________________________________________________________ @@ -236,8 +236,8 @@ def opimpl_goto(self, target): self.pc = target - @arguments("label", "box") - def opimpl_goto_if_not(self, target, box): + @arguments("box", "label") + def opimpl_goto_if_not(self, box, target): switchcase = box.getint() if switchcase: opnum = rop.GUARD_TRUE @@ -249,19 +249,19 @@ for _opimpl in ['int_is_zero', 'ptr_iszero', 'ptr_nonzero']: exec py.code.Source(''' - @arguments("label", "box") - def opimpl_goto_if_not_%s(self, target, box): + @arguments("box", "label") + def opimpl_goto_if_not_%s(self, box, target): condbox = self.execute(rop.%s, box) - self.opimpl_goto_if_not(target, condbox) + self.opimpl_goto_if_not(condbox, target) ''' % (_opimpl, _opimpl.upper())).compile() for _opimpl in ['int_lt', 'int_le', 'int_eq', 'int_ne', 'int_gt', 'int_ge', 'ptr_eq', 'ptr_ne']: exec py.code.Source(''' - @arguments("label", "box", "box") - def opimpl_goto_if_not_%s(self, target, b1, b2): + @arguments("box", "box", "label") + def opimpl_goto_if_not_%s(self, b1, b2, target): condbox = self.execute(rop.%s, b1, b2) - self.opimpl_goto_if_not(target, condbox) + self.opimpl_goto_if_not(condbox, target) ''' % (_opimpl, _opimpl.upper())).compile() def follow_jump(self): @@ -316,18 +316,18 @@ def opimpl_new_with_vtable(self, sizevtabledescr): return self.execute_with_descr(rop.NEW_WITH_VTABLE, sizevtabledescr) - @XXX #arguments("box") + @FixME #arguments("box") def opimpl_runtimenew(self, classbox): self.execute(rop.RUNTIMENEW, classbox) - @XXX #arguments("orgpc", "box", "descr") + @FixME #arguments("orgpc", "box", "descr") def opimpl_instanceof(self, pc, objbox, typedescr): clsbox = self.cls_of_box(objbox) if isinstance(objbox, Box): self.generate_guard(pc, rop.GUARD_CLASS, objbox, [clsbox]) self.execute_with_descr(rop.INSTANCEOF, typedescr, objbox) - @XXX #arguments("box", "box") + @FixME #arguments("box", "box") def opimpl_subclassof(self, box1, box2): self.execute(rop.SUBCLASSOF, box1, box2) @@ -335,11 +335,11 @@ def opimpl_new_array(self, itemsizedescr, countbox): return self.execute_with_descr(rop.NEW_ARRAY, itemsizedescr, countbox) - @XXX #arguments("box", "descr", "box") + @FixME #arguments("box", "descr", "box") def opimpl_getarrayitem_gc(self, arraybox, arraydesc, indexbox): self.execute_with_descr(rop.GETARRAYITEM_GC, arraydesc, arraybox, indexbox) - @XXX #arguments("box", "descr", "box") + @FixME #arguments("box", "descr", "box") def opimpl_getarrayitem_gc_pure(self, arraybox, arraydesc, indexbox): self.execute_with_descr(rop.GETARRAYITEM_GC_PURE, arraydesc, arraybox, indexbox) @@ -352,18 +352,18 @@ opimpl_setarrayitem_gc_r = _opimpl_setarrayitem_gc opimpl_setarrayitem_gc_f = _opimpl_setarrayitem_gc - @XXX #arguments("box", "descr") + @FixME #arguments("box", "descr") def opimpl_arraylen_gc(self, arraybox, arraydesc): self.execute_with_descr(rop.ARRAYLEN_GC, arraydesc, arraybox) - @XXX #arguments("descr", "box", "box", "box", "box", "box", "box", "descr") + @FixME #arguments("descr", "box", "box", "box", "box", "box", "box", "descr") def opimpl_arraycopy(self, calldescr, fnptr, sourcebox, destbox, source_startbox, dest_startbox, lengthbox, arraydescr): self.execute_with_descr(rop.ARRAYCOPY, arraydescr, calldescr, fnptr, sourcebox, destbox, source_startbox, dest_startbox, lengthbox) - @XXX #arguments("orgpc", "box", "descr", "box") + @FixME #arguments("orgpc", "box", "descr", "box") def opimpl_check_neg_index(self, pc, arraybox, arraydesc, indexbox): negbox = self.metainterp.execute_and_record( rop.INT_LT, None, indexbox, ConstInt(0)) @@ -376,7 +376,7 @@ rop.INT_ADD, None, indexbox, lenbox) self.make_result_box(indexbox) - @XXX #arguments("descr", "descr", "descr", "descr", "box") + @FixME #arguments("descr", "descr", "descr", "descr", "box") def opimpl_newlist(self, structdescr, lengthdescr, itemsdescr, arraydescr, sizebox): sbox = self.metainterp.execute_and_record(rop.NEW, structdescr) @@ -388,20 +388,20 @@ sbox, abox) self.make_result_box(sbox) - @XXX #arguments("box", "descr", "descr", "box") + @FixME #arguments("box", "descr", "descr", "box") def opimpl_getlistitem_gc(self, listbox, itemsdescr, arraydescr, indexbox): arraybox = self.metainterp.execute_and_record(rop.GETFIELD_GC, itemsdescr, listbox) self.execute_with_descr(rop.GETARRAYITEM_GC, arraydescr, arraybox, indexbox) - @XXX #arguments("box", "descr", "descr", "box", "box") + @FixME #arguments("box", "descr", "descr", "box", "box") def opimpl_setlistitem_gc(self, listbox, itemsdescr, arraydescr, indexbox, valuebox): arraybox = self.metainterp.execute_and_record(rop.GETFIELD_GC, itemsdescr, listbox) self.execute_with_descr(rop.SETARRAYITEM_GC, arraydescr, arraybox, indexbox, valuebox) - @XXX #arguments("orgpc", "box", "descr", "box") + @FixME #arguments("orgpc", "box", "descr", "box") def opimpl_check_resizable_neg_index(self, pc, listbox, lengthdesc, indexbox): negbox = self.metainterp.execute_and_record( @@ -415,7 +415,7 @@ rop.INT_ADD, None, indexbox, lenbox) self.make_result_box(indexbox) - @XXX #arguments("orgpc", "box") + @FixME #arguments("orgpc", "box") def opimpl_check_zerodivisionerror(self, pc, box): nonzerobox = self.metainterp.execute_and_record( rop.INT_NE, None, box, ConstInt(0)) @@ -426,7 +426,7 @@ # division by zero! return self.metainterp.raise_zero_division_error() - @XXX #arguments("orgpc", "box", "box") + @FixME #arguments("orgpc", "box", "box") def opimpl_check_div_overflow(self, pc, box1, box2): # detect the combination "box1 = -sys.maxint-1, box2 = -1". import sys @@ -443,11 +443,11 @@ # division overflow! return self.metainterp.raise_overflow_error() - @XXX #arguments() + @FixME #arguments() def opimpl_overflow_error(self): return self.metainterp.raise_overflow_error() - @XXX #arguments("orgpc", "box") + @FixME #arguments("orgpc", "box") def opimpl_int_abs(self, pc, box): nonneg = self.metainterp.execute_and_record( rop.INT_GE, None, box, ConstInt(0)) @@ -457,7 +457,7 @@ else: self.execute(rop.INT_NEG, box) - @XXX #arguments("orgpc", "box") + @FixME #arguments("orgpc", "box") def opimpl_oononnull(self, pc, box): value = box.nonnull() if value: @@ -469,7 +469,7 @@ self.generate_guard(pc, opnum, box, []) self.make_result_box(res) - @XXX #arguments("orgpc", "box") + @FixME #arguments("orgpc", "box") def opimpl_ooisnull(self, pc, box): value = box.nonnull() if value: @@ -481,11 +481,11 @@ self.generate_guard(pc, opnum, box, []) self.make_result_box(res) - @XXX #arguments("box", "box") + @FixME #arguments("box", "box") def opimpl_ptr_eq(self, box1, box2): self.execute(rop.OOIS, box1, box2) - @XXX #arguments("box", "box") + @FixME #arguments("box", "box") def opimpl_ptr_ne(self, box1, box2): self.execute(rop.OOISNOT, box1, box2) @@ -557,7 +557,7 @@ vinfo = self.metainterp.staticdata.virtualizable_info return vinfo.array_descrs[index] - @XXX #arguments("orgpc", "box", "int") + @FixME #arguments("orgpc", "box", "int") def opimpl_getfield_vable(self, pc, basebox, index): if self._nonstandard_virtualizable(pc, basebox): self.execute_with_descr(rop.GETFIELD_GC, self._get_virtualizable_field_descr(index), basebox) @@ -565,7 +565,7 @@ self.metainterp.check_synchronized_virtualizable() resbox = self.metainterp.virtualizable_boxes[index] self.make_result_box(resbox) - @XXX #arguments("orgpc", "box", "int", "box") + @FixME #arguments("orgpc", "box", "int", "box") def opimpl_setfield_vable(self, pc, basebox, index, valuebox): if self._nonstandard_virtualizable(pc, basebox): self.execute_with_descr(rop.SETFIELD_GC, self._get_virtualizable_field_descr(index), basebox, valuebox) @@ -585,7 +585,7 @@ assert 0 <= index < vinfo.get_array_length(virtualizable, arrayindex) return vinfo.get_index_in_array(virtualizable, arrayindex, index) - @XXX #arguments("orgpc", "box", "int", "box") + @FixME #arguments("orgpc", "box", "int", "box") def opimpl_getarrayitem_vable(self, pc, basebox, arrayindex, indexbox): if self._nonstandard_virtualizable(pc, basebox): descr = self._get_virtualizable_array_field_descr(arrayindex) @@ -599,7 +599,7 @@ index = self._get_arrayitem_vable_index(pc, arrayindex, indexbox) resbox = self.metainterp.virtualizable_boxes[index] self.make_result_box(resbox) - @XXX #arguments("orgpc", "box", "int", "box", "box") + @FixME #arguments("orgpc", "box", "int", "box", "box") def opimpl_setarrayitem_vable(self, pc, basebox, arrayindex, indexbox, valuebox): if self._nonstandard_virtualizable(pc, basebox): @@ -614,7 +614,7 @@ self.metainterp.virtualizable_boxes[index] = valuebox self.metainterp.synchronize_virtualizable() # XXX only the index'th field needs to be synchronized, really - @XXX #arguments("orgpc", "box", "int") + @FixME #arguments("orgpc", "box", "int") def opimpl_arraylen_vable(self, pc, basebox, arrayindex): if self._nonstandard_virtualizable(pc, basebox): descr = self._get_virtualizable_array_field_descr(arrayindex) @@ -671,11 +671,11 @@ opimpl_residual_call_irf_f = _opimpl_residual_call3 opimpl_residual_call_irf_v = _opimpl_residual_call3 - @XXX #arguments("descr", "varargs") + @FixME #arguments("descr", "varargs") def opimpl_residual_call_loopinvariant(self, calldescr, varargs): return self.execute_varargs(rop.CALL_LOOPINVARIANT, varargs, calldescr, exc=True) - @XXX #arguments("orgpc", "descr", "varargs") + @FixME #arguments("orgpc", "descr", "varargs") def opimpl_recursive_call(self, pc, calldescr, varargs): warmrunnerstate = self.metainterp.staticdata.state token = None @@ -710,7 +710,7 @@ call_position) return res - @XXX #arguments("orgpc", "methdescr", "varargs") + @FixME #arguments("orgpc", "methdescr", "varargs") def opimpl_oosend(self, pc, methdescr, varargs): objbox = varargs[0] clsbox = self.cls_of_box(objbox) @@ -758,15 +758,15 @@ def opimpl_newunicode(self, lengthbox): return self.execute(rop.NEWUNICODE, lengthbox) - @XXX #arguments("descr", "varargs") + @FixME #arguments("descr", "varargs") def opimpl_residual_oosend_canraise(self, methdescr, varargs): return self.execute_varargs(rop.OOSEND, varargs, descr=methdescr, exc=True) - @XXX #arguments("descr", "varargs") + @FixME #arguments("descr", "varargs") def opimpl_residual_oosend_noraise(self, methdescr, varargs): self.execute_varargs(rop.OOSEND, varargs, descr=methdescr, exc=False) - @XXX #arguments("descr", "varargs") + @FixME #arguments("descr", "varargs") def opimpl_residual_oosend_pure(self, methdescr, boxes): self.execute_varargs(rop.OOSEND_PURE, boxes, descr=methdescr, exc=False) @@ -784,17 +784,17 @@ self.generate_guard(rop.GUARD_CLASS, box, [clsbox]) return clsbox -## @XXX #arguments("orgpc", "box", "builtin") +## @FixME #arguments("orgpc", "box", "builtin") ## def opimpl_guard_builtin(self, pc, box, builtin): ## self.generate_guard(pc, "guard_builtin", box, [builtin]) -## @XXX #arguments("orgpc", "box", "builtin") +## @FixME #arguments("orgpc", "box", "builtin") ## def opimpl_guard_len(self, pc, box, builtin): ## intbox = self.metainterp.cpu.execute_operation( ## 'len', [builtin.len_func, box], 'int') ## self.generate_guard(pc, "guard_len", box, [intbox]) - @XXX #arguments("box") + @FixME #arguments("box") def opimpl_keepalive(self, box): pass # xxx? @@ -873,7 +873,7 @@ self.metainterp.popframe() self.metainterp.finishframe_exception() - @XXX #arguments("box") + @FixME #arguments("box") def opimpl_virtual_ref(self, box): # Details on the content of metainterp.virtualref_boxes: # @@ -907,7 +907,7 @@ metainterp.virtualref_boxes.append(resbox) self.make_result_box(resbox) - @XXX #arguments("box") + @FixME #arguments("box") def opimpl_virtual_ref_finish(self, box): # virtual_ref_finish() assumes that we have a stack-like, last-in # first-out order. @@ -1134,6 +1134,9 @@ RESULT = codewriter.portal_graph.getreturnvar().concretetype self.result_type = history.getkind(RESULT) # + from pypy.jit.metainterp.blackhole import BlackholeInterpBuilder + self.blackholeinterpbuilder = BlackholeInterpBuilder(codewriter, self) + # warmrunnerdesc = self.warmrunnerdesc if warmrunnerdesc is not None: self.num_green_args = warmrunnerdesc.num_green_args @@ -1460,6 +1463,7 @@ self.last_exc_value_box = None def switch_to_blackhole(self, reason): + XXX self.staticdata.profiler.count(reason) debug_print('~~~ ABORTING TRACING') debug_stop('jit-tracing') @@ -1513,10 +1517,7 @@ try: return self._compile_and_run_once(*args) finally: - if self.history is None: - debug_stop('jit-blackhole') - else: - debug_stop('jit-tracing') + debug_stop('jit-tracing') def _compile_and_run_once(self, *args): original_boxes = self.initialize_state_from_start(*args) @@ -1539,10 +1540,7 @@ try: return self._handle_guard_failure(key) finally: - if self.history is None: - debug_stop('jit-blackhole') - else: - debug_stop('jit-tracing') + debug_stop('jit-tracing') def _handle_guard_failure(self, key): from pypy.jit.metainterp.warmspot import ContinueRunningNormallyBase @@ -1553,6 +1551,7 @@ self.current_merge_points = [(original_greenkey, -1)] self.resumekey = key self.seen_can_enter_jit = False + xxx started_as_blackhole = self.is_blackholing() try: self.prepare_resume_from_failure(key.guard_opnum) @@ -1741,6 +1740,7 @@ return original_boxes def initialize_state_from_guard_failure(self, resumedescr): + XXX # guard failure: rebuild a complete MIFrame stack self.in_recursion = -1 # always one portal around inputargs_and_holes = self.cpu.make_boxes_from_latest_values( Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py Tue May 18 17:04:44 2010 @@ -1,10 +1,10 @@ import sys, os -from pypy.jit.metainterp.history import Box, Const, ConstInt, INT, REF +from pypy.jit.metainterp.history import Box, Const, ConstInt, INT, REF, FLOAT from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp import jitprof from pypy.rpython.lltypesystem import rffi from pypy.rlib import rarithmetic -from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.rlib.debug import have_debug_prints from pypy.rlib.debug import debug_start, debug_stop, debug_print @@ -390,7 +390,9 @@ class AbstractVirtualInfo(object): def allocate(self, metainterp): raise NotImplementedError - def setfields(self, metainterp, box, fn_decode_box): + #def setfields(self, decoder, struct): + # raise NotImplementedError + def bh_allocate(self, cpu): raise NotImplementedError def equals(self, fieldnums): return tagged_list_eq(self.fieldnums, fieldnums) @@ -405,12 +407,14 @@ self.fielddescrs = fielddescrs #self.fieldnums = ... - def setfields(self, metainterp, box, fn_decode_box): + @specialize.argtype(1) + def setfields(self, decoder, struct): for i in range(len(self.fielddescrs)): - fieldbox = fn_decode_box(self.fieldnums[i]) - metainterp.execute_and_record(rop.SETFIELD_GC, - self.fielddescrs[i], - box, fieldbox) + descr = self.fielddescrs[i] + decoder.setfield(descr, struct, self.fieldnums[i]) +## fieldbox = fn_decode_box(self.fieldnums[i], kind) +## metainterp.execute_and_record(rop.SETFIELD_GC, +## descr, box, fieldbox) def debug_prints(self): assert len(self.fielddescrs) == len(self.fieldnums) @@ -455,12 +459,11 @@ self.arraydescr, ConstInt(length)) - def setfields(self, metainterp, box, fn_decode_box): + @specialize.argtype(1) + def setfields(self, decoder, array): + arraydescr = self.arraydescr for i in range(len(self.fieldnums)): - itembox = fn_decode_box(self.fieldnums[i]) - metainterp.execute_and_record(rop.SETARRAYITEM_GC, - self.arraydescr, - box, ConstInt(i), itembox) + decoder.setarrayitem(descr, array, i, self.fieldnums[i]) def debug_prints(self): debug_print("\tvarrayinfo", self.arraydescr) @@ -496,7 +499,7 @@ return virtualizable_boxes, virtualref_boxes, resumereader.virtuals -class ResumeDataReader(object): +class ResumeDataBoxReader(object): virtuals = None def __init__(self, storage, liveboxes, metainterp=None): @@ -545,7 +548,7 @@ self.cur_numb = numb.prev return boxes - def _decode_box(self, tagged): + def _decode_box(self, tagged, kind): num, tag = untag(tagged) if tag == TAGCONST: if tagged_eq(tagged, NULLREF): @@ -561,6 +564,164 @@ assert tag == TAGBOX return self.liveboxes[num] + +def blackhole_from_resumedata(blackholeinterpbuilder, storage, + expects_virtualizables): + resumereader = ResumeDataDirectReader(storage, blackholeinterpbuilder.cpu) + if expects_virtualizables: + XXX + #virtualref_boxes = resumereader.consume_boxes() + resumereader.consume_one_section(None) # XXX + # + # First get a chain of blackhole interpreters whose length is given + # by the depth of rd_frame_info_list. The first one we get must be + # the bottom one, i.e. the last one in the chain, in order to make + # the comment in BlackholeInterpreter.setposition() valid. + nextbh = None + frameinfo = storage.rd_frame_info_list + while True: + curbh = blackholeinterpbuilder.acquire_interp() + curbh.nextblackholeinterp = nextbh + nextbh = curbh + frameinfo = frameinfo.prev + if frameinfo is None: + break + firstbh = nextbh + # + # Now fill the blackhole interpreters with resume data. + curbh = firstbh + frameinfo = storage.rd_frame_info_list + while True: + curbh.setposition(frameinfo.jitcode, frameinfo.pc) + resumereader.consume_one_section(curbh) + curbh = curbh.nextblackholeinterp + frameinfo = frameinfo.prev + if frameinfo is None: + break + resumereader.done() + return firstbh + +class ResumeDataDirectReader(object): + virtuals = None + + def __init__(self, storage, cpu): + self.cur_numb = storage.rd_numb + self.consts = storage.rd_consts + self.cpu = cpu + self._prepare_virtuals(storage.rd_virtuals) + self._prepare_pendingfields(storage.rd_pendingfields) + + def _prepare_virtuals(self, virtuals): + if virtuals: + self.virtuals = [None] * len(virtuals) + for i in range(len(virtuals)): + vinfo = virtuals[i] + if vinfo is not None: + self.virtuals[i] = vinfo.bh_allocate(self.cpu) + for i in range(len(virtuals)): + vinfo = virtuals[i] + if vinfo is not None: + vinfo.setfields(self, self.virtuals[i]) + + def _prepare_pendingfields(self, pendingfields): + if pendingfields is not None: + for descr, num, fieldnum in pendingfields: + struct = self._decode_ref(num) + self.setfield(descr, struct, fieldnum) + + def setfield(self, descr, struct, fieldnum): + if descr.is_pointer_field(): + newvalue = self._decode_ref(fieldnum) + self.cpu.bh_setfield_gc_r(struct, descr, newfield) + elif descr.is_float_field(): + newvalue = self._decode_float(fieldnum) + self.cpu.bh_setfield_gc_f(struct, descr, newfield) + else: + newvalue = self._decode_int(fieldnum) + self.cpu.bh_setfield_gc_i(struct, descr, newfield) + + def setarrayitem(self, arraydescr, array, index, fieldnum): + if arraydescr.is_array_of_pointers(): + newvalue = self._decode_ref(fieldnum) + self.cpu.bh_setarrayitem_gc_r(arraydescr, array, index, newvalue) + elif arraydescr.is_array_of_floats(): + newvalue = self._decode_float(fieldnum) + self.cpu.bh_setarrayitem_gc_f(arraydescr, array, index, newvalue) + else: + newvalue = self._decode_int(fieldnum) + self.cpu.bh_setarrayitem_gc_i(arraydescr, array, index, newvalue) + + def consume_one_section(self, blackholeinterp): + numb = self.cur_numb + count_i = count_r = count_f = 0 + for num in numb.nums: + kind = self._decode_kind(num) + if kind == INT: + blackholeinterp.setarg_i(count_i, self._decode_int(num)) + count_i += 1 + elif kind == REF: + blackholeinterp.setarg_r(count_r, self._decode_ref(num)) + count_r += 1 + elif kind == FLOAT: + blackholeinterp.setarg_f(count_f, self._decode_float(num)) + count_f += 1 + self.cur_numb = numb.prev + + def _decode_kind(self, tagged): + num, tag = untag(tagged) + if tag == TAGCONST: + if tagged_eq(tagged, NULLREF): + return history.REF + return self.consts[num].type + elif tag == TAGVIRTUAL: + return history.REF + elif tag == TAGINT: + return history.INT + else: + assert tag == TAGBOX + return self.cpu.get_latest_value_kind(num) + + def _decode_int(self, tagged): + num, tag = untag(tagged) + if tag == TAGCONST: + return self.consts[num].getint() + elif tag == TAGINT: + return num + else: + assert tag == TAGBOX + if num < 0: + num += self.cpu.get_latest_value_count() + return self.cpu.get_latest_value_int(num) + + def _decode_ref(self, tagged): + num, tag = untag(tagged) + if tag == TAGCONST: + if tagged_eq(tagged, NULLREF): + return self.cpu.ts.NULLREF + return self.consts[num].getptr_base() + elif tag == TAGVIRTUAL: + virtuals = self.virtuals + assert virtuals is not None + return virtuals[num] + else: + assert tag == TAGBOX + if num < 0: + num += self.cpu.get_latest_value_count() + return self.cpu.get_latest_value_ref(num) + + def _decode_float(self, tagged): + num, tag = untag(tagged) + if tag == TAGCONST: + return self.consts[num].getfloat() + else: + assert tag == TAGBOX + if num < 0: + num += self.cpu.get_latest_value_count() + return self.cpu.get_latest_value_float(num) + + def done(self): + self.cpu.clear_latest_values() + # ____________________________________________________________ def dump_storage(storage, liveboxes): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Tue May 18 17:04:44 2010 @@ -43,7 +43,8 @@ count_f += 1 else: raise TypeError(T) - blackholeinterp.run(cw.mainjitcode, 0) + blackholeinterp.setposition(cw.mainjitcode, 0) + blackholeinterp.run() return blackholeinterp._get_result_anytype() def _run_with_pyjitpl(cw, args, testself): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/typesystem.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/typesystem.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/typesystem.py Tue May 18 17:04:44 2010 @@ -45,7 +45,8 @@ BoxRef = history.BoxPtr ConstRef = history.ConstPtr loops_done_with_this_frame_ref = None # patched by compile.py - CONST_NULL = history.ConstPtr(history.ConstPtr.value) + NULLREF = history.ConstPtr.value + CONST_NULL = history.ConstPtr(NULLREF) CVAL_NULLREF = None # patched by optimizeopt.py def new_ConstRef(self, x): @@ -150,7 +151,8 @@ BoxRef = history.BoxObj ConstRef = history.ConstObj loops_done_with_this_frame_ref = None # patched by compile.py - CONST_NULL = history.ConstObj(history.ConstObj.value) + NULLREF = history.ConstObj.value + CONST_NULL = history.ConstObj(NULLREF) CVAL_NULLREF = None # patched by optimizeopt.py def new_ConstRef(self, x): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py Tue May 18 17:04:44 2010 @@ -267,8 +267,7 @@ warmrunnerdesc=self) def make_exception_classes(self): - portalfunc_ARGS = unrolling_iterable( - [(i, 'arg%d' % i, ARG) for i, ARG in enumerate(self.PORTAL_FUNCTYPE.ARGS)]) + class DoneWithThisFrameVoid(JitException): def __str__(self): return 'DoneWithThisFrameVoid()' @@ -302,18 +301,19 @@ return 'ExitFrameWithExceptionRef(%s)' % (self.value,) class ContinueRunningNormally(ContinueRunningNormallyBase): - def __init__(self, argboxes): - # accepts boxes as argument, but unpacks them immediately - # before we raise the exception -- the boxes' values will - # be modified in a 'finally' by restore_patched_boxes(). - from pypy.jit.metainterp.warmstate import unwrap - for i, name, ARG in portalfunc_ARGS: - v = unwrap(ARG, argboxes[i]) - setattr(self, name, v) - + def __init__(self, gi, gr, gf, ri, rr, rf): + # the six arguments are: lists of green ints, greens refs, + # green floats, red ints, red refs, and red floats. + self.green_int = gi + self.green_ref = gr + self.green_float = gf + self.red_int = ri + self.red_ref = rr + self.red_float = rf def __str__(self): - return 'ContinueRunningNormally(%s)' % ( - ', '.join(map(str, self.args)),) + return 'ContinueRunningNormally(%s, %s, %s, %s, %s, %s)' % ( + self.green_int, self.green_ref, self.green_float, + self.red_int, self.red_ref, self.red_float) self.DoneWithThisFrameVoid = DoneWithThisFrameVoid self.DoneWithThisFrameInt = DoneWithThisFrameInt @@ -327,6 +327,7 @@ self.metainterp_sd.DoneWithThisFrameFloat = DoneWithThisFrameFloat self.metainterp_sd.ExitFrameWithExceptionRef = ExitFrameWithExceptionRef self.metainterp_sd.ContinueRunningNormally = ContinueRunningNormally + def make_enter_function(self): from pypy.jit.metainterp.warmstate import WarmEnterState state = WarmEnterState(self) @@ -497,13 +498,24 @@ # ____________________________________________________________ # Prepare the portal_runner() helper # + from pypy.jit.metainterp.warmstate import specialize_value portal_ptr = self.cpu.ts.functionptr(PORTALFUNC, 'portal', graph = portalgraph) self.portal_ptr = portal_ptr - portalfunc_ARGS = unrolling_iterable( - [(i, 'arg%d' % i, ARG) for i, ARG in enumerate(PORTALFUNC.ARGS)]) - - + # + portalfunc_ARGS = [] + nums = {} + for i, ARG in enumerate(PORTALFUNC.ARGS): + if i < len(self.jitdriver.greens): + color = 'green' + else: + color = 'red' + attrname = '%s_%s' % (color, history.getkind(ARG)) + count = nums.get(attrname, 0) + nums[attrname] = count + 1 + portalfunc_ARGS.append((ARG, attrname, count)) + portalfunc_ARGS = unrolling_iterable(portalfunc_ARGS) + # rtyper = self.translator.rtyper RESULT = PORTALFUNC.RESULT result_kind = history.getkind(RESULT) @@ -517,21 +529,22 @@ portal_ptr)(*args) except self.ContinueRunningNormally, e: args = () - for _, name, _ in portalfunc_ARGS: - v = getattr(e, name) - args = args + (v,) + for ARGTYPE, attrname, count in portalfunc_ARGS: + x = getattr(e, attrname)[count] + x = specialize_value(ARGTYPE, x) + args = args + (x,) except self.DoneWithThisFrameVoid: assert result_kind == 'void' return except self.DoneWithThisFrameInt, e: assert result_kind == 'int' - return lltype.cast_primitive(RESULT, e.result) + return specialize_value(RESULT, e.result) except self.DoneWithThisFrameRef, e: assert result_kind == 'ref' - return ts.cast_from_ref(RESULT, e.result) + return specialize_value(RESULT, e.result) except self.DoneWithThisFrameFloat, e: assert result_kind == 'float' - return e.result + return specialize_value(RESULT, e.result) except self.ExitFrameWithExceptionRef, e: value = ts.cast_to_baseclass(e.value) if not we_are_translated(): @@ -561,6 +574,7 @@ loop_token = fail_descr.handle_fail(self.metainterp_sd) fail_descr = self.cpu.execute_token(loop_token) except self.ContinueRunningNormally, e: + xxxxxxxxx args = () for _, name, _ in portalfunc_ARGS: v = getattr(e, name) @@ -571,13 +585,13 @@ return except self.DoneWithThisFrameInt, e: assert result_kind == 'int' - return lltype.cast_primitive(RESULT, e.result) + return specialize_value(RESULT, e.result) except self.DoneWithThisFrameRef, e: assert result_kind == 'ref' - return ts.cast_from_ref(RESULT, e.result) + return specialize_value(RESULT, e.result) except self.DoneWithThisFrameFloat, e: assert result_kind == 'float' - return e.result + return specialize_value(RESULT, e.result) except self.ExitFrameWithExceptionRef, e: value = ts.cast_to_baseclass(e.value) if not we_are_translated(): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmstate.py Tue May 18 17:04:44 2010 @@ -17,6 +17,20 @@ # ____________________________________________________________ @specialize.arg(0) +def specialize_value(TYPE, x): + """'x' must be a Signed, a GCREF or a Float. + This function casts it to a more specialized type, like Char or Ptr(..). + """ + INPUT = lltype.typeOf(x) + if INPUT is lltype.Signed: + return lltype.cast_primitive(TYPE, x) # XXX missing: Ptr(non-gc) + elif INPUT is lltype.Float: + assert TYPE is lltype.Float + return x + else: + return lltype.cast_opaque_ptr(TYPE, x) + + at specialize.arg(0) def unwrap(TYPE, box): if TYPE is lltype.Void: return None From jcreigh at codespeak.net Tue May 18 18:21:22 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Tue, 18 May 2010 18:21:22 +0200 (CEST) Subject: [pypy-svn] r74544 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/tool Message-ID: <20100518162122.97E35282BE0@codespeak.net> Author: jcreigh Date: Tue May 18 18:21:20 2010 New Revision: 74544 Added: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/tool/instruction_encoding.sh (contents, props changed) Log: add tool to easily check how the GNU assembler encodes a given instruction Added: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/tool/instruction_encoding.sh ============================================================================== --- (empty file) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/tool/instruction_encoding.sh Tue May 18 18:21:20 2010 @@ -0,0 +1,12 @@ +#!/bin/bash + +# Tool to quickly see how the GNU assembler encodes an instruction +# (AT&T syntax only for now) + +while :; do + echo -n '? ' + read instruction + echo "$instruction" | as + objdump --disassemble ./a.out | tail -n +8 + rm -f ./a.out +done From arigo at codespeak.net Tue May 18 18:38:25 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 18 May 2010 18:38:25 +0200 (CEST) Subject: [pypy-svn] r74545 - in pypy/branch/blackhole-improvement/pypy/jit: backend backend/llgraph codewriter codewriter/test metainterp Message-ID: <20100518163825.0E8EE282BE0@codespeak.net> Author: arigo Date: Tue May 18 18:38:23 2010 New Revision: 74545 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py pypy/branch/blackhole-improvement/pypy/jit/backend/model.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmstate.py Log: Add in heaptracker.py a bidirectional correspondance between vtable pointers and Descrs. Needed because for now the simplest seems to be to have the NEW_WITH_VTABLE ResOperation contain just the vtable pointer as a constant, but have the 'new_with_vtable' jitcode specify the descr instead. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py Tue May 18 18:38:23 2010 @@ -21,13 +21,17 @@ pass +NO_VTABLE = lltype.nullptr(rclass.CLASSTYPE.TO) + class Descr(history.AbstractDescr): - def __init__(self, ofs, typeinfo, extrainfo=None, name=None): + def __init__(self, ofs, typeinfo, extrainfo=None, name=None, + vtable=NO_VTABLE): self.ofs = ofs self.typeinfo = typeinfo self.extrainfo = extrainfo self.name = name + self.vtable = vtable def get_return_type(self): return self.typeinfo @@ -93,18 +97,18 @@ llimpl._llinterp = LLInterpreter(self.rtyper) self._future_values = [] self._descrs = {} - self.class_vtables_as_int = {} def _freeze_(self): assert self.translate_support_code return False - def getdescr(self, ofs, typeinfo='?', extrainfo=None, name=None): - key = (ofs, typeinfo, extrainfo, name) + def getdescr(self, ofs, typeinfo='?', extrainfo=None, name=None, + vtable=NO_VTABLE): + key = (ofs, typeinfo, extrainfo, name, vtable._obj) try: return self._descrs[key] except KeyError: - descr = Descr(ofs, typeinfo, extrainfo, name) + descr = Descr(ofs, typeinfo, extrainfo, name, vtable) self._descrs[key] = descr return descr @@ -258,17 +262,12 @@ # ---------- - def sizeof(self, S): + def sizeof(self, S, vtable=None): assert not isinstance(S, lltype.Ptr) - return self.getdescr(symbolic.get_size(S)) - - def sizevtableof(self, S, vtable): - assert isinstance(S, lltype.GcStruct) - descr = self.getdescr(symbolic.get_size(S)) - vtable_adr = llmemory.cast_ptr_to_adr(vtable) - vtable_int = llmemory.cast_adr_to_int(vtable_adr) - self.class_vtables_as_int[descr] = vtable_int - return descr + if vtable is None: + return self.getdescr(symbolic.get_size(S)) + else: + return self.getdescr(symbolic.get_size(S), vtable=vtable) class LLtypeCPU(BaseCPU): @@ -396,10 +395,10 @@ assert isinstance(sizedescr, Descr) return llimpl.do_new(sizedescr.ofs) - def bh_new_with_vtable(self, sizevtabledescr): - result = llimpl.do_new(sizevtabledescr.ofs) - llimpl.do_setfield_gc_int(result, self.fielddescrof_vtable.ofs, - self.class_vtables_as_int[sizevtabledescr]) + def bh_new_with_vtable(self, sizedescr, vtable): + assert isinstance(sizedescr, Descr) + result = llimpl.do_new(sizedescr.ofs) + llimpl.do_setfield_gc_int(result, self.fielddescrof_vtable.ofs, vtable) return result def bh_classof(self, struct): Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/model.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/model.py Tue May 18 18:38:23 2010 @@ -121,11 +121,7 @@ raise NotImplementedError @staticmethod - def sizeof(S): - raise NotImplementedError - - @staticmethod - def sizevtableof(S, vtable): + def sizeof(S, vtable=None): raise NotImplementedError @staticmethod @@ -181,7 +177,7 @@ def bh_new(self, sizedescr): raise NotImplementedError - def bh_new_with_vtable(self, sizevtabledescr): + def bh_new_with_vtable(self, sizedescr, vtable): raise NotImplementedError def bh_new_array(self, arraydescr, length): raise NotImplementedError Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Tue May 18 18:38:23 2010 @@ -309,9 +309,8 @@ elif isinstance(v, ListOfKind): lst = [self.getcolor(x) for x in v] v = ListOfKind(v.kind, lst) - elif isinstance(v, AbstractDescr): - pass - elif isinstance(v, IndirectCallTargets): + elif isinstance(v, (AbstractDescr, + IndirectCallTargets)): pass else: raise NotImplementedError(type(v)) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py Tue May 18 18:38:23 2010 @@ -1,4 +1,5 @@ from pypy.rpython.lltypesystem import lltype, rclass +from pypy.rlib.objectmodel import we_are_translated def get_vtable_for_gcstruct(cpu, GCSTRUCT): @@ -33,3 +34,37 @@ testing_gcstruct2vtable[GCSTRUCT] = vtable testing_gcstruct2vtable = {} + +# ____________________________________________________________ + +VTABLETYPE = rclass.CLASSTYPE + +def register_known_gctype(cpu, vtable, STRUCT): + # register the correspondance 'vtable' <-> 'STRUCT' in the cpu + sizedescr = cpu.sizeof(STRUCT) + if hasattr(sizedescr, '_corresponding_vtable'): + assert sizedescr._corresponding_vtable == vtable + else: + assert lltype.typeOf(vtable) == VTABLETYPE + if not hasattr(cpu, '_all_size_descrs'): + cpu._all_size_descrs = [] + cpu._all_size_descrs.append(sizedescr) + sizedescr._corresponding_vtable = vtable + +def vtable2descr(cpu, vtable): + assert lltype.typeOf(vtable) == VTABLETYPE + if we_are_translated(): + # Build the dict {vtable: sizedescr} at runtime. + # This is necessary because the 'vtables' are just pointers to + # static data, so they can't be used as keys in prebuilt dicts. + XXX + else: + for descr in cpu._all_size_descrs: + if descr._corresponding_vtable == vtable: + return descr + raise KeyError(vtable) + +def descr2vtable(cpu, descr): + from pypy.jit.metainterp import history + assert isinstance(descr, history.AbstractDescr) + return descr._corresponding_vtable Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Tue May 18 18:38:23 2010 @@ -278,18 +278,21 @@ lst.append(jitcode) return self.handle_residual_call(op, [IndirectCallTargets(lst)]) - def _prepare_builtin_call(self, op, oopspec_name, args, extra=None): + def _prepare_builtin_call(self, op, oopspec_name, args, + extra=None, extrakey=None): argtypes = [v.concretetype for v in args] resulttype = op.result.concretetype c_func, TP = support.builtin_func_for_spec(self.cpu.rtyper, oopspec_name, argtypes, - resulttype, extra) + resulttype, extra, extrakey) return SpaceOperation('direct_call', [c_func] + args, op.result) - def _do_builtin_call(self, op, oopspec_name=None, args=None, extra=None): + def _do_builtin_call(self, op, oopspec_name=None, args=None, + extra=None, extrakey=None): if oopspec_name is None: oopspec_name = op.opname if args is None: args = op.args - op1 = self._prepare_builtin_call(op, oopspec_name, args, extra) + op1 = self._prepare_builtin_call(op, oopspec_name, args, + extra, extrakey) return self.rewrite_op_direct_call(op1) rewrite_op_int_floordiv_ovf_zer = _do_builtin_call @@ -447,17 +450,15 @@ if hasattr(rtti._obj, 'destructor_funcptr'): RESULT = lltype.Ptr(STRUCT) assert RESULT == op.result.concretetype - return self._do_builtin_call(op, 'alloc_with_del', - [], extra = (RESULT, vtable)) - # store the vtable as an address -- that's fine, because the - # GC doesn't need to follow them - #self.codewriter.register_known_gctype(vtable, STRUCT) - sizevtabledescr = self.cpu.sizevtableof(STRUCT, vtable) - return SpaceOperation('new_with_vtable', [sizevtabledescr], - op.result) + return self._do_builtin_call(op, 'alloc_with_del', [], + extra = (RESULT, vtable), + extrakey = STRUCT) + heaptracker.register_known_gctype(self.cpu, vtable, STRUCT) + opname = 'new_with_vtable' else: - sizedescr = self.cpu.sizeof(STRUCT) - return SpaceOperation('new', [sizedescr], op.result) + opname = 'new' + sizedescr = self.cpu.sizeof(STRUCT) + return SpaceOperation(opname, [sizedescr], op.result) def rewrite_op_getinteriorarraysize(self, op): # only supports strings and unicodes Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py Tue May 18 18:38:23 2010 @@ -400,8 +400,10 @@ else: raise ValueError(op.opname) -def builtin_func_for_spec(rtyper, oopspec_name, ll_args, ll_res, extra=None): - key = (oopspec_name, tuple(ll_args), ll_res, extra) +def builtin_func_for_spec(rtyper, oopspec_name, ll_args, ll_res, + extra=None, extrakey=None): + assert (extra is None) == (extrakey is None) + key = (oopspec_name, tuple(ll_args), ll_res, extrakey) try: return rtyper._builtin_func_for_spec_cache[key] except (KeyError, AttributeError): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Tue May 18 18:38:23 2010 @@ -17,10 +17,11 @@ return ('calldescr', FUNC, ARGS, RESULT) def fielddescrof(self, STRUCT, name): return ('fielddescr', STRUCT, name) - def sizeof(self, STRUCT): - return ('sizedescr', STRUCT) - def sizevtableof(self, STRUCT, vtable): - return ('sizevtabledescr', STRUCT, vtable) + def sizeof(self, STRUCT, vtable=None): + if vtable is None: + return ('sizedescr', STRUCT) + else: + return ('sizedescr', STRUCT, vtable) class FakeLink: args = [] @@ -326,7 +327,7 @@ assert op1.args == [('sizedescr', S)] def test_malloc_new_with_vtable(): - class vtable: pass + vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) S = lltype.GcStruct('S', ('parent', rclass.OBJECT)) heaptracker.set_testing_vtable_for_gcstruct(S, vtable, 'S') v = varoftype(lltype.Ptr(S)) @@ -334,10 +335,10 @@ Constant({'flavor': 'gc'}, lltype.Void)], v) op1 = Transformer(FakeCPU()).rewrite_operation(op) assert op1.opname == 'new_with_vtable' - assert op1.args == [('sizevtabledescr', S, vtable)] + assert op1.args == [('sizedescr', S, vtable)] def test_malloc_new_with_destructor(): - class vtable: pass + vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) S = lltype.GcStruct('S', ('parent', rclass.OBJECT)) DESTRUCTOR = lltype.FuncType([lltype.Ptr(S)], lltype.Void) destructor = lltype.functionptr(DESTRUCTOR, 'destructor') Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Tue May 18 18:38:23 2010 @@ -7,6 +7,7 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.llinterp import LLException from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr +from pypy.jit.codewriter import heaptracker def arguments(*argtypes, **kwds): @@ -56,8 +57,9 @@ def __init__(self, codewriter, metainterp_sd=None): self.cpu = codewriter.cpu - self.setup_insns(codewriter.assembler.insns) - self.setup_descrs(codewriter.assembler.descrs) + asm = codewriter.assembler + self.setup_insns(asm.insns) + self.setup_descrs(asm.descrs) self.metainterp_sd = metainterp_sd self._freeze_() @@ -167,7 +169,7 @@ args = args + (value,) if verbose and not we_are_translated(): - print '\t', name, list(args), + print '\tbh:', name, list(args), # call the method bhimpl_xxx() try: @@ -902,7 +904,9 @@ @arguments("cpu", "d", returns="r") def bhimpl_new_with_vtable(cpu, descr): - return cpu.bh_new_with_vtable(descr) + vtable = heaptracker.descr2vtable(cpu, descr) + vtable = llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(vtable)) + return cpu.bh_new_with_vtable(descr, vtable) @arguments("cpu", "r", returns="i") def bhimpl_guard_class(cpu, struct): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py Tue May 18 18:38:23 2010 @@ -2,13 +2,13 @@ """ import py -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.ootypesystem import ootype from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.rarithmetic import ovfcheck, r_uint, intmask from pypy.rlib.unroll import unrolling_iterable from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, check_descr -from pypy.jit.metainterp.history import INT, REF, FLOAT +from pypy.jit.metainterp.history import INT, REF, FLOAT, AbstractDescr from pypy.jit.metainterp import resoperation from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.blackhole import BlackholeInterpreter, NULL @@ -148,6 +148,18 @@ else: cpu.bh_setfield_raw_i(struct, fielddescr, itembox.getint()) +def exec_new_with_vtable(cpu, clsbox): + from pypy.jit.codewriter import heaptracker + vtable = clsbox.getint() + vtableadr = llmemory.cast_int_to_adr(vtable) + vtableptr = llmemory.cast_adr_to_ptr(vtableadr, heaptracker.VTABLETYPE) + descr = heaptracker.vtable2descr(cpu, vtableptr) + return cpu.bh_new_with_vtable(descr, vtable) + +def do_new_with_vtable(metainterp, clsbox): + cpu = metainterp.cpu + return BoxPtr(exec_new_with_vtable(cpu, clsbox)) + def do_int_add_ovf(metainterp, box1, box2): a = box1.getint() b = box2.getint() @@ -279,6 +291,7 @@ value = metainterp.cpu elif argtype == 'd': value = argboxes[-1] + assert isinstance(value, AbstractDescr) argboxes = argboxes[:-1] else: argbox = argboxes[0] Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Tue May 18 18:38:23 2010 @@ -313,8 +313,13 @@ return self.execute_with_descr(rop.NEW, sizedescr) @arguments("descr") - def opimpl_new_with_vtable(self, sizevtabledescr): - return self.execute_with_descr(rop.NEW_WITH_VTABLE, sizevtabledescr) + def opimpl_new_with_vtable(self, sizedescr): + from pypy.jit.codewriter import heaptracker + from pypy.jit.metainterp.warmstate import wrap + cpu = self.metainterp.cpu + cls = heaptracker.descr2vtable(cpu, sizedescr) + clsbox = wrap(cpu, cls, in_const_box=True) + return self.execute(rop.NEW_WITH_VTABLE, clsbox) @FixME #arguments("box") def opimpl_runtimenew(self, classbox): @@ -2117,7 +2122,7 @@ self.pc = position + num_return_args # if not we_are_translated(): - print '\tjitcode: %s(%s)' % (name, ', '.join(map(repr, args))), + print '\tpyjitpl: %s(%s)' % (name, ', '.join(map(repr, args))), try: resultbox = unboundmethod(self, *args) except Exception, e: Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py Tue May 18 18:38:23 2010 @@ -207,7 +207,7 @@ 'GETFIELD_GC/1d', 'GETFIELD_RAW/1d', 'NEW/0d', - 'NEW_WITH_VTABLE/0d', + 'NEW_WITH_VTABLE/1', 'NEW_ARRAY/1d', 'FORCE_TOKEN/0', 'VIRTUAL_REF/2', Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py Tue May 18 18:38:23 2010 @@ -388,12 +388,10 @@ class AbstractVirtualInfo(object): - def allocate(self, metainterp): - raise NotImplementedError + #def allocate(self, metainterp): + # raise NotImplementedError #def setfields(self, decoder, struct): # raise NotImplementedError - def bh_allocate(self, cpu): - raise NotImplementedError def equals(self, fieldnums): return tagged_list_eq(self.fieldnums, fieldnums) def set_content(self, fieldnums): @@ -428,9 +426,9 @@ AbstractVirtualStructInfo.__init__(self, fielddescrs) self.known_class = known_class - def allocate(self, metainterp): - return metainterp.execute_and_record(rop.NEW_WITH_VTABLE, - None, self.known_class) + def allocate(self, cpu): + from pypy.jit.metainterp.executor import exec_new_with_vtable + return exec_new_with_vtable(cpu, self.known_class) def debug_prints(self): debug_print("\tvirtualinfo", self.known_class.repr_rpython()) @@ -617,7 +615,7 @@ for i in range(len(virtuals)): vinfo = virtuals[i] if vinfo is not None: - self.virtuals[i] = vinfo.bh_allocate(self.cpu) + self.virtuals[i] = vinfo.allocate(self.cpu) for i in range(len(virtuals)): vinfo = virtuals[i] if vinfo is not None: @@ -632,13 +630,13 @@ def setfield(self, descr, struct, fieldnum): if descr.is_pointer_field(): newvalue = self._decode_ref(fieldnum) - self.cpu.bh_setfield_gc_r(struct, descr, newfield) + self.cpu.bh_setfield_gc_r(struct, descr, newvalue) elif descr.is_float_field(): newvalue = self._decode_float(fieldnum) - self.cpu.bh_setfield_gc_f(struct, descr, newfield) + self.cpu.bh_setfield_gc_f(struct, descr, newvalue) else: newvalue = self._decode_int(fieldnum) - self.cpu.bh_setfield_gc_i(struct, descr, newfield) + self.cpu.bh_setfield_gc_i(struct, descr, newvalue) def setarrayitem(self, arraydescr, array, index, fieldnum): if arraydescr.is_array_of_pointers(): @@ -671,12 +669,12 @@ num, tag = untag(tagged) if tag == TAGCONST: if tagged_eq(tagged, NULLREF): - return history.REF + return REF return self.consts[num].type elif tag == TAGVIRTUAL: - return history.REF + return REF elif tag == TAGINT: - return history.INT + return INT else: assert tag == TAGBOX return self.cpu.get_latest_value_kind(num) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmstate.py Tue May 18 18:38:23 2010 @@ -54,7 +54,7 @@ return history.BoxPtr(value) else: adr = llmemory.cast_ptr_to_adr(value) - value = cpu.cast_adr_to_int(adr) + value = llmemory.cast_adr_to_int(adr) # fall through to the end of the function elif isinstance(lltype.typeOf(value), ootype.OOType): value = ootype.cast_to_object(value) From arigo at codespeak.net Tue May 18 18:49:19 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 18 May 2010 18:49:19 +0200 (CEST) Subject: [pypy-svn] r74546 - pypy/branch/blackhole-improvement/pypy/jit/codewriter/test Message-ID: <20100518164919.4AC55282BE0@codespeak.net> Author: arigo Date: Tue May 18 18:49:17 2010 New Revision: 74546 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Log: Fix test. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Tue May 18 18:49:17 2010 @@ -17,11 +17,11 @@ return ('calldescr', FUNC, ARGS, RESULT) def fielddescrof(self, STRUCT, name): return ('fielddescr', STRUCT, name) - def sizeof(self, STRUCT, vtable=None): - if vtable is None: - return ('sizedescr', STRUCT) - else: - return ('sizedescr', STRUCT, vtable) + def sizeof(self, STRUCT): + return FakeDescr(('sizedescr', STRUCT)) + +class FakeDescr(tuple): + pass class FakeLink: args = [] @@ -333,9 +333,12 @@ v = varoftype(lltype.Ptr(S)) op = SpaceOperation('malloc', [Constant(S, lltype.Void), Constant({'flavor': 'gc'}, lltype.Void)], v) - op1 = Transformer(FakeCPU()).rewrite_operation(op) + cpu = FakeCPU() + op1 = Transformer(cpu).rewrite_operation(op) assert op1.opname == 'new_with_vtable' - assert op1.args == [('sizedescr', S, vtable)] + assert op1.args == [('sizedescr', S)] + #assert heaptracker.descr2vtable(cpu, op1.args[0]) == vtable [type check] + assert heaptracker.vtable2descr(cpu, vtable) == op1.args[0] def test_malloc_new_with_destructor(): vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) From arigo at codespeak.net Tue May 18 18:53:10 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 18 May 2010 18:53:10 +0200 (CEST) Subject: [pypy-svn] r74547 - in pypy/branch/blackhole-improvement/pypy/jit: backend backend/llgraph codewriter Message-ID: <20100518165310.84EBF282BE0@codespeak.net> Author: arigo Date: Tue May 18 18:53:09 2010 New Revision: 74547 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py pypy/branch/blackhole-improvement/pypy/jit/backend/model.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Log: Minor fixes. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py Tue May 18 18:53:09 2010 @@ -21,17 +21,13 @@ pass -NO_VTABLE = lltype.nullptr(rclass.CLASSTYPE.TO) - class Descr(history.AbstractDescr): - def __init__(self, ofs, typeinfo, extrainfo=None, name=None, - vtable=NO_VTABLE): + def __init__(self, ofs, typeinfo, extrainfo=None, name=None): self.ofs = ofs self.typeinfo = typeinfo self.extrainfo = extrainfo self.name = name - self.vtable = vtable def get_return_type(self): return self.typeinfo @@ -102,13 +98,12 @@ assert self.translate_support_code return False - def getdescr(self, ofs, typeinfo='?', extrainfo=None, name=None, - vtable=NO_VTABLE): - key = (ofs, typeinfo, extrainfo, name, vtable._obj) + def getdescr(self, ofs, typeinfo='?', extrainfo=None, name=None): + key = (ofs, typeinfo, extrainfo, name) try: return self._descrs[key] except KeyError: - descr = Descr(ofs, typeinfo, extrainfo, name, vtable) + descr = Descr(ofs, typeinfo, extrainfo, name) self._descrs[key] = descr return descr @@ -262,12 +257,9 @@ # ---------- - def sizeof(self, S, vtable=None): + def sizeof(self, S): assert not isinstance(S, lltype.Ptr) - if vtable is None: - return self.getdescr(symbolic.get_size(S)) - else: - return self.getdescr(symbolic.get_size(S), vtable=vtable) + return self.getdescr(symbolic.get_size(S)) class LLtypeCPU(BaseCPU): Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/model.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/model.py Tue May 18 18:53:09 2010 @@ -121,7 +121,7 @@ raise NotImplementedError @staticmethod - def sizeof(S, vtable=None): + def sizeof(S): raise NotImplementedError @staticmethod Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Tue May 18 18:53:09 2010 @@ -33,9 +33,17 @@ if block.operations == (): return renamings = {} + renamings_constants = {} # subset of 'renamings', {Var:Const} only newoperations = [] # + def do_rename(var, var_or_const): + renamings[var] = var_or_const + if isinstance(var_or_const, Constant): + renamings_constants[var] = var_or_const + # for op in block.operations: + if renamings_constants: + op = self._do_renaming(renamings_constants, op) oplist = self.rewrite_operation(op) # count_before_last_operation = len(newoperations) @@ -48,10 +56,10 @@ # rewrite_operation() returns None to mean "has no real # effect, the result should just be renamed to args[0]" if op.result is not None: - renamings[op.result] = renamings.get(op.args[0], - op.args[0]) + do_rename(op.result, renamings.get(op.args[0], + op.args[0])) elif isinstance(op1, Constant): - renamings[op.result] = op1 + do_rename(op.result, op1) else: raise TypeError(repr(op1)) # From arigo at codespeak.net Tue May 18 18:56:22 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 18 May 2010 18:56:22 +0200 (CEST) Subject: [pypy-svn] r74548 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100518165622.4A2D6282BE0@codespeak.net> Author: arigo Date: Tue May 18 18:56:20 2010 New Revision: 74548 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Log: Fix. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Tue May 18 18:56:20 2010 @@ -254,7 +254,7 @@ # many of them -- a chain of int_eq comparisons is fine assert kind == 'int' # XXX color = self.getcolor(block.exitswitch) - self.emitline('int_guard_value', color, color) + self.emitline('int_guard_value', color) for switch in switches: # make the case described by 'switch' self.emitline('goto_if_not_int_eq', Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Tue May 18 18:56:20 2010 @@ -209,7 +209,7 @@ elif n == 7: return 1212 else: return 42 self.encoding_test(f, [65], """ - int_guard_value %i0, %i0 + int_guard_value %i0 goto_if_not_int_eq %i0, $-5, L1 int_return $12 L1: From arigo at codespeak.net Tue May 18 19:01:48 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 18 May 2010 19:01:48 +0200 (CEST) Subject: [pypy-svn] r74549 - in pypy/branch/blackhole-improvement/pypy/jit: backend/llgraph backend/llgraph/test metainterp/test Message-ID: <20100518170148.B0A88282BE0@codespeak.net> Author: arigo Date: Tue May 18 19:01:47 2010 New Revision: 74549 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/test/test_llgraph.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/oparser.py Log: Random fixes. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py Tue May 18 19:01:47 2010 @@ -230,19 +230,19 @@ else: return repr(x) -def repr_list(lst, types, memocast): +def repr_list(lst, types): res_l = [] if types and types[-1] == 'varargs': types = types[:-1] + ('int',) * (len(lst) - len(types) + 1) assert len(types) == len(lst) for elem, tp in zip(lst, types): if isinstance(elem, Constant): - res_l.append('(%s)' % repr1(elem, tp, memocast)) + res_l.append('(%s)' % repr1(elem, tp)) else: - res_l.append(repr1(elem, tp, memocast)) + res_l.append(repr1(elem, tp)) return '[%s]' % (', '.join(res_l)) -def repr1(x, tp, memocast): +def repr1(x, tp): if tp == "intorptr": TYPE = lltype.typeOf(x) if isinstance(TYPE, lltype.Ptr) and TYPE.TO._gckind == 'gc': @@ -258,7 +258,7 @@ return '(* None)' if isinstance(x, int): # XXX normalize? - ptr = str(cast_int_to_adr(memocast, x)) + ptr = str(llmemory.cast_int_to_adr(x)) elif isinstance(ootype.typeOf(x), ootype.OOType): return repr(x) else: @@ -520,10 +520,9 @@ elif res is NotImplemented: resdata = '*fail*' else: - resdata = '-> ' + repr1(res, restype, self.memocast) + resdata = '-> ' + repr1(res, restype) # fish the types - log.cpu('\t%s %s %s' % (opname, repr_list(values, argtypes, - self.memocast), + log.cpu('\t%s %s %s' % (opname, repr_list(values, argtypes), resdata)) return res @@ -592,7 +591,7 @@ def op_guard_class(self, _, value, expected_class): value = lltype.cast_opaque_ptr(rclass.OBJECTPTR, value) expected_class = llmemory.cast_adr_to_ptr( - cast_int_to_adr(self.memocast, expected_class), + llmemory.cast_int_to_adr(expected_class), rclass.CLASSTYPE) if value.typeptr != expected_class: raise GuardFailed @@ -626,7 +625,7 @@ def _cast_exception(self, exception): return llmemory.cast_adr_to_ptr( - cast_int_to_adr(self.memocast, exception), + llmemory.cast_int_to_adr(exception), rclass.CLASSTYPE) def _issubclass(self, cls1, cls2): @@ -692,7 +691,7 @@ if arraydescr.typeinfo == REF: return do_getarrayitem_gc_ptr(array, index) elif arraydescr.typeinfo == INT: - return do_getarrayitem_gc_int(array, index, self.memocast) + return do_getarrayitem_gc_int(array, index) elif arraydescr.typeinfo == FLOAT: return do_getarrayitem_gc_float(array, index) else: @@ -704,7 +703,7 @@ if fielddescr.typeinfo == REF: return do_getfield_gc_ptr(struct, fielddescr.ofs) elif fielddescr.typeinfo == INT: - return do_getfield_gc_int(struct, fielddescr.ofs, self.memocast) + return do_getfield_gc_int(struct, fielddescr.ofs) elif fielddescr.typeinfo == FLOAT: return do_getfield_gc_float(struct, fielddescr.ofs) else: @@ -714,11 +713,11 @@ def op_getfield_raw(self, fielddescr, struct): if fielddescr.typeinfo == REF: - return do_getfield_raw_ptr(struct, fielddescr.ofs, self.memocast) + return do_getfield_raw_ptr(struct, fielddescr.ofs) elif fielddescr.typeinfo == INT: - return do_getfield_raw_int(struct, fielddescr.ofs, self.memocast) + return do_getfield_raw_int(struct, fielddescr.ofs) elif fielddescr.typeinfo == FLOAT: - return do_getfield_raw_float(struct, fielddescr.ofs, self.memocast) + return do_getfield_raw_float(struct, fielddescr.ofs) else: raise NotImplementedError @@ -729,6 +728,7 @@ def op_new_with_vtable(self, descr, vtable): assert descr is None + xxxxxxxxxxxx size = get_class_size(self.memocast, vtable) result = do_new(size) value = lltype.cast_opaque_ptr(rclass.OBJECTPTR, result) @@ -749,8 +749,7 @@ if fielddescr.typeinfo == REF: do_setfield_gc_ptr(struct, fielddescr.ofs, newvalue) elif fielddescr.typeinfo == INT: - do_setfield_gc_int(struct, fielddescr.ofs, newvalue, - self.memocast) + do_setfield_gc_int(struct, fielddescr.ofs, newvalue) elif fielddescr.typeinfo == FLOAT: do_setfield_gc_float(struct, fielddescr.ofs, newvalue) else: @@ -758,14 +757,11 @@ def op_setfield_raw(self, fielddescr, struct, newvalue): if fielddescr.typeinfo == REF: - do_setfield_raw_ptr(struct, fielddescr.ofs, newvalue, - self.memocast) + do_setfield_raw_ptr(struct, fielddescr.ofs, newvalue) elif fielddescr.typeinfo == INT: - do_setfield_raw_int(struct, fielddescr.ofs, newvalue, - self.memocast) + do_setfield_raw_int(struct, fielddescr.ofs, newvalue) elif fielddescr.typeinfo == FLOAT: - do_setfield_raw_float(struct, fielddescr.ofs, newvalue, - self.memocast) + do_setfield_raw_float(struct, fielddescr.ofs, newvalue) else: raise NotImplementedError @@ -1185,13 +1181,13 @@ ## assert 0 <= int < len(memocast.addresses) ## return memocast.addresses[int] -def get_class_size(memocast, vtable): - memocast = _from_opaque(memocast) - return memocast.vtable_to_size[vtable] - -def set_class_size(memocast, vtable, size): - memocast = _from_opaque(memocast) - memocast.vtable_to_size[vtable] = size +##def get_class_size(memocast, vtable): +## memocast = _from_opaque(memocast) +## return memocast.vtable_to_size[vtable] + +##def set_class_size(memocast, vtable, size): +## memocast = _from_opaque(memocast) +## memocast.vtable_to_size[vtable] = size class GuardFailed(Exception): pass @@ -1219,9 +1215,9 @@ uni = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string) return ord(uni.chars[index]) -def do_getarrayitem_gc_int(array, index, memocast): +def do_getarrayitem_gc_int(array, index): array = array._obj.container - return cast_to_int(array.getitem(index), memocast) + return cast_to_int(array.getitem(index)) def do_getarrayitem_gc_float(array, index): array = array._obj.container @@ -1269,10 +1265,10 @@ x = lltype.malloc(TYPE, count, zero=True) return cast_to_ptr(x) -def do_setarrayitem_gc_int(array, index, newvalue, memocast): +def do_setarrayitem_gc_int(array, index, newvalue): array = array._obj.container ITEMTYPE = lltype.typeOf(array).OF - newvalue = cast_from_int(ITEMTYPE, newvalue, memocast) + newvalue = cast_from_int(ITEMTYPE, newvalue) array.setitem(index, newvalue) def do_setarrayitem_gc_float(array, index, newvalue): @@ -1308,23 +1304,23 @@ newvalue = cast_from_ptr(FIELDTYPE, newvalue) setattr(ptr, fieldname, newvalue) -def do_setfield_raw_int(struct, fieldnum, newvalue, memocast): +def do_setfield_raw_int(struct, fieldnum, newvalue): STRUCT, fieldname = symbolic.TokenToField[fieldnum] - ptr = cast_from_int(lltype.Ptr(STRUCT), struct, memocast) + ptr = cast_from_int(lltype.Ptr(STRUCT), struct) FIELDTYPE = getattr(STRUCT, fieldname) - newvalue = cast_from_int(FIELDTYPE, newvalue, memocast) + newvalue = cast_from_int(FIELDTYPE, newvalue) setattr(ptr, fieldname, newvalue) -def do_setfield_raw_float(struct, fieldnum, newvalue, memocast): +def do_setfield_raw_float(struct, fieldnum, newvalue): STRUCT, fieldname = symbolic.TokenToField[fieldnum] - ptr = cast_from_int(lltype.Ptr(STRUCT), struct, memocast) + ptr = cast_from_int(lltype.Ptr(STRUCT), struct) FIELDTYPE = getattr(STRUCT, fieldname) newvalue = cast_from_float(FIELDTYPE, newvalue) setattr(ptr, fieldname, newvalue) -def do_setfield_raw_ptr(struct, fieldnum, newvalue, memocast): +def do_setfield_raw_ptr(struct, fieldnum, newvalue): STRUCT, fieldname = symbolic.TokenToField[fieldnum] - ptr = cast_from_int(lltype.Ptr(STRUCT), struct, memocast) + ptr = cast_from_int(lltype.Ptr(STRUCT), struct) FIELDTYPE = getattr(STRUCT, fieldname) newvalue = cast_from_ptr(FIELDTYPE, newvalue) setattr(ptr, fieldname, newvalue) Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/test/test_llgraph.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/test/test_llgraph.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/test/test_llgraph.py Tue May 18 19:01:47 2010 @@ -18,19 +18,18 @@ def setup_method(self, _): self.cpu = self.cpu_type(None) - def test_cast_adr_to_int_and_back(self): - cpu = self.cpu - X = lltype.Struct('X', ('foo', lltype.Signed)) - x = lltype.malloc(X, immortal=True) - x.foo = 42 - a = llmemory.cast_ptr_to_adr(x) - i = cpu.cast_adr_to_int(a) - assert isinstance(i, int) - a2 = cpu.cast_int_to_adr(i) - assert llmemory.cast_adr_to_ptr(a2, lltype.Ptr(X)) == x - assert cpu.cast_adr_to_int(llmemory.NULL) == 0 - assert cpu.cast_int_to_adr(0) == llmemory.NULL +def test_cast_adr_to_int_and_back(): + X = lltype.Struct('X', ('foo', lltype.Signed)) + x = lltype.malloc(X, immortal=True) + x.foo = 42 + a = llmemory.cast_ptr_to_adr(x) + i = llmemory.cast_adr_to_int(a) + assert lltype.typeOf(i) is lltype.Signed + a2 = llmemory.cast_int_to_adr(i) + assert llmemory.cast_adr_to_ptr(a2, lltype.Ptr(X)) == x + assert llmemory.cast_adr_to_int(llmemory.NULL) == 0 + assert llmemory.cast_int_to_adr(0) == llmemory.NULL ## these tests never worked ## class TestOOTypeLLGraph(LLGraphTest): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/oparser.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/oparser.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/oparser.py Tue May 18 19:01:47 2010 @@ -4,7 +4,7 @@ """ from pypy.jit.metainterp.history import TreeLoop, BoxInt, ConstInt,\ - ConstAddr, ConstObj, ConstPtr, Box, BasicFailDescr, BoxFloat, ConstFloat,\ + ConstObj, ConstPtr, Box, BasicFailDescr, BoxFloat, ConstFloat,\ LoopToken from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.metainterp.typesystem import llhelper @@ -74,8 +74,8 @@ return ConstPtr(obj) else: assert typ == 'class' - return ConstAddr(llmemory.cast_ptr_to_adr(obj), - self.cpu) + return ConstInt(llmemory.cast_adr_to_int( + llmemory.cast_ptr_to_adr(obj))) else: if typ == 'ptr': return ConstObj(obj) From arigo at codespeak.net Tue May 18 22:05:23 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 18 May 2010 22:05:23 +0200 (CEST) Subject: [pypy-svn] r74550 - in pypy/branch/blackhole-improvement/pypy/jit: backend/llgraph codewriter metainterp Message-ID: <20100518200523.72B4E282BE0@codespeak.net> Author: arigo Date: Tue May 18 22:05:20 2010 New Revision: 74550 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/effectinfo.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizeopt.py Log: Fixes. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py Tue May 18 22:05:20 2010 @@ -766,18 +766,18 @@ raise NotImplementedError def op_call(self, calldescr, func, *args): - _call_args[:] = args - if calldescr.typeinfo == 'v': - err_result = None - elif calldescr.typeinfo == REF: - err_result = lltype.nullptr(llmemory.GCREF.TO) - elif calldescr.typeinfo == INT: - err_result = 0 - elif calldescr.typeinfo == FLOAT: - err_result = 0.0 - else: - raise NotImplementedError - return _do_call_common(func, err_result) + assert _call_args_i == _call_args_r == _call_args_f == [] + for x in args: + T = lltype.typeOf(x) + if T is lltype.Signed: + _call_args_i.append(x) + elif T == llmemory.GCREF: + _call_args_r.append(x) + elif T is lltype.Float: + _call_args_f.append(x) + else: + raise TypeError(x) + return _do_call_common(func) op_call_pure = op_call Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/effectinfo.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/effectinfo.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/effectinfo.py Tue May 18 22:05:20 2010 @@ -32,6 +32,9 @@ cls._cache[key] = result return result + def check_forces_virtual_or_virtualizable(self): + return self.extraeffect >= EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE + def effectinfo_from_writeanalyze(effects, cpu, extraeffect=EF_CAN_RAISE): from pypy.translator.backendopt.writeanalyze import top_set if effects is top_set: Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizeopt.py Tue May 18 22:05:20 2010 @@ -1091,7 +1091,7 @@ del self.cached_arrayitems[arraydescr] except KeyError: pass - if effectinfo.forces_virtual_or_virtualizable: + if effectinfo.check_forces_virtual_or_virtualizable(): vrefinfo = self.optimizer.metainterp_sd.virtualref_info self.force_lazy_setfield(vrefinfo.descr_forced) # ^^^ we only need to force this field; the other fields From arigo at codespeak.net Tue May 18 22:20:43 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 18 May 2010 22:20:43 +0200 (CEST) Subject: [pypy-svn] r74551 - in pypy/branch/blackhole-improvement/pypy/jit: backend/llgraph codewriter metainterp Message-ID: <20100518202043.4CD76282BE0@codespeak.net> Author: arigo Date: Tue May 18 22:20:41 2010 New Revision: 74551 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: Tweaks. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py Tue May 18 22:20:41 2010 @@ -156,8 +156,6 @@ llimpl.compile_add_int_const(c, x.value) elif isinstance(x, self.ts.ConstRef): llimpl.compile_add_ref_const(c, x.value, self.ts.BASETYPE) - elif isinstance(x, history.ConstAddr): - llimpl.compile_add_int_const(c, x.getint()) elif isinstance(x, history.ConstFloat): llimpl.compile_add_float_const(c, x.value) else: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Tue May 18 22:20:41 2010 @@ -303,6 +303,8 @@ extra, extrakey) return self.rewrite_op_direct_call(op1) + # XXX some of the following functions should not become residual calls + # but be really compiled rewrite_op_int_floordiv_ovf_zer = _do_builtin_call rewrite_op_int_floordiv_ovf = _do_builtin_call rewrite_op_int_floordiv_zer = _do_builtin_call @@ -310,6 +312,7 @@ rewrite_op_int_mod_ovf = _do_builtin_call rewrite_op_int_mod_zer = _do_builtin_call rewrite_op_int_lshift_ovf = _do_builtin_call + rewrite_op_int_abs = _do_builtin_call rewrite_op_gc_identityhash = _do_builtin_call # ---------- Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py Tue May 18 22:20:41 2010 @@ -164,6 +164,9 @@ def _ll_2_int_lshift_ovf(x, y): return llop.int_lshift_ovf(lltype.Signed, x, y) +def _ll_1_int_abs(x): + return abs(x) + class LLtypeHelpers: Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py Tue May 18 22:20:41 2010 @@ -255,7 +255,7 @@ return llmemory.cast_int_to_adr(self.value) def _get_hash_(self): - return self.value + return integer_hash(self.value) def set_future_value(self, cpu, j): cpu.set_future_value_int(j, self.value) @@ -492,7 +492,7 @@ return llmemory.cast_int_to_adr(self.value) def _get_hash_(self): - return self.value + return integer_hash(self.value) def set_future_value(self, cpu, j): cpu.set_future_value_int(j, self.value) @@ -655,6 +655,13 @@ except lltype.DelayedPointer: return -2 # xxx risk of changing hash... +def integer_hash(i): + if not we_are_translated() and isinstance(i, llmemory.AddressAsInt): + # Warning: such a hash changes at the time of translation + adr = llmemory.cast_int_to_adr(i) + return lltype.cast_ptr_to_int(adr.ptr) + return i + # ____________________________________________________________ # The TreeLoop class contains a loop or a generalized loop, i.e. a tree Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Tue May 18 22:20:41 2010 @@ -159,6 +159,7 @@ 'float_add', 'float_sub', 'float_mul', 'float_truediv', 'float_lt', 'float_le', 'float_eq', 'float_ne', 'float_gt', 'float_ge', + 'ptr_eq', 'ptr_ne', ]: exec py.code.Source(''' @arguments("box", "box") @@ -175,9 +176,10 @@ return resbox ''' % (_opimpl, _opimpl.upper())).compile() - for _opimpl in ['int_is_true', 'int_neg', 'int_invert', 'bool_not', + for _opimpl in ['int_is_true', 'int_is_zero', 'int_neg', 'int_invert', 'cast_ptr_to_int', 'cast_float_to_int', 'cast_int_to_float', 'float_neg', 'float_abs', + 'ptr_iszero', 'ptr_nonzero', ]: exec py.code.Source(''' @arguments("box") @@ -452,48 +454,6 @@ def opimpl_overflow_error(self): return self.metainterp.raise_overflow_error() - @FixME #arguments("orgpc", "box") - def opimpl_int_abs(self, pc, box): - nonneg = self.metainterp.execute_and_record( - rop.INT_GE, None, box, ConstInt(0)) - nonneg = self.implement_guard_value(pc, nonneg) - if nonneg.getint(): - self.make_result_box(box) - else: - self.execute(rop.INT_NEG, box) - - @FixME #arguments("orgpc", "box") - def opimpl_oononnull(self, pc, box): - value = box.nonnull() - if value: - opnum = rop.GUARD_NONNULL - res = ConstInt(1) - else: - opnum = rop.GUARD_ISNULL - res = ConstInt(0) - self.generate_guard(pc, opnum, box, []) - self.make_result_box(res) - - @FixME #arguments("orgpc", "box") - def opimpl_ooisnull(self, pc, box): - value = box.nonnull() - if value: - opnum = rop.GUARD_NONNULL - res = ConstInt(0) - else: - opnum = rop.GUARD_ISNULL - res = ConstInt(1) - self.generate_guard(pc, opnum, box, []) - self.make_result_box(res) - - @FixME #arguments("box", "box") - def opimpl_ptr_eq(self, box1, box2): - self.execute(rop.OOIS, box1, box2) - - @FixME #arguments("box", "box") - def opimpl_ptr_ne(self, box1, box2): - self.execute(rop.OOISNOT, box1, box2) - @arguments("box", "descr") def _opimpl_getfield_gc_any(self, box, fielddescr): return self.execute_with_descr(rop.GETFIELD_GC, fielddescr, box) From arigo at codespeak.net Tue May 18 22:55:34 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 18 May 2010 22:55:34 +0200 (CEST) Subject: [pypy-svn] r74552 - pypy/trunk/pypy/lib/app_test Message-ID: <20100518205534.5CBB3282BE0@codespeak.net> Author: arigo Date: Tue May 18 22:55:32 2010 New Revision: 74552 Added: pypy/trunk/pypy/lib/app_test/test_pickle_extra.py (contents, props changed) Log: Add a test for pickle._pickle_moduledict(). It passes. Added: pypy/trunk/pypy/lib/app_test/test_pickle_extra.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/lib/app_test/test_pickle_extra.py Tue May 18 22:55:32 2010 @@ -0,0 +1,15 @@ +import pickle + +def test_pickle_moduledict(): + if "_pickle_moduledict" not in pickle.Pickler.__dict__: + import py + py.test.skip("test the _pickle_moduledict() addition to pickle.py") + # + s1 = pickle.dumps(pickle.__dict__) + import gc; gc.collect() + s2 = pickle.dumps(pickle.__dict__) + # + d1 = pickle.loads(s1) + assert d1 is pickle.__dict__ + d2 = pickle.loads(s2) + assert d2 is pickle.__dict__ From jcreigh at codespeak.net Tue May 18 22:55:54 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Tue, 18 May 2010 22:55:54 +0200 (CEST) Subject: [pypy-svn] r74553 - in pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86: . test Message-ID: <20100518205554.DDED6282BE0@codespeak.net> Author: jcreigh Date: Tue May 18 22:55:53 2010 New Revision: 74553 Removed: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/ri386.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/ri386setup.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_ri386.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py Log: remove ri386 files From afa at codespeak.net Wed May 19 01:55:14 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 19 May 2010 01:55:14 +0200 (CEST) Subject: [pypy-svn] r74554 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100518235514.2C092282BE0@codespeak.net> Author: afa Date: Wed May 19 01:55:11 2010 New Revision: 74554 Modified: pypy/trunk/pypy/module/cpyext/api.py pypy/trunk/pypy/module/cpyext/modsupport.py pypy/trunk/pypy/module/cpyext/pyobject.py pypy/trunk/pypy/module/cpyext/state.py pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Log: Add a new way to borrow objects: borrow_from(None, w_obj) will keep the object alive until the end of the C function call (managed by generic_cpy_call()). >From what I understand, this is very similar to JNI "Local References". Use this whenever there is no container managed by user code: PyErr_Occurred(), PySys_GetObject() Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Wed May 19 01:55:11 2010 @@ -811,6 +811,7 @@ @specialize.memo() def make_generic_cpy_call(FT, decref_args, expect_null): from pypy.module.cpyext.pyobject import make_ref, from_ref, Py_DecRef + from pypy.module.cpyext.pyobject import RefcountState from pypy.module.cpyext.pyerrors import PyErr_Occurred unrolling_arg_types = unrolling_iterable(enumerate(FT.ARGS)) RESULT_TYPE = FT.RESULT @@ -856,8 +857,17 @@ boxed_args += (arg,) else: boxed_args += (arg,) - result = call_external_function(func, *boxed_args) + try: + # create a new container for borrowed references + state = space.fromcache(RefcountState) + old_container = state.swap_borrow_container(None) + try: + # Call the function + result = call_external_function(func, *boxed_args) + finally: + state.swap_borrow_container(old_container) + if RESULT_TYPE is PyObject: if result is None: ret = result Modified: pypy/trunk/pypy/module/cpyext/modsupport.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/modsupport.py (original) +++ pypy/trunk/pypy/module/cpyext/modsupport.py Wed May 19 01:55:11 2010 @@ -58,8 +58,7 @@ if doc: space.setattr(w_mod, space.wrap("__doc__"), space.wrap(rffi.charp2str(doc))) - # we cannot borrow here - return w_mod + return borrow_from(None, w_mod) def convert_method_defs(space, dict_w, methods, w_type, w_self=None): Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Wed May 19 01:55:11 2010 @@ -151,8 +151,14 @@ self.space = space self.py_objects_w2r = {} # { w_obj -> raw PyObject } self.py_objects_r2w = {} # { addr of raw PyObject -> w_obj } - self.borrow_mapping = {} # { w_container -> { w_containee -> None } } - self.borrowed_objects = {} # { addr of containee -> None } + + self.borrow_mapping = {None: {}} + # { w_container -> { w_containee -> None } } + # the None entry manages references borrowed during a call to + # generic_cpy_call() + self.borrowed_objects = {} + # { addr of containee -> None } + # For tests self.non_heaptypes_w = [] @@ -173,12 +179,69 @@ for w_obj, obj in self.py_objects_w2r.items(): print "%r: %i" % (w_obj, obj.c_ob_refcnt) + def make_borrowed(self, w_container, w_borrowed): + """ + Create a borrowed reference, which will live as long as the container + has a living reference (as a PyObject!) + """ + ref = make_ref(self.space, w_borrowed) + obj_ptr = rffi.cast(ADDR, ref) + if obj_ptr not in self.borrowed_objects: + # borrowed_objects owns the reference + self.borrowed_objects[obj_ptr] = None + else: + Py_DecRef(self.space, ref) # already in borrowed list + + borrowees = self.borrow_mapping.setdefault(w_container, {}) + borrowees[w_borrowed] = None + return ref + def reset_borrowed_references(self): + "Used in tests" while self.borrowed_objects: addr, _ = self.borrowed_objects.popitem() w_obj = self.py_objects_r2w[addr] Py_DecRef(self.space, w_obj) - self.borrow_mapping = {} + self.borrow_mapping = {None: {}} + + def delete_borrower(self, w_obj): + """ + Called when a potential container for borrowed references has lost its + last reference. Removes the borrowed references it contains. + """ + if w_obj in self.borrow_mapping: # move to lifeline __del__ + for w_containee in self.borrow_mapping[w_obj]: + self.forget_borrowee(w_containee) + del self.borrow_mapping[w_obj] + + def swap_borrow_container(self, container): + """switch the current default contained with the given one.""" + if container is None: + old_container = self.borrow_mapping[None] + self.borrow_mapping[None] = {} + return old_container + else: + old_container = self.borrow_mapping[None] + self.borrow_mapping[None] = container + for w_containee in old_container: + self.forget_borrowee(w_containee) + + def forget_borrowee(self, w_obj): + "De-register an object from the list of borrowed references" + ref = self.py_objects_w2r.get(w_obj, lltype.nullptr(PyObject.TO)) + if not ref: + if DEBUG_REFCOUNT: + print >>sys.stderr, "Borrowed object is already gone:", \ + hex(containee) + return + + containee_ptr = rffi.cast(ADDR, ref) + try: + del self.borrowed_objects[containee_ptr] + except KeyError: + pass + else: + Py_DecRef(self.space, ref) class InvalidPointerException(Exception): pass @@ -293,7 +356,7 @@ _Py_Dealloc(space, obj) del state.py_objects_w2r[w_obj] # if the object was a container for borrowed references - delete_borrower(space, w_obj) + state.delete_borrower(w_obj) else: if not we_are_translated() and obj.c_ob_refcnt < 0: message = "Negative refcount for obj %s with type %s" % ( @@ -329,23 +392,11 @@ Create a borrowed reference, which will live as long as the container has a living reference (as a PyObject!) """ - ref = make_ref(space, w_borrowed) - if not ref: - return ref + if w_borrowed is None: + return lltype.nullptr(PyObject.TO) - # state.borrowed_objects owns the reference state = space.fromcache(RefcountState) - obj_ptr = rffi.cast(ADDR, ref) - if obj_ptr not in state.borrowed_objects: - state.borrowed_objects[obj_ptr] = None - else: - Py_DecRef(space, ref) # already in borrowed list - - if w_container is None: # self-managed - return ref - borrowees = state.borrow_mapping.setdefault(w_container, {}) - borrowees[w_borrowed] = None - return ref + return state.make_borrowed(w_container, w_borrowed) class BorrowPair: """ @@ -361,36 +412,6 @@ def borrow_from(container, borrowed): return BorrowPair(container, borrowed) -def forget_borrowee(space, w_obj): - "De-register an object from the list of borrowed references" - state = space.fromcache(RefcountState) - ref = state.py_objects_w2r.get(w_obj, lltype.nullptr(PyObject.TO)) - if not ref: - if DEBUG_REFCOUNT: - print >>sys.stderr, "Borrowed object is already gone:", \ - hex(containee) - return - - containee_ptr = rffi.cast(ADDR, ref) - try: - del state.borrowed_objects[containee_ptr] - except KeyError: - pass - else: - Py_DecRef(space, ref) - -def delete_borrower(space, w_obj): - """ - Called when a potential container for borrowed references has lost its - last reference. Removes the borrowed references it contains. - """ - state = space.fromcache(RefcountState) - if w_obj in state.borrow_mapping: # move to lifeline __del__ - for w_containee in state.borrow_mapping[w_obj]: - forget_borrowee(space, w_containee) - del state.borrow_mapping[w_obj] - - #___________________________________________________________ @cpython_api([rffi.VOIDP_real], lltype.Signed, error=CANNOT_FAIL) Modified: pypy/trunk/pypy/module/cpyext/state.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/state.py (original) +++ pypy/trunk/pypy/module/cpyext/state.py Wed May 19 01:55:11 2010 @@ -29,17 +29,9 @@ self.operror = operror def clear_exception(self): - """Clear the current exception state, and return the operror. - Also frees the borrowed reference returned by PyErr_Occurred() - """ + """Clear the current exception state, and return the operror.""" from pypy.module.cpyext.api import ADDR - # handling of borrowed objects, remove when we have - # a weakkeydict operror = self.operror - if operror is not None: - # we have no explicit container - from pypy.module.cpyext.pyobject import forget_borrowee - forget_borrowee(self.space, operror.w_type) self.operror = None return operror Modified: pypy/trunk/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Wed May 19 01:55:11 2010 @@ -249,14 +249,11 @@ def unimport_module(self, name): """ - Remove the named module from the space's sys.modules and discard the - reference (owned by "the test") to it. + Remove the named module from the space's sys.modules. """ w_modules = self.space.sys.get('modules') w_name = self.space.wrap(name) - w_mod = self.space.getitem(w_modules, w_name) self.space.delitem(w_modules, w_name) - Py_DecRef(self.space, w_mod) def teardown_method(self, func): for name in self.imported_module_names: From afa at codespeak.net Wed May 19 02:26:00 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 19 May 2010 02:26:00 +0200 (CEST) Subject: [pypy-svn] r74555 - pypy/trunk/pypy/module/cpyext Message-ID: <20100519002600.7A40A282BE0@codespeak.net> Author: afa Date: Wed May 19 02:25:58 2010 New Revision: 74555 Modified: pypy/trunk/pypy/module/cpyext/pyobject.py pypy/trunk/pypy/module/cpyext/state.py pypy/trunk/pypy/module/cpyext/typeobjectdefs.py Log: remove unused imports Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Wed May 19 02:25:58 2010 @@ -169,7 +169,6 @@ def init_r2w_from_w2r(self): """Rebuilds the dict py_objects_r2w on startup""" - from pypy.module.cpyext.api import ADDR for w_obj, obj in self.py_objects_w2r.items(): ptr = rffi.cast(ADDR, obj) self.py_objects_r2w[ptr] = w_obj Modified: pypy/trunk/pypy/module/cpyext/state.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/state.py (original) +++ pypy/trunk/pypy/module/cpyext/state.py Wed May 19 02:25:58 2010 @@ -30,7 +30,6 @@ def clear_exception(self): """Clear the current exception state, and return the operror.""" - from pypy.module.cpyext.api import ADDR operror = self.operror self.operror = None return operror Modified: pypy/trunk/pypy/module/cpyext/typeobjectdefs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobjectdefs.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobjectdefs.py Wed May 19 02:25:58 2010 @@ -2,7 +2,7 @@ from pypy.rpython.lltypesystem.lltype import Ptr, FuncType, Void from pypy.module.cpyext.api import cpython_struct, \ PyVarObjectFields, Py_ssize_t, Py_TPFLAGS_READYING, \ - Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE, ADDR, \ + Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE, \ PyTypeObject, PyTypeObjectPtr, PyBufferProcs from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref from pypy.module.cpyext.modsupport import PyMethodDef From afa at codespeak.net Wed May 19 02:27:32 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 19 May 2010 02:27:32 +0200 (CEST) Subject: [pypy-svn] r74556 - pypy/trunk/pypy/module/cpyext Message-ID: <20100519002732.D455E282BE0@codespeak.net> Author: afa Date: Wed May 19 02:27:31 2010 New Revision: 74556 Modified: pypy/trunk/pypy/module/cpyext/sysmodule.py Log: Don't borrow a reference from an object which has no reference Modified: pypy/trunk/pypy/module/cpyext/sysmodule.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/sysmodule.py (original) +++ pypy/trunk/pypy/module/cpyext/sysmodule.py Wed May 19 02:27:31 2010 @@ -10,7 +10,7 @@ name = rffi.charp2str(name) w_dict = space.sys.getdict() w_obj = space.finditem_str(w_dict, name) - return borrow_from(w_dict, w_obj) + return borrow_from(None, w_obj) @cpython_api([CONST_STRING, PyObject], rffi.INT_real, error=-1) def PySys_SetObject(space, name, w_obj): From agaynor at codespeak.net Wed May 19 02:49:25 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Wed, 19 May 2010 02:49:25 +0200 (CEST) Subject: [pypy-svn] r74557 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100519004925.16A88282BE0@codespeak.net> Author: agaynor Date: Wed May 19 02:49:23 2010 New Revision: 74557 Modified: pypy/trunk/pypy/module/cpyext/pyerrors.py pypy/trunk/pypy/module/cpyext/stubs.py pypy/trunk/pypy/module/cpyext/stubsactive.py pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py Log: Implemented PyErr_Print Modified: pypy/trunk/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyerrors.py (original) +++ pypy/trunk/pypy/module/cpyext/pyerrors.py Wed May 19 02:49:23 2010 @@ -53,10 +53,11 @@ if operror: ptype[0] = make_ref(space, operror.w_type) pvalue[0] = make_ref(space, operror.get_w_value(space)) + ptraceback[0] = make_ref(space, space.wrap(operror.application_traceback)) else: ptype[0] = lltype.nullptr(PyObject.TO) pvalue[0] = lltype.nullptr(PyObject.TO) - ptraceback[0] = lltype.nullptr(PyObject.TO) + ptraceback[0] = lltype.nullptr(PyObject.TO) @cpython_api([PyObject, PyObject, PyObject], lltype.Void) def PyErr_Restore(space, w_type, w_value, w_traceback): @@ -208,4 +209,36 @@ Deprecated; use PyErr_WarnEx() instead.""" return PyErr_WarnEx(space, w_category, message, 1) + at cpython_api([rffi.INT_real], lltype.Void) +def PyErr_PrintEx(space, set_sys_last_vars): + """Print a standard traceback to sys.stderr and clear the error indicator. + Call this function only when the error indicator is set. (Otherwise it will + cause a fatal error!) + + If set_sys_last_vars is nonzero, the variables sys.last_type, + sys.last_value and sys.last_traceback will be set to the + type, value and traceback of the printed exception, respectively.""" + if not PyErr_Occurred(space): + PyErr_BadInternalCall(space) + state = space.fromcache(State) + operror = state.clear_exception() + + w_type = operror.w_type + w_value = operror.get_w_value(space) + w_tb = space.wrap(operror.application_traceback) + space.call_function(space.sys.get("excepthook"), + w_type, + w_value, + w_tb + ) + + if set_sys_last_vars: + w_dict = space.sys.getdict() + w_dict.setitem_str("last_type", w_type) + w_dict.setitem_str("last_value", w_value) + w_dict.setitem_str("last_traceback", w_tb) + at cpython_api([], lltype.Void) +def PyErr_Print(space): + """Alias for PyErr_PrintEx(1).""" + PyErr_PrintEx(space, 1) Modified: pypy/trunk/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/stubs.py (original) +++ pypy/trunk/pypy/module/cpyext/stubs.py Wed May 19 02:49:23 2010 @@ -826,17 +826,6 @@ """ raise NotImplementedError - at cpython_api([rffi.INT_real], lltype.Void) -def PyErr_PrintEx(space, set_sys_last_vars): - """Print a standard traceback to sys.stderr and clear the error indicator. - Call this function only when the error indicator is set. (Otherwise it will - cause a fatal error!) - - If set_sys_last_vars is nonzero, the variables sys.last_type, - sys.last_value and sys.last_traceback will be set to the - type, value and traceback of the printed exception, respectively.""" - raise NotImplementedError - @cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void) def PyErr_NormalizeException(space, exc, val, tb): """Under certain circumstances, the values returned by PyErr_Fetch() below Modified: pypy/trunk/pypy/module/cpyext/stubsactive.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/stubsactive.py (original) +++ pypy/trunk/pypy/module/cpyext/stubsactive.py Wed May 19 02:49:23 2010 @@ -23,11 +23,6 @@ instead of the repr().""" raise NotImplementedError - at cpython_api([], lltype.Void) -def PyErr_Print(space): - """Alias for PyErr_PrintEx(1).""" - raise NotImplementedError - @cpython_api([PyInterpreterState], PyThreadState, error=CANNOT_FAIL) def PyThreadState_New(space, interp): """Create a new thread state object belonging to the given interpreter object. Modified: pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py Wed May 19 02:49:23 2010 @@ -1,3 +1,6 @@ +import sys +import StringIO + from pypy.module.cpyext.state import State from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase @@ -79,6 +82,13 @@ assert ": UserWarning: this is a warning" in err rffi.free_charp(message) + def test_print_err(self, space, api, capfd): + api.PyErr_SetObject(space.w_Exception, space.wrap("cpyext is cool")) + api.PyErr_Print() + out, err = capfd.readouterr() + assert "cpyext is cool" in err + assert not api.PyErr_Occurred() + class AppTestFetch(AppTestCpythonExtensionBase): def setup_class(cls): AppTestCpythonExtensionBase.setup_class.im_func(cls) From fijal at codespeak.net Wed May 19 06:15:00 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 19 May 2010 06:15:00 +0200 (CEST) Subject: [pypy-svn] r74558 - pypy/branch/multilist Message-ID: <20100519041500.1A4FD282BE0@codespeak.net> Author: fijal Date: Wed May 19 06:14:57 2010 New Revision: 74558 Added: pypy/branch/multilist/ (props changed) - copied from r74557, pypy/trunk/ Log: Yet another approach for multilists From fijal at codespeak.net Wed May 19 06:24:11 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 19 May 2010 06:24:11 +0200 (CEST) Subject: [pypy-svn] r74559 - pypy/trunk/pypy/objspace/std/test Message-ID: <20100519042411.A654B282BE0@codespeak.net> Author: fijal Date: Wed May 19 06:24:10 2010 New Revision: 74559 Modified: pypy/trunk/pypy/objspace/std/test/test_listobject.py Log: Improve tests a little bit (untested code path) Modified: pypy/trunk/pypy/objspace/std/test/test_listobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_listobject.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_listobject.py Wed May 19 06:24:10 2010 @@ -5,7 +5,7 @@ from pypy.conftest import gettestobjspace -class TestW_ListObject: +class TestW_ListObject(object): def test_is_true(self): w = self.space.wrap @@ -342,7 +342,7 @@ self.space.w_True) -class AppTestW_ListObject: +class AppTestW_ListObject(object): def test_call_list(self): assert list('') == [] assert list('abc') == ['a', 'b', 'c'] @@ -576,6 +576,12 @@ l *= 2 assert l == [0, 1, 0, 1] + def test_mul_errors(self): + try: + [1, 2, 3] * (3,) + except TypeError: + pass + def test_index(self): c = range(10) assert c.index(0) == 0 From fijal at codespeak.net Wed May 19 06:57:27 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 19 May 2010 06:57:27 +0200 (CEST) Subject: [pypy-svn] r74560 - in pypy/branch/multilist/pypy: config objspace/std objspace/std/test Message-ID: <20100519045727.A2958282BE0@codespeak.net> Author: fijal Date: Wed May 19 06:57:24 2010 New Revision: 74560 Added: pypy/branch/multilist/pypy/objspace/std/listmultiobject.py (contents, props changed) pypy/branch/multilist/pypy/objspace/std/test/test_listmultiobject.py (contents, props changed) Modified: pypy/branch/multilist/pypy/config/pypyoption.py pypy/branch/multilist/pypy/objspace/std/model.py Log: Don't do anything, just boilerplate for now Modified: pypy/branch/multilist/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/multilist/pypy/config/pypyoption.py (original) +++ pypy/branch/multilist/pypy/config/pypyoption.py Wed May 19 06:57:24 2010 @@ -217,6 +217,8 @@ BoolOption("withropeunicode", "use ropes for the unicode implementation", default=False, requires=[("objspace.std.withrope", True)]), + BoolOption("withmultilist", "use multilist implementation instead" + " of normal list", default=False), BoolOption("withcelldict", "use dictionaries that are optimized for being used as module dicts", Added: pypy/branch/multilist/pypy/objspace/std/listmultiobject.py ============================================================================== --- (empty file) +++ pypy/branch/multilist/pypy/objspace/std/listmultiobject.py Wed May 19 06:57:24 2010 @@ -0,0 +1,538 @@ +from pypy.interpreter.argument import Signature +from pypy.objspace.std.model import registerimplementation, W_Object +from pypy.interpreter import gateway, baseobjspace +from pypy.rlib.listsort import TimSort +from pypy.objspace.std.register_all import register_all +from pypy.objspace.std.sliceobject import W_SliceObject, normalize_simple_slice +from pypy.objspace.std.listtype import get_list_index +from pypy.objspace.std.inttype import wrapint + +class W_ListMultiObject(W_Object): + """ A multilist object. It works as normal W_ListObject (contains + wrappeditems as a list of W_Root), but if wrappeditems is None, we + fall back to fallbackimpl which implements some sort of interface. + + That way we can have advantages of multilist with minimal overhead, + which can be neglected by the JIT + """ + + from pypy.objspace.std.listtype import list_typedef as typedef + + def __init__(self, wrappeditems, fallbackimpl=None): + # invariant + if wrappeditems is None: + assert fallbackimpl is not None + else: + assert fallbackimpl is None + self.wrappeditems = wrappeditems + self.fallbackimpl = fallbackimpl + + def __repr__(w_self): + """ representation for debugging purposes """ + if w_self.wrappeditems is not None: + return "%s(%s)" % (w_self.__class__.__name__, w_self.wrappeditems) + else: + return "W_ListMultiObject(%r)" % self.fallbackimpl + +registerimplementation(W_ListMultiObject) + + +init_signature = Signature(['sequence'], None, None) +init_defaults = [None] + +def init__ListMulti(space, w_list, __args__): + # this is on the silly side + w_iterable, = __args__.parse_obj( + None, 'list', init_signature, init_defaults) + # + # this is the old version of the loop at the end of this function: + # + # w_list.wrappeditems = space.unpackiterable(w_iterable) + # + # This is commented out to avoid assigning a new RPython list to + # 'wrappeditems', which defeats the W_FastSeqIterObject optimization. + # + items_w = w_list.wrappeditems + del items_w[:] + if w_iterable is not None: + w_iterator = space.iter(w_iterable) + while True: + try: + w_item = space.next(w_iterator) + except OperationError, e: + if not e.match(space, space.w_StopIteration): + raise + break # done + items_w.append(w_item) + +def len__ListMulti(space, w_list): + result = len(w_list.wrappeditems) + return wrapint(space, result) + +def getitem__ListMulti_ANY(space, w_list, w_index): + try: + return w_list.wrappeditems[get_list_index(space, w_index)] + except IndexError: + raise OperationError(space.w_IndexError, + space.wrap("list index out of range")) + +def getitem__ListMulti_Slice(space, w_list, w_slice): + # XXX consider to extend rlist's functionality? + length = len(w_list.wrappeditems) + start, stop, step, slicelength = w_slice.indices4(space, length) + assert slicelength >= 0 + if step == 1 and 0 <= start <= stop: + return W_ListMultiObject(w_list.wrappeditems[start:stop]) + w_res = W_ListMultiObject([None] * slicelength) + items_w = w_list.wrappeditems + subitems_w = w_res.wrappeditems + for i in range(slicelength): + subitems_w[i] = items_w[start] + start += step + return w_res + +def getslice__ListMulti_ANY_ANY(space, w_list, w_start, w_stop): + length = len(w_list.wrappeditems) + start, stop = normalize_simple_slice(space, length, w_start, w_stop) + return W_ListMultiObject(w_list.wrappeditems[start:stop]) + +def setslice__ListMulti_ANY_ANY_ANY(space, w_list, w_start, w_stop, w_sequence): + length = len(w_list.wrappeditems) + start, stop = normalize_simple_slice(space, length, w_start, w_stop) + _setitem_slice_helper(space, w_list, start, 1, stop-start, w_sequence) + +def delslice__ListMulti_ANY_ANY(space, w_list, w_start, w_stop): + length = len(w_list.wrappeditems) + start, stop = normalize_simple_slice(space, length, w_start, w_stop) + _delitem_slice_helper(space, w_list, start, 1, stop-start) + +def contains__ListMulti_ANY(space, w_list, w_obj): + # needs to be safe against eq_w() mutating the w_list behind our back + i = 0 + items_w = w_list.wrappeditems + while i < len(items_w): # intentionally always calling len! + if space.eq_w(items_w[i], w_obj): + return space.w_True + i += 1 + return space.w_False + +def iter__ListMulti(space, w_list): + from pypy.objspace.std import iterobject + return iterobject.W_FastListIterObject(w_list, w_list.wrappeditems) + +def add__ListMulti_ListMulti(space, w_list1, w_list2): + return W_ListMultiObject(w_list1.wrappeditems + w_list2.wrappeditems) + + +def inplace_add__ListMulti_ANY(space, w_list1, w_iterable2): + list_extend__ListMulti_ANY(space, w_list1, w_iterable2) + return w_list1 + +def inplace_add__ListMulti_ListMulti(space, w_list1, w_list2): + list_extend__ListMulti_ListMulti(space, w_list1, w_list2) + return w_list1 + +def mul_list_times(space, w_list, w_times): + try: + times = space.getindex_w(w_times, space.w_OverflowError) + except OperationError, e: + if e.match(space, space.w_TypeError): + raise FailedToImplement + raise + return W_ListMultiObject(w_list.wrappeditems * times) + +def mul__ListMulti_ANY(space, w_list, w_times): + return mul_list_times(space, w_list, w_times) + +def mul__ANY_ListMulti(space, w_times, w_list): + return mul_list_times(space, w_list, w_times) + +def inplace_mul__ListMulti_ANY(space, w_list, w_times): + try: + times = space.getindex_w(w_times, space.w_OverflowError) + except OperationError, e: + if e.match(space, space.w_TypeError): + raise FailedToImplement + raise + w_list.wrappeditems *= times + return w_list + +def eq__ListMulti_ListMulti(space, w_list1, w_list2): + # needs to be safe against eq_w() mutating the w_lists behind our back + items1_w = w_list1.wrappeditems + items2_w = w_list2.wrappeditems + return equal_wrappeditems(space, items1_w, items2_w) + +def equal_wrappeditems(space, items1_w, items2_w): + if len(items1_w) != len(items2_w): + return space.w_False + i = 0 + while i < len(items1_w) and i < len(items2_w): + if not space.eq_w(items1_w[i], items2_w[i]): + return space.w_False + i += 1 + return space.w_True + +def lessthan_unwrappeditems(space, items1_w, items2_w): + # needs to be safe against eq_w() mutating the w_lists behind our back + # Search for the first index where items are different + i = 0 + while i < len(items1_w) and i < len(items2_w): + w_item1 = items1_w[i] + w_item2 = items2_w[i] + if not space.eq_w(w_item1, w_item2): + return space.lt(w_item1, w_item2) + i += 1 + # No more items to compare -- compare sizes + return space.newbool(len(items1_w) < len(items2_w)) + +def greaterthan_unwrappeditems(space, items1_w, items2_w): + # needs to be safe against eq_w() mutating the w_lists behind our back + # Search for the first index where items are different + i = 0 + while i < len(items1_w) and i < len(items2_w): + w_item1 = items1_w[i] + w_item2 = items2_w[i] + if not space.eq_w(w_item1, w_item2): + return space.gt(w_item1, w_item2) + i += 1 + # No more items to compare -- compare sizes + return space.newbool(len(items1_w) > len(items2_w)) + +def lt__ListMulti_ListMulti(space, w_list1, w_list2): + return lessthan_unwrappeditems(space, w_list1.wrappeditems, + w_list2.wrappeditems) + +def gt__ListMulti_ListMulti(space, w_list1, w_list2): + return greaterthan_unwrappeditems(space, w_list1.wrappeditems, + w_list2.wrappeditems) + +def delitem__ListMulti_ANY(space, w_list, w_idx): + idx = get_list_index(space, w_idx) + try: + del w_list.wrappeditems[idx] + except IndexError: + raise OperationError(space.w_IndexError, + space.wrap("list deletion index out of range")) + return space.w_None + + +def delitem__ListMulti_Slice(space, w_list, w_slice): + start, stop, step, slicelength = w_slice.indices4(space, + len(w_list.wrappeditems)) + _delitem_slice_helper(space, w_list, start, step, slicelength) + +def _delitem_slice_helper(space, w_list, start, step, slicelength): + if slicelength==0: + return + + if step < 0: + start = start + step * (slicelength-1) + step = -step + + if step == 1: + assert start >= 0 + assert slicelength >= 0 + del w_list.wrappeditems[start:start+slicelength] + else: + items = w_list.wrappeditems + n = len(items) + i = start + + for discard in range(1, slicelength): + j = i+1 + i += step + while j < i: + items[j-discard] = items[j] + j += 1 + + j = i+1 + while j < n: + items[j-slicelength] = items[j] + j += 1 + start = n - slicelength + assert start >= 0 # annotator hint + del items[start:] + +def setitem__ListMulti_ANY_ANY(space, w_list, w_index, w_any): + idx = get_list_index(space, w_index) + try: + w_list.wrappeditems[idx] = w_any + except IndexError: + raise OperationError(space.w_IndexError, + space.wrap("list index out of range")) + return space.w_None + +def setitem__ListMulti_Slice_ANY(space, w_list, w_slice, w_iterable): + oldsize = len(w_list.wrappeditems) + start, stop, step, slicelength = w_slice.indices4(space, oldsize) + _setitem_slice_helper(space, w_list, start, step, slicelength, w_iterable) + +def _setitem_slice_helper(space, w_list, start, step, slicelength, w_iterable): + sequence2 = space.listview(w_iterable) + assert slicelength >= 0 + items = w_list.wrappeditems + oldsize = len(items) + len2 = len(sequence2) + if step == 1: # Support list resizing for non-extended slices + delta = slicelength - len2 + if delta < 0: + delta = -delta + newsize = oldsize + delta + # XXX support this in rlist! + items += [None] * delta + lim = start+len2 + i = newsize - 1 + while i >= lim: + items[i] = items[i-delta] + i -= 1 + elif start >= 0: + del items[start:start+delta] + else: + assert delta==0 + elif len2 != slicelength: # No resize for extended slices + raise operationerrfmt(space.w_ValueError, "attempt to " + "assign sequence of size %d to extended slice of size %d", + len2, slicelength) + + if sequence2 is items: + if step > 0: + # Always copy starting from the right to avoid + # having to make a shallow copy in the case where + # the source and destination lists are the same list. + i = len2 - 1 + start += i*step + while i >= 0: + items[start] = sequence2[i] + start -= step + i -= 1 + return + else: + # Make a shallow copy to more easily handle the reversal case + sequence2 = list(sequence2) + for i in range(len2): + items[start] = sequence2[i] + start += step + +app = gateway.applevel(""" + def listrepr(currently_in_repr, l): + 'The app-level part of repr().' + list_id = id(l) + if list_id in currently_in_repr: + return '[...]' + currently_in_repr[list_id] = 1 + try: + return "[" + ", ".join([repr(x) for x in l]) + ']' + finally: + try: + del currently_in_repr[list_id] + except: + pass +""", filename=__file__) + +listrepr = app.interphook("listrepr") + +def repr__ListMulti(space, w_list): + if len(w_list.wrappeditems) == 0: + return space.wrap('[]') + ec = space.getexecutioncontext() + w_currently_in_repr = ec._py_repr + if w_currently_in_repr is None: + w_currently_in_repr = ec._py_repr = space.newdict() + return listrepr(space, w_currently_in_repr, w_list) + +def list_insert__ListMulti_ANY_ANY(space, w_list, w_where, w_any): + where = space.int_w(w_where) + length = len(w_list.wrappeditems) + if where < 0: + where += length + if where < 0: + where = 0 + elif where > length: + where = length + w_list.wrappeditems.insert(where, w_any) + return space.w_None + +def list_append__ListMulti_ANY(space, w_list, w_any): + w_list.wrappeditems.append(w_any) + return space.w_None + +def list_extend__ListMulti_ListMulti(space, w_list, w_other): + w_list.wrappeditems += w_other.wrappeditems + return space.w_None + +def list_extend__ListMulti_ANY(space, w_list, w_any): + w_list.wrappeditems += space.listview(w_any) + return space.w_None + +# note that the default value will come back wrapped!!! +def list_pop__ListMulti_ANY(space, w_list, w_idx=-1): + items = w_list.wrappeditems + if len(items)== 0: + raise OperationError(space.w_IndexError, + space.wrap("pop from empty list")) + idx = space.int_w(w_idx) + try: + return items.pop(idx) + except IndexError: + raise OperationError(space.w_IndexError, + space.wrap("pop index out of range")) + +def list_remove__ListMulti_ANY(space, w_list, w_any): + # needs to be safe against eq_w() mutating the w_list behind our back + items = w_list.wrappeditems + i = 0 + while i < len(items): + if space.eq_w(items[i], w_any): + if i < len(items): # if this is wrong the list was changed + del items[i] + return space.w_None + i += 1 + raise OperationError(space.w_ValueError, + space.wrap("list.remove(x): x not in list")) + +def list_index__ListMulti_ANY_ANY_ANY(space, w_list, w_any, w_start, w_stop): + # needs to be safe against eq_w() mutating the w_list behind our back + items = w_list.wrappeditems + size = len(items) + i = slicetype.adapt_bound(space, size, w_start) + stop = slicetype.adapt_bound(space, size, w_stop) + while i < stop and i < len(items): + if space.eq_w(items[i], w_any): + return space.wrap(i) + i += 1 + raise OperationError(space.w_ValueError, + space.wrap("list.index(x): x not in list")) + +def list_count__ListMulti_ANY(space, w_list, w_any): + # needs to be safe against eq_w() mutating the w_list behind our back + count = 0 + i = 0 + items = w_list.wrappeditems + while i < len(items): + if space.eq_w(items[i], w_any): + count += 1 + i += 1 + return space.wrap(count) + +def list_reverse__ListMulti(space, w_list): + w_list.wrappeditems.reverse() + return space.w_None + +# ____________________________________________________________ +# Sorting + +# Reverse a slice of a list in place, from lo up to (exclusive) hi. +# (used in sort) + +class KeyContainer(baseobjspace.W_Root): + def __init__(self, w_key, w_item): + self.w_key = w_key + self.w_item = w_item + +# NOTE: all the subclasses of TimSort should inherit from a common subclass, +# so make sure that only SimpleSort inherits directly from TimSort. +# This is necessary to hide the parent method TimSort.lt() from the +# annotator. +class SimpleSort(TimSort): + def lt(self, a, b): + space = self.space + return space.is_true(space.lt(a, b)) + +class CustomCompareSort(SimpleSort): + def lt(self, a, b): + space = self.space + w_cmp = self.w_cmp + w_result = space.call_function(w_cmp, a, b) + try: + result = space.int_w(w_result) + except OperationError, e: + if e.match(space, space.w_TypeError): + raise OperationError(space.w_TypeError, + space.wrap("comparison function must return int")) + raise + return result < 0 + +class CustomKeySort(SimpleSort): + def lt(self, a, b): + assert isinstance(a, KeyContainer) + assert isinstance(b, KeyContainer) + space = self.space + return space.is_true(space.lt(a.w_key, b.w_key)) + +class CustomKeyCompareSort(CustomCompareSort): + def lt(self, a, b): + assert isinstance(a, KeyContainer) + assert isinstance(b, KeyContainer) + return CustomCompareSort.lt(self, a.w_key, b.w_key) + +def list_sort__ListMulti_ANY_ANY_ANY(space, w_list, w_cmp, w_keyfunc, w_reverse): + has_cmp = not space.is_w(w_cmp, space.w_None) + has_key = not space.is_w(w_keyfunc, space.w_None) + has_reverse = space.is_true(w_reverse) + + # create and setup a TimSort instance + if has_cmp: + if has_key: + sorterclass = CustomKeyCompareSort + else: + sorterclass = CustomCompareSort + else: + if has_key: + sorterclass = CustomKeySort + else: + sorterclass = SimpleSort + items = w_list.wrappeditems + sorter = sorterclass(items, len(items)) + sorter.space = space + sorter.w_cmp = w_cmp + + try: + # The list is temporarily made empty, so that mutations performed + # by comparison functions can't affect the slice of memory we're + # sorting (allowing mutations during sorting is an IndexError or + # core-dump factory, since wrappeditems may change). + w_list.wrappeditems = [] + + # wrap each item in a KeyContainer if needed + if has_key: + for i in range(sorter.listlength): + w_item = sorter.list[i] + w_key = space.call_function(w_keyfunc, w_item) + sorter.list[i] = KeyContainer(w_key, w_item) + + # Reverse sort stability achieved by initially reversing the list, + # applying a stable forward sort, then reversing the final result. + if has_reverse: + sorter.list.reverse() + + # perform the sort + sorter.sort() + + # reverse again + if has_reverse: + sorter.list.reverse() + + finally: + # unwrap each item if needed + if has_key: + for i in range(sorter.listlength): + w_obj = sorter.list[i] + if isinstance(w_obj, KeyContainer): + sorter.list[i] = w_obj.w_item + + # check if the user mucked with the list during the sort + mucked = len(w_list.wrappeditems) > 0 + + # put the items back into the list + w_list.wrappeditems = sorter.list + + if mucked: + raise OperationError(space.w_ValueError, + space.wrap("list modified during sort")) + + return space.w_None + + +from pypy.objspace.std import listtype +register_all(vars(), listtype) Modified: pypy/branch/multilist/pypy/objspace/std/model.py ============================================================================== --- pypy/branch/multilist/pypy/objspace/std/model.py (original) +++ pypy/branch/multilist/pypy/objspace/std/model.py Wed May 19 06:57:24 2010 @@ -26,6 +26,7 @@ "rangeobject.W_RangeIterObject"], "withtproxy" : ["proxyobject.W_TransparentList", "proxyobject.W_TransparentDict"], + "withmultilist" : ["listmultiobject.W_ListMultiObject"], } class StdTypeModel: @@ -69,6 +70,7 @@ from pypy.objspace.std import smallintobject from pypy.objspace.std import tupleobject from pypy.objspace.std import listobject + from pypy.objspace.std import listmultiobject from pypy.objspace.std import dictmultiobject from pypy.objspace.std import stringobject from pypy.objspace.std import ropeobject Added: pypy/branch/multilist/pypy/objspace/std/test/test_listmultiobject.py ============================================================================== --- (empty file) +++ pypy/branch/multilist/pypy/objspace/std/test/test_listmultiobject.py Wed May 19 06:57:24 2010 @@ -0,0 +1,13 @@ + +from pypy.conftest import gettestobjspace +from pypy.objspace.std.test.test_listobject import TestW_ListObject,\ + AppTestW_ListObject + +class TestW_ListMultiObject(TestW_ListObject): + def setup_class(cls): + cls.space = gettestobjspace(**{'objspace.std.withmultilist': True}) + +class AppTestMultiList(AppTestW_ListObject): + def setup_class(cls): + cls.space = gettestobjspace(**{'objspace.std.withmultilist': True}) + From afa at codespeak.net Wed May 19 09:49:21 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 19 May 2010 09:49:21 +0200 (CEST) Subject: [pypy-svn] r74561 - pypy/trunk/pypy/module/cpyext Message-ID: <20100519074921.9DB03282BE0@codespeak.net> Author: afa Date: Wed May 19 09:49:18 2010 New Revision: 74561 Modified: pypy/trunk/pypy/module/cpyext/dictobject.py Log: PyDict_Next is a stub, but gives it at least the correct signature Modified: pypy/trunk/pypy/module/cpyext/dictobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/dictobject.py (original) +++ pypy/trunk/pypy/module/cpyext/dictobject.py Wed May 19 09:49:18 2010 @@ -1,7 +1,8 @@ from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, build_type_checkers,\ - Py_ssize_t, PyObjectP, CONST_STRING -from pypy.module.cpyext.pyobject import PyObject, borrow_from +from pypy.module.cpyext.api import ( + cpython_api, CANNOT_FAIL, build_type_checkers, Py_ssize_t, + Py_ssize_tP, CONST_STRING) +from pypy.module.cpyext.pyobject import PyObject, PyObjectP, borrow_from from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.interpreter.error import OperationError @@ -102,7 +103,7 @@ dictionary, as in the dictionary method dict.items().""" return space.call_method(w_obj, "items") - at cpython_api([PyObject, Py_ssize_t, PyObjectP, PyObjectP], rffi.INT_real, error=CANNOT_FAIL) + at cpython_api([PyObject, Py_ssize_tP, PyObjectP, PyObjectP], rffi.INT_real, error=CANNOT_FAIL) def PyDict_Next(space, w_obj, ppos, pkey, pvalue): """Iterate over all key-value pairs in the dictionary p. The Py_ssize_t referred to by ppos must be initialized to 0 From afa at codespeak.net Wed May 19 09:50:16 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 19 May 2010 09:50:16 +0200 (CEST) Subject: [pypy-svn] r74562 - pypy/trunk/pypy/module/cpyext Message-ID: <20100519075016.566DF282BE0@codespeak.net> Author: afa Date: Wed May 19 09:50:14 2010 New Revision: 74562 Modified: pypy/trunk/pypy/module/cpyext/stubsactive.py Log: Fix signature of stub function for PyFile_AsFile() Modified: pypy/trunk/pypy/module/cpyext/stubsactive.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/stubsactive.py (original) +++ pypy/trunk/pypy/module/cpyext/stubsactive.py Wed May 19 09:50:14 2010 @@ -4,7 +4,9 @@ from pypy.module.cpyext.pystate import PyThreadState, PyInterpreterState - at cpython_api([PyObject], rffi.VOIDP, error=CANNOT_FAIL) #XXX +FILE = rffi.VOIDP_real.TO +FILEP = lltype.Ptr(FILE) + at cpython_api([PyObject], FILEP, error=CANNOT_FAIL) def PyFile_AsFile(space, p): """Return the file object associated with p as a FILE*. @@ -13,8 +15,6 @@ PyFile_DecUseCount() functions described below as appropriate.""" raise NotImplementedError -FILE = rffi.VOIDP_real.TO -FILEP = lltype.Ptr(FILE) @cpython_api([PyObject, FILEP, rffi.INT_real], rffi.INT_real, error=-1) def PyObject_Print(space, o, fp, flags): """Print an object o, on file fp. Returns -1 on error. The flags argument From afa at codespeak.net Wed May 19 09:51:22 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 19 May 2010 09:51:22 +0200 (CEST) Subject: [pypy-svn] r74563 - pypy/trunk/pypy/module/cpyext/include Message-ID: <20100519075122.BFE19282C07@codespeak.net> Author: afa Date: Wed May 19 09:51:21 2010 New Revision: 74563 Modified: pypy/trunk/pypy/module/cpyext/include/Python.h Log: This belongs to the previous commit: stdlib.h is now needed before function declarations Modified: pypy/trunk/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/trunk/pypy/module/cpyext/include/Python.h (original) +++ pypy/trunk/pypy/module/cpyext/include/Python.h Wed May 19 09:51:21 2010 @@ -66,7 +66,7 @@ # error "Python needs a typedef for Py_uintptr_t in pyport.h." #endif /* HAVE_UINTPTR_T */ - +#include /* Convert a possibly signed character to a nonnegative int */ /* XXX This assumes characters are 8 bits wide */ @@ -98,7 +98,6 @@ #include #include #include -#include #include "pyconfig.h" From afa at codespeak.net Wed May 19 09:53:25 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 19 May 2010 09:53:25 +0200 (CEST) Subject: [pypy-svn] r74564 - pypy/trunk/pypy/module/cpyext/include Message-ID: <20100519075325.27CE7282BE0@codespeak.net> Author: afa Date: Wed May 19 09:53:23 2010 New Revision: 74564 Modified: pypy/trunk/pypy/module/cpyext/include/unicodeobject.h Log: Add Py_UNICODE_REPLACEMENT_CHARACTER Modified: pypy/trunk/pypy/module/cpyext/include/unicodeobject.h ============================================================================== --- pypy/trunk/pypy/module/cpyext/include/unicodeobject.h (original) +++ pypy/trunk/pypy/module/cpyext/include/unicodeobject.h Wed May 19 09:53:23 2010 @@ -16,6 +16,7 @@ #endif typedef PY_UNICODE_TYPE Py_UNICODE; +#define Py_UNICODE_REPLACEMENT_CHARACTER ((Py_UNICODE) 0xFFFD) typedef struct { PyObject_HEAD From afa at codespeak.net Wed May 19 09:58:18 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 19 May 2010 09:58:18 +0200 (CEST) Subject: [pypy-svn] r74565 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100519075818.8376A282BE0@codespeak.net> Author: afa Date: Wed May 19 09:58:17 2010 New Revision: 74565 Modified: pypy/trunk/pypy/module/cpyext/eval.py pypy/trunk/pypy/module/cpyext/test/test_eval.py Log: Implement PyEval_GetBuiltins Modified: pypy/trunk/pypy/module/cpyext/eval.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/eval.py (original) +++ pypy/trunk/pypy/module/cpyext/eval.py Wed May 19 09:58:17 2010 @@ -1,12 +1,27 @@ from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import ( - cpython_api, PyObject, CANNOT_FAIL, CONST_STRING) +from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING +from pypy.module.cpyext.pyobject import PyObject, borrow_from @cpython_api([PyObject, PyObject, PyObject], PyObject) def PyEval_CallObjectWithKeywords(space, w_obj, w_arg, w_kwds): return space.call(w_obj, w_arg, w_kwds) + at cpython_api([], PyObject) +def PyEval_GetBuiltins(space): + """Return a dictionary of the builtins in the current execution + frame, or the interpreter of the thread state if no frame is + currently executing.""" + caller = space.getexecutioncontext().gettopframe_nohidden() + if caller is not None: + w_globals = caller.w_globals + w_builtins = space.getitem(w_globals, space.wrap('__builtins__')) + if not space.isinstance_w(w_builtins, space.w_dict): + w_builtins = w_builtins.getdict() + else: + w_builtins = space.builtin.getdict() + return borrow_from(None, w_builtins) + @cpython_api([PyObject, PyObject], PyObject) def PyObject_CallObject(space, w_obj, w_arg): """ Modified: pypy/trunk/pypy/module/cpyext/test/test_eval.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_eval.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_eval.py Wed May 19 09:58:17 2010 @@ -76,6 +76,26 @@ assert 42 * 43 == space.unwrap( api.PyObject_GetItem(w_globals, space.wrap("a"))) + def test_getbuiltins(self, space, api): + assert api.PyEval_GetBuiltins() is space.builtin.w_dict + + from pypy.interpreter.gateway import ObjSpace, interp2app + def cpybuiltins(space): + return api.PyEval_GetBuiltins() + cpybuiltins.unwrap_spec = [ObjSpace] + w_cpybuiltins = space.wrap(interp2app(cpybuiltins)) + + w_result = space.appexec([w_cpybuiltins], """(cpybuiltins): + return cpybuiltins() is __builtins__.__dict__ + """) + assert space.is_true(w_result) + + w_result = space.appexec([w_cpybuiltins], """(cpybuiltins): + d = dict(__builtins__={'len':len}, cpybuiltins=cpybuiltins) + return eval("cpybuiltins()", d, d) + """) + assert space.int_w(space.len(w_result)) == 1 + class AppTestCall(AppTestCpythonExtensionBase): def test_CallFunction(self): From afa at codespeak.net Wed May 19 10:09:31 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 19 May 2010 10:09:31 +0200 (CEST) Subject: [pypy-svn] r74566 - pypy/trunk/pypy/module/cpyext Message-ID: <20100519080931.62251282BE0@codespeak.net> Author: afa Date: Wed May 19 10:09:29 2010 New Revision: 74566 Modified: pypy/trunk/pypy/module/cpyext/pyobject.py Log: Fix translation Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Wed May 19 10:09:29 2010 @@ -163,7 +163,8 @@ self.non_heaptypes_w = [] def _freeze_(self): - assert not self.borrowed_objects and not self.borrow_mapping + assert not self.borrowed_objects + assert self.borrow_mapping == {None: {}} self.py_objects_r2w.clear() # is not valid anymore after translation return False From afa at codespeak.net Wed May 19 10:26:04 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 19 May 2010 10:26:04 +0200 (CEST) Subject: [pypy-svn] r74567 - pypy/trunk/pypy/module/cpyext Message-ID: <20100519082604.6EDA4282BE0@codespeak.net> Author: afa Date: Wed May 19 10:26:02 2010 New Revision: 74567 Modified: pypy/trunk/pypy/module/cpyext/pyerrors.py Log: Fix translation: don't use a method specific to the w_dict implementation, but instead the generic w_object interface. Also save sys.last_* before calling sys.excepthook. Modified: pypy/trunk/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyerrors.py (original) +++ pypy/trunk/pypy/module/cpyext/pyerrors.py Wed May 19 10:26:02 2010 @@ -214,7 +214,7 @@ """Print a standard traceback to sys.stderr and clear the error indicator. Call this function only when the error indicator is set. (Otherwise it will cause a fatal error!) - + If set_sys_last_vars is nonzero, the variables sys.last_type, sys.last_value and sys.last_traceback will be set to the type, value and traceback of the printed exception, respectively.""" @@ -222,21 +222,18 @@ PyErr_BadInternalCall(space) state = space.fromcache(State) operror = state.clear_exception() - + w_type = operror.w_type w_value = operror.get_w_value(space) w_tb = space.wrap(operror.application_traceback) - space.call_function(space.sys.get("excepthook"), - w_type, - w_value, - w_tb - ) - + if set_sys_last_vars: - w_dict = space.sys.getdict() - w_dict.setitem_str("last_type", w_type) - w_dict.setitem_str("last_value", w_value) - w_dict.setitem_str("last_traceback", w_tb) + space.sys.setdictvalue(space, "last_type", w_type) + space.sys.setdictvalue(space, "last_value", w_value) + space.sys.setdictvalue(space, "last_traceback", w_tb) + + space.call_function(space.sys.get("excepthook"), + w_type, w_value, w_tb) @cpython_api([], lltype.Void) def PyErr_Print(space): From afa at codespeak.net Wed May 19 10:31:54 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 19 May 2010 10:31:54 +0200 (CEST) Subject: [pypy-svn] r74568 - pypy/trunk/pypy/module/cpyext Message-ID: <20100519083154.A3B91282BE0@codespeak.net> Author: afa Date: Wed May 19 10:31:53 2010 New Revision: 74568 Modified: pypy/trunk/pypy/module/cpyext/pyerrors.py Log: Grumble. rffi.INT values are even not comparable to zero. Modified: pypy/trunk/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyerrors.py (original) +++ pypy/trunk/pypy/module/cpyext/pyerrors.py Wed May 19 10:31:53 2010 @@ -227,7 +227,7 @@ w_value = operror.get_w_value(space) w_tb = space.wrap(operror.application_traceback) - if set_sys_last_vars: + if rffi.cast(lltype.Signed, set_sys_last_vars): space.sys.setdictvalue(space, "last_type", w_type) space.sys.setdictvalue(space, "last_value", w_value) space.sys.setdictvalue(space, "last_traceback", w_tb) From arigo at codespeak.net Wed May 19 11:10:06 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 May 2010 11:10:06 +0200 (CEST) Subject: [pypy-svn] r74569 - in pypy/branch/blackhole-improvement/pypy: jit/backend jit/backend/llgraph jit/codewriter jit/metainterp rpython/lltypesystem Message-ID: <20100519091006.886DF282BE0@codespeak.net> Author: arigo Date: Wed May 19 11:10:04 2010 New Revision: 74569 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py pypy/branch/blackhole-improvement/pypy/jit/backend/model.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizeopt.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py Log: General progress. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py Wed May 19 11:10:04 2010 @@ -1060,31 +1060,25 @@ def frame_int_getvalue(frame, num): frame = _from_opaque(frame) assert num >= 0 - return frame.fail_args[num] + x = frame.fail_args[num] + assert lltype.typeOf(x) is lltype.Signed + return x def frame_float_getvalue(frame, num): frame = _from_opaque(frame) assert num >= 0 - return frame.fail_args[num] + x = frame.fail_args[num] + assert lltype.typeOf(x) is lltype.Float + return x def frame_ptr_getvalue(frame, num): frame = _from_opaque(frame) assert num >= 0 - return frame.fail_args[num] + x = frame.fail_args[num] + assert lltype.typeOf(x) == llmemory.GCREF + return x -def frame_get_value_kind(frame, num): - frame = _from_opaque(frame) - assert num >= 0 - TYPE = lltype.typeOf(frame.fail_args[num]) - if TYPE is lltype.Signed: - return INT - if TYPE == llmemory.GCREF: - return REF - if TYPE is lltype.Float: - return FLOAT - raise TypeError("frame.fail_args[%d] is of type %r" % (num, TYPE)) - -def get_latest_value_count(frame): +def frame_get_value_count(frame): frame = _from_opaque(frame) return len(frame.fail_args) Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py Wed May 19 11:10:04 2010 @@ -240,9 +240,6 @@ def get_latest_value_float(self, index): return llimpl.frame_float_getvalue(self.latest_frame, index) - def get_latest_value_kind(self, index): - return llimpl.frame_get_value_kind(self.latest_frame, index) - def get_latest_value_count(self): return llimpl.frame_get_value_count(self.latest_frame) Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/model.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/model.py Wed May 19 11:10:04 2010 @@ -85,11 +85,6 @@ or from 'args' if it was a FINISH). Returns a ptr or an obj.""" raise NotImplementedError - def get_latest_value_kind(self, index): - """Return the kind (history.INT, REF or FLOAT) of the index'th - argument to the last executed operation.""" - raise NotImplementedError - def get_latest_value_count(self): """Return how many values are ready to be returned by get_latest_value_xxx().""" Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py Wed May 19 11:10:04 2010 @@ -97,6 +97,18 @@ def _missing_liveness(self, pc): raise MissingLiveness("missing liveness[%d]\n%s" % (pc, self.dump())) + def follow_jump(self, position): + """Assuming that 'position' points just after a bytecode + instruction that ends with a label, follow that label.""" + code = self.code + position -= 2 + assert position >= 0 + if not we_are_translated(): + assert position in self._alllabels + labelvalue = ord(code[position]) | (ord(code[position+1])<<8) + assert labelvalue < len(code) + return labelvalue + def dump(self): if self._ssarepr is None: return '' Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Wed May 19 11:10:04 2010 @@ -61,6 +61,7 @@ self.setup_insns(asm.insns) self.setup_descrs(asm.descrs) self.metainterp_sd = metainterp_sd + self.num_interpreters = 0 self._freeze_() def _freeze_(self): @@ -234,7 +235,8 @@ if len(self.blackholeinterps) > 0: return self.blackholeinterps.pop() else: - return BlackholeInterpreter(self) + self.num_interpreters += 1 + return BlackholeInterpreter(self, self.num_interpreters) def release_interp(self, interp): interp.cleanup_registers() @@ -243,12 +245,13 @@ class BlackholeInterpreter(object): - def __init__(self, builder): + def __init__(self, builder, count_interpreter): self.builder = builder self.cpu = builder.cpu self.dispatch_loop = builder.dispatch_loop self.descrs = builder.descrs self.op_catch_exception = builder.op_catch_exception + self.count_interpreter = count_interpreter # if we_are_translated(): default_i = 0 @@ -263,6 +266,9 @@ self.registers_f = [default_f] * 256 self.jitcode = None + def __repr__(self): + return '' % self.count_interpreter + def setposition(self, jitcode, position): if jitcode is not self.jitcode: # the real performance impact of the following code is unclear, @@ -356,18 +362,6 @@ registers[j] = constants[i] i -= 1 - def follow_jump(self): - """Assuming that self.position points just after a bytecode - instruction that ends with a label, follow that label.""" - code = self.jitcode.code - position = self.position - 2 - assert position >= 0 - if not we_are_translated(): - assert position in self.jitcode._alllabels - labelvalue = ord(code[position]) | (ord(code[position+1])<<8) - assert labelvalue < len(code) - self.position = labelvalue - # ---------- @arguments("i", "i", returns="i") @@ -956,8 +950,7 @@ # XXX virtualizable _prepare_resume_from_failure(blackholeinterp, resumedescr.guard_opnum) try: - blackholeinterp = _resume_mainloop( - metainterp_sd.blackholeinterpbuilder, blackholeinterp) + blackholeinterp = _resume_mainloop(blackholeinterp) finally: metainterp_sd.profiler.end_blackhole() debug_stop('jit-blackhole') @@ -965,12 +958,12 @@ # normally (in general we get a ContinueRunningNormally exception). _done_with_this_frame(blackholeinterp) -def _resume_mainloop(blackholeinterpbuilder, blackholeinterp): +def _resume_mainloop(blackholeinterp): while True: try: blackholeinterp.run() finally: - blackholeinterpbuilder.release_interp(blackholeinterp) + blackholeinterp.builder.release_interp(blackholeinterp) #...x.x.x... assert blackholeinterp.nextblackholeinterp is None # XXX break @@ -980,7 +973,8 @@ def _prepare_resume_from_failure(blackholeinterp, opnum): from pypy.jit.metainterp.resoperation import rop if opnum == rop.GUARD_TRUE: # a goto_if_not_xxx that jumps only now - blackholeinterp.follow_jump() + blackholeinterp.position = blackholeinterp.jitcode.follow_jump( + blackholeinterp.position) elif opnum == rop.GUARD_FALSE: # a goto_if_not that stops jumping pass else: Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py Wed May 19 11:10:04 2010 @@ -239,12 +239,17 @@ self._counter = ~i # use ~(index_of_guarded_box_in_fail_args) def handle_fail(self, metainterp_sd): - from pypy.jit.metainterp.blackhole import resume_in_blackhole - return resume_in_blackhole(metainterp_sd, self) - XXX + if self.must_compile(metainterp_sd): + return self._trace_and_compile_from_bridge(metainterp_sd) + else: + from pypy.jit.metainterp.blackhole import resume_in_blackhole + resume_in_blackhole(metainterp_sd, self) + + def _trace_and_compile_from_bridge(self, metainterp_sd): from pypy.jit.metainterp.pyjitpl import MetaInterp metainterp = MetaInterp(metainterp_sd) return metainterp.handle_guard_failure(self) + _trace_and_compile_from_bridge._dont_inline_ = True def must_compile(self, metainterp_sd): trace_eagerness = metainterp_sd.state.trace_eagerness Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py Wed May 19 11:10:04 2010 @@ -85,6 +85,9 @@ return None raise AssertionError("bad rettype") +do_call_pure = do_call +do_call_loopinvariant = do_call + def do_getarrayitem_gc(metainterp, arraybox, indexbox, arraydescr): cpu = metainterp.cpu array = arraybox.getref_base() Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py Wed May 19 11:10:04 2010 @@ -255,7 +255,7 @@ return llmemory.cast_int_to_adr(self.value) def _get_hash_(self): - return integer_hash(self.value) + return make_hashable_int(self.value) def set_future_value(self, cpu, j): cpu.set_future_value_int(j, self.value) @@ -492,7 +492,7 @@ return llmemory.cast_int_to_adr(self.value) def _get_hash_(self): - return integer_hash(self.value) + return make_hashable_int(self.value) def set_future_value(self, cpu, j): cpu.set_future_value_int(j, self.value) @@ -655,7 +655,7 @@ except lltype.DelayedPointer: return -2 # xxx risk of changing hash... -def integer_hash(i): +def make_hashable_int(i): if not we_are_translated() and isinstance(i, llmemory.AddressAsInt): # Warning: such a hash changes at the time of translation adr = llmemory.cast_int_to_adr(i) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizeopt.py Wed May 19 11:10:04 2010 @@ -16,7 +16,7 @@ from pypy.jit.metainterp.typesystem import llhelper, oohelper from pypy.rlib.objectmodel import we_are_translated from pypy.rpython.lltypesystem import lltype -from pypy.jit.metainterp.history import AbstractDescr +from pypy.jit.metainterp.history import AbstractDescr, make_hashable_int def optimize_loop_1(metainterp_sd, loop): """Optimize loop.operations to make it match the input of loop.specnodes @@ -953,7 +953,8 @@ if not funcvalue.is_constant(): self.optimize_default(op) return - resvalue = self.loop_invariant_results.get(op.args[0].getint(), None) + key = make_hashable_int(op.args[0].getint()) + resvalue = self.loop_invariant_results.get(key, None) if resvalue is not None: self.make_equal_to(op.result, resvalue) return @@ -962,7 +963,7 @@ op.opnum = rop.CALL self.optimize_default(op) resvalue = self.getvalue(op.result) - self.loop_invariant_results[op.args[0].getint()] = resvalue + self.loop_invariant_results[key] = resvalue optimize_ops = _findall(Optimizer, 'optimize_') Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Wed May 19 11:10:04 2010 @@ -26,10 +26,6 @@ # ____________________________________________________________ -def check_args(*args): - for arg in args: - assert isinstance(arg, (Box, Const)) - def arguments(*args): def decorate(func): func.argtypes = args @@ -83,6 +79,13 @@ i -= 1 copy_constants._annspecialcase_ = 'specialize:arg(3)' + def cleanup_registers(self): + # To avoid keeping references alive, this cleans up the registers_r. + # It does not clear the references set by copy_constants(), but + # these are all prebuilt constants anyway. + for i in range(self.jitcode.num_regs_r()): + self.registers_r[i] = None + # ------------------------------ # Decoding of the JitCode @@ -266,39 +269,39 @@ self.opimpl_goto_if_not(condbox, target) ''' % (_opimpl, _opimpl.upper())).compile() - def follow_jump(self): - _op_goto_if_not = self.metainterp.staticdata._op_goto_if_not - assert ord(self.bytecode[self.pc]) == _op_goto_if_not - self.pc += 1 # past the bytecode for 'goto_if_not' - target = self.load_3byte() # load the 'target' argument - self.pc = target # jump - - def ignore_next_guard_nullness(self, opnum): - _op_ooisnull = self.metainterp.staticdata._op_ooisnull - _op_oononnull = self.metainterp.staticdata._op_oononnull - bc = ord(self.bytecode[self.pc]) - if bc == _op_ooisnull: - if opnum == rop.GUARD_ISNULL: - res = ConstInt(0) - else: - res = ConstInt(1) - else: - assert bc == _op_oononnull - if opnum == rop.GUARD_ISNULL: - res = ConstInt(1) - else: - res = ConstInt(0) - self.pc += 1 # past the bytecode for ptr_iszero/ptr_nonzero - self.load_int() # past the 'box' argument - self.make_result_box(res) - - def dont_follow_jump(self): - _op_goto_if_not = self.metainterp.staticdata._op_goto_if_not - assert ord(self.bytecode[self.pc]) == _op_goto_if_not - self.pc += 1 # past the bytecode for 'goto_if_not' - self.load_3byte() # past the 'target' argument - self.load_int() # past the 'box' argument - self.ignore_varargs() # past the 'livelist' argument +## def follow_jump(self): +## _op_goto_if_not = self.metainterp.staticdata._op_goto_if_not +## assert ord(self.bytecode[self.pc]) == _op_goto_if_not +## self.pc += 1 # past the bytecode for 'goto_if_not' +## target = self.load_3byte() # load the 'target' argument +## self.pc = target # jump + +## def ignore_next_guard_nullness(self, opnum): +## _op_ooisnull = self.metainterp.staticdata._op_ooisnull +## _op_oononnull = self.metainterp.staticdata._op_oononnull +## bc = ord(self.bytecode[self.pc]) +## if bc == _op_ooisnull: +## if opnum == rop.GUARD_ISNULL: +## res = ConstInt(0) +## else: +## res = ConstInt(1) +## else: +## assert bc == _op_oononnull +## if opnum == rop.GUARD_ISNULL: +## res = ConstInt(1) +## else: +## res = ConstInt(0) +## self.pc += 1 # past the bytecode for ptr_iszero/ptr_nonzero +## self.load_int() # past the 'box' argument +## self.make_result_box(res) + +## def dont_follow_jump(self): +## _op_goto_if_not = self.metainterp.staticdata._op_goto_if_not +## assert ord(self.bytecode[self.pc]) == _op_goto_if_not +## self.pc += 1 # past the bytecode for 'goto_if_not' +## self.load_3byte() # past the 'target' argument +## self.load_int() # past the 'box' argument +## self.ignore_varargs() # past the 'livelist' argument @arguments("box", "descr") def opimpl_switch(self, valuebox, switchdict): @@ -905,12 +908,8 @@ else: raise AssertionError(box.type) - def setup_resume_at_op(self, pc, env): - if not we_are_translated(): - check_args(*env) + def setup_resume_at_op(self, pc): self.pc = pc - xxxxxxxxxxxxxxxxxxxxxxx - self.env = env ## values = ' '.join([box.repr_rpython() for box in self.env]) ## log('setup_resume_at_op %s:%d [%s] %d' % (self.jitcode.name, ## self.pc, values, @@ -1213,7 +1212,6 @@ class MetaInterp(object): in_recursion = 0 - _already_allocated_resume_virtuals = None def __init__(self, staticdata): self.staticdata = staticdata @@ -1255,6 +1253,7 @@ # we save the freed MIFrames to avoid needing to re-create new # MIFrame objects all the time; they are a bit big, with their # 3*256 register entries. + frame.cleanup_registers() self.free_frames_list.append(frame) def finishframe(self, resultbox): @@ -1516,8 +1515,6 @@ self.current_merge_points = [(original_greenkey, -1)] self.resumekey = key self.seen_can_enter_jit = False - xxx - started_as_blackhole = self.is_blackholing() try: self.prepare_resume_from_failure(key.guard_opnum) self.interpret() @@ -1525,8 +1522,7 @@ except GenerateMergePoint, gmp: return self.designate_target_loop(gmp) except ContinueRunningNormallyBase: - if not started_as_blackhole: - key.reset_counter_from_failure(self) + key.reset_counter_from_failure(self) raise def remove_consts_and_duplicates(self, boxes, endindex, duplicates): @@ -1599,17 +1595,20 @@ return loop_token def prepare_resume_from_failure(self, opnum): + frame = self.framestack[-1] if opnum == rop.GUARD_TRUE: # a goto_if_not that jumps only now - self.framestack[-1].follow_jump() + frame.pc = frame.jitcode.follow_jump(frame.pc) elif opnum == rop.GUARD_FALSE: # a goto_if_not that stops jumping - self.framestack[-1].dont_follow_jump() + pass elif (opnum == rop.GUARD_NO_EXCEPTION or opnum == rop.GUARD_EXCEPTION or opnum == rop.GUARD_NOT_FORCED): - self.handle_exception() + xxx #self.handle_exception() elif opnum == rop.GUARD_NO_OVERFLOW: # an overflow now detected - self.raise_overflow_error() + xxx #self.raise_overflow_error() elif opnum == rop.GUARD_NONNULL or opnum == rop.GUARD_ISNULL: - self.framestack[-1].ignore_next_guard_nullness(opnum) + xxx #self.framestack[-1].ignore_next_guard_nullness(opnum) + else: + raise NotImplementedError(opnum) def compile(self, original_boxes, live_arg_boxes, start): num_green_args = self.staticdata.num_green_args @@ -1705,23 +1704,13 @@ return original_boxes def initialize_state_from_guard_failure(self, resumedescr): - XXX # guard failure: rebuild a complete MIFrame stack + debug_start('jit-tracing') + self.staticdata.profiler.start_tracing() self.in_recursion = -1 # always one portal around - inputargs_and_holes = self.cpu.make_boxes_from_latest_values( - resumedescr) - must_compile = resumedescr.must_compile(self.staticdata, - inputargs_and_holes) - if must_compile: - debug_start('jit-tracing') - self.history = history.History() - self.history.inputargs = [box for box in inputargs_and_holes if box] - self.staticdata.profiler.start_tracing() - else: - debug_start('jit-blackhole') - self.staticdata.profiler.start_blackhole() - self.history = None # this means that is_blackholing() is true - self.rebuild_state_after_failure(resumedescr, inputargs_and_holes) + self.history = history.History() + inputargs_and_holes = self.rebuild_state_after_failure(resumedescr) + self.history.inputargs = [box for box in inputargs_and_holes if box] def initialize_virtualizable(self, original_boxes): vinfo = self.staticdata.virtualizable_info @@ -1834,12 +1823,15 @@ def assert_no_exception(self): assert not self.last_exc_value_box - def rebuild_state_after_failure(self, resumedescr, newboxes): + def rebuild_state_after_failure(self, resumedescr): vinfo = self.staticdata.virtualizable_info self.framestack = [] expect_virtualizable = vinfo is not None - virtualizable_boxes, virtualref_boxes = resume.rebuild_from_resumedata( - self, newboxes, resumedescr, expect_virtualizable) + boxlists = resume.rebuild_from_resumedata(self, resumedescr, + expect_virtualizable) + #inputargs_and_holes, virtualizable_boxes, virtualref_boxes = boxlists + inputargs_and_holes = boxlists # XXX + virtualref_boxes = [] # XXX # # virtual refs: make the vrefs point to the freshly allocated virtuals self.virtualref_boxes = virtualref_boxes @@ -1854,7 +1846,7 @@ # boxes, in whichever direction is appropriate if expect_virtualizable: self.virtualizable_boxes = virtualizable_boxes - if self._already_allocated_resume_virtuals is not None: + if 0: ## self._already_allocated_resume_virtuals is not None: # resuming from a ResumeGuardForcedDescr: load the new values # currently stored on the virtualizable fields self.load_fields_from_virtualizable() @@ -1866,13 +1858,14 @@ virtualizable_box = self.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) assert not virtualizable.vable_token - if self._already_allocated_resume_virtuals is not None: + if 0: ## self._already_allocated_resume_virtuals is not None: # resuming from a ResumeGuardForcedDescr: load the new values # currently stored on the virtualizable fields self.load_fields_from_virtualizable() else: # normal case: fill the virtualizable with the local boxes self.synchronize_virtualizable() + return inputargs_and_holes def check_synchronized_virtualizable(self): if not we_are_translated(): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py Wed May 19 11:10:04 2010 @@ -134,7 +134,7 @@ '_NOSIDEEFFECT_FIRST', # ----- start of no_side_effect operations ----- '_ALWAYS_PURE_FIRST', # ----- start of always_pure operations ----- - 'OOSEND_PURE', # ootype operation + #'OOSEND_PURE', # ootype operation 'CALL_PURE', # 'CAST_PTR_TO_INT/1', @@ -198,8 +198,8 @@ 'UNICODEGETITEM/2', # # ootype operations - 'INSTANCEOF/1db', - 'SUBCLASSOF/2b', + #'INSTANCEOF/1db', + #'SUBCLASSOF/2b', # '_ALWAYS_PURE_LAST', # ----- end of always_pure operations ----- @@ -222,7 +222,7 @@ 'STRSETITEM/3', 'UNICODESETITEM/3', 'NEWUNICODE/1', - 'RUNTIMENEW/1', # ootype operation + #'RUNTIMENEW/1', # ootype operation 'COND_CALL_GC_WB', # [objptr, newvalue] (for the write barrier) 'DEBUG_MERGE_POINT/1', # debugging only 'VIRTUAL_REF_FINISH/2', @@ -232,7 +232,7 @@ 'CALL_ASSEMBLER', 'CALL_MAY_FORCE', 'CALL_LOOPINVARIANT', - 'OOSEND', # ootype operation + #'OOSEND', # ootype operation '_CANRAISE_LAST', # ----- end of can_raise operations ----- '_OVF_FIRST', # ----- start of is_ovf operations ----- Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py Wed May 19 11:10:04 2010 @@ -1,5 +1,7 @@ import sys, os -from pypy.jit.metainterp.history import Box, Const, ConstInt, INT, REF, FLOAT +from pypy.jit.metainterp.history import Box, Const, ConstInt +from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat +from pypy.jit.metainterp.history import INT, REF, FLOAT, HOLE from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp import jitprof from pypy.rpython.lltypesystem import rffi @@ -98,6 +100,9 @@ UNASSIGNED = tag(-1<<13, TAGBOX) UNASSIGNEDVIRTUAL = tag(-1<<13, TAGVIRTUAL) NULLREF = tag(-1, TAGCONST) +TYPEBARRIER = tag(-2, TAGCONST) + # nums = [..BoxInts.., TYPEBARRIER, ..BoxPtrs..] + # and optionally an extra [TYPEBARRIER, ..BoxFloats..] class ResumeDataLoopMemo(object): @@ -161,8 +166,19 @@ numb1, liveboxes, v = self.number(values, snapshot.prev) n = len(liveboxes)-v boxes = snapshot.boxes + # + if not we_are_translated(): + # verifies that 'boxes' are in order: all INTs, then all REFs, + # and finally all FLOATs. + _kind2count = {INT: 1, REF: 2, FLOAT: 3} + kinds = [_kind2count[box.type] for box in boxes] + assert kinds == sorted(kinds) + # length = len(boxes) - nums = [UNASSIGNED] * length + numslength = length + 1 + (length > 0 and boxes[-1].type == FLOAT) + nums = [UNASSIGNED] * numslength + prevkind = INT + j = 0 for i in range(length): box = boxes[i] value = values.get(box, None) @@ -181,7 +197,23 @@ tagged = tag(n, TAGBOX) n += 1 liveboxes[box] = tagged - nums[i] = tagged + # + if box.type != prevkind: + nums[j] = TYPEBARRIER + j += 1 + if prevkind == INT and box.type == FLOAT: + nums[j] = TYPEBARRIER + j += 1 + prevkind = box.type + # + nums[j] = tagged + j += 1 + # + if j == length: + nums[j] = TYPEBARRIER + j += 1 + assert j == numslength + # numb = Numbering(numb1, nums) self.numberings[snapshot] = numb, liveboxes, v return numb, liveboxes.copy(), v @@ -426,9 +458,9 @@ AbstractVirtualStructInfo.__init__(self, fielddescrs) self.known_class = known_class - def allocate(self, cpu): - from pypy.jit.metainterp.executor import exec_new_with_vtable - return exec_new_with_vtable(cpu, self.known_class) + @specialize.argtype(1) + def allocate(self, decoder): + return decoder.allocate_with_vtable(self.known_class) def debug_prints(self): debug_print("\tvirtualinfo", self.known_class.repr_rpython()) @@ -468,24 +500,94 @@ for i in self.fieldnums: debug_print("\t\t", str(untag(i))) +# ____________________________________________________________ + +class AbstractResumeDataReader(object): + """A base mixin containing the logic to reconstruct virtuals out of + guard failure. There are two implementations of this mixin: + ResumeDataBoxReader for when we are compiling (i.e. when we have a + metainterp), and ResumeDataDirectReader for when we are merely + blackholing and want the best performance. + """ + _mixin_ = True + virtuals = None + + def _prepare(self, cpu, storage): + self.cpu = cpu + self.cur_numb = storage.rd_numb + self.consts = storage.rd_consts + self._prepare_virtuals(storage.rd_virtuals) + self._prepare_pendingfields(storage.rd_pendingfields) + + def _prepare_virtuals(self, virtuals): + if virtuals: + self.virtuals = [None] * len(virtuals) + for i in range(len(virtuals)): + vinfo = virtuals[i] + if vinfo is not None: + self.virtuals[i] = vinfo.allocate(self) + for i in range(len(virtuals)): + vinfo = virtuals[i] + if vinfo is not None: + vinfo.setfields(self, self.virtuals[i]) -def rebuild_from_resumedata(metainterp, newboxes, storage, + def _prepare_pendingfields(self, pendingfields): + if pendingfields is not None: + for descr, num, fieldnum in pendingfields: + struct = self.decode_ref(num) + self.setfield(descr, struct, fieldnum) + + def _prepare_next_section(self): + numb = self.cur_numb + self.cur_numb = numb.prev + nums = numb.nums + length = len(nums) + count_i = count_r = count_f = 0 + i = 0 + while True: + num = nums[i] + i += 1 + if tagged_eq(num, TYPEBARRIER): + break + self.write_an_int(count_i, self.decode_int(num)) + count_i += 1 + while i < length: + num = nums[i] + i += 1 + if tagged_eq(num, TYPEBARRIER): + break + self.write_a_ref(count_r, self.decode_ref(num)) + count_r += 1 + while i < length: + num = nums[i] + i += 1 + assert not tagged_eq(num, TYPEBARRIER) + self.write_a_float(count_f, self.decode_float(num)) + count_f += 1 + + def done(self): + self.cpu.clear_latest_values() + +# ---------- when resuming for pyjitpl.py, make boxes ---------- + +def rebuild_from_resumedata(metainterp, storage, expects_virtualizables): - resumereader = ResumeDataReader(storage, newboxes, metainterp) + resumereader = ResumeDataBoxReader(storage, metainterp) virtualizable_boxes = None if expects_virtualizables: - virtualizable_boxes = resumereader.consume_boxes() - virtualref_boxes = resumereader.consume_boxes() + XXX # virtualizable_boxes = resumereader.consume_boxes() + resumereader.consume_boxes([], [], []) # XXX virtualref frameinfo = storage.rd_frame_info_list while True: - env = resumereader.consume_boxes() f = metainterp.newframe(frameinfo.jitcode) - f.setup_resume_at_op(frameinfo.pc, frameinfo.exception_target, env) + f.setup_resume_at_op(frameinfo.pc) + resumereader.consume_boxes(f.registers_i, f.registers_r, f.registers_f) frameinfo = frameinfo.prev if frameinfo is None: break metainterp.framestack.reverse() - return virtualizable_boxes, virtualref_boxes + resumereader.done() + return resumereader.liveboxes def force_from_resumedata(metainterp, newboxes, storage, expects_virtualizables): @@ -496,57 +598,34 @@ virtualref_boxes = resumereader.consume_boxes() return virtualizable_boxes, virtualref_boxes, resumereader.virtuals +class ResumeDataBoxReader(AbstractResumeDataReader): -class ResumeDataBoxReader(object): - virtuals = None - - def __init__(self, storage, liveboxes, metainterp=None): - self.cur_numb = storage.rd_numb - self.consts = storage.rd_consts - self.liveboxes = liveboxes - self.cpu = metainterp.cpu - self._prepare_virtuals(metainterp, storage.rd_virtuals) - self._prepare_pendingfields(metainterp, storage.rd_pendingfields) + def __init__(self, storage, metainterp): + self.metainterp = metainterp + self.liveboxes = [None] * metainterp.cpu.get_latest_value_count() + self._prepare(metainterp.cpu, storage) + + def consume_boxes(self, boxes_i, boxes_r, boxes_f): + self.boxes_i = boxes_i + self.boxes_r = boxes_r + self.boxes_f = boxes_f + self._prepare_next_section() - def _prepare_virtuals(self, metainterp, virtuals): - if virtuals: - v = metainterp._already_allocated_resume_virtuals - if v is not None: - self.virtuals = v - return - self.virtuals = [None] * len(virtuals) - for i in range(len(virtuals)): - vinfo = virtuals[i] - if vinfo is not None: - self.virtuals[i] = vinfo.allocate(metainterp) - for i in range(len(virtuals)): - vinfo = virtuals[i] - if vinfo is not None: - vinfo.setfields(metainterp, self.virtuals[i], - self._decode_box) + def allocate_with_vtable(self, known_class): + xxx + def setfield(self, descr, struct, fieldnum): + xxx + def setarrayitem(self, arraydescr, array, index, fieldnum): + xxx - def _prepare_pendingfields(self, metainterp, pendingfields): - if pendingfields: - if metainterp._already_allocated_resume_virtuals is not None: - return - for descr, num, fieldnum in pendingfields: - box = self._decode_box(num) - fieldbox = self._decode_box(fieldnum) - metainterp.execute_and_record(rop.SETFIELD_GC, - descr, box, fieldbox) + def decode_int(self, tagged): + return self.decode_box(tagged, INT) + def decode_ref(self, tagged): + return self.decode_box(tagged, REF) + def decode_float(self, tagged): + return self.decode_box(tagged, FLOAT) - def consume_boxes(self): - numb = self.cur_numb - assert numb is not None - nums = numb.nums - n = len(nums) - boxes = [None] * n - for i in range(n): - boxes[i] = self._decode_box(nums[i]) - self.cur_numb = numb.prev - return boxes - - def _decode_box(self, tagged, kind): + def decode_box(self, tagged, kind): num, tag = untag(tagged) if tag == TAGCONST: if tagged_eq(tagged, NULLREF): @@ -560,16 +639,42 @@ return ConstInt(num) else: assert tag == TAGBOX - return self.liveboxes[num] + box = self.liveboxes[num] + if box is None: + box = self.load_box_from_cpu(num, kind) + return box + + def load_box_from_cpu(self, num, kind): + if num < 0: + num += len(self.liveboxes) + assert num >= 0 + if kind == INT: + box = BoxInt(self.cpu.get_latest_value_int(num)) + elif kind == REF: + box = BoxPtr(self.cpu.get_latest_value_ref(num)) + elif kind == FLOAT: + box = BoxFloat(self.cpu.get_latest_value_float(num)) + else: + assert 0, "bad kind: %d" % ord(kind) + self.liveboxes[num] = box + return box + + def write_an_int(self, index, box): + self.boxes_i[index] = box + def write_a_ref(self, index, box): + self.boxes_r[index] = box + def write_a_float(self, index, box): + self.boxes_f[index] = box +# ---------- when resuming for blackholing, get direct values ---------- def blackhole_from_resumedata(blackholeinterpbuilder, storage, expects_virtualizables): - resumereader = ResumeDataDirectReader(storage, blackholeinterpbuilder.cpu) + resumereader = ResumeDataDirectReader(blackholeinterpbuilder.cpu, storage) if expects_virtualizables: XXX #virtualref_boxes = resumereader.consume_boxes() - resumereader.consume_one_section(None) # XXX + resumereader.consume_one_section(None) # virtualref XXX # # First get a chain of blackhole interpreters whose length is given # by the depth of rd_frame_info_list. The first one we get must be @@ -599,87 +704,42 @@ resumereader.done() return firstbh -class ResumeDataDirectReader(object): - virtuals = None +class ResumeDataDirectReader(AbstractResumeDataReader): - def __init__(self, storage, cpu): - self.cur_numb = storage.rd_numb - self.consts = storage.rd_consts - self.cpu = cpu - self._prepare_virtuals(storage.rd_virtuals) - self._prepare_pendingfields(storage.rd_pendingfields) + def __init__(self, cpu, storage): + self._prepare(cpu, storage) - def _prepare_virtuals(self, virtuals): - if virtuals: - self.virtuals = [None] * len(virtuals) - for i in range(len(virtuals)): - vinfo = virtuals[i] - if vinfo is not None: - self.virtuals[i] = vinfo.allocate(self.cpu) - for i in range(len(virtuals)): - vinfo = virtuals[i] - if vinfo is not None: - vinfo.setfields(self, self.virtuals[i]) + def consume_one_section(self, blackholeinterp): + self.blackholeinterp = blackholeinterp + self._prepare_next_section() - def _prepare_pendingfields(self, pendingfields): - if pendingfields is not None: - for descr, num, fieldnum in pendingfields: - struct = self._decode_ref(num) - self.setfield(descr, struct, fieldnum) + def allocate_with_vtable(self, known_class): + from pypy.jit.metainterp.executor import exec_new_with_vtable + return exec_new_with_vtable(self.cpu, self.known_class) def setfield(self, descr, struct, fieldnum): if descr.is_pointer_field(): - newvalue = self._decode_ref(fieldnum) + newvalue = self.decode_ref(fieldnum) self.cpu.bh_setfield_gc_r(struct, descr, newvalue) elif descr.is_float_field(): - newvalue = self._decode_float(fieldnum) + newvalue = self.decode_float(fieldnum) self.cpu.bh_setfield_gc_f(struct, descr, newvalue) else: - newvalue = self._decode_int(fieldnum) + newvalue = self.decode_int(fieldnum) self.cpu.bh_setfield_gc_i(struct, descr, newvalue) def setarrayitem(self, arraydescr, array, index, fieldnum): if arraydescr.is_array_of_pointers(): - newvalue = self._decode_ref(fieldnum) + newvalue = self.decode_ref(fieldnum) self.cpu.bh_setarrayitem_gc_r(arraydescr, array, index, newvalue) elif arraydescr.is_array_of_floats(): - newvalue = self._decode_float(fieldnum) + newvalue = self.decode_float(fieldnum) self.cpu.bh_setarrayitem_gc_f(arraydescr, array, index, newvalue) else: - newvalue = self._decode_int(fieldnum) + newvalue = self.decode_int(fieldnum) self.cpu.bh_setarrayitem_gc_i(arraydescr, array, index, newvalue) - def consume_one_section(self, blackholeinterp): - numb = self.cur_numb - count_i = count_r = count_f = 0 - for num in numb.nums: - kind = self._decode_kind(num) - if kind == INT: - blackholeinterp.setarg_i(count_i, self._decode_int(num)) - count_i += 1 - elif kind == REF: - blackholeinterp.setarg_r(count_r, self._decode_ref(num)) - count_r += 1 - elif kind == FLOAT: - blackholeinterp.setarg_f(count_f, self._decode_float(num)) - count_f += 1 - self.cur_numb = numb.prev - - def _decode_kind(self, tagged): - num, tag = untag(tagged) - if tag == TAGCONST: - if tagged_eq(tagged, NULLREF): - return REF - return self.consts[num].type - elif tag == TAGVIRTUAL: - return REF - elif tag == TAGINT: - return INT - else: - assert tag == TAGBOX - return self.cpu.get_latest_value_kind(num) - - def _decode_int(self, tagged): + def decode_int(self, tagged): num, tag = untag(tagged) if tag == TAGCONST: return self.consts[num].getint() @@ -691,7 +751,7 @@ num += self.cpu.get_latest_value_count() return self.cpu.get_latest_value_int(num) - def _decode_ref(self, tagged): + def decode_ref(self, tagged): num, tag = untag(tagged) if tag == TAGCONST: if tagged_eq(tagged, NULLREF): @@ -707,7 +767,7 @@ num += self.cpu.get_latest_value_count() return self.cpu.get_latest_value_ref(num) - def _decode_float(self, tagged): + def decode_float(self, tagged): num, tag = untag(tagged) if tag == TAGCONST: return self.consts[num].getfloat() @@ -717,8 +777,14 @@ num += self.cpu.get_latest_value_count() return self.cpu.get_latest_value_float(num) - def done(self): - self.cpu.clear_latest_values() + def write_an_int(self, index, int): + self.blackholeinterp.setarg_i(index, int) + + def write_a_ref(self, index, ref): + self.blackholeinterp.setarg_r(index, ref) + + def write_a_float(self, index, float): + self.blackholeinterp.setarg_f(index, float) # ____________________________________________________________ Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py Wed May 19 11:10:04 2010 @@ -7,6 +7,7 @@ import weakref from pypy.rlib.objectmodel import Symbolic from pypy.rpython.lltypesystem import lltype +from pypy.tool.uid import uid class AddressOffset(Symbolic): @@ -507,6 +508,11 @@ if isinstance(other, AddressAsInt): return cmp(self.adr, other.adr) return Symbolic.__cmp__(self, other) + def __repr__(self): + try: + return '' % (self.adr.ptr,) + except AttributeError: + return '' % (uid(self),) # ____________________________________________________________ From arigo at codespeak.net Wed May 19 11:28:32 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 May 2010 11:28:32 +0200 (CEST) Subject: [pypy-svn] r74570 - in pypy/branch/blackhole-improvement/pypy/jit: backend/llgraph codewriter metainterp metainterp/test Message-ID: <20100519092832.73ED8282BE0@codespeak.net> Author: arigo Date: Wed May 19 11:28:30 2010 New Revision: 74570 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Log: new_with_vtable. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py Wed May 19 11:28:30 2010 @@ -10,6 +10,7 @@ from pypy.jit.metainterp.history import (ConstInt, ConstPtr, BoxInt, BoxPtr, BoxObj, BoxFloat, REF, INT, FLOAT) +from pypy.jit.codewriter import heaptracker from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr from pypy.rpython.ootypesystem import ootype from pypy.rpython.module.support import LLSupport, OOSupport @@ -728,9 +729,8 @@ def op_new_with_vtable(self, descr, vtable): assert descr is None - xxxxxxxxxxxx - size = get_class_size(self.memocast, vtable) - result = do_new(size) + descr = heaptracker.vtable2descr(self.cpu, vtable) + result = do_new(descr.ofs) value = lltype.cast_opaque_ptr(rclass.OBJECTPTR, result) value.typeptr = cast_from_int(rclass.CLASSTYPE, vtable) return result Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py Wed May 19 11:28:30 2010 @@ -1,4 +1,4 @@ -from pypy.rpython.lltypesystem import lltype, rclass +from pypy.rpython.lltypesystem import lltype, llmemory, rclass from pypy.rlib.objectmodel import we_are_translated @@ -52,7 +52,9 @@ sizedescr._corresponding_vtable = vtable def vtable2descr(cpu, vtable): - assert lltype.typeOf(vtable) == VTABLETYPE + assert lltype.typeOf(vtable) is lltype.Signed + vtable = llmemory.cast_int_to_adr(vtable) + vtable = llmemory.cast_adr_to_ptr(vtable, VTABLETYPE) if we_are_translated(): # Build the dict {vtable: sizedescr} at runtime. # This is necessary because the 'vtables' are just pointers to @@ -67,4 +69,7 @@ def descr2vtable(cpu, descr): from pypy.jit.metainterp import history assert isinstance(descr, history.AbstractDescr) - return descr._corresponding_vtable + vtable = descr._corresponding_vtable + vtable = llmemory.cast_ptr_to_adr(vtable) + vtable = llmemory.cast_adr_to_int(vtable) + return vtable Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Wed May 19 11:28:30 2010 @@ -899,7 +899,6 @@ @arguments("cpu", "d", returns="r") def bhimpl_new_with_vtable(cpu, descr): vtable = heaptracker.descr2vtable(cpu, descr) - vtable = llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(vtable)) return cpu.bh_new_with_vtable(descr, vtable) @arguments("cpu", "r", returns="i") Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py Wed May 19 11:28:30 2010 @@ -154,9 +154,7 @@ def exec_new_with_vtable(cpu, clsbox): from pypy.jit.codewriter import heaptracker vtable = clsbox.getint() - vtableadr = llmemory.cast_int_to_adr(vtable) - vtableptr = llmemory.cast_adr_to_ptr(vtableadr, heaptracker.VTABLETYPE) - descr = heaptracker.vtable2descr(cpu, vtableptr) + descr = heaptracker.vtable2descr(cpu, vtable) return cpu.bh_new_with_vtable(descr, vtable) def do_new_with_vtable(metainterp, clsbox): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Wed May 19 11:28:30 2010 @@ -320,11 +320,9 @@ @arguments("descr") def opimpl_new_with_vtable(self, sizedescr): from pypy.jit.codewriter import heaptracker - from pypy.jit.metainterp.warmstate import wrap cpu = self.metainterp.cpu cls = heaptracker.descr2vtable(cpu, sizedescr) - clsbox = wrap(cpu, cls, in_const_box=True) - return self.execute(rop.NEW_WITH_VTABLE, clsbox) + return self.execute(rop.NEW_WITH_VTABLE, ConstInt(cls)) @FixME #arguments("box") def opimpl_runtimenew(self, classbox): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py Wed May 19 11:28:30 2010 @@ -715,7 +715,7 @@ def allocate_with_vtable(self, known_class): from pypy.jit.metainterp.executor import exec_new_with_vtable - return exec_new_with_vtable(self.cpu, self.known_class) + return exec_new_with_vtable(self.cpu, known_class) def setfield(self, descr, struct, fieldnum): if descr.is_pointer_field(): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Wed May 19 11:28:30 2010 @@ -636,10 +636,9 @@ self.check_tree_loop_count(2) # the loop and the entry path # we get: # ENTER - compile the new loop - # ENTER (BlackHole) - leave # ENTER - compile the entry bridge # ENTER - compile the leaving path - self.check_enter_count(4) + self.check_enter_count(3) def test_bridge_from_interpreter_2(self): # one case for backend - computing of framesize on guard failure From afa at codespeak.net Wed May 19 12:07:16 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 19 May 2010 12:07:16 +0200 (CEST) Subject: [pypy-svn] r74571 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100519100716.F2559282BE0@codespeak.net> Author: afa Date: Wed May 19 12:07:12 2010 New Revision: 74571 Modified: pypy/trunk/pypy/module/cpyext/dictobject.py pypy/trunk/pypy/module/cpyext/test/test_dictobject.py Log: Implement (very inefficiently) PyDict_Next Modified: pypy/trunk/pypy/module/cpyext/dictobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/dictobject.py (original) +++ pypy/trunk/pypy/module/cpyext/dictobject.py Wed May 19 12:07:12 2010 @@ -3,6 +3,7 @@ cpython_api, CANNOT_FAIL, build_type_checkers, Py_ssize_t, Py_ssize_tP, CONST_STRING) from pypy.module.cpyext.pyobject import PyObject, PyObjectP, borrow_from +from pypy.module.cpyext.pyobject import RefcountState from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.interpreter.error import OperationError @@ -104,7 +105,7 @@ return space.call_method(w_obj, "items") @cpython_api([PyObject, Py_ssize_tP, PyObjectP, PyObjectP], rffi.INT_real, error=CANNOT_FAIL) -def PyDict_Next(space, w_obj, ppos, pkey, pvalue): +def PyDict_Next(space, w_dict, ppos, pkey, pvalue): """Iterate over all key-value pairs in the dictionary p. The Py_ssize_t referred to by ppos must be initialized to 0 prior to the first call to this function to start the iteration; the @@ -145,8 +146,30 @@ } Py_DECREF(o); }""" - if w_obj is None: + if w_dict is None: return 0 - raise NotImplementedError + + # Note: this is not efficient. Storing an iterator would probably + # work, but we can't work out how to not leak it if iteration does + # not complete. + + try: + w_iter = space.call_method(w_dict, "iteritems") + pos = ppos[0] + while pos: + space.call_method(w_iter, "next") + pos -= 1 + + w_item = space.call_method(w_iter, "next") + w_key, w_value = space.fixedview(w_item, 2) + state = space.fromcache(RefcountState) + pkey[0] = state.make_borrowed(w_dict, w_key) + pvalue[0] = state.make_borrowed(w_dict, w_value) + ppos[0] += 1 + except OperationError, e: + if not e.match(space, space.w_StopIteration): + raise + return 0 + return 1 Modified: pypy/trunk/pypy/module/cpyext/test/test_dictobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_dictobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_dictobject.py Wed May 19 12:07:12 2010 @@ -1,5 +1,7 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.api import Py_ssize_tP, PyObjectP +from pypy.module.cpyext.pyobject import make_ref, from_ref class TestDictObject(BaseApiTest): def test_dict(self, space, api): @@ -71,3 +73,28 @@ api.PyDict_Update(w_d, w_d2) assert space.unwrap(w_d) == dict(a='b', c='d', e='f') + + def test_iter(self, space, api): + w_dict = space.sys.getdict() + py_dict = make_ref(space, w_dict) + + ppos = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw') + ppos[0] = 0 + pkey = lltype.malloc(PyObjectP.TO, 1, flavor='raw') + pvalue = lltype.malloc(PyObjectP.TO, 1, flavor='raw') + + try: + w_copy = space.newdict() + while api.PyDict_Next(w_dict, ppos, pkey, pvalue): + w_key = from_ref(space, pkey[0]) + w_value = from_ref(space, pvalue[0]) + space.setitem(w_copy, w_key, w_value) + finally: + lltype.free(ppos, flavor='raw') + lltype.free(pkey, flavor='raw') + lltype.free(pvalue, flavor='raw') + + api.Py_DecRef(py_dict) # release borrowed references + + assert space.eq_w(space.len(w_copy), space.len(w_dict)) + assert space.eq_w(w_copy, w_dict) From arigo at codespeak.net Wed May 19 15:37:11 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 May 2010 15:37:11 +0200 (CEST) Subject: [pypy-svn] r74572 - in pypy/branch/blackhole-improvement/pypy/jit: backend backend/llgraph metainterp Message-ID: <20100519133711.F0878282BE0@codespeak.net> Author: arigo Date: Wed May 19 15:37:09 2010 New Revision: 74572 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py pypy/branch/blackhole-improvement/pypy/jit/backend/model.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: Whack at exception handling until test_div_overflow passes. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py Wed May 19 15:37:09 2010 @@ -766,6 +766,8 @@ raise NotImplementedError def op_call(self, calldescr, func, *args): + global _last_exception + assert _last_exception is None, "exception left behind" assert _call_args_i == _call_args_r == _call_args_f == [] for x in args: T = lltype.typeOf(x) @@ -777,7 +779,15 @@ _call_args_f.append(x) else: raise TypeError(x) - return _do_call_common(func) + try: + return _do_call_common(func) + except LLException, lle: + _last_exception = lle + d = {'v': None, + REF: lltype.nullptr(llmemory.GCREF.TO), + INT: 0, + FLOAT: 0.0} + return d[calldescr.typeinfo] op_call_pure = op_call @@ -1088,55 +1098,38 @@ _last_exception = None -def get_exception(): - if _last_exception: - return llmemory.cast_ptr_to_adr(_last_exception.args[0]) - else: - return llmemory.NULL - -def get_exc_value(): - if _last_exception: - return lltype.cast_opaque_ptr(llmemory.GCREF, _last_exception.args[1]) +def grab_exc_value(): + global _last_exception + if _last_exception is not None: + result = _last_exception.args[1] + _last_exception = None + return lltype.cast_opaque_ptr(llmemory.GCREF, result) else: return lltype.nullptr(llmemory.GCREF.TO) -def clear_exception(): - global _last_exception - _last_exception = None - -_pseudo_exceptions = {} +##_pseudo_exceptions = {} -def _get_error(Class): - if _llinterp.typer is not None: - llframe = _llinterp.frame_class(None, None, _llinterp) - try: - llframe.make_llexception(Class()) - except LLException, e: - return e - else: - assert 0, "should have raised" - else: - # for tests, a random emulated ll_inst will do - if Class not in _pseudo_exceptions: - ll_inst = lltype.malloc(rclass.OBJECT, zero=True) - ll_inst.typeptr = lltype.malloc(rclass.OBJECT_VTABLE, - immortal=True) - _pseudo_exceptions[Class] = LLException(ll_inst.typeptr, ll_inst) - return _pseudo_exceptions[Class] - -def get_overflow_error(): - return llmemory.cast_ptr_to_adr(_get_error(OverflowError).args[0]) - -def get_overflow_error_value(): - return lltype.cast_opaque_ptr(llmemory.GCREF, - _get_error(OverflowError).args[1]) - -def get_zero_division_error(): - return llmemory.cast_ptr_to_adr(_get_error(ZeroDivisionError).args[0]) - -def get_zero_division_error_value(): - return lltype.cast_opaque_ptr(llmemory.GCREF, - _get_error(ZeroDivisionError).args[1]) +##def _get_error(Class): +## if _llinterp.typer is not None: +## llframe = _llinterp.frame_class(None, None, _llinterp) +## try: +## llframe.make_llexception(Class()) +## except LLException, e: +## return e +## else: +## assert 0, "should have raised" +## else: +## # for tests, a random emulated ll_inst will do +## if Class not in _pseudo_exceptions: +## ll_inst = lltype.malloc(rclass.OBJECT, zero=True) +## ll_inst.typeptr = lltype.malloc(rclass.OBJECT_VTABLE, +## immortal=True) +## _pseudo_exceptions[Class] = LLException(ll_inst.typeptr, ll_inst) +## return _pseudo_exceptions[Class] + +##def get_overflow_error_value(): +## return lltype.cast_opaque_ptr(llmemory.GCREF, +## _get_error(OverflowError).args[1]) def force(opaque_frame): frame = _from_opaque(opaque_frame) @@ -1513,13 +1506,7 @@ setannotation(frame_ptr_getvalue, annmodel.SomePtr(llmemory.GCREF)) setannotation(frame_float_getvalue, annmodel.SomeFloat()) -setannotation(get_exception, annmodel.SomeAddress()) -setannotation(get_exc_value, annmodel.SomePtr(llmemory.GCREF)) -setannotation(clear_exception, annmodel.s_None) -setannotation(get_overflow_error, annmodel.SomeAddress()) -setannotation(get_overflow_error_value, annmodel.SomePtr(llmemory.GCREF)) -setannotation(get_zero_division_error, annmodel.SomeAddress()) -setannotation(get_zero_division_error_value, annmodel.SomePtr(llmemory.GCREF)) +setannotation(grab_exc_value, annmodel.SomePtr(llmemory.GCREF)) setannotation(force, annmodel.SomeInteger()) setannotation(get_forced_token_frame, s_Frame) setannotation(get_frame_forced_token, annmodel.SomeAddress()) Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py Wed May 19 15:37:09 2010 @@ -274,11 +274,8 @@ token = history.getkind(RESULT) return self.getdescr(0, token[0], extrainfo=extrainfo) - def get_exception(self): - return self.cast_adr_to_int(llimpl.get_exception()) - - def get_exc_value(self): - return llimpl.get_exc_value() + def grab_exc_value(self): + return llimpl.grab_exc_value() def arraydescrof(self, A): assert isinstance(A, lltype.GcArray) Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/model.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/model.py Wed May 19 15:37:09 2010 @@ -100,19 +100,8 @@ they no longer keep objects alive).""" raise NotImplementedError - def get_exception(self): - raise NotImplementedError - - def get_exc_value(self): - raise NotImplementedError - - def clear_exception(self): - raise NotImplementedError - - def get_overflow_error(self): - raise NotImplementedError - - def get_zero_division_error(self): + def grab_exc_value(self): + """Return and clear the exception set by the latest execute_token().""" raise NotImplementedError @staticmethod Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Wed May 19 15:37:09 2010 @@ -32,21 +32,19 @@ NULL = lltype.nullptr(llmemory.GCREF.TO) -def get_standard_error_llexception(rtyper, Class): +def _get_standard_error(rtyper, Class): exdata = rtyper.getexceptiondata() clsdef = rtyper.annotator.bookkeeper.getuniqueclassdef(Class) evalue = exdata.get_standard_ll_exc_instance(rtyper, clsdef) - etype = rclass.ll_type(evalue) - return LLException(etype, evalue) + return evalue def get_llexception(cpu, e): if we_are_translated(): return XXX(e) if isinstance(e, LLException): - return e # ok + return e.args[1] # ok if isinstance(e, OverflowError): - return get_standard_error_llexception(cpu.rtyper, - OverflowError) + return _get_standard_error(cpu.rtyper, OverflowError) raise # leave other exceptions to be propagated # ____________________________________________________________ @@ -180,7 +178,7 @@ print '-> %s!' % (e.__class__.__name__,) if resulttype == 'i' or resulttype == 'r' or resulttype == 'f': position += 1 - self.exception_pc = position + self.position = position raise if verbose and not we_are_translated(): @@ -291,21 +289,21 @@ self.registers_f[index] = value def run(self): - code = self.jitcode.code - position = self.position while True: try: - self.dispatch_loop(self, code, position) + self.dispatch_loop(self, self.jitcode.code, self.position) except LeaveFrame: break except Exception, e: e = get_llexception(self.cpu, e) - position = self.handle_exception_in_frame(e, code) + self.handle_exception_in_frame(e) def get_result_i(self): + assert self._return_type == 'i' return self.tmpreg_i def get_result_r(self): + assert self._return_type == 'r' result = self.tmpreg_r if we_are_translated(): self.tmpreg_r = NULL @@ -314,14 +312,18 @@ return result def get_result_f(self): + assert self._return_type == 'f' return self.tmpreg_f + def get_result_v(self): + assert self._return_type == 'v' + def _get_result_anytype(self): "NOT_RPYTHON" - if self._return_type == 'int': return self.get_result_i() - if self._return_type == 'ref': return self.get_result_r() - if self._return_type == 'float': return self.get_result_f() - if self._return_type == 'void': return None + if self._return_type == 'i': return self.get_result_i() + if self._return_type == 'r': return self.get_result_r() + if self._return_type == 'f': return self.get_result_f() + if self._return_type == 'v': return None raise ValueError(self._return_type) def cleanup_registers(self): @@ -332,21 +334,24 @@ self.registers_r[i] = NULL self.exception_last_value = None - def handle_exception_in_frame(self, e, code): + def handle_exception_in_frame(self, e): # This frame raises an exception. First try to see if # the exception is handled in the frame itself. - position = self.exception_pc # <-- just after the insn that raised + code = self.jitcode.code + position = self.position opcode = ord(code[position]) if opcode != self.op_catch_exception: # no 'catch_exception' insn follows: just reraise - raise Exception, e + if we_are_translated(): + raise Exception, e + else: + etype = rclass.ll_type(e) + raise LLException(etype, e) else: # else store the exception on 'self', and jump to the handler - if not we_are_translated(): # get the lltyped exception - e = e.args[1] # object out of the LLException self.exception_last_value = e target = ord(code[position+1]) | (ord(code[position+2])<<8) - return target + self.position = target # XXX must be specialized def copy_constants(self, registers, constants): @@ -586,28 +591,24 @@ @arguments("self", "i") def bhimpl_int_return(self, a): self.tmpreg_i = a - if not we_are_translated(): - self._return_type = "int" + self._return_type = 'i' raise LeaveFrame @arguments("self", "r") def bhimpl_ref_return(self, a): self.tmpreg_r = a - if not we_are_translated(): - self._return_type = "ref" + self._return_type = 'r' raise LeaveFrame @arguments("self", "f") def bhimpl_float_return(self, a): self.tmpreg_f = a - if not we_are_translated(): - self._return_type = "float" + self._return_type = 'f' raise LeaveFrame @arguments("self") def bhimpl_void_return(self): - if not we_are_translated(): - self._return_type = "void" + self._return_type = 'v' raise LeaveFrame @arguments("i", "L", "pc", returns="L") @@ -935,6 +936,80 @@ def bhimpl_unicodesetitem(cpu, unicode, index, newchr): cpu.bh_unicodesetitem(unicode, index, newchr) + # ---------- + # helpers to resume running in blackhole mode when a guard failed + + def _resume_mainloop(self, current_exc): + try: + # if there is a current exception, raise it now + # (it may be caught by a catch_operation in this frame) + if current_exc: + self.handle_exception_in_frame(current_exc) + # unless the call above raised again the exception, + # we now proceed to interpret the bytecode in this frame + self.run() + # + except Exception, e: + # if we get an exception, return it to the caller frame + current_exc = get_llexception(self.cpu, e) + if not self.nextblackholeinterp: + self._exit_frame_with_exception(current_exc) + return current_exc + # + # pass the frame's return value to the caller + caller = self.nextblackholeinterp + if not caller: + self._done_with_this_frame() + kind = self._return_type + if kind == 'i': + caller._setup_return_value_i(self.get_result_i()) + elif kind == 'r': + caller._setup_return_value_r(self.get_result_r()) + elif kind == 'f': + caller._setup_return_value_f(self.get_result_f()) + else: + assert kind == 'v' + return NULL + + def _prepare_resume_from_failure(self, opnum): + from pypy.jit.metainterp.resoperation import rop + if opnum == rop.GUARD_TRUE: # a goto_if_not_xxx that jumps only now + self.position = self.jitcode.follow_jump(self.position) + elif opnum == rop.GUARD_FALSE: # a goto_if_not that stops jumping + pass + elif opnum == rop.GUARD_NO_EXCEPTION or opnum == rop.GUARD_EXCEPTION: + pass + else: + raise NotImplementedError(opnum) + + # connect the return of values from the called frame to the + # 'xxx_call_yyy' instructions from the caller frame + def _setup_return_value_i(self, result): + self.registers_i[ord(self.jitcode.code[position-1])] = result + def _setup_return_value_r(self, result): + self.registers_r[ord(self.jitcode.code[position-1])] = result + def _setup_return_value_f(self, result): + self.registers_f[ord(self.jitcode.code[position-1])] = result + + def _exit_frame_with_exception(self, e): + xxx + + def _done_with_this_frame(self): + # rare case: we only get there if the blackhole interps all returned + # normally (in general we get a ContinueRunningNormally exception). + sd = self.builder.metainterp_sd + if sd.result_type == 'void': + self.get_result_v() + raise sd.DoneWithThisFrameVoid() + elif sd.result_type == 'int': + raise sd.DoneWithThisFrameInt(self.get_result_i()) + elif sd.result_type == 'ref': + raise sd.DoneWithThisFrameRef(self.get_result_r()) + elif sd.result_type == 'float': + raise sd.DoneWithThisFrameFloat(self.get_result_f()) + else: + assert False + # ____________________________________________________________ def resume_in_blackhole(metainterp_sd, resumedescr): @@ -947,47 +1022,13 @@ False) # XXX # XXX virtualrefs # XXX virtualizable - _prepare_resume_from_failure(blackholeinterp, resumedescr.guard_opnum) + blackholeinterp._prepare_resume_from_failure(resumedescr.guard_opnum) try: - blackholeinterp = _resume_mainloop(blackholeinterp) + current_exc = blackholeinterp.cpu.grab_exc_value() + current_exc = lltype.cast_opaque_ptr(rclass.OBJECTPTR, current_exc) + while True: + current_exc = blackholeinterp._resume_mainloop(current_exc) + blackholeinterp = blackholeinterp.nextblackholeinterp finally: metainterp_sd.profiler.end_blackhole() debug_stop('jit-blackhole') - # rare case: we only get there if the blackhole interps all returned - # normally (in general we get a ContinueRunningNormally exception). - _done_with_this_frame(blackholeinterp) - -def _resume_mainloop(blackholeinterp): - while True: - try: - blackholeinterp.run() - finally: - blackholeinterp.builder.release_interp(blackholeinterp) - #...x.x.x... - assert blackholeinterp.nextblackholeinterp is None # XXX - break - xxx - return blackholeinterp - -def _prepare_resume_from_failure(blackholeinterp, opnum): - from pypy.jit.metainterp.resoperation import rop - if opnum == rop.GUARD_TRUE: # a goto_if_not_xxx that jumps only now - blackholeinterp.position = blackholeinterp.jitcode.follow_jump( - blackholeinterp.position) - elif opnum == rop.GUARD_FALSE: # a goto_if_not that stops jumping - pass - else: - raise NotImplementedError(opnum) - -def _done_with_this_frame(blackholeinterp): - sd = blackholeinterp.builder.metainterp_sd - if sd.result_type == 'void': - raise sd.DoneWithThisFrameVoid() - elif sd.result_type == 'int': - raise sd.DoneWithThisFrameInt(blackholeinterp.get_result_i()) - elif sd.result_type == 'ref': - raise sd.DoneWithThisFrameRef(blackholeinterp.get_result_r()) - elif sd.result_type == 'float': - raise sd.DoneWithThisFrameFloat(blackholeinterp.get_result_f()) - else: - assert False Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Wed May 19 15:37:09 2010 @@ -1413,8 +1413,6 @@ # to generate either GUARD_EXCEPTION or GUARD_NO_EXCEPTION, and also # to handle the following opcodes 'goto_if_exception_mismatch'. llexception = get_llexception(self.cpu, exception) - if not we_are_translated(): - llexception = llexception.args[1] llexception = self.cpu.ts.cast_to_ref(llexception) exc_value_box = self.cpu.ts.get_exc_value_box(llexception) if constant: @@ -1600,7 +1598,12 @@ pass elif (opnum == rop.GUARD_NO_EXCEPTION or opnum == rop.GUARD_EXCEPTION or opnum == rop.GUARD_NOT_FORCED): - xxx #self.handle_exception() + exception = self.cpu.grab_exc_value() + if exception: + self.execute_raised(exception) + else: + self.execute_did_not_raise() + self.handle_possible_exception() elif opnum == rop.GUARD_NO_OVERFLOW: # an overflow now detected xxx #self.raise_overflow_error() elif opnum == rop.GUARD_NONNULL or opnum == rop.GUARD_ISNULL: From arigo at codespeak.net Wed May 19 15:39:24 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 May 2010 15:39:24 +0200 (CEST) Subject: [pypy-svn] r74573 - pypy/branch/blackhole-improvement/pypy/jit/codewriter/test Message-ID: <20100519133924.81F9A282BE0@codespeak.net> Author: arigo Date: Wed May 19 15:39:23 2010 New Revision: 74573 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Log: Fix test. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Wed May 19 15:39:23 2010 @@ -3,7 +3,7 @@ from pypy.objspace.flow.model import SpaceOperation, Variable, Constant from pypy.jit.codewriter.jtransform import Transformer from pypy.jit.metainterp.history import getkind -from pypy.rpython.lltypesystem import lltype, rclass, rstr +from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr from pypy.translator.unsimplify import varoftype from pypy.jit.codewriter import heaptracker @@ -338,7 +338,8 @@ assert op1.opname == 'new_with_vtable' assert op1.args == [('sizedescr', S)] #assert heaptracker.descr2vtable(cpu, op1.args[0]) == vtable [type check] - assert heaptracker.vtable2descr(cpu, vtable) == op1.args[0] + vtable_int = llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(vtable)) + assert heaptracker.vtable2descr(cpu, vtable_int) == op1.args[0] def test_malloc_new_with_destructor(): vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) From afa at codespeak.net Wed May 19 15:45:14 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 19 May 2010 15:45:14 +0200 (CEST) Subject: [pypy-svn] r74574 - in pypy/trunk/pypy: module/sys module/sys/test rlib Message-ID: <20100519134514.C9924282BE0@codespeak.net> Author: afa Date: Wed May 19 15:45:13 2010 New Revision: 74574 Modified: pypy/trunk/pypy/module/sys/__init__.py pypy/trunk/pypy/module/sys/test/test_sysmodule.py pypy/trunk/pypy/module/sys/version.py pypy/trunk/pypy/module/sys/vm.py pypy/trunk/pypy/rlib/rwin32.py Log: Add sys.winver: version of the python DLL, and sys.getwindowsversion(): OS version info. Modified: pypy/trunk/pypy/module/sys/__init__.py ============================================================================== --- pypy/trunk/pypy/module/sys/__init__.py (original) +++ pypy/trunk/pypy/module/sys/__init__.py Wed May 19 15:45:13 2010 @@ -1,5 +1,6 @@ from pypy.interpreter.mixedmodule import MixedModule from pypy.interpreter.error import OperationError +import sys class Module(MixedModule): """Sys Builtin Module. """ @@ -71,6 +72,10 @@ 'setdefaultencoding' : 'interp_encoding.setdefaultencoding', 'getfilesystemencoding' : 'interp_encoding.getfilesystemencoding', } + + if sys.platform == 'win32': + interpleveldefs['winver'] = 'version.get_winver(space)' + interpleveldefs['getwindowsversion'] = 'vm.getwindowsversion' appleveldefs = { 'excepthook' : 'app.excepthook', Modified: pypy/trunk/pypy/module/sys/test/test_sysmodule.py ============================================================================== --- pypy/trunk/pypy/module/sys/test/test_sysmodule.py (original) +++ pypy/trunk/pypy/module/sys/test/test_sysmodule.py Wed May 19 15:45:13 2010 @@ -331,6 +331,10 @@ assert isinstance(v[3], int) assert isinstance(v[4], str) + def test_winver(self): + if hasattr(sys, "winver"): + assert sys.winver == sys.version[:3] + def test_dlopenflags(self): if hasattr(sys, "setdlopenflags"): assert hasattr(sys, "getdlopenflags") Modified: pypy/trunk/pypy/module/sys/version.py ============================================================================== --- pypy/trunk/pypy/module/sys/version.py (original) +++ pypy/trunk/pypy/module/sys/version.py Wed May 19 15:45:13 2010 @@ -58,6 +58,11 @@ PYPY_VERSION[1], PYPY_VERSION[2])) +def get_winver(space): + return space.wrap("%d.%d" % ( + CPYTHON_VERSION[0], + CPYTHON_VERSION[1])) + def get_hexversion(space): return space.wrap(tuple2hex(CPYTHON_VERSION)) Modified: pypy/trunk/pypy/module/sys/vm.py ============================================================================== --- pypy/trunk/pypy/module/sys/vm.py (original) +++ pypy/trunk/pypy/module/sys/vm.py Wed May 19 15:45:13 2010 @@ -117,3 +117,12 @@ saved, and restored afterwards. This is intended to be called from a debugger from a checkpoint, to recursively debug some other code.""" return space.getexecutioncontext().call_tracing(w_func, w_args) + +def getwindowsversion(space): + from pypy.rlib import rwin32 + info = rwin32.GetVersionEx() + return space.newtuple([space.wrap(info[0]), + space.wrap(info[1]), + space.wrap(info[2]), + space.wrap(info[3]), + space.wrap(info[4])]) Modified: pypy/trunk/pypy/rlib/rwin32.py ============================================================================== --- pypy/trunk/pypy/rlib/rwin32.py (original) +++ pypy/trunk/pypy/rlib/rwin32.py Wed May 19 15:45:13 2010 @@ -50,6 +50,15 @@ SYSTEMTIME = rffi_platform.Struct('SYSTEMTIME', []) + OSVERSIONINFO = rffi_platform.Struct( + 'OSVERSIONINFO', + [('dwOSVersionInfoSize', rffi.UINT), + ('dwMajorVersion', rffi.UINT), + ('dwMinorVersion', rffi.UINT), + ('dwBuildNumber', rffi.UINT), + ('dwPlatformId', rffi.UINT), + ('szCSDVersion', rffi.CFixedArray(lltype.Char, 1))]) + LPSECURITY_ATTRIBUTES = rffi_platform.SimpleType( "LPSECURITY_ATTRIBUTES", rffi.CCHARP) @@ -184,3 +193,22 @@ return ''.join([buf[i] for i in range(res)]) finally: lltype.free(buf, flavor='raw') + + _GetVersionEx = winexternal('GetVersionExA', + [lltype.Ptr(OSVERSIONINFO)], + DWORD) + + def GetVersionEx(): + info = lltype.malloc(OSVERSIONINFO, flavor='raw') + rffi.setintfield(info, 'c_dwOSVersionInfoSize', + rffi.sizeof(OSVERSIONINFO)) + try: + if not _GetVersionEx(info): + raise lastWindowsError() + return (rffi.cast(lltype.Signed, info.c_dwMajorVersion), + rffi.cast(lltype.Signed, info.c_dwMinorVersion), + rffi.cast(lltype.Signed, info.c_dwBuildNumber), + rffi.cast(lltype.Signed, info.c_dwPlatformId), + rffi.charp2str(info.c_szCSDVersion)) + finally: + lltype.free(info, flavor='raw') From arigo at codespeak.net Wed May 19 16:08:03 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 May 2010 16:08:03 +0200 (CEST) Subject: [pypy-svn] r74575 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter metainterp Message-ID: <20100519140803.2CD6E282BE0@codespeak.net> Author: arigo Date: Wed May 19 16:08:01 2010 New Revision: 74575 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py Log: Move decode_hp_hint_args() into support.py and use it systematically. It now returns the args in the standard order INT-REF-FLOATs. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py Wed May 19 16:08:01 2010 @@ -69,14 +69,33 @@ portalblock = link.target portalop = portalblock.operations[0] # split again, this time enforcing the order of the live vars - # specified by the user in the jit_merge_point() call + # specified by decode_hp_hint_args(). assert portalop.opname == 'jit_marker' assert portalop.args[0].value == 'jit_merge_point' - livevars = [v for v in portalop.args[2:] - if v.concretetype is not lltype.Void] - link = split_block(None, portalblock, 0, livevars) + greens_v, reds_v = decode_hp_hint_args(portalop) + link = split_block(None, portalblock, 0, greens_v + reds_v) return link.target +def decode_hp_hint_args(op): + # Returns (list-of-green-vars, list-of-red-vars) without Voids. + # Both lists are sorted: first INT, then REF, then FLOAT. + assert op.opname == 'jit_marker' + jitdriver = op.args[1].value + numgreens = len(jitdriver.greens) + numreds = len(jitdriver.reds) + greens_v = op.args[2:2+numgreens] + reds_v = op.args[2+numgreens:] + assert len(reds_v) == numreds + # + def _sort(args_v): + from pypy.jit.metainterp.history import getkind + lst = [v for v in args_v if v.concretetype is not lltype.Void] + _kind2count = {'int': 1, 'ref': 2, 'float': 3} + lst.sort(key=lambda v: _kind2count[getkind(v.concretetype)]) + return lst + # + return (_sort(greens_v), _sort(reds_v)) + def maybe_on_top_of_llinterp(rtyper, fnptr): # Run a generated graph on top of the llinterp for testing. # When translated, this just returns the fnptr. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Wed May 19 16:08:01 2010 @@ -1538,12 +1538,13 @@ duplicates) live_arg_boxes = greenboxes + redboxes if self.staticdata.virtualizable_info is not None: - # we use ':-1' to remove the last item, which is the virtualizable + # we use pop() to remove the last item, which is the virtualizable # itself self.remove_consts_and_duplicates(self.virtualizable_boxes, len(self.virtualizable_boxes)-1, duplicates) - live_arg_boxes += self.virtualizable_boxes[:-1] + live_arg_boxes += self.virtualizable_boxes + live_arg_boxes.pop() assert len(self.virtualref_boxes) == 0, "missing virtual_ref_finish()?" # Called whenever we reach the 'can_enter_jit' hint. # First, attempt to make a bridge: Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py Wed May 19 16:08:01 2010 @@ -409,17 +409,10 @@ def make_args_specification(self): graph, block, index = self.jit_merge_point_pos op = block.operations[index] - args = op.args[2:] - ALLARGS = [] - self.green_args_spec = [] - self.red_args_types = [] - for i, v in enumerate(args): - TYPE = v.concretetype - ALLARGS.append(TYPE) - if i < len(self.jitdriver.greens): - self.green_args_spec.append(TYPE) - else: - self.red_args_types.append(history.getkind(TYPE)) + greens_v, reds_v = support.decode_hp_hint_args(op) + ALLARGS = [v.concretetype for v in (greens_v + reds_v)] + self.green_args_spec = [v.concretetype for v in greens_v] + self.red_args_types = [history.getkind(v.concretetype) for v in reds_v] self.num_green_args = len(self.green_args_spec) RESTYPE = graph.getreturnvar().concretetype (self.JIT_ENTER_FUNCTYPE, @@ -441,7 +434,7 @@ continue op = block.operations[index] - greens_v, reds_v = decode_hp_hint_args(op) + greens_v, reds_v = support.decode_hp_hint_args(op) args_v = greens_v + reds_v vlist = [Constant(jit_enter_fnptr, FUNCPTR)] + args_v @@ -618,7 +611,7 @@ op = origblock.operations[origindex] assert op.opname == 'jit_marker' assert op.args[0].value == 'jit_merge_point' - greens_v, reds_v = decode_hp_hint_args(op) + greens_v, reds_v = support.decode_hp_hint_args(op) vlist = [Constant(self.portal_runner_ptr, self.PTR_PORTAL_FUNCTYPE)] vlist += greens_v vlist += reds_v @@ -668,16 +661,3 @@ all_graphs = self.translator.graphs vrefinfo = self.metainterp_sd.virtualref_info vrefinfo.replace_force_virtual_with_call(all_graphs) - - -def decode_hp_hint_args(op): - # Returns (list-of-green-vars, list-of-red-vars) without Voids. - assert op.opname == 'jit_marker' - jitdriver = op.args[1].value - numgreens = len(jitdriver.greens) - numreds = len(jitdriver.reds) - greens_v = op.args[2:2+numgreens] - reds_v = op.args[2+numgreens:] - assert len(reds_v) == numreds - return ([v for v in greens_v if v.concretetype is not lltype.Void], - [v for v in reds_v if v.concretetype is not lltype.Void]) From arigo at codespeak.net Wed May 19 16:18:11 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 May 2010 16:18:11 +0200 (CEST) Subject: [pypy-svn] r74576 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100519141811.0613D282BE0@codespeak.net> Author: arigo Date: Wed May 19 16:18:10 2010 New Revision: 74576 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: In-progress. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Wed May 19 16:18:10 2010 @@ -146,6 +146,9 @@ if argtype == 'I': reg = self.registers_i[index] elif argtype == 'R': reg = self.registers_r[index] elif argtype == 'F': reg = self.registers_f[index] + if not we_are_translated(): + assert not isinstance(reg, MissingValue), ( + name, self.jitcode, position) value.append(reg) make_sure_not_resized(value) position += length @@ -165,6 +168,9 @@ position += 2 else: raise AssertionError("bad argtype: %r" % (argtype,)) + if not we_are_translated(): + assert not isinstance(value, MissingValue), ( + name, self.jitcode, position) args = args + (value,) if verbose and not we_are_translated(): @@ -979,17 +985,19 @@ pass elif opnum == rop.GUARD_NO_EXCEPTION or opnum == rop.GUARD_EXCEPTION: pass + elif opnum == rop.GUARD_CLASS: + pass else: raise NotImplementedError(opnum) # connect the return of values from the called frame to the # 'xxx_call_yyy' instructions from the caller frame def _setup_return_value_i(self, result): - self.registers_i[ord(self.jitcode.code[position-1])] = result + self.registers_i[ord(self.jitcode.code[self.position-1])] = result def _setup_return_value_r(self, result): - self.registers_r[ord(self.jitcode.code[position-1])] = result + self.registers_r[ord(self.jitcode.code[self.position-1])] = result def _setup_return_value_f(self, result): - self.registers_f[ord(self.jitcode.code[position-1])] = result + self.registers_f[ord(self.jitcode.code[self.position-1])] = result def _exit_frame_with_exception(self, e): xxx Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Wed May 19 16:18:10 2010 @@ -744,7 +744,7 @@ opimpl_ref_guard_value = _opimpl_guard_value opimpl_float_guard_value = _opimpl_guard_value - @arguments("box") + @arguments("box", "force_initial_pc") def opimpl_guard_class(self, box): clsbox = self.cls_of_box(box) self.generate_guard(rop.GUARD_CLASS, box, [clsbox]) @@ -2005,6 +2005,7 @@ assert position >= 0 args = () next_argcode = 0 + force_initial_pc = False code = self.bytecode position += 1 for argtype in argtypes: @@ -2068,13 +2069,17 @@ argcodes[next_argcode + 2]) next_argcode = next_argcode + 3 position = position3 + 1 + length3 + elif argtype == "force_initial_pc": + force_initial_pc = True + continue else: raise AssertionError("bad argtype: %r" % (argtype,)) args += (value,) # num_return_args = len(argcodes) - next_argcode assert num_return_args == 0 or num_return_args == 1 - self.pc = position + num_return_args + if not force_initial_pc: + self.pc = position + num_return_args # if not we_are_translated(): print '\tpyjitpl: %s(%s)' % (name, ', '.join(map(repr, args))), @@ -2095,6 +2100,8 @@ else: resultbox = unboundmethod(self, *args) # + if force_initial_pc: + self.pc = position + num_return_args if resultbox is not None: self.make_result_of_lastop(resultbox) # From arigo at codespeak.net Wed May 19 16:18:44 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 May 2010 16:18:44 +0200 (CEST) Subject: [pypy-svn] r74577 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100519141844.EE4A2282BE0@codespeak.net> Author: arigo Date: Wed May 19 16:18:43 2010 New Revision: 74577 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: 'test_isinstance_2' passes. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Wed May 19 16:18:43 2010 @@ -1609,6 +1609,8 @@ xxx #self.raise_overflow_error() elif opnum == rop.GUARD_NONNULL or opnum == rop.GUARD_ISNULL: xxx #self.framestack[-1].ignore_next_guard_nullness(opnum) + elif opnum == rop.GUARD_CLASS: + pass else: raise NotImplementedError(opnum) From arigo at codespeak.net Wed May 19 16:31:11 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 May 2010 16:31:11 +0200 (CEST) Subject: [pypy-svn] r74578 - in pypy/branch/blackhole-improvement/pypy/jit/metainterp: . test Message-ID: <20100519143111.56926282BE0@codespeak.net> Author: arigo Date: Wed May 19 16:31:09 2010 New Revision: 74578 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizefindnode.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizeopt.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Log: 'test_pass_around' passes. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py Wed May 19 16:31:09 2010 @@ -15,7 +15,7 @@ # ____________________________________________________________ -def do_call(metainterp, argboxes, descr): +def do_call(cpu, metainterp, argboxes, descr): # count the number of arguments of the different types count_i = count_r = count_f = 0 for i in range(1, len(argboxes)): @@ -45,7 +45,6 @@ count_f += 1 # get the function address as an integer func = argboxes[0].getint() - cpu = metainterp.cpu # do the call using the correct function from the cpu rettype = descr.get_return_type() if rettype == INT: @@ -88,8 +87,7 @@ do_call_pure = do_call do_call_loopinvariant = do_call -def do_getarrayitem_gc(metainterp, arraybox, indexbox, arraydescr): - cpu = metainterp.cpu +def do_getarrayitem_gc(cpu, _, arraybox, indexbox, arraydescr): array = arraybox.getref_base() index = indexbox.getint() if arraydescr.is_array_of_pointers(): @@ -99,8 +97,7 @@ else: return BoxInt(cpu.bh_setarrayitem_gc_i(arraydescr, array, index)) -def do_setarrayitem_gc(metainterp, arraybox, indexbox, itembox, arraydescr): - cpu = metainterp.cpu +def do_setarrayitem_gc(cpu, _, arraybox, indexbox, itembox, arraydescr): array = arraybox.getref_base() index = indexbox.getint() if arraydescr.is_array_of_pointers(): @@ -111,8 +108,7 @@ else: cpu.bh_setarrayitem_gc_i(arraydescr, array, index, itembox.getint()) -def do_getfield_gc(metainterp, structbox, fielddescr): - cpu = metainterp.cpu +def do_getfield_gc(cpu, _, structbox, fielddescr): struct = structbox.getref_base() if fielddescr.is_pointer_field(): return BoxPtr(cpu.bh_getfield_gc_r(struct, fielddescr)) @@ -121,8 +117,7 @@ else: return BoxInt(cpu.bh_getfield_gc_i(struct, fielddescr)) -def do_getfield_raw(metainterp, structbox, fielddescr): - cpu = metainterp.cpu +def do_getfield_raw(cpu, _, structbox, fielddescr): struct = structbox.getint() if fielddescr.is_pointer_field(): return BoxPtr(cpu.bh_getfield_raw_r(struct, fielddescr)) @@ -131,8 +126,7 @@ else: return BoxInt(cpu.bh_getfield_raw_i(struct, fielddescr)) -def do_setfield_gc(metainterp, structbox, itembox, fielddescr): - cpu = metainterp.cpu +def do_setfield_gc(cpu, _, structbox, itembox, fielddescr): struct = structbox.getref_base() if fielddescr.is_pointer_field(): cpu.bh_setfield_gc_r(struct, fielddescr, itembox.getref_base()) @@ -141,8 +135,7 @@ else: cpu.bh_setfield_gc_i(struct, fielddescr, itembox.getint()) -def do_setfield_raw(metainterp, structbox, itembox, fielddescr): - cpu = metainterp.cpu +def do_setfield_raw(cpu, _, structbox, itembox, fielddescr): struct = structbox.getint() if fielddescr.is_pointer_field(): cpu.bh_setfield_raw_r(struct, fielddescr, itembox.getref_base()) @@ -157,11 +150,10 @@ descr = heaptracker.vtable2descr(cpu, vtable) return cpu.bh_new_with_vtable(descr, vtable) -def do_new_with_vtable(metainterp, clsbox): - cpu = metainterp.cpu +def do_new_with_vtable(cpu, _, clsbox): return BoxPtr(exec_new_with_vtable(cpu, clsbox)) -def do_int_add_ovf(metainterp, box1, box2): +def do_int_add_ovf(cpu, metainterp, box1, box2): a = box1.getint() b = box2.getint() try: @@ -173,7 +165,7 @@ metainterp.execute_did_not_raise() return BoxInt(z) -def do_int_sub_ovf(metainterp, box1, box2): +def do_int_sub_ovf(cpu, metainterp, box1, box2): a = box1.getint() b = box2.getint() try: @@ -185,7 +177,7 @@ metainterp.execute_did_not_raise() return BoxInt(z) -def do_int_mul_ovf(metainterp, box1, box2): +def do_int_mul_ovf(cpu, metainterp, box1, box2): a = box1.getint() b = box2.getint() try: @@ -197,6 +189,9 @@ metainterp.execute_did_not_raise() return BoxInt(z) +def do_same_as(cpu, _, box): + return box.clonebox() + # ____________________________________________________________ ##def do_force_token(cpu): @@ -285,11 +280,11 @@ argtypes = unrolling_iterable(func.argtypes) resulttype = func.resulttype # - def do(metainterp, *argboxes): + def do(cpu, _, *argboxes): newargs = () for argtype in argtypes: if argtype == 'cpu': - value = metainterp.cpu + value = cpu elif argtype == 'd': value = argboxes[-1] assert isinstance(value, AbstractDescr) @@ -334,7 +329,7 @@ has_descr._annspecialcase_ = 'specialize:memo' -def execute(metainterp, opnum, descr, *argboxes): +def execute(cpu, metainterp, opnum, descr, *argboxes): # only for opnums with a fixed arity if has_descr(opnum): check_descr(descr) @@ -342,51 +337,50 @@ else: assert descr is None func = get_execute_function(opnum, len(argboxes)) - return func(metainterp, *argboxes) # note that the 'argboxes' tuple - # optionally ends with the descr + return func(cpu, metainterp, *argboxes) # note that the 'argboxes' tuple + # optionally ends with the descr execute._annspecialcase_ = 'specialize:arg(1)' -def execute_varargs(metainterp, opnum, argboxes, descr): +def execute_varargs(cpu, metainterp, opnum, argboxes, descr): # only for opnums with a variable arity (calls, typically) check_descr(descr) func = get_execute_function(opnum, -1) - return func(metainterp, argboxes, descr) + return func(cpu, metainterp, argboxes, descr) execute_varargs._annspecialcase_ = 'specialize:arg(1)' -def execute_nonspec(metainterp, opnum, argboxes, descr=None): - assert metainterp.__class__.__name__ == 'MetaInterp' # XXX kill me +def execute_nonspec(cpu, metainterp, opnum, argboxes, descr=None): arity = resoperation.oparity[opnum] assert arity == -1 or len(argboxes) == arity if resoperation.opwithdescr[opnum]: check_descr(descr) if arity == -1: func = get_execute_funclist(-1)[opnum] - return func(metainterp, argboxes, descr) + return func(cpu, metainterp, argboxes, descr) if arity == 0: func = get_execute_funclist(1)[opnum] - return func(metainterp, descr) + return func(cpu, metainterp, descr) if arity == 1: func = get_execute_funclist(2)[opnum] - return func(metainterp, argboxes[0], descr) + return func(cpu, metainterp, argboxes[0], descr) if arity == 2: func = get_execute_funclist(3)[opnum] - return func(metainterp, argboxes[0], argboxes[1], descr) + return func(cpu, metainterp, argboxes[0], argboxes[1], descr) if arity == 3: func = get_execute_funclist(4)[opnum] - return func(metainterp, argboxes[0], argboxes[1], argboxes[2], + return func(cpu, metainterp, argboxes[0], argboxes[1], argboxes[2], descr) else: assert descr is None if arity == 1: func = get_execute_funclist(1)[opnum] - return func(metainterp, argboxes[0]) + return func(cpu, metainterp, argboxes[0]) if arity == 2: func = get_execute_funclist(2)[opnum] - return func(metainterp, argboxes[0], argboxes[1]) + return func(cpu, metainterp, argboxes[0], argboxes[1]) if arity == 3: func = get_execute_funclist(3)[opnum] - return func(metainterp, argboxes[0], argboxes[1], argboxes[2]) + return func(cpu, metainterp, argboxes[0], argboxes[1], argboxes[2]) raise NotImplementedError Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizefindnode.py Wed May 19 16:31:09 2010 @@ -159,7 +159,8 @@ else: # all constant arguments: we can constant-fold argboxes = [self.get_constant_box(arg) for arg in op.args] - resbox = execute_nonspec(self.cpu, op.opnum, argboxes, op.descr) + resbox = execute_nonspec(self.cpu, None, + op.opnum, argboxes, op.descr) self.set_constant_node(op.result, resbox.constbox()) # default case: mark the arguments as escaping for box in op.args: Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizeopt.py Wed May 19 16:31:09 2010 @@ -603,7 +603,8 @@ else: # all constant arguments: constant-fold away argboxes = [self.get_constant_box(arg) for arg in op.args] - resbox = execute_nonspec(self.cpu, op.opnum, argboxes, op.descr) + resbox = execute_nonspec(self.cpu, None, + op.opnum, argboxes, op.descr) self.make_constant(op.result, resbox.constbox()) return Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Wed May 19 16:31:09 2010 @@ -1348,7 +1348,7 @@ # execute the operation profiler = self.staticdata.profiler profiler.count_ops(opnum) - resbox = executor.execute(self, opnum, descr, *argboxes) + resbox = executor.execute(self.cpu, self, opnum, descr, *argboxes) if rop._ALWAYS_PURE_FIRST <= opnum <= rop._ALWAYS_PURE_LAST: return self._record_helper_pure(opnum, resbox, descr, *argboxes) else: @@ -1361,7 +1361,8 @@ # execute the operation profiler = self.staticdata.profiler profiler.count_ops(opnum) - resbox = executor.execute_varargs(self, opnum, argboxes, descr) + resbox = executor.execute_varargs(self.cpu, self, + opnum, argboxes, descr) if self.is_blackholing(): profiler.count_ops(opnum, BLACKHOLED_OPS) else: Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Wed May 19 16:31:09 2010 @@ -895,7 +895,9 @@ return obj.a res = self.interp_operations(fn, [1]) assert res == 1 - self.check_operations_history(guard_class=0, instanceof=0) + self.check_operations_history(guard_class=0) + if self.type_system == 'ootype': + self.check_operations_history(instanceof=0) def test_r_dict(self): from pypy.rlib.objectmodel import r_dict From arigo at codespeak.net Wed May 19 16:35:53 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 May 2010 16:35:53 +0200 (CEST) Subject: [pypy-svn] r74579 - pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph Message-ID: <20100519143553.C1C06282BE0@codespeak.net> Author: arigo Date: Wed May 19 16:35:52 2010 New Revision: 74579 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py Log: Fix. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py Wed May 19 16:35:52 2010 @@ -554,7 +554,11 @@ except AttributeError: try: impl = globals()['do_' + opname.lower()] # do_arraylen_gc etc. - op = staticmethod(impl) + def op(self, descr, *args): + if descr is None: + return impl(*args) + else: + return impl(descr, *args) except KeyError: op = cls._make_impl_from_blackhole_interp(opname) cls.OPHANDLERS[opnum] = op @@ -1182,6 +1186,9 @@ # ____________________________________________________________ +def do_same_as(x): + return x + def do_arraylen_gc(arraydescr, array): array = array._obj.container return array.getlength() From arigo at codespeak.net Wed May 19 16:49:41 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 May 2010 16:49:41 +0200 (CEST) Subject: [pypy-svn] r74580 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter codewriter/test metainterp Message-ID: <20100519144941.CAB99282C0B@codespeak.net> Author: arigo Date: Wed May 19 16:49:40 2010 New Revision: 74580 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: Do something reasonable for keepalives. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Wed May 19 16:49:40 2010 @@ -98,6 +98,12 @@ self.get_liveness_info(insn, 'ref'), self.get_liveness_info(insn, 'float')) return + if insn[0] == 'keepalive': + # The 'keepalive' instruction is useful when doing register + # allocation, to ensure that the register holding the value is + # not overwritten too early. But it does not need to be + # present at all in the final code. + return startposition = len(self.code) self.code.append("temporary placeholder") # Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py Wed May 19 16:49:40 2010 @@ -149,6 +149,13 @@ assembler.assemble(ssarepr) assert assembler.indirectcalltargets == set(lst1).union(lst2) +def test_assemble_keepalive(): + ssarepr = SSARepr("test") + ssarepr.insns = [('keepalive', Register('ref', 2))] + assembler = Assembler() + jitcode = assembler.assemble(ssarepr) + assert jitcode.code == "" # the keepalive is removed + def test_num_regs(): assembler = Assembler() ssarepr = SSARepr("test") Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Wed May 19 16:49:40 2010 @@ -9,6 +9,7 @@ from pypy.translator.unsimplify import varoftype from pypy.rlib.rarithmetic import ovfcheck from pypy.rlib.jit import dont_look_inside, _we_are_jitted, JitDriver +from pypy.rlib.objectmodel import keepalive_until_here class FakeRegAlloc: @@ -462,3 +463,21 @@ can_enter_jit void_return """, transform=True) + + def test_keepalive(self): + S = lltype.GcStruct('S') + def g(): + return lltype.malloc(S) + def f(x): + p = g() + q = g() + keepalive_until_here(p) + keepalive_until_here(q) + return x + self.encoding_test(f, [5], """ + G_residual_call_r_r $<* fn g>, , R[], %r0 + G_residual_call_r_r $<* fn g>, , R[], %r1 + keepalive %r0 + keepalive %r1 + int_return %i0 + """, transform=True) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Wed May 19 16:49:40 2010 @@ -8,6 +8,7 @@ from pypy.objspace.flow.model import c_last_exception from pypy.rpython.lltypesystem import lltype, rclass from pypy.rlib.rarithmetic import ovfcheck +from pypy.rlib.objectmodel import keepalive_until_here class TestRegAlloc: Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Wed May 19 16:49:40 2010 @@ -760,10 +760,6 @@ ## 'len', [builtin.len_func, box], 'int') ## self.generate_guard(pc, "guard_len", box, [intbox]) - @FixME #arguments("box") - def opimpl_keepalive(self, box): - pass # xxx? - def verify_green_args(self, varargs): num_green_args = self.metainterp.staticdata.num_green_args for i in range(num_green_args): From arigo at codespeak.net Wed May 19 17:26:35 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 May 2010 17:26:35 +0200 (CEST) Subject: [pypy-svn] r74581 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100519152635.BCACE282C0D@codespeak.net> Author: arigo Date: Wed May 19 17:26:34 2010 New Revision: 74581 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py Log: Oups. Missing an important case. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py Wed May 19 17:26:34 2010 @@ -1,4 +1,5 @@ from pypy.objspace.flow.model import Variable, SpaceOperation, c_last_exception +from pypy.jit.codewriter.flatten import ListOfKind # Some instruction require liveness information (the ones that can end up @@ -34,7 +35,10 @@ if op.opname.startswith('G_'): block.operations.insert(i, _livespaceop(alive)) for v in op.args: - alive.add(v) + if isinstance(v, ListOfKind): + alive.update(v) + else: + alive.add(v) def _livespaceop(alive): livevars = [v for v in alive if isinstance(v, Variable)] Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py Wed May 19 17:26:34 2010 @@ -1,7 +1,7 @@ from pypy.jit.codewriter import support from pypy.jit.codewriter.liveness import compute_liveness from pypy.jit.codewriter.test.test_flatten import fake_regallocs -from pypy.jit.codewriter.flatten import flatten_graph +from pypy.jit.codewriter.flatten import flatten_graph, ListOfKind from pypy.jit.codewriter.format import assert_format from pypy.objspace.flow.model import SpaceOperation @@ -13,10 +13,15 @@ return self.rtyper.annotator.translator.graphs def add_G_prefix(self, graph): - """Add a 'G_' prefix to the opnames 'int_add' and 'int_mul'.""" + """Add a 'G_' prefix to the opnames 'int_add' and 'int_mul'. + Turn the arguments of float_add into a ListOfKind().""" def with_prefix(op): if op.opname in ('int_add', 'int_mul'): return SpaceOperation('G_' + op.opname, op.args, op.result) + if op.opname == 'float_add': + return SpaceOperation(op.opname, + [ListOfKind('float', op.args)], + op.result) return op # for block in graph.iterblocks(): @@ -134,3 +139,15 @@ L1: int_return %i1 """, switches_require_liveness=True) + + def test_list_of_kind(self): + def f(x, y, z, t): + return (x + y) * (z + t) + self.encoding_test(f, [5, 6, 3.2, 4.3], """ + -live- %f0, %f1 + G_int_add %i0, %i1, %i2 + float_add F[%f0, %f1], %f2 + cast_int_to_float %i2, %f3 + float_mul %f3, %f2, %f4 + float_return %f4 + """) From arigo at codespeak.net Wed May 19 17:27:37 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 May 2010 17:27:37 +0200 (CEST) Subject: [pypy-svn] r74582 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter codewriter/test metainterp Message-ID: <20100519152737.2D335282BFF@codespeak.net> Author: arigo Date: Wed May 19 17:27:35 2010 New Revision: 74582 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py Log: Fix, in-progress. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Wed May 19 17:27:35 2010 @@ -444,8 +444,10 @@ def handle_getfield_typeptr(self, op): # note: the 'G_' prefix tells that the operation might generate - # a guard in pyjitpl (see liveness.py) - return SpaceOperation('G_guard_class', [op.args[0]], op.result) + # a guard in pyjitpl (see liveness.py). The following 'keepalive' + # is needed to ensure that op.args[0] is restored on guard failure. + return [SpaceOperation('G_guard_class', [op.args[0]], op.result), + SpaceOperation('keepalive', [op.args[0]], None)] def rewrite_op_malloc(self, op): assert op.args[1].value == {'flavor': 'gc'} Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Wed May 19 17:27:35 2010 @@ -283,10 +283,13 @@ c_name = Constant('typeptr', lltype.Void) v_result = varoftype(rclass.OBJECT.typeptr) op = SpaceOperation('getfield', [v_parent, c_name], v_result) - op1 = Transformer(FakeCPU()).rewrite_operation(op) + op1, op2 = Transformer(FakeCPU()).rewrite_operation(op) assert op1.opname == 'G_guard_class' assert op1.args == [v_parent] assert op1.result == v_result + assert op2.opname == 'keepalive' + assert op2.args == [v_parent] + assert op2.result == None def test_setfield(): # XXX a more compact encoding would be possible; see test_getfield() Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Wed May 19 17:27:35 2010 @@ -186,3 +186,25 @@ L2: reraise """) + + def test_regalloc_lists(self): + v1 = Variable(); v1.concretetype = lltype.Signed + v2 = Variable(); v2.concretetype = lltype.Signed + v3 = Variable(); v3.concretetype = lltype.Signed + v4 = Variable(); v4.concretetype = lltype.Signed + v5 = Variable(); v5.concretetype = lltype.Signed + block = Block([v1]) + block.operations = [ + SpaceOperation('int_add', [v1, Constant(1, lltype.Signed)], v2), + SpaceOperation('rescall', [ListOfKind('int', [v1, v2])], v5), + SpaceOperation('rescall', [ListOfKind('int', [v1, v2])], v3), + ] + graph = FunctionGraph('f', block, v4) + block.closeblock(Link([v3], graph.returnblock)) + # + self.check_assembler(graph, """ + int_add %i0, $1, %i1 + rescall I[%i0, %i1], %i2 + rescall I[%i0, %i1], %i0 + int_return %i0 + """) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Wed May 19 17:27:35 2010 @@ -744,10 +744,10 @@ opimpl_ref_guard_value = _opimpl_guard_value opimpl_float_guard_value = _opimpl_guard_value - @arguments("box", "force_initial_pc") - def opimpl_guard_class(self, box): + @arguments("box", "orgpc") + def opimpl_guard_class(self, box, orgpc): clsbox = self.cls_of_box(box) - self.generate_guard(rop.GUARD_CLASS, box, [clsbox]) + self.generate_guard(rop.GUARD_CLASS, box, [clsbox], resumepc=orgpc) return clsbox ## @FixME #arguments("orgpc", "box", "builtin") @@ -924,7 +924,7 @@ except ChangeFrame: pass - def generate_guard(self, opnum, box=None, extraargs=[]): + def generate_guard(self, opnum, box=None, extraargs=[], resumepc=-1): if isinstance(box, Const): # no need for a guard return metainterp = self.metainterp @@ -947,8 +947,11 @@ virtualizable_boxes = None if metainterp.staticdata.virtualizable_info is not None: virtualizable_boxes = metainterp.virtualizable_boxes + if resumepc < 0: + resumepc = self.pc resume.capture_resumedata(metainterp.framestack, virtualizable_boxes, - metainterp.virtualref_boxes, resumedescr) + metainterp.virtualref_boxes, resumedescr, + resumepc) self.metainterp.staticdata.profiler.count_ops(opnum, GUARDS) # count metainterp.attach_debug_info(guard_op) @@ -2004,8 +2007,8 @@ assert position >= 0 args = () next_argcode = 0 - force_initial_pc = False code = self.bytecode + orgpc = position position += 1 for argtype in argtypes: if argtype == "box": # a box, of whatever type @@ -2068,17 +2071,15 @@ argcodes[next_argcode + 2]) next_argcode = next_argcode + 3 position = position3 + 1 + length3 - elif argtype == "force_initial_pc": - force_initial_pc = True - continue + elif argtype == "orgpc": + value = orgpc else: raise AssertionError("bad argtype: %r" % (argtype,)) args += (value,) # num_return_args = len(argcodes) - next_argcode assert num_return_args == 0 or num_return_args == 1 - if not force_initial_pc: - self.pc = position + num_return_args + self.pc = position + num_return_args # if not we_are_translated(): print '\tpyjitpl: %s(%s)' % (name, ', '.join(map(repr, args))), @@ -2099,8 +2100,6 @@ else: resultbox = unboundmethod(self, *args) # - if force_initial_pc: - self.pc = position + num_return_args if resultbox is not None: self.make_result_of_lastop(resultbox) # Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py Wed May 19 17:27:35 2010 @@ -25,10 +25,10 @@ class FrameInfo(object): __slots__ = ('prev', 'jitcode', 'pc') - def __init__(self, prev, frame): + def __init__(self, prev, jitcode, pc): self.prev = prev - self.jitcode = frame.jitcode - self.pc = frame.pc + self.jitcode = jitcode + self.pc = pc def _ensure_parent_resumedata(framestack, n): target = framestack[n] @@ -38,18 +38,19 @@ back = framestack[n-1] target.parent_resumedata_frame_info_list = FrameInfo( back.parent_resumedata_frame_info_list, - back) + back.jitcode, + back.pc) target.parent_resumedata_snapshot = Snapshot( back.parent_resumedata_snapshot, back.get_list_of_active_boxes()) def capture_resumedata(framestack, virtualizable_boxes, virtualref_boxes, - storage): + storage, pc): n = len(framestack)-1 top = framestack[n] _ensure_parent_resumedata(framestack, n) frame_info_list = FrameInfo(top.parent_resumedata_frame_info_list, - top) + top.jitcode, pc) storage.rd_frame_info_list = frame_info_list snapshot = Snapshot(top.parent_resumedata_snapshot, top.get_list_of_active_boxes()) From arigo at codespeak.net Wed May 19 18:04:00 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 May 2010 18:04:00 +0200 (CEST) Subject: [pypy-svn] r74583 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter codewriter/test metainterp Message-ID: <20100519160400.5F400282BFF@codespeak.net> Author: arigo Date: Wed May 19 18:03:58 2010 New Revision: 74583 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: In guard_class and guard_value, the value being guarded on must also be included in -live-, and we must resume failures at the start of the operation, not at the end. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Wed May 19 18:03:58 2010 @@ -284,7 +284,9 @@ jitcode = self.callcontrol.get_jitcode(targetgraph, called_from=self.graph) lst.append(jitcode) - return self.handle_residual_call(op, [IndirectCallTargets(lst)]) + op0 = SpaceOperation('G_int_guard_value', [op.args[0]], None) + op1 = self.handle_residual_call(op, [IndirectCallTargets(lst)]) + return [op0, op1] def _prepare_builtin_call(self, op, oopspec_name, args, extra=None, extrakey=None): @@ -324,12 +326,14 @@ assert op.args[0].concretetype != lltype.Ptr(rstr.STR) kind = getkind(op.args[0].concretetype) # note: the 'G_' prefix tells that the operation might generate - # a guard in pyjitpl (see liveness.py) - op = SpaceOperation('G_%s_guard_value' % kind, - [op.args[0]], None) + # a guard in pyjitpl (see liveness.py). The following 'keepalive' + # is needed to ensure that op.args[0] is restored on guard failure. + op1 = SpaceOperation('G_%s_guard_value' % kind, + [op.args[0]], None) + op2 = SpaceOperation('keepalive', [op.args[0]], None) # this special return value forces op.result to be considered # equal to op.args[0] - return [op, None] + return [op1, op2, None] else: log.WARNING('ignoring hint %r at %r' % (hints, self.graph)) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Wed May 19 18:03:58 2010 @@ -45,7 +45,15 @@ assert graph == 'somegraph' return 'somejitcode' -class FakeIndirectCallControl: +class FakeResidualIndirectCallControl: + def guess_call_kind(self, op): + return 'residual' + def getcalldescr(self, op): + return 'calldescr' + def calldescr_canraise(self, calldescr): + return True + +class FakeRegularIndirectCallControl: def guess_call_kind(self, op): return 'regular' def graphs_from(self, op): @@ -178,7 +186,8 @@ else: expectedkind = 'r' # only references yield residual_call_test, ARGS, RESTYPE, expectedkind yield direct_call_test, ARGS, RESTYPE, expectedkind - yield indirect_call_test, ARGS, RESTYPE, expectedkind + yield indirect_residual_call_test, ARGS, RESTYPE, expectedkind + yield indirect_regular_call_test, ARGS, RESTYPE, expectedkind def get_direct_call_op(argtypes, restype): FUNC = lltype.FuncType(argtypes, restype) @@ -225,25 +234,58 @@ kind = getkind(v.concretetype) assert kind == 'void' or kind[0] in expectedkind -def indirect_call_test(argtypes, restype, expectedkind): +def indirect_residual_call_test(argtypes, restype, expectedkind): + # an indirect call that is residual in all cases is very similar to + # a residual direct call + op = get_direct_call_op(argtypes, restype) + op.opname = 'indirect_call' + op.args[0] = varoftype(op.args[0].concretetype) + op.args.append(Constant(['somegraph1', 'somegraph2'], lltype.Void)) + tr = Transformer(FakeCPU(), FakeResidualIndirectCallControl()) + tr.graph = 'someinitialgraph' + op1 = tr.rewrite_operation(op) + reskind = getkind(restype)[0] + assert op1.opname == 'G_residual_call_%s_%s' % (expectedkind, reskind) + assert op1.result == op.result + assert op1.args[0] == op.args[0] + assert op1.args[1] == 'calldescr' + assert len(op1.args) == 2 + len(expectedkind) + for sublist, kind1 in zip(op1.args[2:], expectedkind): + assert sublist.kind.startswith(kind1) + assert list(sublist) == [v for v in op.args[1:] + if getkind(v.concretetype)==sublist.kind] + for v in op.args[1:]: + kind = getkind(v.concretetype) + assert kind == 'void' or kind[0] in expectedkind + +def indirect_regular_call_test(argtypes, restype, expectedkind): + # a regular indirect call is preceded by a guard_value on the + # function address, so that pyjitpl can know which jitcode to follow + from pypy.jit.codewriter.flatten import IndirectCallTargets op = get_direct_call_op(argtypes, restype) op.opname = 'indirect_call' op.args[0] = varoftype(op.args[0].concretetype) op.args.append(Constant(['somegraph1', 'somegraph2'], lltype.Void)) - tr = Transformer(FakeCPU(), FakeIndirectCallControl()) + tr = Transformer(FakeCPU(), FakeRegularIndirectCallControl()) tr.graph = 'someinitialgraph' op1 = tr.rewrite_operation(op) + assert type(op1) is list + op0, op1 = op1 + assert op0.opname == 'G_int_guard_value' + assert op0.args[0] == op.args[0] + assert op0.result is None reskind = getkind(restype)[0] assert op1.opname == 'G_residual_call_%s_%s' % (expectedkind, reskind) assert op1.result == op.result assert op1.args[0] == op.args[0] assert op1.args[1] == 'calldescr' + assert isinstance(op1.args[2], IndirectCallTargets) assert op1.args[2].lst == ['somejitcode1', 'somejitcode2'] assert len(op1.args) == 3 + len(expectedkind) for sublist, kind1 in zip(op1.args[3:], expectedkind): assert sublist.kind.startswith(kind1) assert list(sublist) == [v for v in op.args[1:] - if getkind(v.concretetype) == sublist.kind] + if getkind(v.concretetype)==sublist.kind] for v in op.args[1:]: kind = getkind(v.concretetype) assert kind == 'void' or kind[0] in expectedkind @@ -532,11 +574,13 @@ [v1, Constant({'promote': True}, lltype.Void)], v2) oplist = Transformer().rewrite_operation(op) - assert len(oplist) == 2 - assert oplist[1] is None + assert len(oplist) == 3 assert oplist[0].opname == 'G_int_guard_value' assert oplist[0].args == [v1] assert oplist[0].result is None + assert oplist[1].opname == 'keepalive' + assert oplist[1].args == [v1] + assert oplist[2] is None def test_promote_2(): v1 = varoftype(lltype.Signed) @@ -550,8 +594,10 @@ block.operations = [op] block.closeblock(Link([v2], returnblock)) Transformer().optimize_block(block) - assert len(block.operations) == 1 + assert len(block.operations) == 2 assert block.operations[0].opname == 'G_int_guard_value' assert block.operations[0].args == [v1] assert block.operations[0].result is None + assert block.operations[1].opname == 'keepalive' + assert block.operations[1].args == [v1] assert block.exits[0].args == [v1] Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Wed May 19 18:03:58 2010 @@ -303,9 +303,9 @@ ## self.load_int() # past the 'box' argument ## self.ignore_varargs() # past the 'livelist' argument - @arguments("box", "descr") - def opimpl_switch(self, valuebox, switchdict): - box = self.implement_guard_value(valuebox) + @arguments("box", "descr", "orgpc") + def opimpl_switch(self, valuebox, switchdict, orgpc): + box = self.implement_guard_value(orgpc, valuebox) switchvalue = box.getint() assert isinstance(switchdict, SwitchDictDescr) try: @@ -371,90 +371,6 @@ sourcebox, destbox, source_startbox, dest_startbox, lengthbox) - @FixME #arguments("orgpc", "box", "descr", "box") - def opimpl_check_neg_index(self, pc, arraybox, arraydesc, indexbox): - negbox = self.metainterp.execute_and_record( - rop.INT_LT, None, indexbox, ConstInt(0)) - negbox = self.implement_guard_value(pc, negbox) - if negbox.getint(): - # the index is < 0; add the array length to it - lenbox = self.metainterp.execute_and_record( - rop.ARRAYLEN_GC, arraydesc, arraybox) - indexbox = self.metainterp.execute_and_record( - rop.INT_ADD, None, indexbox, lenbox) - self.make_result_box(indexbox) - - @FixME #arguments("descr", "descr", "descr", "descr", "box") - def opimpl_newlist(self, structdescr, lengthdescr, itemsdescr, arraydescr, - sizebox): - sbox = self.metainterp.execute_and_record(rop.NEW, structdescr) - self.metainterp.execute_and_record(rop.SETFIELD_GC, lengthdescr, - sbox, sizebox) - abox = self.metainterp.execute_and_record(rop.NEW_ARRAY, arraydescr, - sizebox) - self.metainterp.execute_and_record(rop.SETFIELD_GC, itemsdescr, - sbox, abox) - self.make_result_box(sbox) - - @FixME #arguments("box", "descr", "descr", "box") - def opimpl_getlistitem_gc(self, listbox, itemsdescr, arraydescr, indexbox): - arraybox = self.metainterp.execute_and_record(rop.GETFIELD_GC, - itemsdescr, listbox) - self.execute_with_descr(rop.GETARRAYITEM_GC, arraydescr, arraybox, indexbox) - - @FixME #arguments("box", "descr", "descr", "box", "box") - def opimpl_setlistitem_gc(self, listbox, itemsdescr, arraydescr, indexbox, - valuebox): - arraybox = self.metainterp.execute_and_record(rop.GETFIELD_GC, - itemsdescr, listbox) - self.execute_with_descr(rop.SETARRAYITEM_GC, arraydescr, arraybox, indexbox, valuebox) - - @FixME #arguments("orgpc", "box", "descr", "box") - def opimpl_check_resizable_neg_index(self, pc, listbox, lengthdesc, - indexbox): - negbox = self.metainterp.execute_and_record( - rop.INT_LT, None, indexbox, ConstInt(0)) - negbox = self.implement_guard_value(pc, negbox) - if negbox.getint(): - # the index is < 0; add the array length to it - lenbox = self.metainterp.execute_and_record( - rop.GETFIELD_GC, lengthdesc, listbox) - indexbox = self.metainterp.execute_and_record( - rop.INT_ADD, None, indexbox, lenbox) - self.make_result_box(indexbox) - - @FixME #arguments("orgpc", "box") - def opimpl_check_zerodivisionerror(self, pc, box): - nonzerobox = self.metainterp.execute_and_record( - rop.INT_NE, None, box, ConstInt(0)) - nonzerobox = self.implement_guard_value(pc, nonzerobox) - if nonzerobox.getint(): - return False - else: - # division by zero! - return self.metainterp.raise_zero_division_error() - - @FixME #arguments("orgpc", "box", "box") - def opimpl_check_div_overflow(self, pc, box1, box2): - # detect the combination "box1 = -sys.maxint-1, box2 = -1". - import sys - tmp1 = self.metainterp.execute_and_record( # combination to detect: - rop.INT_ADD, None, box1, ConstInt(sys.maxint)) # tmp1=-1, box2=-1 - tmp2 = self.metainterp.execute_and_record( - rop.INT_AND, None, tmp1, box2) # tmp2=-1 - tmp3 = self.metainterp.execute_and_record( - rop.INT_EQ, None, tmp2, ConstInt(-1)) # tmp3? - tmp4 = self.implement_guard_value(pc, tmp3) # tmp4? - if not tmp4.getint(): - return False - else: - # division overflow! - return self.metainterp.raise_overflow_error() - - @FixME #arguments() - def opimpl_overflow_error(self): - return self.metainterp.raise_overflow_error() - @arguments("box", "descr") def _opimpl_getfield_gc_any(self, box, fielddescr): return self.execute_with_descr(rop.GETFIELD_GC, fielddescr, box) @@ -736,16 +652,16 @@ def opimpl_residual_oosend_pure(self, methdescr, boxes): self.execute_varargs(rop.OOSEND_PURE, boxes, descr=methdescr, exc=False) - @arguments("box") - def _opimpl_guard_value(self, box): - self.implement_guard_value(box) + @arguments("orgpc", "box",) + def _opimpl_guard_value(self, orgpc, box): + self.implement_guard_value(orgpc, box) opimpl_int_guard_value = _opimpl_guard_value opimpl_ref_guard_value = _opimpl_guard_value opimpl_float_guard_value = _opimpl_guard_value - @arguments("box", "orgpc") - def opimpl_guard_class(self, box, orgpc): + @arguments("orgpc", "box") + def opimpl_guard_class(self, orgpc, box): clsbox = self.cls_of_box(box) self.generate_guard(rop.GUARD_CLASS, box, [clsbox], resumepc=orgpc) return clsbox @@ -957,7 +873,7 @@ metainterp.attach_debug_info(guard_op) return guard_op - def implement_guard_value(self, box): + def implement_guard_value(self, orgpc, box): """Promote the given Box into a Const. Note: be careful, it's a bit unclear what occurs if a single opcode needs to generate several ones and/or ones not near the beginning.""" @@ -965,7 +881,8 @@ return box # no promotion needed, already a Const else: promoted_box = box.constbox() - self.generate_guard(rop.GUARD_VALUE, box, [promoted_box]) + self.generate_guard(rop.GUARD_VALUE, box, [promoted_box], + resumepc=orgpc) self.metainterp.replace_box(box, promoted_box) return promoted_box @@ -1024,7 +941,7 @@ """The 'residual_call' operation is emitted in two cases: when we have to generate a residual CALL operation, but also to handle an indirect_call that may need to be inlined.""" - funcbox = self.implement_guard_value(funcbox) + assert isinstance(funcbox, Const) sd = self.metainterp.staticdata key = sd.cpu.ts.getaddr_for_box(funcbox) jitcode = sd.bytecode_for_address(key) From arigo at codespeak.net Wed May 19 18:25:00 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 May 2010 18:25:00 +0200 (CEST) Subject: [pypy-svn] r74584 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100519162500.A25D8282BFF@codespeak.net> Author: arigo Date: Wed May 19 18:24:58 2010 New Revision: 74584 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: guard_value, the logic in compile.py. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Wed May 19 18:24:58 2010 @@ -985,7 +985,7 @@ pass elif opnum == rop.GUARD_NO_EXCEPTION or opnum == rop.GUARD_EXCEPTION: pass - elif opnum == rop.GUARD_CLASS: + elif opnum == rop.GUARD_CLASS or opnum == rop.GUARD_VALUE: pass else: raise NotImplementedError(opnum) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py Wed May 19 18:24:58 2010 @@ -4,6 +4,7 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.debug import debug_start, debug_stop from pypy.conftest import option +from pypy.tool.sourcetools import func_with_new_name from pypy.jit.metainterp.resoperation import ResOperation, rop from pypy.jit.metainterp.history import TreeLoop, Box, History, LoopToken @@ -220,6 +221,11 @@ rd_virtuals = None rd_pendingfields = None + CNT_INT = -0x20000000 + CNT_REF = -0x40000000 + CNT_FLOAT = -0x60000000 + CNT_MASK = 0x1FFFFFFF + def __init__(self, metainterp_sd, original_greenkey): ResumeDescr.__init__(self, original_greenkey) self.metainterp_sd = metainterp_sd @@ -236,7 +242,17 @@ except ValueError: return # xxx probably very rare else: - self._counter = ~i # use ~(index_of_guarded_box_in_fail_args) + if box.type == history.INT: + cnt = self.CNT_INT + elif box.type == history.REF: + cnt = self.CNT_REF + elif box.type == history.FLOAT: + cnt = self.CNT_FLOAT + else: + assert 0, box.type + # we build the following value for _counter, which is always + # a negative value + self._counter = cnt | i def handle_fail(self, metainterp_sd): if self.must_compile(metainterp_sd): @@ -257,11 +273,25 @@ self._counter += 1 return self._counter >= trace_eagerness else: - XXX - box = inputargs_and_holes[~self._counter] - if self._counters is None: - self._counters = ResumeGuardCounters() - counter = self._counters.see(box) + index = self._counter & self.CNT_MASK + typetag = self._counter & ~ self.CNT_MASK + if typetag == self.CNT_INT: + intvalue = metainterp_sd.cpu.get_latest_value_int(index) + if self._counters is None: + self._counters = ResumeGuardCountersInt() + counter = self._counters.see_int(intvalue) + elif typetag == self.CNT_REF: + refvalue = metainterp_sd.cpu.get_latest_value_ref(index) + if self._counters is None: + self._counters = ResumeGuardCountersRef() + counter = self._counters.see_ref(refvalue) + elif typetag == self.CNT_FLOAT: + floatvalue = metainterp_sd.cpu.get_latest_value_float(index) + if self._counters is None: + self._counters = ResumeGuardCountersFloat() + counter = self._counters.see_float(floatvalue) + else: + assert 0, typetag return counter >= trace_eagerness def reset_counter_from_failure(self, metainterp): @@ -357,50 +387,64 @@ return data -class ResumeGuardCounters(object): - # Completely custom algorithm for now: keep 5 pairs (box, counter), +class AbstractResumeGuardCounters(object): + # Completely custom algorithm for now: keep 5 pairs (value, counter), # and when we need more, we discard the middle pair (middle in the # current value of the counter). That way, we tend to keep the - # boxes with a high counter, but also we avoid always throwing away - # the most recently added box. **THIS ALGO MUST GO AWAY AT SOME POINT** + # values with a high counter, but also we avoid always throwing away + # the most recently added value. **THIS ALGO MUST GO AWAY AT SOME POINT** + pass + +def _see(self, newvalue): + # find and update an existing counter + unused = -1 + for i in range(5): + cnt = self.counters[i] + if cnt: + if self.values[i] == newvalue: + cnt += 1 + self.counters[i] = cnt + return cnt + else: + unused = i + # not found. Use a previously unused entry, if there is one + if unused >= 0: + self.counters[unused] = 1 + self.values[unused] = newvalue + return 1 + # no unused entry. Overwrite the middle one. Computed with indices + # a, b, c meaning the highest, second highest, and third highest + # entries. + a = 0 + b = c = -1 + for i in range(1, 5): + if self.counters[i] > self.counters[a]: + c = b; b = a; a = i + elif b < 0 or self.counters[i] > self.counters[b]: + c = b; b = i + elif c < 0 or self.counters[i] > self.counters[c]: + c = i + self.counters[c] = 1 + self.values[c] = newvalue + return 1 +class ResumeGuardCountersInt(AbstractResumeGuardCounters): def __init__(self): self.counters = [0] * 5 - self.boxes = [None] * 5 + self.values = [0] * 5 + see_int = func_with_new_name(_see, 'see_int') - def see(self, newbox): - newbox = newbox.constbox() - # find and update an existing counter - unused = -1 - for i in range(5): - cnt = self.counters[i] - if cnt: - if newbox.same_constant(self.boxes[i]): - cnt += 1 - self.counters[i] = cnt - return cnt - else: - unused = i - # not found. Use a previously unused entry, if there is one - if unused >= 0: - self.counters[unused] = 1 - self.boxes[unused] = newbox - return 1 - # no unused entry. Overwrite the middle one. Computed with indices - # a, b, c meaning the highest, second highest, and third highest - # entries. - a = 0 - b = c = -1 - for i in range(1, 5): - if self.counters[i] > self.counters[a]: - c = b; b = a; a = i - elif b < 0 or self.counters[i] > self.counters[b]: - c = b; b = i - elif c < 0 or self.counters[i] > self.counters[c]: - c = i - self.counters[c] = 1 - self.boxes[c] = newbox - return 1 +class ResumeGuardCountersRef(AbstractResumeGuardCounters): + def __init__(self): + self.counters = [0] * 5 + self.values = [history.ConstPtr.value] * 5 + see_ref = func_with_new_name(_see, 'see_ref') + +class ResumeGuardCountersFloat(AbstractResumeGuardCounters): + def __init__(self): + self.counters = [0] * 5 + self.values = [0.0] * 5 + see_float = func_with_new_name(_see, 'see_float') class ResumeFromInterpDescr(ResumeDescr): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Wed May 19 18:24:58 2010 @@ -1526,7 +1526,7 @@ xxx #self.raise_overflow_error() elif opnum == rop.GUARD_NONNULL or opnum == rop.GUARD_ISNULL: xxx #self.framestack[-1].ignore_next_guard_nullness(opnum) - elif opnum == rop.GUARD_CLASS: + elif opnum == rop.GUARD_CLASS or opnum == rop.GUARD_VALUE: pass else: raise NotImplementedError(opnum) From wildchild at codespeak.net Wed May 19 18:50:32 2010 From: wildchild at codespeak.net (wildchild at codespeak.net) Date: Wed, 19 May 2010 18:50:32 +0200 (CEST) Subject: [pypy-svn] r74585 - in pypy/trunk/pypy/interpreter: . test Message-ID: <20100519165032.635F7282BFF@codespeak.net> Author: wildchild Date: Wed May 19 18:50:30 2010 New Revision: 74585 Modified: pypy/trunk/pypy/interpreter/module.py pypy/trunk/pypy/interpreter/test/test_zzpickle_and_slow.py Log: issue539 resolved Unpickling of submodule didn't work. __import__ needs a non-empty value for the "fromlist" parameter or it returns a reference to the top-level package instead of the module itself. Modified: pypy/trunk/pypy/interpreter/module.py ============================================================================== --- pypy/trunk/pypy/interpreter/module.py (original) +++ pypy/trunk/pypy/interpreter/module.py Wed May 19 18:50:30 2010 @@ -73,7 +73,17 @@ ]) #already imported case w_import = space.builtin.get('__import__') - return space.newtuple([w_import, space.newtuple([w_name])]) + tup_return = [ + w_import, + space.newtuple([ + w_name, + space.w_None, + space.w_None, + space.newtuple([space.wrap('')]) + ]) + ] + + return space.newtuple(tup_return) def descr_module__repr__(self, space): from pypy.interpreter.mixedmodule import MixedModule Modified: pypy/trunk/pypy/interpreter/test/test_zzpickle_and_slow.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_zzpickle_and_slow.py (original) +++ pypy/trunk/pypy/interpreter/test/test_zzpickle_and_slow.py Wed May 19 18:50:30 2010 @@ -468,3 +468,18 @@ l = [] unbound_meth2(l, 1) assert l == [1] + + def test_pickle_submodule(self): + import pickle + import sys, new + + mod = new.module('pack.mod') + sys.modules['pack.mod'] = mod + pack = new.module('pack') + pack.mod = mod + sys.modules['pack'] = pack + + import pack.mod + pckl = pickle.dumps(pack.mod) + result = pickle.loads(pckl) + assert pack.mod is result From afa at codespeak.net Wed May 19 18:51:57 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 19 May 2010 18:51:57 +0200 (CEST) Subject: [pypy-svn] r74586 - pypy/trunk/pypy/rlib Message-ID: <20100519165157.4AC88282BFF@codespeak.net> Author: afa Date: Wed May 19 18:51:55 2010 New Revision: 74586 Modified: pypy/trunk/pypy/rlib/rwin32.py Log: Fix translation on Windows Modified: pypy/trunk/pypy/rlib/rwin32.py ============================================================================== --- pypy/trunk/pypy/rlib/rwin32.py (original) +++ pypy/trunk/pypy/rlib/rwin32.py Wed May 19 18:51:55 2010 @@ -209,6 +209,7 @@ rffi.cast(lltype.Signed, info.c_dwMinorVersion), rffi.cast(lltype.Signed, info.c_dwBuildNumber), rffi.cast(lltype.Signed, info.c_dwPlatformId), - rffi.charp2str(info.c_szCSDVersion)) + rffi.charp2str(rffi.cast(rffi.CCHARP, + info.c_szCSDVersion))) finally: lltype.free(info, flavor='raw') From fijal at codespeak.net Thu May 20 01:03:55 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 20 May 2010 01:03:55 +0200 (CEST) Subject: [pypy-svn] r74587 - pypy/extradoc/talk/oopsla2010 Message-ID: <20100519230355.DDD3A282C01@codespeak.net> Author: fijal Date: Thu May 20 01:03:52 2010 New Revision: 74587 Modified: pypy/extradoc/talk/oopsla2010/paper.txt Log: Write an example of using INTROSPECTION in our little language Modified: pypy/extradoc/talk/oopsla2010/paper.txt ============================================================================== --- pypy/extradoc/talk/oopsla2010/paper.txt (original) +++ pypy/extradoc/talk/oopsla2010/paper.txt Thu May 20 01:03:52 2010 @@ -9,21 +9,27 @@ Python language presents a serious challenge for optimizations. The main complexity comes from the dynamism of Python (where you can overload virtually any operation to have effects unknown until runtime) as well as frame -introspection capabilities achievable from within language. We implemented +introspection capabilities achievable from within the language. We implemented a flexible just in time compiler generation (XXX cite) as a first step -to overcome problems presented by Python. In our approach, we automatically -generate the just in time compiler from the interpreter specification written -in a high level language (RPython - a static subset of Python). This language -can also be directly run or, by compilation to lower level language such as C, +to overcome performance problems presented by Python. In our approach, +we automatically generate the just in time compiler from the interpreter +specification written +in a high level language. This language +can also be directly run (it's a static subset of Python) or, by compilation +to lower level language such as C, run efficiently. -The idea of frame introspection is very useful for debugging, especially when -building custom debuggers or traceback representation. An example of such -idea is found with web frameworks written in python - they usually try to -present more information than classic traceback representation. - -Example of a py.test traceback, using local variable introspection to find out -what happened: +This paper treats about mitigating costs related to allocating and managing +frames within the interpreter. Those frames are either necessary to preserve +Python semantics (user program frame introspection) or need to be reconstructed +on demand when asked. Our approach eliminates most of costs associated with +frames and reconstructs them on demand, even though interpreter is written +with frames in mind. + +Let's look at the example how frame introspection is used to improve language +debugging capabilities. In the example below, py.test, a testing tool is using +frame introspection for accessing and listing local variables which are +on the stack when exception happened. =================================== FAILURES =================================== ___________________________________ test_one ___________________________________ @@ -40,15 +46,8 @@ > return i / j E ZeroDivisionError: integer division or modulo by zero -In this example we can see that py.test (after the test failure) looked up -the traceback for introspecting where in source the error has happened. -It also printed out contents of local variables, which greatly improve -the visibility of an error. - -In this paper we present the way we deal with frames in order to completely -(or mostly) avoid costs associated with them. Our approach is giving -us serious (XXX how serious) performance benefit on top of just in time -compiler without the frame optimizations. +After the exception was generated, the whole chain of frames was stored in +a user accessible place, from which the testing tool read local variables. The performance evaluation of our approach is based on a fact that if programmer doesn't use frame introspection, he's not interested in paying @@ -92,3 +91,26 @@ language itself is not of any use, but provides very similar problems to those encountered on a larger scale. +Our small interpreter is a register based interpreter implementing nine +bytecodes, most of them straightforward (XXX reference source). +The important addition here is the INTROSPECT bytecode which takes +a register as argument (holding integer) and will write a value of register +named by a number in argument to the result register. So the code: + +LOAD 0 => r3 +LOAD 1 => r2 +LOAD 2 => r1 +INTROSPECT r0 => r4 +PRINT r4 + +assuming r0 is the argument of function, it'll print a value 2 (from +register r1) if argument value is 1 etc. When creating an interpreter, +the most obvious implementation would be to have a frame object with +registers in an array, and this is exactly what our simple interpreter does. + +The problem here lays in details. XXX + +XXX Tracing JIT intro, maybe, depends on audience + +Solution +======== From fijal at codespeak.net Thu May 20 02:15:27 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 20 May 2010 02:15:27 +0200 (CEST) Subject: [pypy-svn] r74588 - pypy/branch/multilist/pypy/objspace/std Message-ID: <20100520001527.4E0B4282BFF@codespeak.net> Author: fijal Date: Thu May 20 02:15:24 2010 New Revision: 74588 Modified: pypy/branch/multilist/pypy/objspace/std/listmultiobject.py Log: Add imports. Note - those cases are completely untested it seems Modified: pypy/branch/multilist/pypy/objspace/std/listmultiobject.py ============================================================================== --- pypy/branch/multilist/pypy/objspace/std/listmultiobject.py (original) +++ pypy/branch/multilist/pypy/objspace/std/listmultiobject.py Thu May 20 02:15:24 2010 @@ -1,3 +1,6 @@ +from pypy.objspace.std.multimethod import FailedToImplement +from pypy.interpreter.error import OperationError, operationerrfmt + from pypy.interpreter.argument import Signature from pypy.objspace.std.model import registerimplementation, W_Object from pypy.interpreter import gateway, baseobjspace From benjamin at codespeak.net Thu May 20 05:00:07 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 20 May 2010 05:00:07 +0200 (CEST) Subject: [pypy-svn] r74589 - pypy/trunk/pypy/module/__builtin__ Message-ID: <20100520030007.C7EB1282BFF@codespeak.net> Author: benjamin Date: Thu May 20 05:00:05 2010 New Revision: 74589 Modified: pypy/trunk/pypy/module/__builtin__/functional.py Log: two small simplications Modified: pypy/trunk/pypy/module/__builtin__/functional.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/functional.py (original) +++ pypy/trunk/pypy/module/__builtin__/functional.py Thu May 20 05:00:05 2010 @@ -186,7 +186,7 @@ while True: cont = False args_w = [space.w_None] * num_iterators - for i in range(len(iterators_w)): + for i in range(num_iterators): try: args_w[i] = space.next(iterators_w[i]) except OperationError, e: @@ -194,8 +194,8 @@ raise else: cont = True - w_args = space.newtuple(args_w) if cont: + w_args = space.newtuple(args_w) if none_func: result_w.append(w_args) else: From wildchild at codespeak.net Thu May 20 05:43:45 2010 From: wildchild at codespeak.net (wildchild at codespeak.net) Date: Thu, 20 May 2010 05:43:45 +0200 (CEST) Subject: [pypy-svn] r74590 - pypy/branch/build-problems-x86_64-stackless Message-ID: <20100520034345.9FC93282B9C@codespeak.net> Author: wildchild Date: Thu May 20 05:43:44 2010 New Revision: 74590 Added: pypy/branch/build-problems-x86_64-stackless/ (props changed) - copied from r74589, pypy/trunk/ Log: To test some solutions to build problems with --stackless on x86_64. From wildchild at codespeak.net Thu May 20 05:53:50 2010 From: wildchild at codespeak.net (wildchild at codespeak.net) Date: Thu, 20 May 2010 05:53:50 +0200 (CEST) Subject: [pypy-svn] r74591 - in pypy/branch/build-problems-x86_64-stackless/pypy: module/pyexpat rlib translator/c translator/tool Message-ID: <20100520035350.DD259282B9C@codespeak.net> Author: wildchild Date: Thu May 20 05:53:49 2010 New Revision: 74591 Modified: pypy/branch/build-problems-x86_64-stackless/pypy/module/pyexpat/interp_pyexpat.py pypy/branch/build-problems-x86_64-stackless/pypy/rlib/rlocale.py pypy/branch/build-problems-x86_64-stackless/pypy/translator/c/database.py pypy/branch/build-problems-x86_64-stackless/pypy/translator/tool/cbuild.py Log: - Adding some includes to build under FreeBSD. Some cleanup needed... - Based on a huge comment in the complete() method in database.py, steps 4 and 6 can be switched. It seems it "solves" a build problem with --stackless enabled on x86_64 platforms. Some more tests are needed here... Modified: pypy/branch/build-problems-x86_64-stackless/pypy/module/pyexpat/interp_pyexpat.py ============================================================================== --- pypy/branch/build-problems-x86_64-stackless/pypy/module/pyexpat/interp_pyexpat.py (original) +++ pypy/branch/build-problems-x86_64-stackless/pypy/module/pyexpat/interp_pyexpat.py Thu May 20 05:53:49 2010 @@ -17,9 +17,18 @@ libname = 'libexpat' else: libname = 'expat' + +include_dirs = [] +library_dirs = [] +if sys.platform == "freebsd7": + include_dirs = ['/usr/local/include'] + library_dirs = ['/usr/local/lib'] + eci = ExternalCompilationInfo( libraries=[libname], includes=['expat.h'], + include_dirs=include_dirs, + library_dirs=library_dirs, pre_include_bits=[ '#define XML_COMBINED_VERSION' + ' (10000*XML_MAJOR_VERSION+100*XML_MINOR_VERSION+XML_MICRO_VERSION)', Modified: pypy/branch/build-problems-x86_64-stackless/pypy/rlib/rlocale.py ============================================================================== --- pypy/branch/build-problems-x86_64-stackless/pypy/rlib/rlocale.py (original) +++ pypy/branch/build-problems-x86_64-stackless/pypy/rlib/rlocale.py Thu May 20 05:53:49 2010 @@ -27,6 +27,8 @@ includes += ['windows.h'] if sys.platform == 'darwin': include_dirs += ['/opt/local/include'] + if sys.platform == 'freebsd7': + include_dirs += ['/usr/local/include'] _compilation_info_ = ExternalCompilationInfo( includes=includes, include_dirs=include_dirs Modified: pypy/branch/build-problems-x86_64-stackless/pypy/translator/c/database.py ============================================================================== --- pypy/branch/build-problems-x86_64-stackless/pypy/translator/c/database.py (original) +++ pypy/branch/build-problems-x86_64-stackless/pypy/translator/c/database.py Thu May 20 05:53:49 2010 @@ -287,12 +287,12 @@ if self.gctransformer: finish_callbacks.append(('GC transformer: finished helpers', self.gctransformer.finish_helpers)) - if self.stacklesstransformer: - finish_callbacks.append(('Stackless transformer: finished', - self.stacklesstransformer.finish)) if self.gctransformer: finish_callbacks.append(('GC transformer: finished tables', self.gctransformer.get_finish_tables())) + if self.stacklesstransformer: + finish_callbacks.append(('Stackless transformer: finished', + self.stacklesstransformer.finish)) def add_dependencies(newdependencies): for value in newdependencies: Modified: pypy/branch/build-problems-x86_64-stackless/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/branch/build-problems-x86_64-stackless/pypy/translator/tool/cbuild.py (original) +++ pypy/branch/build-problems-x86_64-stackless/pypy/translator/tool/cbuild.py Thu May 20 05:53:49 2010 @@ -282,7 +282,7 @@ for sym in self.export_symbols: f.write("%s;\n" % (sym,)) f.write("};") - d['link_extra'] += ("-Wl,--dynamic-list=" + str(file_name), ) + d['link_extra'] += ("-Wl,--version-script=" + str(file_name), ) f.close() d['export_symbols'] = () return ExternalCompilationInfo(**d) From arigo at codespeak.net Thu May 20 11:18:06 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 20 May 2010 11:18:06 +0200 (CEST) Subject: [pypy-svn] r74592 - pypy/trunk/pypy/rpython/lltypesystem/test Message-ID: <20100520091806.BE3AD282C0B@codespeak.net> Author: arigo Date: Thu May 20 11:18:04 2010 New Revision: 74592 Modified: pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Log: Add another test about COpaquePtr, this time allocated and freed externally. Modified: pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Thu May 20 11:18:04 2010 @@ -343,6 +343,18 @@ lltype.free(ll_timevalp, flavor='raw') assert not ALLOCATED # detects memory leaks in the test + def test_opaque_obj_2(self): + FILEP = rffi.COpaquePtr('FILE') + fopen = rffi.llexternal('fopen', [rffi.CCHARP, rffi.CCHARP], FILEP) + fclose = rffi.llexternal('fopen', [FILEP], rffi.INT) + tmppath = udir.join('test_ll2ctypes.test_opaque_obj_2') + ll_file = fopen(str(tmppath), "w") + assert ll_file + fclose(ll_file) + assert tmppath.check(file=1) + #assert not ALLOCATED --- fails, because ll2ctypes misses the + # fact that fclose() frees 'll_file' + def test_simple_cast(self): assert rffi.cast(rffi.SIGNEDCHAR, 0x123456) == 0x56 assert rffi.cast(rffi.SIGNEDCHAR, 0x123481) == -127 From afa at codespeak.net Thu May 20 13:26:03 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 20 May 2010 13:26:03 +0200 (CEST) Subject: [pypy-svn] r74593 - pypy/trunk/pypy/rpython/lltypesystem/test Message-ID: <20100520112603.BC961282B90@codespeak.net> Author: afa Date: Thu May 20 13:26:01 2010 New Revision: 74593 Modified: pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Log: "fix" the test, it now segfaults with exactly the same error as the one I get in PyObject_Print (WindowsError: access violation writing 0x00050012) Modified: pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Thu May 20 13:26:01 2010 @@ -344,9 +344,10 @@ assert not ALLOCATED # detects memory leaks in the test def test_opaque_obj_2(self): + py.test.skip("FIXME") FILEP = rffi.COpaquePtr('FILE') fopen = rffi.llexternal('fopen', [rffi.CCHARP, rffi.CCHARP], FILEP) - fclose = rffi.llexternal('fopen', [FILEP], rffi.INT) + fclose = rffi.llexternal('fclose', [FILEP], rffi.INT) tmppath = udir.join('test_ll2ctypes.test_opaque_obj_2') ll_file = fopen(str(tmppath), "w") assert ll_file From hpk at codespeak.net Thu May 20 14:37:56 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 20 May 2010 14:37:56 +0200 (CEST) Subject: [pypy-svn] r74595 - pypy/branch/py12 Message-ID: <20100520123756.BA10B282B90@codespeak.net> Author: hpk Date: Thu May 20 14:37:55 2010 New Revision: 74595 Removed: pypy/branch/py12/ Log: remove old py12 porting branch From hpk at codespeak.net Thu May 20 15:06:45 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 20 May 2010 15:06:45 +0200 (CEST) Subject: [pypy-svn] r74596 - pypy/branch/py131 Message-ID: <20100520130645.7269C282B90@codespeak.net> Author: hpk Date: Thu May 20 15:06:43 2010 New Revision: 74596 Added: pypy/branch/py131/ - copied from r74594, pypy/trunk/ Log: starting a py131 pypy/py.test integration branch From hpk at codespeak.net Thu May 20 15:55:27 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 20 May 2010 15:55:27 +0200 (CEST) Subject: [pypy-svn] r74597 - in pypy/branch/py131: lib-python py py/_cmdline py/_code py/_io py/_plugin py/_test Message-ID: <20100520135527.6C788282B90@codespeak.net> Author: hpk Date: Thu May 20 15:55:24 2010 New Revision: 74597 Removed: pypy/branch/py131/py/__init__.py.orig pypy/branch/py131/py/_plugin/pytest_pytester.py.orig pypy/branch/py131/py/_plugin/pytest_terminal.py.orig Modified: pypy/branch/py131/lib-python/conftest.py pypy/branch/py131/py/__init__.py pypy/branch/py131/py/_builtin.py pypy/branch/py131/py/_cmdline/pytest.py pypy/branch/py131/py/_code/code.py pypy/branch/py131/py/_code/source.py pypy/branch/py131/py/_io/capture.py pypy/branch/py131/py/_plugin/pytest__pytest.py pypy/branch/py131/py/_plugin/pytest_capture.py pypy/branch/py131/py/_plugin/pytest_genscript.py pypy/branch/py131/py/_plugin/pytest_junitxml.py pypy/branch/py131/py/_plugin/pytest_resultlog.py pypy/branch/py131/py/_plugin/pytest_runner.py pypy/branch/py131/py/_plugin/pytest_skipping.py pypy/branch/py131/py/_plugin/pytest_terminal.py pypy/branch/py131/py/_test/cmdline.py pypy/branch/py131/py/_test/pycollect.py Log: upgrading to current py-trunk 1.3.1a1 series Modified: pypy/branch/py131/lib-python/conftest.py ============================================================================== --- pypy/branch/py131/lib-python/conftest.py (original) +++ pypy/branch/py131/lib-python/conftest.py Thu May 20 15:55:24 2010 @@ -14,7 +14,6 @@ # the following adds command line options as a side effect! from pypy.conftest import gettestobjspace, option as pypy_option -from test import pystone from pypy.tool.pytest import appsupport from pypy.tool.pytest.confpath import pypydir, libpythondir, \ @@ -40,6 +39,7 @@ option = py.test.config.option def gettimeout(): + from test import pystone timeout = option.timeout.lower() if timeout.endswith('mp'): megapystone = float(timeout[:-2]) Modified: pypy/branch/py131/py/__init__.py ============================================================================== --- pypy/branch/py131/py/__init__.py (original) +++ pypy/branch/py131/py/__init__.py Thu May 20 15:55:24 2010 @@ -8,7 +8,7 @@ (c) Holger Krekel and others, 2004-2010 """ -__version__ = version = "1.3.0" +__version__ = version = "1.3.1a1" import py.apipkg @@ -25,7 +25,6 @@ 'pytest': '._cmdline.pytest:main', 'pylookup': '._cmdline.pylookup:main', 'pycountloc': '._cmdline.pycountlog:main', - 'pytest': '._test.cmdline:main', 'pylookup': '._cmdline.pylookup:main', 'pycountloc': '._cmdline.pycountloc:main', 'pycleanup': '._cmdline.pycleanup:main', Modified: pypy/branch/py131/py/_builtin.py ============================================================================== --- pypy/branch/py131/py/_builtin.py (original) +++ pypy/branch/py131/py/_builtin.py Thu May 20 15:55:24 2010 @@ -151,7 +151,10 @@ return getattr(function, "__dict__", None) def _getcode(function): - return getattr(function, "func_code", None) + try: + return getattr(function, "__code__") + except AttributeError: + return getattr(function, "func_code", None) def print_(*args, **kwargs): """ minimal backport of py3k print statement. """ @@ -175,6 +178,7 @@ def exec_(obj, globals=None, locals=None): """ minimal backport of py3k exec statement. """ + __tracebackhide__ = True if globals is None: frame = sys._getframe(1) globals = frame.f_globals @@ -187,14 +191,17 @@ if sys.version_info >= (3,0): exec (""" def _reraise(cls, val, tb): + __tracebackhide__ = True assert hasattr(val, '__traceback__') raise val """) else: exec (""" def _reraise(cls, val, tb): + __tracebackhide__ = True raise cls, val, tb def exec2(obj, globals, locals): + __tracebackhide__ = True exec obj in globals, locals """) Modified: pypy/branch/py131/py/_cmdline/pytest.py ============================================================================== --- pypy/branch/py131/py/_cmdline/pytest.py (original) +++ pypy/branch/py131/py/_cmdline/pytest.py Thu May 20 15:55:24 2010 @@ -1,5 +1,5 @@ #!/usr/bin/env python import py -def main(args): - py.test.cmdline.main(args) +def main(args=None): + raise SystemExit(py.test.cmdline.main(args)) Modified: pypy/branch/py131/py/_code/code.py ============================================================================== --- pypy/branch/py131/py/_code/code.py (original) +++ pypy/branch/py131/py/_code/code.py Thu May 20 15:55:24 2010 @@ -23,64 +23,14 @@ def __ne__(self, other): return not self == other - def new(self, rec=False, **kwargs): - """ return new code object with modified attributes. - if rec-cursive is true then dive into code - objects contained in co_consts. - """ - if sys.platform.startswith("java"): - # XXX jython does not support the below co_filename hack - return self.raw - names = [x for x in dir(self.raw) if x[:3] == 'co_'] - for name in kwargs: - if name not in names: - raise TypeError("unknown code attribute: %r" %(name, )) - if rec and hasattr(self.raw, 'co_consts'): # jython - newconstlist = [] - co = self.raw - cotype = type(co) - for c in co.co_consts: - if isinstance(c, cotype): - c = self.__class__(c).new(rec=True, **kwargs) - newconstlist.append(c) - return self.new(rec=False, co_consts=tuple(newconstlist), **kwargs) - for name in names: - if name not in kwargs: - kwargs[name] = getattr(self.raw, name) - arglist = [ - kwargs['co_argcount'], - kwargs['co_nlocals'], - kwargs.get('co_stacksize', 0), # jython - kwargs.get('co_flags', 0), # jython - kwargs.get('co_code', ''), # jython - kwargs.get('co_consts', ()), # jython - kwargs.get('co_names', []), # - kwargs['co_varnames'], - kwargs['co_filename'], - kwargs['co_name'], - kwargs['co_firstlineno'], - kwargs.get('co_lnotab', ''), #jython - kwargs.get('co_freevars', None), #jython - kwargs.get('co_cellvars', None), # jython - ] - if sys.version_info >= (3,0): - arglist.insert(1, kwargs['co_kwonlyargcount']) - return self.raw.__class__(*arglist) - else: - return py.std.new.code(*arglist) - def path(self): """ return a path object pointing to source code""" - fn = self.raw.co_filename - try: - return fn.__path__ - except AttributeError: - p = py.path.local(self.raw.co_filename) - if not p.check(): - # XXX maybe try harder like the weird logic - # in the standard lib [linecache.updatecache] does? - p = self.raw.co_filename - return p + p = py.path.local(self.raw.co_filename) + if not p.check(): + # XXX maybe try harder like the weird logic + # in the standard lib [linecache.updatecache] does? + p = self.raw.co_filename + return p path = property(path, None, None, "path of this code object") Modified: pypy/branch/py131/py/_code/source.py ============================================================================== --- pypy/branch/py131/py/_code/source.py (original) +++ pypy/branch/py131/py/_code/source.py Thu May 20 15:55:24 2010 @@ -2,6 +2,7 @@ import sys import inspect, tokenize import py +from types import ModuleType cpy_compile = compile try: @@ -212,10 +213,17 @@ else: if flag & _AST_FLAG: return co - co_filename = MyStr(filename) - co_filename.__source__ = self - return py.code.Code(co).new(rec=1, co_filename=co_filename) - #return newcode_withfilename(co, co_filename) + lines = [(x + "\n") for x in self.lines] + if sys.version_info[0] >= 3: + # XXX py3's inspect.getsourcefile() checks for a module + # and a pep302 __loader__ ... we don't have a module + # at code compile-time so we need to fake it here + m = ModuleType("_pycodecompile_pseudo_module") + py.std.inspect.modulesbyfile[filename] = None + py.std.sys.modules[None] = m + m.__loader__ = 1 + py.std.linecache.cache[filename] = (1, None, lines, filename) + return co # # public API shortcut functions @@ -224,11 +232,9 @@ def compile_(source, filename=None, mode='exec', flags= generators.compiler_flag, dont_inherit=0): """ compile the given source to a raw code object, - which points back to the source code through - "co_filename.__source__". All code objects - contained in the code object will recursively - also have this special subclass-of-string - filename. + and maintain an internal cache which allows later + retrieval of the source code for the code object + and any recursively created code objects. """ if _ast is not None and isinstance(source, _ast.AST): # XXX should Source support having AST? @@ -262,44 +268,26 @@ # # helper functions # -class MyStr(str): - """ custom string which allows to add attributes. """ def findsource(obj): - obj = py.code.getrawcode(obj) try: - fullsource = obj.co_filename.__source__ - except AttributeError: - try: - sourcelines, lineno = py.std.inspect.findsource(obj) - except (KeyboardInterrupt, SystemExit): - raise - except: - return None, None - source = Source() - source.lines = [line.rstrip() for line in sourcelines] - return source, lineno - else: - lineno = obj.co_firstlineno - 1 - return fullsource, lineno - + sourcelines, lineno = py.std.inspect.findsource(obj) + except (KeyboardInterrupt, SystemExit): + raise + except: + return None, None + source = Source() + source.lines = [line.rstrip() for line in sourcelines] + return source, lineno def getsource(obj, **kwargs): obj = py.code.getrawcode(obj) try: - fullsource = obj.co_filename.__source__ - except AttributeError: - try: - strsrc = inspect.getsource(obj) - except IndentationError: - strsrc = "\"Buggy python version consider upgrading, cannot get source\"" - assert isinstance(strsrc, str) - return Source(strsrc, **kwargs) - else: - lineno = obj.co_firstlineno - 1 - end = fullsource.getblockend(lineno) - return Source(fullsource[lineno:end+1], deident=True) - + strsrc = inspect.getsource(obj) + except IndentationError: + strsrc = "\"Buggy python version consider upgrading, cannot get source\"" + assert isinstance(strsrc, str) + return Source(strsrc, **kwargs) def deindent(lines, offset=None): if offset is None: Modified: pypy/branch/py131/py/_io/capture.py ============================================================================== --- pypy/branch/py131/py/_io/capture.py (original) +++ pypy/branch/py131/py/_io/capture.py Thu May 20 15:55:24 2010 @@ -26,46 +26,56 @@ raise TypeError("not a byte value: %r" %(data,)) StringIO.write(self, data) +patchsysdict = {0: 'stdin', 1: 'stdout', 2: 'stderr'} + class FDCapture: """ Capture IO to/from a given os-level filedescriptor. """ - def __init__(self, targetfd, tmpfile=None): + def __init__(self, targetfd, tmpfile=None, now=True, patchsys=False): """ save targetfd descriptor, and open a new temporary file there. If no tmpfile is specified a tempfile.Tempfile() will be opened in text mode. """ self.targetfd = targetfd - if tmpfile is None: + if tmpfile is None and targetfd != 0: f = tempfile.TemporaryFile('wb+') tmpfile = dupfile(f, encoding="UTF-8") f.close() self.tmpfile = tmpfile - self._savefd = os.dup(targetfd) - os.dup2(self.tmpfile.fileno(), targetfd) - self._patched = [] - - def setasfile(self, name, module=sys): - """ patch . to self.tmpfile - """ - key = (module, name) - self._patched.append((key, getattr(module, name))) - setattr(module, name, self.tmpfile) + self._savefd = os.dup(self.targetfd) + if patchsys: + self._oldsys = getattr(sys, patchsysdict[targetfd]) + if now: + self.start() - def unsetfiles(self): - """ unpatch all patched items - """ - while self._patched: - (module, name), value = self._patched.pop() - setattr(module, name, value) + def start(self): + try: + os.fstat(self._savefd) + except OSError: + raise ValueError("saved filedescriptor not valid, " + "did you call start() twice?") + if self.targetfd == 0 and not self.tmpfile: + fd = os.open(devnullpath, os.O_RDONLY) + os.dup2(fd, 0) + os.close(fd) + if hasattr(self, '_oldsys'): + setattr(sys, patchsysdict[self.targetfd], DontReadFromInput()) + else: + fd = self.tmpfile.fileno() + os.dup2(self.tmpfile.fileno(), self.targetfd) + if hasattr(self, '_oldsys'): + setattr(sys, patchsysdict[self.targetfd], self.tmpfile) def done(self): """ unpatch and clean up, returns the self.tmpfile (file object) """ os.dup2(self._savefd, self.targetfd) - self.unsetfiles() os.close(self._savefd) - self.tmpfile.seek(0) + if self.targetfd != 0: + self.tmpfile.seek(0) + if hasattr(self, '_oldsys'): + setattr(sys, patchsysdict[self.targetfd], self._oldsys) return self.tmpfile def writeorg(self, data): @@ -100,7 +110,7 @@ if encoding is not None: mode = mode.replace("b", "") buffering = True - return os.fdopen(newfd, mode, buffering, encoding, closefd=False) + return os.fdopen(newfd, mode, buffering, encoding, closefd=True) else: f = os.fdopen(newfd, mode, buffering) if encoding is not None: @@ -146,90 +156,94 @@ def reset(self): """ reset sys.stdout/stderr and return captured output as strings. """ - if hasattr(self, '_suspended'): - outfile = self._kwargs['out'] - errfile = self._kwargs['err'] - del self._kwargs - else: - outfile, errfile = self.done() + outfile, errfile = self.done() out, err = "", "" - if outfile: + if outfile and not outfile.closed: out = outfile.read() outfile.close() - if errfile and errfile != outfile: + if errfile and errfile != outfile and not errfile.closed: err = errfile.read() errfile.close() return out, err def suspend(self): """ return current snapshot captures, memorize tempfiles. """ - assert not hasattr(self, '_suspended') - self._suspended = True outerr = self.readouterr() outfile, errfile = self.done() - self._kwargs['out'] = outfile - self._kwargs['err'] = errfile return outerr - def resume(self): - """ resume capturing with original temp files. """ - assert self._suspended - self._initialize(**self._kwargs) - del self._suspended - class StdCaptureFD(Capture): """ This class allows to capture writes to FD1 and FD2 and may connect a NULL file to FD0 (and prevent - reads from sys.stdin) + reads from sys.stdin). If any of the 0,1,2 file descriptors + is invalid it will not be captured. """ - def __init__(self, out=True, err=True, - mixed=False, in_=True, patchsys=True): - self._kwargs = locals().copy() - del self._kwargs['self'] - self._initialize(**self._kwargs) - - def _initialize(self, out=True, err=True, - mixed=False, in_=True, patchsys=True): + def __init__(self, out=True, err=True, mixed=False, + in_=True, patchsys=True, now=True): + self._options = locals() + self._save() + if now: + self.startall() + + def _save(self): + in_ = self._options['in_'] + out = self._options['out'] + err = self._options['err'] + mixed = self._options['mixed'] + patchsys = self._options['patchsys'] if in_: - self._oldin = (sys.stdin, os.dup(0)) - sys.stdin = DontReadFromInput() - fd = os.open(devnullpath, os.O_RDONLY) - os.dup2(fd, 0) - os.close(fd) - if out: + try: + self.in_ = FDCapture(0, tmpfile=None, now=False, + patchsys=patchsys) + except OSError: + pass + if out: tmpfile = None if hasattr(out, 'write'): tmpfile = out - self.out = py.io.FDCapture(1, tmpfile=tmpfile) - if patchsys: - self.out.setasfile('stdout') - if err: - if mixed and out: + try: + self.out = FDCapture(1, tmpfile=tmpfile, + now=False, patchsys=patchsys) + self._options['out'] = self.out.tmpfile + except OSError: + pass + if err: + if out and mixed: tmpfile = self.out.tmpfile elif hasattr(err, 'write'): tmpfile = err else: tmpfile = None - self.err = py.io.FDCapture(2, tmpfile=tmpfile) - if patchsys: - self.err.setasfile('stderr') + try: + self.err = FDCapture(2, tmpfile=tmpfile, + now=False, patchsys=patchsys) + self._options['err'] = self.err.tmpfile + except OSError: + pass + + def startall(self): + if hasattr(self, 'in_'): + self.in_.start() + if hasattr(self, 'out'): + self.out.start() + if hasattr(self, 'err'): + self.err.start() + + def resume(self): + """ resume capturing with original temp files. """ + self.startall() def done(self): """ return (outfile, errfile) and stop capturing. """ - if hasattr(self, 'out'): + outfile = errfile = None + if hasattr(self, 'out') and not self.out.tmpfile.closed: outfile = self.out.done() - else: - outfile = None - if hasattr(self, 'err'): + if hasattr(self, 'err') and not self.err.tmpfile.closed: errfile = self.err.done() - else: - errfile = None - if hasattr(self, '_oldin'): - oldsys, oldfd = self._oldin - os.dup2(oldfd, 0) - os.close(oldfd) - sys.stdin = oldsys + if hasattr(self, 'in_'): + tmpfile = self.in_.done() + self._save() return outfile, errfile def readouterr(self): @@ -252,69 +266,61 @@ modifies sys.stdout|stderr|stdin attributes and does not touch underlying File Descriptors (use StdCaptureFD for that). """ - def __init__(self, out=True, err=True, in_=True, mixed=False): - self._kwargs = locals().copy() - del self._kwargs['self'] - self._initialize(**self._kwargs) - - def _initialize(self, out, err, in_, mixed): - self._out = out - self._err = err - self._in = in_ - if out: - self._oldout = sys.stdout - if not hasattr(out, 'write'): - out = TextIO() - sys.stdout = self.out = out - if err: - self._olderr = sys.stderr - if out and mixed: - err = self.out + def __init__(self, out=True, err=True, in_=True, mixed=False, now=True): + self._oldout = sys.stdout + self._olderr = sys.stderr + self._oldin = sys.stdin + if out and not hasattr(out, 'file'): + out = TextIO() + self.out = out + if err: + if mixed: + err = out elif not hasattr(err, 'write'): err = TextIO() - sys.stderr = self.err = err - if in_: - self._oldin = sys.stdin - sys.stdin = self.newin = DontReadFromInput() + self.err = err + self.in_ = in_ + if now: + self.startall() + + def startall(self): + if self.out: + sys.stdout = self.out + if self.err: + sys.stderr = self.err + if self.in_: + sys.stdin = self.in_ = DontReadFromInput() def done(self): """ return (outfile, errfile) and stop capturing. """ - o,e = sys.stdout, sys.stderr - if self._out: - try: - sys.stdout = self._oldout - except AttributeError: - raise IOError("stdout capturing already reset") - del self._oldout + outfile = errfile = None + if self.out and not self.out.closed: + sys.stdout = self._oldout outfile = self.out outfile.seek(0) - else: - outfile = None - if self._err: - try: - sys.stderr = self._olderr - except AttributeError: - raise IOError("stderr capturing already reset") - del self._olderr + if self.err and not self.err.closed: + sys.stderr = self._olderr errfile = self.err errfile.seek(0) - else: - errfile = None - if self._in: + if self.in_: sys.stdin = self._oldin return outfile, errfile + def resume(self): + """ resume capturing with original temp files. """ + self.startall() + def readouterr(self): """ return snapshot value of stdout/stderr capturings. """ out = err = "" - if self._out: - out = sys.stdout.getvalue() - sys.stdout.truncate(0) - sys.stdout.seek(0) - if self._err: - err = sys.stderr.getvalue() - sys.stderr.truncate(0) - sys.stderr.seek(0) + if self.out: + out = self.out.getvalue() + self.out.truncate(0) + self.out.seek(0) + if self.err: + err = self.err.getvalue() + self.err.truncate(0) + self.err.seek(0) return out, err class DontReadFromInput: @@ -344,5 +350,3 @@ devnullpath = 'NUL' else: devnullpath = '/dev/null' - - Modified: pypy/branch/py131/py/_plugin/pytest__pytest.py ============================================================================== --- pypy/branch/py131/py/_plugin/pytest__pytest.py (original) +++ pypy/branch/py131/py/_plugin/pytest__pytest.py Thu May 20 15:55:24 2010 @@ -46,7 +46,8 @@ recorder = RecordCalls() self._recorders[hookspec] = recorder self._registry.register(recorder) - self.hook = HookRelay(hookspecs, registry=self._registry) + self.hook = HookRelay(hookspecs, registry=self._registry, + prefix="pytest_") def finish_recording(self): for recorder in self._recorders.values(): Modified: pypy/branch/py131/py/_plugin/pytest_capture.py ============================================================================== --- pypy/branch/py131/py/_plugin/pytest_capture.py (original) +++ pypy/branch/py131/py/_plugin/pytest_capture.py Thu May 20 15:55:24 2010 @@ -106,6 +106,14 @@ def pytest_configure(config): config.pluginmanager.register(CaptureManager(), 'capturemanager') +class NoCapture: + def startall(self): + pass + def resume(self): + pass + def suspend(self): + return "", "" + class CaptureManager: def __init__(self): self._method2capture = {} @@ -118,15 +126,17 @@ def _makestringio(self): return py.io.TextIO() - def _startcapture(self, method): + def _getcapture(self, method): if method == "fd": - return py.io.StdCaptureFD( + return py.io.StdCaptureFD(now=False, out=self._maketempfile(), err=self._maketempfile() ) elif method == "sys": - return py.io.StdCapture( + return py.io.StdCapture(now=False, out=self._makestringio(), err=self._makestringio() ) + elif method == "no": + return NoCapture() else: raise ValueError("unknown capturing method: %r" % method) @@ -152,27 +162,25 @@ if hasattr(self, '_capturing'): raise ValueError("cannot resume, already capturing with %r" % (self._capturing,)) - if method != "no": - cap = self._method2capture.get(method) - if cap is None: - cap = self._startcapture(method) - self._method2capture[method] = cap - else: - cap.resume() + cap = self._method2capture.get(method) self._capturing = method + if cap is None: + self._method2capture[method] = cap = self._getcapture(method) + cap.startall() + else: + cap.resume() def suspendcapture(self, item=None): self.deactivate_funcargs() if hasattr(self, '_capturing'): method = self._capturing - if method != "no": - cap = self._method2capture[method] + cap = self._method2capture.get(method) + if cap is not None: outerr = cap.suspend() - else: - outerr = "", "" del self._capturing if item: - outerr = (item.outerr[0] + outerr[0], item.outerr[1] + outerr[1]) + outerr = (item.outerr[0] + outerr[0], + item.outerr[1] + outerr[1]) return outerr return "", "" @@ -180,19 +188,17 @@ if not hasattr(pyfuncitem, 'funcargs'): return assert not hasattr(self, '_capturing_funcargs') - l = [] - for name, obj in pyfuncitem.funcargs.items(): - if name == 'capfd' and not hasattr(os, 'dup'): - py.test.skip("capfd funcarg needs os.dup") + self._capturing_funcargs = capturing_funcargs = [] + for name, capfuncarg in pyfuncitem.funcargs.items(): if name in ('capsys', 'capfd'): - obj._start() - l.append(obj) - if l: - self._capturing_funcargs = l + capturing_funcargs.append(capfuncarg) + capfuncarg._start() def deactivate_funcargs(self): - if hasattr(self, '_capturing_funcargs'): - for capfuncarg in self._capturing_funcargs: + capturing_funcargs = getattr(self, '_capturing_funcargs', None) + if capturing_funcargs is not None: + while capturing_funcargs: + capfuncarg = capturing_funcargs.pop() capfuncarg._finalize() del self._capturing_funcargs @@ -256,16 +262,19 @@ platform does not have ``os.dup`` (e.g. Jython) tests using this funcarg will automatically skip. """ + if not hasattr(os, 'dup'): + py.test.skip("capfd funcarg needs os.dup") return CaptureFuncarg(request, py.io.StdCaptureFD) class CaptureFuncarg: def __init__(self, request, captureclass): self._cclass = captureclass + self.capture = self._cclass(now=False) #request.addfinalizer(self._finalize) def _start(self): - self.capture = self._cclass() + self.capture.startall() def _finalize(self): if hasattr(self, 'capture'): @@ -276,6 +285,4 @@ return self.capture.readouterr() def close(self): - self.capture.reset() - del self.capture - + self._finalize() Modified: pypy/branch/py131/py/_plugin/pytest_genscript.py ============================================================================== --- pypy/branch/py131/py/_plugin/pytest_genscript.py (original) +++ pypy/branch/py131/py/_plugin/pytest_genscript.py Thu May 20 15:55:24 2010 @@ -4,14 +4,7 @@ """ import os -import zlib -import base64 import sys -try: - import pickle -except Importerror: - import cPickle as pickle - def pytest_addoption(parser): group = parser.getgroup("debugconfig") group.addoption("--genscript", action="store", default=None, @@ -30,6 +23,13 @@ raise SystemExit(0) def main(pybasedir, outfile, infile): + import base64 + import zlib + try: + import pickle + except Importerror: + import cPickle as pickle + outfile = str(outfile) infile = str(infile) assert os.path.isabs(outfile) Modified: pypy/branch/py131/py/_plugin/pytest_junitxml.py ============================================================================== --- pypy/branch/py131/py/_plugin/pytest_junitxml.py (original) +++ pypy/branch/py131/py/_plugin/pytest_junitxml.py Thu May 20 15:55:24 2010 @@ -56,10 +56,15 @@ def append_failure(self, report): self._opentestcase(report) #msg = str(report.longrepr.reprtraceback.extraline) - self.appendlog('%s', - report.longrepr) + if "xfail" in report.keywords: + self.appendlog( + '') + self.skipped += 1 + else: + self.appendlog('%s', + report.longrepr) + self.failed += 1 self._closetestcase() - self.failed += 1 def _opentestcase_collectfailure(self, report): node = report.collector @@ -95,7 +100,12 @@ def append_skipped(self, report): self._opentestcase(report) - self.appendlog("") + if "xfail" in report.keywords: + self.appendlog( + '%s', + report.keywords['xfail']) + else: + self.appendlog("") self._closetestcase() self.skipped += 1 Modified: pypy/branch/py131/py/_plugin/pytest_resultlog.py ============================================================================== --- pypy/branch/py131/py/_plugin/pytest_resultlog.py (original) +++ pypy/branch/py131/py/_plugin/pytest_resultlog.py Thu May 20 15:55:24 2010 @@ -73,7 +73,7 @@ code = report.shortrepr if code == 'x': longrepr = str(report.longrepr) - elif code == 'P': + elif code == 'X': longrepr = '' elif report.passed: longrepr = "" Modified: pypy/branch/py131/py/_plugin/pytest_runner.py ============================================================================== --- pypy/branch/py131/py/_plugin/pytest_runner.py (original) +++ pypy/branch/py131/py/_plugin/pytest_runner.py Thu May 20 15:55:24 2010 @@ -10,6 +10,7 @@ 'skip' : skip, 'importorskip' : importorskip, 'fail' : fail, + 'xfail' : xfail, 'exit' : exit, } @@ -295,6 +296,10 @@ """ raised from an explicit call to py.test.fail() """ __module__ = 'builtins' +class XFailed(OutcomeException): + """ raised from an explicit call to py.test.xfail() """ + __module__ = 'builtins' + class ExceptionFailure(Failed): """ raised by py.test.raises on an exception-assertion mismatch. """ def __init__(self, expr, expected, msg=None, excinfo=None): @@ -335,6 +340,14 @@ fail.Exception = Failed +def xfail(reason=""): + """ xfail an executing test or setup functions, taking an optional + reason string. + """ + __tracebackhide__ = True + raise XFailed(reason) +xfail.Exception = XFailed + def raises(ExpectedException, *args, **kwargs): """ if args[0] is callable: raise AssertionError if calling it with the remaining arguments does not raise the expected exception. Modified: pypy/branch/py131/py/_plugin/pytest_skipping.py ============================================================================== --- pypy/branch/py131/py/_plugin/pytest_skipping.py (original) +++ pypy/branch/py131/py/_plugin/pytest_skipping.py Thu May 20 15:55:24 2010 @@ -82,10 +82,17 @@ depending on platform:: @py.test.mark.xfail("sys.version_info >= (3,0)") - def test_function(): ... +To not run a test and still regard it as "xfailed":: + + @py.test.mark.xfail(..., run=False) + +To specify an explicit reason to be shown with xfailure detail:: + + @py.test.mark.xfail(..., reason="my reason") + skipping on a missing import dependency -------------------------------------------------- @@ -115,11 +122,14 @@ py.test.skip("unsuppored configuration") """ -# XXX py.test.skip, .importorskip and the Skipped class -# should also be defined in this plugin, requires thought/changes import py +def pytest_addoption(parser): + group = parser.getgroup("general") + group.addoption('--runxfail', + action="store_true", dest="runxfail", default=False, + help="run tests even if they are marked xfail") class MarkEvaluator: def __init__(self, item, name): @@ -161,27 +171,53 @@ def pytest_runtest_setup(item): - expr, result = evalexpression(item, 'skipif') - if result: - py.test.skip(expr) + if not isinstance(item, py.test.collect.Function): + return + evalskip = MarkEvaluator(item, 'skipif') + if evalskip.istrue(): + py.test.skip(evalskip.getexplanation()) + item._evalxfail = MarkEvaluator(item, 'xfail') + if not item.config.getvalue("runxfail"): + if item._evalxfail.istrue(): + if not item._evalxfail.get('run', True): + py.test.skip("xfail") def pytest_runtest_makereport(__multicall__, item, call): - if call.when != "call": + if not isinstance(item, py.test.collect.Function): return - expr, result = evalexpression(item, 'xfail') - rep = __multicall__.execute() - if result: - if call.excinfo: - rep.skipped = True - rep.failed = rep.passed = False + if not (call.excinfo and + call.excinfo.errisinstance(py.test.xfail.Exception)): + evalxfail = getattr(item, '_evalxfail', None) + if not evalxfail: + return + if call.excinfo and call.excinfo.errisinstance(py.test.xfail.Exception): + rep = __multicall__.execute() + rep.keywords['xfail'] = "reason: " + call.excinfo.value.msg + rep.skipped = True + rep.failed = False + return rep + if call.when == "setup": + rep = __multicall__.execute() + if rep.skipped and evalxfail.istrue(): + expl = evalxfail.getexplanation() + if not evalxfail.get("run", True): + expl = "[NOTRUN] " + expl + rep.keywords['xfail'] = expl + return rep + elif call.when == "call": + rep = __multicall__.execute() + if not item.config.getvalue("runxfail") and evalxfail.istrue(): + if call.excinfo: + rep.skipped = True + rep.failed = rep.passed = False + else: + rep.skipped = rep.passed = False + rep.failed = True + rep.keywords['xfail'] = evalxfail.getexplanation() else: - rep.skipped = rep.passed = False - rep.failed = True - rep.keywords['xfail'] = expr - else: - if 'xfail' in rep.keywords: - del rep.keywords['xfail'] - return rep + if 'xfail' in rep.keywords: + del rep.keywords['xfail'] + return rep # called by terminalreporter progress reporting def pytest_report_teststatus(report): @@ -189,7 +225,7 @@ if report.skipped: return "xfailed", "x", "xfail" elif report.failed: - return "xpassed", "P", "xpass" + return "xpassed", "X", "XPASS" # called by the terminalreporter instance/plugin def pytest_terminal_summary(terminalreporter): @@ -229,13 +265,8 @@ xfailed = terminalreporter.stats.get("xfailed") if xfailed: for rep in xfailed: - entry = rep.longrepr.reprcrash - modpath = rep.item.getmodpath(includemodule=True) - pos = "%s %s:%d: " %(modpath, entry.path, entry.lineno) - reason = rep.longrepr.reprcrash.message - i = reason.find("\n") - if i != -1: - reason = reason[:i] + pos = terminalreporter.gettestid(rep.item) + reason = rep.keywords['xfail'] lines.append("XFAIL %s %s" %(pos, reason)) def show_xpassed(terminalreporter, lines): @@ -246,24 +277,6 @@ reason = rep.keywords['xfail'] lines.append("XPASS %s %s" %(pos, reason)) - -def evalexpression(item, keyword): - if isinstance(item, py.test.collect.Function): - markholder = getattr(item.obj, keyword, None) - result = False - if markholder: - d = {'os': py.std.os, 'sys': py.std.sys, 'config': item.config} - expr, result = None, True - for expr in markholder.args: - if isinstance(expr, str): - result = cached_eval(item.config, expr, d) - else: - result = expr - if not result: - break - return expr, result - return None, False - def cached_eval(config, expr, d): if not hasattr(config, '_evalcache'): config._evalcache = {} Modified: pypy/branch/py131/py/_plugin/pytest_terminal.py ============================================================================== --- pypy/branch/py131/py/_plugin/pytest_terminal.py (original) +++ pypy/branch/py131/py/_plugin/pytest_terminal.py Thu May 20 15:55:24 2010 @@ -141,6 +141,20 @@ else: return "???", dict(red=True) + def gettestid(self, item, relative=True): + fspath = item.fspath + chain = [x for x in item.listchain() if x.fspath == fspath] + chain = chain[1:] + names = [x.name for x in chain if x.name != "()"] + path = item.fspath + if relative: + relpath = path.relto(self.curdir) + if relpath: + path = relpath + names.insert(0, str(path)) + return "::".join(names) + + def pytest_internalerror(self, excrepr): for line in str(excrepr).split("\n"): self.write_line("INTERNALERROR> " + line) Modified: pypy/branch/py131/py/_test/cmdline.py ============================================================================== --- pypy/branch/py131/py/_test/cmdline.py (original) +++ pypy/branch/py131/py/_test/cmdline.py Thu May 20 15:55:24 2010 @@ -16,8 +16,9 @@ colitems = config.getinitialnodes() exitstatus = session.main(colitems) config.pluginmanager.do_unconfigure(config) - raise SystemExit(exitstatus) except config.Error: e = sys.exc_info()[1] sys.stderr.write("ERROR: %s\n" %(e.args[0],)) - raise SystemExit(3) + exitstatus = 3 + py.test.config = py.test.config.__class__() + return exitstatus Modified: pypy/branch/py131/py/_test/pycollect.py ============================================================================== --- pypy/branch/py131/py/_test/pycollect.py (original) +++ pypy/branch/py131/py/_test/pycollect.py Thu May 20 15:55:24 2010 @@ -393,5 +393,5 @@ def hasinit(obj): init = getattr(obj, '__init__', None) if init: - if not isinstance(init, type(object.__init__)): + if init != object.__init__: return True From hpk at codespeak.net Thu May 20 16:00:41 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 20 May 2010 16:00:41 +0200 (CEST) Subject: [pypy-svn] r74598 - pypy/branch/py131/pypy/jit/metainterp/test Message-ID: <20100520140041.44432282B90@codespeak.net> Author: hpk Date: Thu May 20 16:00:39 2010 New Revision: 74598 Modified: pypy/branch/py131/pypy/jit/metainterp/test/test_warmspot.py Log: use 'capfd' funcarg for capturing output - this also avoid the being-deprecated py.io.StdCaptureFD usage Modified: pypy/branch/py131/pypy/jit/metainterp/test/test_warmspot.py ============================================================================== --- pypy/branch/py131/pypy/jit/metainterp/test/test_warmspot.py (original) +++ pypy/branch/py131/pypy/jit/metainterp/test/test_warmspot.py Thu May 20 16:00:39 2010 @@ -131,7 +131,7 @@ assert warmrunnerdescr.state.optimize_loop is optimize.optimize_loop assert warmrunnerdescr.state.optimize_bridge is optimize.optimize_bridge - def test_static_debug_level(self): + def test_static_debug_level(self, capfd): py.test.skip("debug_level is being deprecated") from pypy.rlib.jit import DEBUG_PROFILE, DEBUG_OFF, DEBUG_STEPS from pypy.jit.metainterp.jitprof import EmptyProfiler, Profiler @@ -144,36 +144,31 @@ n -= 1 return n - outerr = py.io.StdCaptureFD() + capfd.readouterr() self.meta_interp(f, [10], debug_level=DEBUG_OFF, ProfilerClass=Profiler) - out, errf = outerr.done() - err = errf.read() + out, err = capfd.readouterr() assert not 'ENTER' in err assert not 'LEAVE' in err assert not "Running asm" in err - outerr = py.io.StdCaptureFD() self.meta_interp(f, [10], debug_level=DEBUG_PROFILE, ProfilerClass=Profiler) - out, errf = outerr.done() - err = errf.read() + out, err = capfd.readouterr() assert not 'ENTER' in err assert not 'LEAVE' in err assert not 'compiled new' in err assert "Running asm" in err - outerr = py.io.StdCaptureFD() + self.meta_interp(f, [10], debug_level=DEBUG_STEPS, ProfilerClass=Profiler) - out, errf = outerr.done() - err = errf.read() + out, err = capfd.readouterr() assert 'ENTER' in err assert 'LEAVE' in err assert "Running asm" in err - outerr = py.io.StdCaptureFD() + self.meta_interp(f, [10], debug_level=DEBUG_STEPS, ProfilerClass=EmptyProfiler) - out, errf = outerr.done() - err = errf.read() + out, err = capfd.readouterr() assert 'ENTER' in err assert 'LEAVE' in err assert not "Running asm" in err From arigo at codespeak.net Thu May 20 16:25:28 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 20 May 2010 16:25:28 +0200 (CEST) Subject: [pypy-svn] r74600 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100520142528.69DAE282C0C@codespeak.net> Author: arigo Date: Thu May 20 16:25:26 2010 New Revision: 74600 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Log: Optimize a case that occurs in practice (see comment). Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Thu May 20 16:25:26 2010 @@ -123,9 +123,6 @@ if (v == c_last_exception or isinstance(v, tuple) or v.concretetype != lltype.Bool): return False - for link in block.exits: - if v in link.args: - return False # variable escapes to next block for op in block.operations[::-1]: if v in op.args: return False # variable is also used in cur block @@ -139,6 +136,13 @@ # ok! optimize this case block.operations.remove(op) block.exitswitch = (op.opname,) + tuple(op.args) + # if the variable escape to the next block along a link, + # replace it with a constant, because we know its value + for link in block.exits: + while v in link.args: + index = link.args.index(v) + link.args[index] = Constant(link.llexitcase, + lltype.Bool) return True return False Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Thu May 20 16:25:26 2010 @@ -91,15 +91,22 @@ assert not Transformer().optimize_goto_if_not(block) def test_optimize_goto_if_not__exit(): + # this case occurs in practice, e.g. with RPython code like: + # return bool(p) and p.somefield > 0 v1 = Variable() v2 = Variable() v3 = Variable(); v3.concretetype = lltype.Bool block = Block([v1, v2]) block.operations = [SpaceOperation('int_gt', [v1, v2], v3)] block.exitswitch = v3 - block.exits = [FakeLink(False), FakeLink(True)] + block.exits = exits = [FakeLink(False), FakeLink(True)] block.exits[1].args = [v3] - assert not Transformer().optimize_goto_if_not(block) + res = Transformer().optimize_goto_if_not(block) + assert res == True + assert block.operations == [] + assert block.exitswitch == ('int_gt', v1, v2) + assert block.exits == exits + assert exits[1].args == [Constant(True, lltype.Bool)] def test_optimize_goto_if_not__unknownop(): v3 = Variable(); v3.concretetype = lltype.Bool From arigo at codespeak.net Thu May 20 19:30:52 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 20 May 2010 19:30:52 +0200 (CEST) Subject: [pypy-svn] r74604 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100520173052.E7B52282B90@codespeak.net> Author: arigo Date: Thu May 20 19:30:50 2010 New Revision: 74604 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py Log: Change the way I would like liveness.py to be. Breaks all other code for now. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Thu May 20 19:30:50 2010 @@ -40,11 +40,14 @@ self.kind = kind self.content = tuple(content) def __repr__(self): - return '%s%s' % (self.kind[0], self.content) + return '%s%s' % (self.kind[0].upper(), list(self.content)) def __iter__(self): return iter(self.content) def __nonzero__(self): return bool(self.content) + def __eq__(self, other): + return (isinstance(other, ListOfKind) and + self.kind == other.kind and self.content == other.content) class IndirectCallTargets(object): def __init__(self, lst): @@ -146,9 +149,11 @@ self.emitline("reraise") return # done if link.last_exception in link.args: - self.emitline("last_exception", self.getcolor(link.last_exception)) + self.emitline("last_exception", + "->", self.getcolor(link.last_exception)) if link.last_exc_value in link.args: - self.emitline("last_exc_value", self.getcolor(link.last_exc_value)) + self.emitline("last_exc_value", + "->", self.getcolor(link.last_exc_value)) self.make_link(link) def insert_exits(self, block): @@ -292,9 +297,9 @@ if w is None: self.emitline('%s_push' % kind, v) elif v is None: - self.emitline('%s_pop' % kind, w) + self.emitline('%s_pop' % kind, "->", w) else: - self.emitline('%s_copy' % kind, v, w) + self.emitline('%s_copy' % kind, v, "->", w) def emitline(self, *line): self.ssarepr.insns.append(line) @@ -322,6 +327,7 @@ if op.result is not None: kind = getkind(op.result.concretetype) if kind != 'void': + args.append("->") args.append(self.getcolor(op.result)) self.emitline(op.opname, *args) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py Thu May 20 19:30:50 2010 @@ -56,9 +56,16 @@ else: print >> output, asm[0], if len(asm) > 1: - lst = map(repr, asm[1:]) - if asm[0] == '-live-': lst.sort() - print >> output, ', '.join(lst) + if asm[-2] == '->': + if len(asm) == 3: + print >> output, '->', repr(asm[-1]) + else: + lst = map(repr, asm[1:-2]) + print >> output, ', '.join(lst), '->', repr(asm[-1]) + else: + lst = map(repr, asm[1:]) + if asm[0] == '-live-': lst.sort() + print >> output, ', '.join(lst) else: print >> output res = output.getvalue() @@ -71,23 +78,25 @@ explines = expected.split("\n") for asm, exp in zip(asmlines, explines): if asm != exp: - print - print "Got: " + asm - print "Expected: " + exp + msg = [""] + msg.append("Got: " + asm) + msg.append("Expected: " + exp) lgt = 0 for i in range(min(len(asm), len(exp))): if exp[i] == asm[i]: lgt += 1 else: break - print " " + " " * lgt + "^^^^" - raise AssertionError + msg.append(" " + " " * lgt + "^^^^") + raise AssertionError('\n'.join(msg)) assert len(asmlines) == len(explines) def unformat_assembler(text, registers=None): # XXX limited to simple assembler right now # def unformat_arg(s): + if s.endswith(','): + s = s[:-1].rstrip() if s[0] == '%': try: return registers[s] @@ -104,6 +113,21 @@ return Constant(intvalue, lltype.Signed) elif s[0] == 'L': return TLabel(s) + elif s[0] in 'IRF' and s[1] == '[' and s[-1] == ']': + items = split_words(s[2:-1]) + items = map(unformat_arg, items) + return ListOfKind({'I': 'int', 'R': 'ref', 'F': 'float'}[s[0]], + items) + elif s.startswith('') + switchdict = SwitchDictDescr() + switchdict._labels = [] + items = split_words(s[len('' in words: + assert words.index('->') == len(words) - 2 + extra = ['->', unformat_arg(words[-1])] + del words[-2:] + else: + extra = [] + insn = [opname] + [unformat_arg(s) for s in words] + extra ssarepr.insns.append(tuple(insn)) return ssarepr + + +def split_words(line): + word = '' + nested = 0 + for i, c in enumerate(line): + if c == ' ' and nested == 0: + if word: + yield word + word = '' + else: + word += c + if c in '<([': + nested += 1 + if c in '])>' and line[i-2:i+2] != ' -> ': + nested -= 1 + assert nested >= 0 + if word: + yield word + assert nested == 0 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py Thu May 20 19:30:50 2010 @@ -1,45 +1,79 @@ -from pypy.objspace.flow.model import Variable, SpaceOperation, c_last_exception -from pypy.jit.codewriter.flatten import ListOfKind +from pypy.jit.codewriter.flatten import Register, ListOfKind, Label, TLabel +from pypy.jit.codewriter.jitcode import SwitchDictDescr -# Some instruction require liveness information (the ones that can end up -# in generate_guard() in pyjitpl.py); jtransform.py prefixes these opnames -# with a 'G_'. Additionally, boolean and general switches in the flow graph -# will turn in 'goto_if_not_*' operations, which also require liveness info. +# Some instructions require liveness information (the ones that can end up +# in generate_guard() in pyjitpl.py). This is done by putting special +# space operations called '-live-' in the graph. They turn into '-live-' +# operation in the ssarepr. Then this module expands the arguments of +# the '-live-' operations to also include all values that are alive at +# this point: more precisely, all values that are created before the +# '-live-' operation and that are needed afterwards, with the exception +# of the values that are needed only in the very next instruction. These +# are not considered alive any more. You can force them to be alive by +# putting them as args of the '-live-' operation in the first place. + +# For this to work properly, a special operation called '---' must be +# used to mark unreachable places (e.g. just after a 'goto'). # ____________________________________________________________ -def compute_liveness(graph, switches_require_liveness=True): - for block in graph.iterblocks(): - num_operations = len(block.operations) - alive = set() - for link in block.exits: - for v in link.args: - if (v is not link.last_exception and - v is not link.last_exc_value): - alive.add(v) - if switches_require_liveness: - if len(block.exits) > 1 and block.exitswitch != c_last_exception: - block.operations.append(_livespaceop(alive)) - if isinstance(block.exitswitch, tuple): - for v in block.exitswitch[1:]: - alive.add(v) - else: - alive.add(block.exitswitch) - for i in range(num_operations-1, -1, -1): - op = block.operations[i] - try: - alive.remove(op.result) - except KeyError: - pass - if op.opname.startswith('G_'): - block.operations.insert(i, _livespaceop(alive)) - for v in op.args: - if isinstance(v, ListOfKind): - alive.update(v) - else: - alive.add(v) - -def _livespaceop(alive): - livevars = [v for v in alive if isinstance(v, Variable)] - return SpaceOperation('-live-', livevars, None) +def compute_liveness(ssarepr): + label2alive = {} + while _compute_liveness_must_continue(ssarepr, label2alive): + pass + +def _compute_liveness_must_continue(ssarepr, label2alive): + alive = set() + prevalive = None + must_continue = False + + for i in range(len(ssarepr.insns)-1, -1, -1): + insn = ssarepr.insns[i] + + if isinstance(insn[0], Label): + alive_at_point = label2alive.setdefault(insn[0].name, set()) + prevlength = len(alive_at_point) + alive_at_point.update(alive) + if prevlength != len(alive_at_point): + must_continue = True + prevalive = None + continue + + if insn[0] == '-live-': + assert prevalive is not None + for x in insn[1:]: + prevalive.discard(x) + ssarepr.insns[i] = insn + tuple(prevalive) + prevalive = None + continue + + if insn[0] == '---': + alive = set() + prevalive = None + continue + + args = insn[1:] + # + if len(args) >= 2 and args[-2] == '->': + reg = args[-1] + assert isinstance(reg, Register) + alive.discard(reg) + args = args[:-2] + # + prevalive = alive.copy() + # + for x in args: + if isinstance(x, Register): + alive.add(x) + elif isinstance(x, ListOfKind): + alive.update(x) + elif isinstance(x, TLabel): + alive_at_point = label2alive.get(x.name, ()) + alive.update(alive_at_point) + elif isinstance(x, SwitchDictDescr): + for key, label in x._labels: + alive_at_point = label2alive.get(label.name, ()) + alive.update(alive_at_point) + + return must_continue Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py Thu May 20 19:30:50 2010 @@ -4,6 +4,7 @@ from pypy.jit.codewriter.flatten import Label, TLabel, SSARepr, Register from pypy.jit.codewriter.flatten import ListOfKind from pypy.jit.metainterp.history import AbstractDescr +from pypy.jit.codewriter.jitcode import SwitchDictDescr from pypy.rpython.lltypesystem import lltype @@ -11,12 +12,12 @@ ssarepr = SSARepr("test") i0, i1, i2 = Register('int', 0), Register('int', 1), Register('int', 2) ssarepr.insns = [ - ('int_add', i0, i1, i2), + ('int_add', i0, i1, '->', i2), ('int_return', i2), ] asm = format_assembler(ssarepr) expected = """ - int_add %i0, %i1, %i2 + int_add %i0, %i1 -> %i2 int_return %i2 """ assert asm == str(py.code.Source(expected)).strip() + '\n' @@ -39,11 +40,11 @@ s.x = 123 ssarepr = SSARepr("test") ssarepr.insns = [ - ('foobar', Constant(s, lltype.typeOf(s))), + ('foobar', '->', Constant(s, lltype.typeOf(s))), ] asm = format_assembler(ssarepr) expected = """ - foobar $<* struct S> + foobar -> $<* struct S> """ assert asm == str(py.code.Source(expected)).strip() + '\n' @@ -53,8 +54,8 @@ ssarepr.insns = [ (Label('L1'),), ('goto_if_not_int_gt', i0, Constant(0, lltype.Signed), TLabel('L2')), - ('int_add', i1, i0, i1), - ('int_sub', i0, Constant(1, lltype.Signed), i0), + ('int_add', i1, i0, '->', i1), + ('int_sub', i0, Constant(1, lltype.Signed), '->', i0), ('goto', TLabel('L1')), (Label('L2'),), ('int_return', i1), @@ -63,8 +64,8 @@ expected = """ L1: goto_if_not_int_gt %i0, $0, L2 - int_add %i1, %i0, %i1 - int_sub %i0, $1, %i0 + int_add %i1, %i0 -> %i1 + int_sub %i0, $1 -> %i0 goto L1 L2: int_return %i1 @@ -99,7 +100,7 @@ def test_unformat_assembler_simple(): input = """ - int_add %i0, %i1, %i2 + int_add %i0, %i1 -> %i2 int_return %i2 """ regs = {} @@ -107,7 +108,7 @@ assert regs['%i2'].kind == 'int' assert regs['%i2'].index == 2 assert ssarepr.insns == [ - ('int_add', regs['%i0'], regs['%i1'], regs['%i2']), + ('int_add', regs['%i0'], regs['%i1'], '->', regs['%i2']), ('int_return', regs['%i2']), ] @@ -134,3 +135,30 @@ (Label('L2'),), ('bar', TLabel('L1')), ] + +def test_unformat_assembler_lists(): + input = """ + foo F[%f0, %f3] + """ + regs = {} + ssarepr = unformat_assembler(input, regs) + assert ssarepr.insns == [ + ('foo', ListOfKind('float', [regs['%f0'], regs['%f3']])) + ] + +def test_unformat_switchdictdescr(): + input = """ + foo + L1: + L2: + """ + regs = {} + ssarepr = unformat_assembler(input, regs) + sdd = ssarepr.insns[0][1] + assert ssarepr.insns == [ + ('foo', sdd), + (Label('L1'),), + (Label('L2'),), + ] + assert isinstance(sdd, SwitchDictDescr) + assert sdd._labels == [(4, TLabel('L2')), (5, TLabel('L1'))] Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py Thu May 20 19:30:50 2010 @@ -1,153 +1,189 @@ -from pypy.jit.codewriter import support from pypy.jit.codewriter.liveness import compute_liveness -from pypy.jit.codewriter.test.test_flatten import fake_regallocs -from pypy.jit.codewriter.flatten import flatten_graph, ListOfKind -from pypy.jit.codewriter.format import assert_format -from pypy.objspace.flow.model import SpaceOperation +from pypy.jit.codewriter.format import unformat_assembler, assert_format class TestFlatten: - def make_graphs(self, func, values, type_system='lltype'): - self.rtyper = support.annotate(func, values, type_system=type_system) - return self.rtyper.annotator.translator.graphs - - def add_G_prefix(self, graph): - """Add a 'G_' prefix to the opnames 'int_add' and 'int_mul'. - Turn the arguments of float_add into a ListOfKind().""" - def with_prefix(op): - if op.opname in ('int_add', 'int_mul'): - return SpaceOperation('G_' + op.opname, op.args, op.result) - if op.opname == 'float_add': - return SpaceOperation(op.opname, - [ListOfKind('float', op.args)], - op.result) - return op - # - for block in graph.iterblocks(): - if block.operations: - block.operations = map(with_prefix, block.operations) - - def encoding_test(self, func, args, expected, - switches_require_liveness=False): - graphs = self.make_graphs(func, args) - self.add_G_prefix(graphs[0]) - compute_liveness(graphs[0], switches_require_liveness) - ssarepr = flatten_graph(graphs[0], fake_regallocs()) - assert_format(ssarepr, expected) + def liveness_test(self, input, output): + ssarepr = unformat_assembler(input) + compute_liveness(ssarepr) + assert_format(ssarepr, output) def test_simple_no_live(self): - def f(n): - return n + 10 - self.encoding_test(f, [5], """ + self.liveness_test(""" -live- - G_int_add %i0, $10, %i1 - int_return %i1 + int_add %i0, $10 -> %i1 + """, """ + -live- + int_add %i0, $10 -> %i1 """) def test_simple(self): - def f(n): - return (n + 10) * (n + 3) * (n + 6) - self.encoding_test(f, [5], """ + self.liveness_test(""" + -live- + int_add %i0, $10 -> %i1 + -live- + int_add %i0, $3 -> %i2 + -live- + int_mul %i1, %i2 -> %i3 + -live- + int_add %i0, $6 -> %i4 + -live- + int_mul %i3, %i4 -> %i5 + int_return %i5 + """, """ -live- %i0 - G_int_add %i0, $10, %i1 + int_add %i0, $10 -> %i1 -live- %i0, %i1 - G_int_add %i0, $3, %i2 + int_add %i0, $3 -> %i2 -live- %i0 - G_int_mul %i1, %i2, %i3 + int_mul %i1, %i2 -> %i3 -live- %i3 - G_int_add %i0, $6, %i4 + int_add %i0, $6 -> %i4 -live- - G_int_mul %i3, %i4, %i5 + int_mul %i3, %i4 -> %i5 int_return %i5 """) def test_one_path(self): - def f(x, y): - if x+5: - return x+1 - return y+2 - self.encoding_test(f, [5, 6], """ + self.liveness_test(""" + -live- + int_add %i0, $5 -> %i2 + int_is_true %i2 -> %i3 + goto_if_not %i3, L1 + int_copy %i0 -> %i4 + -live- + int_add %i4, $1 -> %i5 + int_return %i5 + --- + L1: + int_copy %i1 -> %i6 + -live- + int_add %i6, $2 -> %i7 + int_return %i7 + """, """ -live- %i0, %i1 - G_int_add %i0, $5, %i2 - int_is_true %i2, %i3 + int_add %i0, $5 -> %i2 + int_is_true %i2 -> %i3 goto_if_not %i3, L1 - int_copy %i0, %i4 + int_copy %i0 -> %i4 -live- - G_int_add %i4, $1, %i5 + int_add %i4, $1 -> %i5 int_return %i5 + --- L1: - int_copy %i1, %i6 + int_copy %i1 -> %i6 -live- - G_int_add %i6, $2, %i7 + int_add %i6, $2 -> %i7 int_return %i7 """) def test_other_path(self): - def f(x, y): - if x+5: - return x+y - return x+2 - self.encoding_test(f, [5, 6], """ + self.liveness_test(""" + -live- + int_add %i0, $5 -> %i2 + int_is_true %i2 -> %i3 + goto_if_not %i3, L1 + int_copy %i0 -> %i4 + int_copy %i1 -> %i5 + -live- + int_add %i4, %i5 -> %i6 + int_return %i6 + --- + L1: + int_copy %i0 -> %i7 + -live- + int_add %i7, $2 -> %i8 + int_return %i8 + """, """ -live- %i0, %i1 - G_int_add %i0, $5, %i2 - int_is_true %i2, %i3 + int_add %i0, $5 -> %i2 + int_is_true %i2 -> %i3 goto_if_not %i3, L1 - int_copy %i0, %i4 - int_copy %i1, %i5 + int_copy %i0 -> %i4 + int_copy %i1 -> %i5 -live- - G_int_add %i4, %i5, %i6 + int_add %i4, %i5 -> %i6 int_return %i6 + --- L1: - int_copy %i0, %i7 + int_copy %i0 -> %i7 -live- - G_int_add %i7, $2, %i8 + int_add %i7, $2 -> %i8 int_return %i8 """) def test_no_path(self): - def f(x, y): - if x+y: - return x+5 - return x+2 - self.encoding_test(f, [5, 6], """ + self.liveness_test(""" + -live- + int_add %i0, %i1 -> %i2 + int_is_true %i2 -> %i3 + goto_if_not %i3, L1 + int_copy %i0 -> %i4 + -live- + int_add %i4, $5 -> %i5 + int_return %i5 + --- + L1: + int_copy %i0 -> %i6 + -live- + int_add %i6, $2 -> %i7 + int_return %i7 + """, """ -live- %i0 - G_int_add %i0, %i1, %i2 - int_is_true %i2, %i3 + int_add %i0, %i1 -> %i2 + int_is_true %i2 -> %i3 goto_if_not %i3, L1 - int_copy %i0, %i4 + int_copy %i0 -> %i4 -live- - G_int_add %i4, $5, %i5 + int_add %i4, $5 -> %i5 int_return %i5 + --- L1: - int_copy %i0, %i6 + int_copy %i0 -> %i6 -live- - G_int_add %i6, $2, %i7 + int_add %i6, $2 -> %i7 int_return %i7 """) - def test_switch_require_liveness(self): - def f(x, y): - if x: - return x - return y - self.encoding_test(f, [5, 6], """ - int_is_true %i0, %i2 - -live- %i0, %i1 - goto_if_not %i2, L1 - int_return %i0 - L1: - int_return %i1 - """, switches_require_liveness=True) - def test_list_of_kind(self): - def f(x, y, z, t): - return (x + y) * (z + t) - self.encoding_test(f, [5, 6, 3.2, 4.3], """ - -live- %f0, %f1 - G_int_add %i0, %i1, %i2 - float_add F[%f0, %f1], %f2 - cast_int_to_float %i2, %f3 - float_mul %f3, %f2, %f4 - float_return %f4 + self.liveness_test(""" + -live- + stub + foobar F[%f0] + """, """ + -live- %f0 + stub + foobar F[%f0] + """) + + def test_switch(self): + self.liveness_test(""" + goto_maybe L1 + -live- + stub + fooswitch + --- + L3: + int_return %i7 + --- + L1: + int_return %i4 + --- + L2: + int_return %i3 + """, """ + goto_maybe L1 + -live- %i3, %i7 + stub + fooswitch + --- + L3: + int_return %i7 + --- + L1: + int_return %i4 + --- + L2: + int_return %i3 """) From arigo at codespeak.net Thu May 20 20:01:23 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 20 May 2010 20:01:23 +0200 (CEST) Subject: [pypy-svn] r74605 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100520180123.24215282B90@codespeak.net> Author: arigo Date: Thu May 20 20:01:21 2010 New Revision: 74605 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py Log: In-progress. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Thu May 20 20:01:21 2010 @@ -105,6 +105,7 @@ return if block in self.seen_blocks: self.emitline("goto", TLabel(block)) + self.emitline("---") return # inserting a goto not necessary, falling through self.seen_blocks[block] = True @@ -130,6 +131,7 @@ self.emitline("raise", self.getcolor(args[1])) else: raise Exception("?") + self.emitline("---") def make_link(self, link): if link.target.exits == (): @@ -167,12 +169,13 @@ # An exception block. See test_exc_exitswitch in test_flatten.py # for an example of what kind of code this makes. lastopname = block.operations[-1].opname - if lastopname == '-live-': - lastopname = block.operations[-2].opname + assert lastopname != '-live-' assert block.exits[0].exitcase is None # is this always True? # if not self._include_all_exc_links: - if not lastopname.startswith('G_'): # cannot actually raise + if (len(block.operations) == 1 or + block.operations[-2].opname != '-live-'): + # cannot actually raise self.make_link(block.exits[0]) return # @@ -182,7 +185,7 @@ for link in block.exits[1:]: if (link.exitcase is Exception or (link.exitcase is OverflowError and - lastopname.startswith('G_int_') and + lastopname.startswith('int_') and lastopname.endswith('_ovf'))): # this link captures all exceptions self.make_exception_link(link) @@ -197,6 +200,7 @@ # no link captures all exceptions, so we have to put a reraise # for the other exceptions self.emitline("reraise") + self.emitline("---") # elif len(block.exits) == 2 and ( isinstance(block.exitswitch, tuple) or @@ -230,7 +234,7 @@ if block.exits[-1].exitcase == 'default': self.make_link(block.exits[-1]) else: - self.emitline('unreachable') + self.emitline("---") # switches = [link for link in block.exits if link.exitcase != 'default'] Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Thu May 20 20:01:21 2010 @@ -198,9 +198,6 @@ rewrite_op_int_gt = _rewrite_symmetric rewrite_op_int_ge = _rewrite_symmetric - rewrite_op_G_int_add_ovf = _rewrite_symmetric - rewrite_op_G_int_mul_ovf = _rewrite_symmetric - rewrite_op_float_add = _rewrite_symmetric rewrite_op_float_mul = _rewrite_symmetric rewrite_op_float_lt = _rewrite_symmetric @@ -208,6 +205,17 @@ rewrite_op_float_gt = _rewrite_symmetric rewrite_op_float_ge = _rewrite_symmetric + def rewrite_op_int_add_ovf(self, op): + op0 = SpaceOperation('-live-', op.args[:], None) + op1 = self._rewrite_symmetric(op) + return [op0, op1] + + rewrite_op_int_mul_ovf = rewrite_op_int_add_ovf + + def rewrite_op_int_sub_ovf(self, op): + op0 = SpaceOperation('-live-', op.args[:], None) + return [op0, op] + # ---------- # Various kinds of calls @@ -259,11 +267,11 @@ is calling a function that we don't want to JIT. The initial args of 'residual_call_xxx' are the function to call, and its calldescr.""" calldescr = self.callcontrol.getcalldescr(op) + op1 = self.rewrite_call(op, 'residual_call', + [op.args[0], calldescr] + extraargs) if self.callcontrol.calldescr_canraise(calldescr): - name = 'G_residual_call' - else: - name = 'residual_call' - return self.rewrite_call(op, name, [op.args[0], calldescr] + extraargs) + op1 = [SpaceOperation('-live-', [], None), op1] + return op1 def handle_regular_call(self, op): """A direct_call turns into the operation 'inline_call_xxx' if it @@ -272,7 +280,9 @@ [targetgraph] = self.callcontrol.graphs_from(op) jitcode = self.callcontrol.get_jitcode(targetgraph, called_from=self.graph) - return self.rewrite_call(op, 'G_inline_call', [jitcode]) + op0 = SpaceOperation('-live-', [], None) + op1 = self.rewrite_call(op, 'inline_call', [jitcode]) + return [op0, op1] def handle_builtin_call(self, op): oopspec_name, args = support.decode_builtin_call(op) @@ -288,9 +298,15 @@ jitcode = self.callcontrol.get_jitcode(targetgraph, called_from=self.graph) lst.append(jitcode) - op0 = SpaceOperation('G_int_guard_value', [op.args[0]], None) - op1 = self.handle_residual_call(op, [IndirectCallTargets(lst)]) - return [op0, op1] + op0 = SpaceOperation('-live-', [op.args[0]], None) + op1 = SpaceOperation('int_guard_value', [op.args[0]], None) + op2 = self.handle_residual_call(op, [IndirectCallTargets(lst)]) + result = [op0, op1] + if isinstance(op2, list): + result += op2 + else: + result.append(op2) + return result def _prepare_builtin_call(self, op, oopspec_name, args, extra=None, extrakey=None): @@ -329,15 +345,11 @@ if hints.get('promote') and op.args[0].concretetype is not lltype.Void: assert op.args[0].concretetype != lltype.Ptr(rstr.STR) kind = getkind(op.args[0].concretetype) - # note: the 'G_' prefix tells that the operation might generate - # a guard in pyjitpl (see liveness.py). The following 'keepalive' - # is needed to ensure that op.args[0] is restored on guard failure. - op1 = SpaceOperation('G_%s_guard_value' % kind, - [op.args[0]], None) - op2 = SpaceOperation('keepalive', [op.args[0]], None) - # this special return value forces op.result to be considered + op0 = SpaceOperation('-live-', [op.args[0]], None) + op1 = SpaceOperation('%s_guard_value' % kind, [op.args[0]], None) + # the special return value None forces op.result to be considered # equal to op.args[0] - return [op1, op2, None] + return [op0, op1, None] else: log.WARNING('ignoring hint %r at %r' % (hints, self.graph)) @@ -451,11 +463,9 @@ op.args[0].concretetype.TO._hints.get('typeptr')) def handle_getfield_typeptr(self, op): - # note: the 'G_' prefix tells that the operation might generate - # a guard in pyjitpl (see liveness.py). The following 'keepalive' - # is needed to ensure that op.args[0] is restored on guard failure. - return [SpaceOperation('G_guard_class', [op.args[0]], op.result), - SpaceOperation('keepalive', [op.args[0]], None)] + op0 = SpaceOperation('-live-', [op.args[0]], None) + op1 = SpaceOperation('guard_class', [op.args[0]], op.result) + return [op0, op1] def rewrite_op_malloc(self, op): assert op.args[1].value == {'flavor': 'gc'} @@ -569,10 +579,7 @@ ('cast_uint_to_float', 'cast_int_to_float'), ('cast_float_to_uint', 'cast_float_to_int'), - ('int_add_ovf', 'G_int_add_ovf'), - ('int_add_nonneg_ovf', 'G_int_add_ovf'), - ('int_sub_ovf', 'G_int_sub_ovf'), - ('int_mul_ovf', 'G_int_mul_ovf'), + ('int_add_nonneg_ovf', 'int_add_ovf'), ('char_lt', 'int_lt'), ('char_le', 'int_le'), @@ -649,7 +656,8 @@ for v in op.args[2:2+num_green_args]: if isinstance(v, Variable) and v.concretetype is not lltype.Void: kind = getkind(v.concretetype) - ops.append(SpaceOperation('G_%s_guard_value' % kind, + ops.append(SpaceOperation('-live-', [v], None)) + ops.append(SpaceOperation('%s_guard_value' % kind, [v], None)) args = (self.make_three_lists(op.args[2:2+num_green_args]) + self.make_three_lists(op.args[2+num_green_args:])) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py Thu May 20 20:01:21 2010 @@ -43,8 +43,9 @@ if insn[0] == '-live-': assert prevalive is not None for x in insn[1:]: - prevalive.discard(x) - ssarepr.insns[i] = insn + tuple(prevalive) + if isinstance(x, Register): + prevalive.add(x) + ssarepr.insns[i] = insn[:1] + tuple(prevalive) prevalive = None continue Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Thu May 20 20:01:21 2010 @@ -152,7 +152,6 @@ 'uint_le': ('int_le', 'int_ge'), 'char_ne': 'int_ne', 'char_lt': ('int_lt', 'int_gt'), - 'int_add_nonneg_ovf': 'G_int_add_ovf', 'uint_xor': 'int_xor', 'float_mul': 'float_mul', 'float_gt': ('float_gt', 'float_lt'), @@ -174,6 +173,24 @@ assert op1.result == v3 assert op1.opname == name2[0] +def test_symmetric_int_add_ovf(): + v3 = varoftype(lltype.Signed) + for v1 in [varoftype(lltype.Signed), Constant(42, lltype.Signed)]: + for v2 in [varoftype(lltype.Signed), Constant(43, lltype.Signed)]: + op = SpaceOperation('int_add_nonneg_ovf', [v1, v2], v3) + oplist = Transformer(FakeCPU()).rewrite_operation(op) + op0, op1 = oplist + assert op0.opname == '-live-' + assert op0.args == [v1, v2] + assert op0.result is None + assert op1.opname == 'int_add_ovf' + if isinstance(v1, Constant) and isinstance(v2, Variable): + assert op1.args == [v2, v1] + assert op1.result == v3 + else: + assert op1.args == [v1, v2] + assert op1.result == v3 + def test_calls(): for RESTYPE in [lltype.Signed, rclass.OBJECTPTR, lltype.Float, lltype.Void]: @@ -208,9 +225,12 @@ def residual_call_test(argtypes, restype, expectedkind): op = get_direct_call_op(argtypes, restype) tr = Transformer(FakeCPU(), FakeResidualCallControl()) - op1 = tr.rewrite_operation(op) + oplist = tr.rewrite_operation(op) + op0, op1 = oplist + assert op0.opname == '-live-' + assert op0.args == [] reskind = getkind(restype)[0] - assert op1.opname == 'G_residual_call_%s_%s' % (expectedkind, reskind) + assert op1.opname == 'residual_call_%s_%s' % (expectedkind, reskind) assert op1.result == op.result assert op1.args[0] == op.args[0] assert op1.args[1] == 'calldescr' @@ -227,9 +247,12 @@ op = get_direct_call_op(argtypes, restype) tr = Transformer(FakeCPU(), FakeRegularCallControl()) tr.graph = 'someinitialgraph' - op1 = tr.rewrite_operation(op) + oplist = tr.rewrite_operation(op) + op0, op1 = oplist + assert op0.opname == '-live-' + assert op0.args == [] reskind = getkind(restype)[0] - assert op1.opname == 'G_inline_call_%s_%s' % (expectedkind, reskind) + assert op1.opname == 'inline_call_%s_%s' % (expectedkind, reskind) assert op1.result == op.result assert op1.args[0] == 'somejitcode' assert len(op1.args) == 1 + len(expectedkind) @@ -250,9 +273,12 @@ op.args.append(Constant(['somegraph1', 'somegraph2'], lltype.Void)) tr = Transformer(FakeCPU(), FakeResidualIndirectCallControl()) tr.graph = 'someinitialgraph' - op1 = tr.rewrite_operation(op) + oplist = tr.rewrite_operation(op) + op0, op1 = oplist + assert op0.opname == '-live-' + assert op0.args == [] reskind = getkind(restype)[0] - assert op1.opname == 'G_residual_call_%s_%s' % (expectedkind, reskind) + assert op1.opname == 'residual_call_%s_%s' % (expectedkind, reskind) assert op1.result == op.result assert op1.args[0] == op.args[0] assert op1.args[1] == 'calldescr' @@ -275,14 +301,18 @@ op.args.append(Constant(['somegraph1', 'somegraph2'], lltype.Void)) tr = Transformer(FakeCPU(), FakeRegularIndirectCallControl()) tr.graph = 'someinitialgraph' - op1 = tr.rewrite_operation(op) - assert type(op1) is list - op0, op1 = op1 - assert op0.opname == 'G_int_guard_value' - assert op0.args[0] == op.args[0] - assert op0.result is None + oplist = tr.rewrite_operation(op) + op0gv, op1gv, op0, op1 = oplist + assert op0gv.opname == '-live-' + assert op0gv.args == [op.args[0]] + assert op1gv.opname == 'int_guard_value' + assert op1gv.args == [op.args[0]] + assert op1gv.result is None + # + assert op0.opname == '-live-' + assert op0.args == [] reskind = getkind(restype)[0] - assert op1.opname == 'G_residual_call_%s_%s' % (expectedkind, reskind) + assert op1.opname == 'residual_call_%s_%s' % (expectedkind, reskind) assert op1.result == op.result assert op1.args[0] == op.args[0] assert op1.args[1] == 'calldescr' @@ -332,13 +362,13 @@ c_name = Constant('typeptr', lltype.Void) v_result = varoftype(rclass.OBJECT.typeptr) op = SpaceOperation('getfield', [v_parent, c_name], v_result) - op1, op2 = Transformer(FakeCPU()).rewrite_operation(op) - assert op1.opname == 'G_guard_class' + oplist = Transformer(FakeCPU()).rewrite_operation(op) + op0, op1 = oplist + assert op0.opname == '-live-' + assert op0.args == [v_parent] + assert op1.opname == 'guard_class' assert op1.args == [v_parent] assert op1.result == v_result - assert op2.opname == 'keepalive' - assert op2.args == [v_parent] - assert op2.result == None def test_setfield(): # XXX a more compact encoding would be possible; see test_getfield() @@ -404,8 +434,11 @@ op = SpaceOperation('malloc', [Constant(S, lltype.Void), Constant({'flavor': 'gc'}, lltype.Void)], v) tr = Transformer(FakeCPU(), FakeResidualCallControl()) - op1 = tr.rewrite_operation(op) - assert op1.opname == 'G_residual_call_r_r' + oplist = tr.rewrite_operation(op) + op0, op1 = oplist + assert op0.opname == '-live-' + assert op0.args == [] + assert op1.opname == 'residual_call_r_r' assert op1.args[0].value == 'alloc_with_del' # pseudo-function as a str assert list(op1.args[2]) == [] @@ -581,13 +614,13 @@ [v1, Constant({'promote': True}, lltype.Void)], v2) oplist = Transformer().rewrite_operation(op) - assert len(oplist) == 3 - assert oplist[0].opname == 'G_int_guard_value' - assert oplist[0].args == [v1] - assert oplist[0].result is None - assert oplist[1].opname == 'keepalive' - assert oplist[1].args == [v1] - assert oplist[2] is None + op0, op1, op2 = oplist + assert op0.opname == '-live-' + assert op0.args == [v1] + assert op1.opname == 'int_guard_value' + assert op1.args == [v1] + assert op1.result is None + assert op2 is None def test_promote_2(): v1 = varoftype(lltype.Signed) @@ -602,9 +635,9 @@ block.closeblock(Link([v2], returnblock)) Transformer().optimize_block(block) assert len(block.operations) == 2 - assert block.operations[0].opname == 'G_int_guard_value' + assert block.operations[0].opname == '-live-' assert block.operations[0].args == [v1] - assert block.operations[0].result is None - assert block.operations[1].opname == 'keepalive' + assert block.operations[1].opname == 'int_guard_value' assert block.operations[1].args == [v1] + assert block.operations[1].result is None assert block.exits[0].args == [v1] Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py Thu May 20 20:01:21 2010 @@ -187,3 +187,14 @@ L2: int_return %i3 """) + + def test_already_some(self): + self.liveness_test(""" + -live- %i0, $52, %i2, %i0 + foo %i0, %i1, %i2 + bar %i3, %i4, %i5 + """, """ + -live- %i0, %i2, %i3, %i4, %i5 + foo %i0, %i1, %i2 + bar %i3, %i4, %i5 + """) From afa at codespeak.net Thu May 20 23:09:34 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 20 May 2010 23:09:34 +0200 (CEST) Subject: [pypy-svn] r74610 - in pypy/trunk/pypy/rpython/lltypesystem: . test Message-ID: <20100520210934.E9E97282B90@codespeak.net> Author: afa Date: Thu May 20 23:09:33 2010 New Revision: 74610 Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Log: Allows ll2ctypes to return a COpaquePtr allocated by an external function. Now you can use fopen(), fwrite() and fclose() in rpython code! Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Thu May 20 23:09:33 2010 @@ -208,6 +208,8 @@ else: restype = get_ctypes_type(T.TO.RESULT) return ctypes.CFUNCTYPE(restype, *argtypes) + elif isinstance(T.TO, lltype.OpaqueType): + return ctypes.c_void_p else: return ctypes.POINTER(get_ctypes_type(T.TO, delayed_builders)) elif T is lltype.Void: @@ -635,10 +637,15 @@ cbuf = ctypes.create_string_buffer(T.TO.hints['getsize']()) else: cbuf = ctypes.create_string_buffer("\x00") + cbuf = ctypes.cast(cbuf, ctypes.c_void_p) add_storage(container, _parentable_mixin, cbuf) else: raise NotImplementedError(T) container._ctypes_storage_was_allocated() + + if isinstance(T.TO, lltype.OpaqueType): + return container._storage + storage = container._storage p = ctypes.pointer(storage) if index: @@ -729,6 +736,7 @@ container = _llgcopaque(cobj) else: container = lltype._opaque(T.TO) + container._storage = ctypes.cast(cobj, ctypes.c_void_p) else: raise NotImplementedError(T) llobj = lltype._ptr(T, container, solid=True) Modified: pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Thu May 20 23:09:33 2010 @@ -344,7 +344,6 @@ assert not ALLOCATED # detects memory leaks in the test def test_opaque_obj_2(self): - py.test.skip("FIXME") FILEP = rffi.COpaquePtr('FILE') fopen = rffi.llexternal('fopen', [rffi.CCHARP, rffi.CCHARP], FILEP) fclose = rffi.llexternal('fclose', [FILEP], rffi.INT) @@ -353,8 +352,7 @@ assert ll_file fclose(ll_file) assert tmppath.check(file=1) - #assert not ALLOCATED --- fails, because ll2ctypes misses the - # fact that fclose() frees 'll_file' + assert not ALLOCATED # detects memory leaks in the test def test_simple_cast(self): assert rffi.cast(rffi.SIGNEDCHAR, 0x123456) == 0x56 From afa at codespeak.net Thu May 20 23:30:58 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 20 May 2010 23:30:58 +0200 (CEST) Subject: [pypy-svn] r74611 - in pypy/trunk/pypy/module/pyexpat: . test Message-ID: <20100520213058.B53EC282B90@codespeak.net> Author: afa Date: Thu May 20 23:30:57 2010 New Revision: 74611 Added: pypy/trunk/pypy/module/pyexpat/test/test_parser.py (contents, props changed) Modified: pypy/trunk/pypy/module/pyexpat/interp_pyexpat.py Log: It's now even possible to interpret the pyexpat module. Start a test. Modified: pypy/trunk/pypy/module/pyexpat/interp_pyexpat.py ============================================================================== --- pypy/trunk/pypy/module/pyexpat/interp_pyexpat.py (original) +++ pypy/trunk/pypy/module/pyexpat/interp_pyexpat.py Thu May 20 23:30:57 2010 @@ -93,6 +93,9 @@ class Storage: "Store objects under a non moving ID" def __init__(self): + self.clear() + + def clear(self): self.next_id = 0 self.storage = {} Added: pypy/trunk/pypy/module/pyexpat/test/test_parser.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/pyexpat/test/test_parser.py Thu May 20 23:30:57 2010 @@ -0,0 +1,17 @@ +from pypy.conftest import gettestobjspace +from pypy.module.pyexpat.interp_pyexpat import global_storage + +class AppTestPyexpat: + def setup_class(cls): + cls.space = gettestobjspace(usemodules=['pyexpat']) + + def teardown_class(cls): + global_storage.clear() + + def test_simple(self): + import pyexpat + p = pyexpat.ParserCreate() + res = p.Parse("") + assert res == 1 + + raises(pyexpat.ExpatError, p.Parse, "3") From wildchild at codespeak.net Fri May 21 00:29:01 2010 From: wildchild at codespeak.net (wildchild at codespeak.net) Date: Fri, 21 May 2010 00:29:01 +0200 (CEST) Subject: [pypy-svn] r74612 - pypy/trunk/pypy/translator/tool Message-ID: <20100520222901.8F3D1282B90@codespeak.net> Author: wildchild Date: Fri May 21 00:28:58 2010 New Revision: 74612 Modified: pypy/trunk/pypy/translator/tool/cbuild.py Log: --dynamic-list doesn't exist under FreeBSD. --export-dynamic and --version-script should be used instead and is still compatible with recent versions of binutils. Modified: pypy/trunk/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/trunk/pypy/translator/tool/cbuild.py (original) +++ pypy/trunk/pypy/translator/tool/cbuild.py Fri May 21 00:28:58 2010 @@ -282,7 +282,7 @@ for sym in self.export_symbols: f.write("%s;\n" % (sym,)) f.write("};") - d['link_extra'] += ("-Wl,--dynamic-list=" + str(file_name), ) + d['link_extra'] += ("-Wl,--export-dynamic,--version-script=" + str(file_name), ) f.close() d['export_symbols'] = () return ExternalCompilationInfo(**d) From wildchild at codespeak.net Fri May 21 00:42:28 2010 From: wildchild at codespeak.net (wildchild at codespeak.net) Date: Fri, 21 May 2010 00:42:28 +0200 (CEST) Subject: [pypy-svn] r74613 - pypy/build/bot2/pypybuildbot Message-ID: <20100520224228.CA463282B90@codespeak.net> Author: wildchild Date: Fri May 21 00:42:25 2010 New Revision: 74613 Modified: pypy/build/bot2/pypybuildbot/master.py Log: Add --stackless and nightly scheduling to FreeBSD x86_64 builds. Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Fri May 21 00:42:25 2010 @@ -116,7 +116,7 @@ STACKLESSAPPLVLLINUX32 = "pypy-c-stackless-app-level-linux-x86-32" APPLVLWIN32 = "pypy-c-app-level-win-32" -APPLVLFREEBSD64 = 'pypy-c-app-level-freebsd-7-x86-64' +STACKLESSAPPLVLFREEBSD64 = 'pypy-c-stackless-app-level-freebsd-7-x86-64' JITLINUX32 = "pypy-c-jit-linux-x86-32" OJITLINUX32 = "pypy-c-Ojit-no-jit-linux-x86-32" @@ -134,7 +134,8 @@ Nightly("nightly-first", [LINUX32, LINUX64], hour=4, minute=44), Nightly("nightly", [APPLVLLINUX32, APPLVLLINUX64, APPLVLWIN32, - STACKLESSAPPLVLLINUX32, JITLINUX32, OJITLINUX32, + STACKLESSAPPLVLLINUX32, STACKLESSAPPLVLFREEBSD64, + JITLINUX32, OJITLINUX32, MACOSX32], hour=4, minute=45), Nightly("nightly-benchmark", [JITBENCH], @@ -195,9 +196,9 @@ "factory": pypyTranslatedAppLevelTestFactoryWin, "category": "windows" }, - {"name" : APPLVLFREEBSD64, + {"name" : STACKLESSAPPLVLFREEBSD64, "slavenames": ['headless'], - 'builddir' : APPLVLFREEBSD64, + 'builddir' : STACKLESSAPPLVLFREEBSD64, 'factory' : pypyTranslatedAppLevelTestFactory, "category": 'other' }, From wildchild at codespeak.net Fri May 21 00:49:15 2010 From: wildchild at codespeak.net (wildchild at codespeak.net) Date: Fri, 21 May 2010 00:49:15 +0200 (CEST) Subject: [pypy-svn] r74614 - pypy/build/bot2/pypybuildbot Message-ID: <20100520224915.A6AFF282B90@codespeak.net> Author: wildchild Date: Fri May 21 00:49:13 2010 New Revision: 74614 Modified: pypy/build/bot2/pypybuildbot/master.py Log: Wrong factory... Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Fri May 21 00:49:13 2010 @@ -199,7 +199,7 @@ {"name" : STACKLESSAPPLVLFREEBSD64, "slavenames": ['headless'], 'builddir' : STACKLESSAPPLVLFREEBSD64, - 'factory' : pypyTranslatedAppLevelTestFactory, + 'factory' : pypyStacklessTranslatedAppLevelTestFactory, "category": 'other' }, {"name" : JITLINUX32, From arigo at codespeak.net Fri May 21 11:42:30 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 May 2010 11:42:30 +0200 (CEST) Subject: [pypy-svn] r74617 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100521094230.54ECE282C0D@codespeak.net> Author: arigo Date: Fri May 21 11:42:28 2010 New Revision: 74617 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py Log: Change again the precise meaning of -live-. Now it's hopefully a simple and clear meaning. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py Fri May 21 11:42:28 2010 @@ -49,7 +49,10 @@ return 'L%d' % seenlabels[lbl.name] # output = StringIO() - for asm in ssarepr.insns: + insns = ssarepr.insns + if insns[-1] == ('---',): + insns = insns[:-1] + for asm in insns: if isinstance(asm[0], Label): if asm[0].name in seenlabels: print >> output, '%s:' % getlabelname(asm[0]) @@ -169,7 +172,7 @@ word += c if c in '<([': nested += 1 - if c in '])>' and line[i-2:i+2] != ' -> ': + if c in '])>' and (' '+line)[i:i+4] != ' -> ': nested -= 1 assert nested >= 0 if word: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py Fri May 21 11:42:28 2010 @@ -5,13 +5,11 @@ # Some instructions require liveness information (the ones that can end up # in generate_guard() in pyjitpl.py). This is done by putting special # space operations called '-live-' in the graph. They turn into '-live-' -# operation in the ssarepr. Then this module expands the arguments of -# the '-live-' operations to also include all values that are alive at -# this point: more precisely, all values that are created before the -# '-live-' operation and that are needed afterwards, with the exception -# of the values that are needed only in the very next instruction. These -# are not considered alive any more. You can force them to be alive by -# putting them as args of the '-live-' operation in the first place. +# operation in the ssarepr. Then the present module expands the arguments +# of the '-live-' operations to also include all values that are alive at +# this point (written to before, and read afterwards). You can also force +# extra variables to be alive by putting them as args of the '-live-' +# operation in the first place. # For this to work properly, a special operation called '---' must be # used to mark unreachable places (e.g. just after a 'goto'). @@ -25,7 +23,6 @@ def _compute_liveness_must_continue(ssarepr, label2alive): alive = set() - prevalive = None must_continue = False for i in range(len(ssarepr.insns)-1, -1, -1): @@ -37,21 +34,17 @@ alive_at_point.update(alive) if prevlength != len(alive_at_point): must_continue = True - prevalive = None continue if insn[0] == '-live-': - assert prevalive is not None for x in insn[1:]: if isinstance(x, Register): - prevalive.add(x) - ssarepr.insns[i] = insn[:1] + tuple(prevalive) - prevalive = None + alive.add(x) + ssarepr.insns[i] = insn[:1] + tuple(alive) continue if insn[0] == '---': alive = set() - prevalive = None continue args = insn[1:] @@ -62,13 +55,13 @@ alive.discard(reg) args = args[:-2] # - prevalive = alive.copy() - # for x in args: if isinstance(x, Register): alive.add(x) elif isinstance(x, ListOfKind): - alive.update(x) + for y in x: + if isinstance(y, Register): + alive.add(y) elif isinstance(x, TLabel): alive_at_point = label2alive.get(x.name, ()) alive.update(alive_at_point) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py Fri May 21 11:42:28 2010 @@ -121,6 +121,16 @@ ('foo', Constant(123, lltype.Signed)), ] +def test_unformat_assembler_single_return(): + input = """ + foo -> %i0 + """ + regs = {} + ssarepr = unformat_assembler(input, regs) + assert ssarepr.insns == [ + ('foo', '->', regs['%i0']), + ] + def test_unformat_assembler_label(): input = """ L1: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py Fri May 21 11:42:28 2010 @@ -13,9 +13,11 @@ self.liveness_test(""" -live- int_add %i0, $10 -> %i1 - """, """ -live- + """, """ + -live- %i0 int_add %i0, $10 -> %i1 + -live- """) def test_simple(self): @@ -30,138 +32,137 @@ int_add %i0, $6 -> %i4 -live- int_mul %i3, %i4 -> %i5 + -live- int_return %i5 """, """ -live- %i0 int_add %i0, $10 -> %i1 -live- %i0, %i1 int_add %i0, $3 -> %i2 - -live- %i0 + -live- %i0, %i1, %i2 int_mul %i1, %i2 -> %i3 - -live- %i3 + -live- %i0, %i3 int_add %i0, $6 -> %i4 - -live- + -live- %i3, %i4 int_mul %i3, %i4 -> %i5 + -live- %i5 int_return %i5 """) def test_one_path(self): self.liveness_test(""" - -live- int_add %i0, $5 -> %i2 + -live- int_is_true %i2 -> %i3 goto_if_not %i3, L1 int_copy %i0 -> %i4 - -live- int_add %i4, $1 -> %i5 + -live- int_return %i5 --- L1: int_copy %i1 -> %i6 - -live- int_add %i6, $2 -> %i7 + -live- int_return %i7 """, """ - -live- %i0, %i1 int_add %i0, $5 -> %i2 + -live- %i0, %i1, %i2 int_is_true %i2 -> %i3 goto_if_not %i3, L1 int_copy %i0 -> %i4 - -live- int_add %i4, $1 -> %i5 + -live- %i5 int_return %i5 --- L1: int_copy %i1 -> %i6 - -live- int_add %i6, $2 -> %i7 + -live- %i7 int_return %i7 """) def test_other_path(self): self.liveness_test(""" - -live- int_add %i0, $5 -> %i2 + -live- %i2 int_is_true %i2 -> %i3 goto_if_not %i3, L1 int_copy %i0 -> %i4 int_copy %i1 -> %i5 - -live- int_add %i4, %i5 -> %i6 + -live- %i6 int_return %i6 --- L1: int_copy %i0 -> %i7 - -live- int_add %i7, $2 -> %i8 + -live- %i8 int_return %i8 """, """ - -live- %i0, %i1 int_add %i0, $5 -> %i2 + -live- %i0, %i1, %i2 int_is_true %i2 -> %i3 goto_if_not %i3, L1 int_copy %i0 -> %i4 int_copy %i1 -> %i5 - -live- int_add %i4, %i5 -> %i6 + -live- %i6 int_return %i6 --- L1: int_copy %i0 -> %i7 - -live- int_add %i7, $2 -> %i8 + -live- %i8 int_return %i8 """) def test_no_path(self): self.liveness_test(""" - -live- int_add %i0, %i1 -> %i2 + -live- %i2 int_is_true %i2 -> %i3 goto_if_not %i3, L1 int_copy %i0 -> %i4 - -live- int_add %i4, $5 -> %i5 + -live- %i5 int_return %i5 --- L1: int_copy %i0 -> %i6 - -live- int_add %i6, $2 -> %i7 + -live- %i7 int_return %i7 """, """ - -live- %i0 int_add %i0, %i1 -> %i2 + -live- %i0, %i2 int_is_true %i2 -> %i3 goto_if_not %i3, L1 int_copy %i0 -> %i4 - -live- int_add %i4, $5 -> %i5 + -live- %i5 int_return %i5 --- L1: int_copy %i0 -> %i6 - -live- int_add %i6, $2 -> %i7 + -live- %i7 int_return %i7 """) def test_list_of_kind(self): self.liveness_test(""" -live- - stub - foobar F[%f0] + foobar I[$25, %i0] """, """ - -live- %f0 - stub - foobar F[%f0] + -live- %i0 + foobar I[$25, %i0] """) def test_switch(self): self.liveness_test(""" goto_maybe L1 -live- - stub fooswitch --- L3: @@ -175,7 +176,6 @@ """, """ goto_maybe L1 -live- %i3, %i7 - stub fooswitch --- L3: @@ -190,11 +190,30 @@ def test_already_some(self): self.liveness_test(""" - -live- %i0, $52, %i2, %i0 foo %i0, %i1, %i2 + -live- %i0, $52, %i2, %i0 bar %i3, %i4, %i5 """, """ - -live- %i0, %i2, %i3, %i4, %i5 foo %i0, %i1, %i2 + -live- %i0, %i2, %i3, %i4, %i5 bar %i3, %i4, %i5 """) + + def test_keepalive(self): + self.liveness_test(""" + -live- + build $1 -> %i6 + -live- + foo %i0, %i1 -> %i2 + -live- + bar %i3, %i2 -> %i5 + -live- %i6 + """, """ + -live- %i0, %i1, %i3 + build $1 -> %i6 + -live- %i0, %i1, %i3, %i6 + foo %i0, %i1 -> %i2 + -live- %i2, %i3, %i6 + bar %i3, %i2 -> %i5 + -live- %i6 + """) From arigo at codespeak.net Fri May 21 11:47:57 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 May 2010 11:47:57 +0200 (CEST) Subject: [pypy-svn] r74618 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100521094757.F2ED1282C0D@codespeak.net> Author: arigo Date: Fri May 21 11:47:56 2010 New Revision: 74618 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Log: Update to the new meaning of -live-. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Fri May 21 11:47:56 2010 @@ -206,15 +206,15 @@ rewrite_op_float_ge = _rewrite_symmetric def rewrite_op_int_add_ovf(self, op): - op0 = SpaceOperation('-live-', op.args[:], None) - op1 = self._rewrite_symmetric(op) + op0 = self._rewrite_symmetric(op) + op1 = SpaceOperation('-live-', [], None) return [op0, op1] rewrite_op_int_mul_ovf = rewrite_op_int_add_ovf def rewrite_op_int_sub_ovf(self, op): - op0 = SpaceOperation('-live-', op.args[:], None) - return [op0, op] + op1 = SpaceOperation('-live-', [], None) + return [op, op1] # ---------- # Various kinds of calls @@ -270,7 +270,7 @@ op1 = self.rewrite_call(op, 'residual_call', [op.args[0], calldescr] + extraargs) if self.callcontrol.calldescr_canraise(calldescr): - op1 = [SpaceOperation('-live-', [], None), op1] + op1 = [op1, SpaceOperation('-live-', [], None)] return op1 def handle_regular_call(self, op): @@ -280,8 +280,8 @@ [targetgraph] = self.callcontrol.graphs_from(op) jitcode = self.callcontrol.get_jitcode(targetgraph, called_from=self.graph) - op0 = SpaceOperation('-live-', [], None) - op1 = self.rewrite_call(op, 'inline_call', [jitcode]) + op0 = self.rewrite_call(op, 'inline_call', [jitcode]) + op1 = SpaceOperation('-live-', [], None) return [op0, op1] def handle_builtin_call(self, op): @@ -298,8 +298,8 @@ jitcode = self.callcontrol.get_jitcode(targetgraph, called_from=self.graph) lst.append(jitcode) - op0 = SpaceOperation('-live-', [op.args[0]], None) - op1 = SpaceOperation('int_guard_value', [op.args[0]], None) + op0 = SpaceOperation('int_guard_value', [op.args[0]], None) + op1 = SpaceOperation('-live-', [op.args[0]], None) op2 = self.handle_residual_call(op, [IndirectCallTargets(lst)]) result = [op0, op1] if isinstance(op2, list): @@ -345,8 +345,8 @@ if hints.get('promote') and op.args[0].concretetype is not lltype.Void: assert op.args[0].concretetype != lltype.Ptr(rstr.STR) kind = getkind(op.args[0].concretetype) - op0 = SpaceOperation('-live-', [op.args[0]], None) - op1 = SpaceOperation('%s_guard_value' % kind, [op.args[0]], None) + op0 = SpaceOperation('%s_guard_value' % kind, [op.args[0]], None) + op1 = SpaceOperation('-live-', [op.args[0]], None) # the special return value None forces op.result to be considered # equal to op.args[0] return [op0, op1, None] @@ -463,8 +463,8 @@ op.args[0].concretetype.TO._hints.get('typeptr')) def handle_getfield_typeptr(self, op): - op0 = SpaceOperation('-live-', [op.args[0]], None) - op1 = SpaceOperation('guard_class', [op.args[0]], op.result) + op0 = SpaceOperation('guard_class', [op.args[0]], op.result) + op1 = SpaceOperation('-live-', [op.args[0]], None) return [op0, op1] def rewrite_op_malloc(self, op): @@ -656,9 +656,9 @@ for v in op.args[2:2+num_green_args]: if isinstance(v, Variable) and v.concretetype is not lltype.Void: kind = getkind(v.concretetype) - ops.append(SpaceOperation('-live-', [v], None)) ops.append(SpaceOperation('%s_guard_value' % kind, [v], None)) + ops.append(SpaceOperation('-live-', [v], None)) args = (self.make_three_lists(op.args[2:2+num_green_args]) + self.make_three_lists(op.args[2+num_green_args:])) ops.append(SpaceOperation('jit_merge_point', args, None)) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Fri May 21 11:47:56 2010 @@ -180,16 +180,16 @@ op = SpaceOperation('int_add_nonneg_ovf', [v1, v2], v3) oplist = Transformer(FakeCPU()).rewrite_operation(op) op0, op1 = oplist - assert op0.opname == '-live-' - assert op0.args == [v1, v2] - assert op0.result is None - assert op1.opname == 'int_add_ovf' + assert op0.opname == 'int_add_ovf' if isinstance(v1, Constant) and isinstance(v2, Variable): - assert op1.args == [v2, v1] - assert op1.result == v3 + assert op0.args == [v2, v1] + assert op0.result == v3 else: - assert op1.args == [v1, v2] - assert op1.result == v3 + assert op0.args == [v1, v2] + assert op0.result == v3 + assert op1.opname == '-live-' + assert op1.args == [] + assert op1.result is None def test_calls(): for RESTYPE in [lltype.Signed, rclass.OBJECTPTR, @@ -227,21 +227,21 @@ tr = Transformer(FakeCPU(), FakeResidualCallControl()) oplist = tr.rewrite_operation(op) op0, op1 = oplist - assert op0.opname == '-live-' - assert op0.args == [] reskind = getkind(restype)[0] - assert op1.opname == 'residual_call_%s_%s' % (expectedkind, reskind) - assert op1.result == op.result - assert op1.args[0] == op.args[0] - assert op1.args[1] == 'calldescr' - assert len(op1.args) == 2 + len(expectedkind) - for sublist, kind1 in zip(op1.args[2:], expectedkind): + assert op0.opname == 'residual_call_%s_%s' % (expectedkind, reskind) + assert op0.result == op.result + assert op0.args[0] == op.args[0] + assert op0.args[1] == 'calldescr' + assert len(op0.args) == 2 + len(expectedkind) + for sublist, kind1 in zip(op0.args[2:], expectedkind): assert sublist.kind.startswith(kind1) assert list(sublist) == [v for v in op.args[1:] if getkind(v.concretetype) == sublist.kind] for v in op.args[1:]: kind = getkind(v.concretetype) assert kind == 'void' or kind[0] in expectedkind + assert op1.opname == '-live-' + assert op1.args == [] def direct_call_test(argtypes, restype, expectedkind): op = get_direct_call_op(argtypes, restype) @@ -249,20 +249,20 @@ tr.graph = 'someinitialgraph' oplist = tr.rewrite_operation(op) op0, op1 = oplist - assert op0.opname == '-live-' - assert op0.args == [] reskind = getkind(restype)[0] - assert op1.opname == 'inline_call_%s_%s' % (expectedkind, reskind) - assert op1.result == op.result - assert op1.args[0] == 'somejitcode' - assert len(op1.args) == 1 + len(expectedkind) - for sublist, kind1 in zip(op1.args[1:], expectedkind): + assert op0.opname == 'inline_call_%s_%s' % (expectedkind, reskind) + assert op0.result == op.result + assert op0.args[0] == 'somejitcode' + assert len(op0.args) == 1 + len(expectedkind) + for sublist, kind1 in zip(op0.args[1:], expectedkind): assert sublist.kind.startswith(kind1) assert list(sublist) == [v for v in op.args[1:] if getkind(v.concretetype) == sublist.kind] for v in op.args[1:]: kind = getkind(v.concretetype) assert kind == 'void' or kind[0] in expectedkind + assert op1.opname == '-live-' + assert op1.args == [] def indirect_residual_call_test(argtypes, restype, expectedkind): # an indirect call that is residual in all cases is very similar to @@ -275,21 +275,21 @@ tr.graph = 'someinitialgraph' oplist = tr.rewrite_operation(op) op0, op1 = oplist - assert op0.opname == '-live-' - assert op0.args == [] reskind = getkind(restype)[0] - assert op1.opname == 'residual_call_%s_%s' % (expectedkind, reskind) - assert op1.result == op.result - assert op1.args[0] == op.args[0] - assert op1.args[1] == 'calldescr' - assert len(op1.args) == 2 + len(expectedkind) - for sublist, kind1 in zip(op1.args[2:], expectedkind): + assert op0.opname == 'residual_call_%s_%s' % (expectedkind, reskind) + assert op0.result == op.result + assert op0.args[0] == op.args[0] + assert op0.args[1] == 'calldescr' + assert len(op0.args) == 2 + len(expectedkind) + for sublist, kind1 in zip(op0.args[2:], expectedkind): assert sublist.kind.startswith(kind1) assert list(sublist) == [v for v in op.args[1:] if getkind(v.concretetype)==sublist.kind] for v in op.args[1:]: kind = getkind(v.concretetype) assert kind == 'void' or kind[0] in expectedkind + assert op1.opname == '-live-' + assert op1.args == [] def indirect_regular_call_test(argtypes, restype, expectedkind): # a regular indirect call is preceded by a guard_value on the @@ -303,29 +303,29 @@ tr.graph = 'someinitialgraph' oplist = tr.rewrite_operation(op) op0gv, op1gv, op0, op1 = oplist - assert op0gv.opname == '-live-' + assert op0gv.opname == 'int_guard_value' assert op0gv.args == [op.args[0]] - assert op1gv.opname == 'int_guard_value' + assert op0gv.result is None + assert op1gv.opname == '-live-' assert op1gv.args == [op.args[0]] - assert op1gv.result is None # - assert op0.opname == '-live-' - assert op0.args == [] reskind = getkind(restype)[0] - assert op1.opname == 'residual_call_%s_%s' % (expectedkind, reskind) - assert op1.result == op.result - assert op1.args[0] == op.args[0] - assert op1.args[1] == 'calldescr' - assert isinstance(op1.args[2], IndirectCallTargets) - assert op1.args[2].lst == ['somejitcode1', 'somejitcode2'] - assert len(op1.args) == 3 + len(expectedkind) - for sublist, kind1 in zip(op1.args[3:], expectedkind): + assert op0.opname == 'residual_call_%s_%s' % (expectedkind, reskind) + assert op0.result == op.result + assert op0.args[0] == op.args[0] + assert op0.args[1] == 'calldescr' + assert isinstance(op0.args[2], IndirectCallTargets) + assert op0.args[2].lst == ['somejitcode1', 'somejitcode2'] + assert len(op0.args) == 3 + len(expectedkind) + for sublist, kind1 in zip(op0.args[3:], expectedkind): assert sublist.kind.startswith(kind1) assert list(sublist) == [v for v in op.args[1:] if getkind(v.concretetype)==sublist.kind] for v in op.args[1:]: kind = getkind(v.concretetype) assert kind == 'void' or kind[0] in expectedkind + assert op1.opname == '-live-' + assert op1.args == [] def test_getfield(): # XXX a more compact encoding would be possible, something along @@ -364,11 +364,11 @@ op = SpaceOperation('getfield', [v_parent, c_name], v_result) oplist = Transformer(FakeCPU()).rewrite_operation(op) op0, op1 = oplist - assert op0.opname == '-live-' + assert op0.opname == 'guard_class' assert op0.args == [v_parent] - assert op1.opname == 'guard_class' + assert op0.result == v_result + assert op1.opname == '-live-' assert op1.args == [v_parent] - assert op1.result == v_result def test_setfield(): # XXX a more compact encoding would be possible; see test_getfield() @@ -436,11 +436,11 @@ tr = Transformer(FakeCPU(), FakeResidualCallControl()) oplist = tr.rewrite_operation(op) op0, op1 = oplist - assert op0.opname == '-live-' - assert op0.args == [] - assert op1.opname == 'residual_call_r_r' - assert op1.args[0].value == 'alloc_with_del' # pseudo-function as a str - assert list(op1.args[2]) == [] + assert op0.opname == 'residual_call_r_r' + assert op0.args[0].value == 'alloc_with_del' # pseudo-function as a str + assert list(op0.args[2]) == [] + assert op1.opname == '-live-' + assert op1.args == [] def test_rename_on_links(): v1 = Variable() @@ -615,11 +615,11 @@ v2) oplist = Transformer().rewrite_operation(op) op0, op1, op2 = oplist - assert op0.opname == '-live-' + assert op0.opname == 'int_guard_value' assert op0.args == [v1] - assert op1.opname == 'int_guard_value' + assert op0.result is None + assert op1.opname == '-live-' assert op1.args == [v1] - assert op1.result is None assert op2 is None def test_promote_2(): @@ -635,9 +635,9 @@ block.closeblock(Link([v2], returnblock)) Transformer().optimize_block(block) assert len(block.operations) == 2 - assert block.operations[0].opname == '-live-' + assert block.operations[0].opname == 'int_guard_value' assert block.operations[0].args == [v1] - assert block.operations[1].opname == 'int_guard_value' + assert block.operations[0].result is None + assert block.operations[1].opname == '-live-' assert block.operations[1].args == [v1] - assert block.operations[1].result is None assert block.exits[0].args == [v1] From afa at codespeak.net Fri May 21 12:02:37 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 21 May 2010 12:02:37 +0200 (CEST) Subject: [pypy-svn] r74619 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100521100237.AFA6C282C0D@codespeak.net> Author: afa Date: Fri May 21 12:02:36 2010 New Revision: 74619 Modified: pypy/trunk/pypy/module/cpyext/object.py pypy/trunk/pypy/module/cpyext/stubsactive.py pypy/trunk/pypy/module/cpyext/test/test_object.py Log: implement PyObject_Print() Modified: pypy/trunk/pypy/module/cpyext/object.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/object.py (original) +++ pypy/trunk/pypy/module/cpyext/object.py Fri May 21 12:02:36 2010 @@ -368,3 +368,31 @@ sizep[0] = size return 0 +FILEP = rffi.COpaquePtr('FILE') +fwrite = rffi.llexternal('fwrite', + [rffi.VOIDP, rffi.SIZE_T, rffi.SIZE_T, FILEP], + rffi.SIZE_T) + +# Also in include/object.h +Py_PRINT_RAW = 1 # No string quotes etc. + + at cpython_api([PyObject, FILEP, rffi.INT_real], rffi.INT_real, error=-1) +def PyObject_Print(space, w_obj, fp, flags): + """Print an object o, on file fp. Returns -1 on error. The flags argument + is used to enable certain printing options. The only option currently + supported is Py_PRINT_RAW; if given, the str() of the object is written + instead of the repr().""" + if rffi.cast(lltype.Signed, flags) & Py_PRINT_RAW: + w_str = space.str(w_obj) + else: + w_str = space.repr(w_obj) + + count = space.int_w(space.len(w_str)) + data = space.str_w(w_str) + buf = rffi.get_nonmovingbuffer(data) + try: + fwrite(buf, 1, count, fp) + finally: + rffi.free_nonmovingbuffer(data, buf) + return 0 + Modified: pypy/trunk/pypy/module/cpyext/stubsactive.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/stubsactive.py (original) +++ pypy/trunk/pypy/module/cpyext/stubsactive.py Fri May 21 12:02:36 2010 @@ -15,14 +15,6 @@ PyFile_DecUseCount() functions described below as appropriate.""" raise NotImplementedError - at cpython_api([PyObject, FILEP, rffi.INT_real], rffi.INT_real, error=-1) -def PyObject_Print(space, o, fp, flags): - """Print an object o, on file fp. Returns -1 on error. The flags argument - is used to enable certain printing options. The only option currently supported - is Py_PRINT_RAW; if given, the str() of the object is written - instead of the repr().""" - raise NotImplementedError - @cpython_api([PyInterpreterState], PyThreadState, error=CANNOT_FAIL) def PyThreadState_New(space, interp): """Create a new thread state object belonging to the given interpreter object. Modified: pypy/trunk/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_object.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_object.py Fri May 21 12:02:36 2010 @@ -1,9 +1,11 @@ import py from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import Py_LT, Py_LE, Py_NE, Py_EQ,\ Py_GE, Py_GT +from pypy.tool.udir import udir class TestObject(BaseApiTest): def test_IsTrue(self, space, api): @@ -178,3 +180,28 @@ assert space.unwrap(api.PyObject_Unicode(space.wrap("e"))) == u"e" assert api.PyObject_Unicode(space.wrap("\xe9")) is None api.PyErr_Clear() + +class AppTestObjectPrint(AppTestCpythonExtensionBase): + def setup_class(cls): + AppTestCpythonExtensionBase.setup_class.im_func(cls) + cls.w_tmpname = cls.space.wrap(str(py.test.ensuretemp("out", dir=0))) + + def test_print(self): + module = self.import_extension('foo', [ + ("dump", "METH_VARARGS", + """ + PyObject *fname = PyTuple_GetItem(args, 0); + PyObject *obj = PyTuple_GetItem(args, 1); + + FILE *fp = fopen(PyString_AsString(fname), "wb"); + int ret; + if (fp == NULL) + Py_RETURN_NONE; + ret = PyObject_Print(obj, fp, Py_PRINT_RAW); + fclose(fp); + if (ret < 0) + return NULL; + Py_RETURN_TRUE; + """)]) + assert module.dump(self.tmpname, None) + assert open(self.tmpname).read() == 'None' From afa at codespeak.net Fri May 21 13:17:23 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 21 May 2010 13:17:23 +0200 (CEST) Subject: [pypy-svn] r74621 - in pypy/trunk/pypy/module/posix: . test Message-ID: <20100521111723.D2910282C0D@codespeak.net> Author: afa Date: Fri May 21 13:17:22 2010 New Revision: 74621 Modified: pypy/trunk/pypy/module/posix/app_posix.py pypy/trunk/pypy/module/posix/test/test_posix2.py Log: issue540 resolved: copy file.fdopen at import time, because setuptools replaces __builtins__.file with its own function. Modified: pypy/trunk/pypy/module/posix/app_posix.py ============================================================================== --- pypy/trunk/pypy/module/posix/app_posix.py (original) +++ pypy/trunk/pypy/module/posix/app_posix.py Fri May 21 13:17:22 2010 @@ -62,13 +62,16 @@ if self.st_ctime is None: self.__dict__['st_ctime'] = self[9] +# Capture file.fdopen at import time, as some code replaces +# __builtins__.file with a custom function. +_fdopen = file.fdopen def fdopen(fd, mode='r', buffering=-1): """fdopen(fd [, mode='r' [, buffering]]) -> file_object Return an open file object connected to a file descriptor.""" - return file.fdopen(fd, mode, buffering) + return _fdopen(fd, mode, buffering) def tmpfile(): Modified: pypy/trunk/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/trunk/pypy/module/posix/test/test_posix2.py (original) +++ pypy/trunk/pypy/module/posix/test/test_posix2.py Fri May 21 13:17:22 2010 @@ -249,6 +249,19 @@ f = posix.fdopen(fd, "r") f.close() + def test_fdopen_hackedbuiltins(self): + "Same test, with __builtins__.file removed" + _file = __builtins__.file + __builtins__.file = None + try: + path = self.path + posix = self.posix + fd = posix.open(path, posix.O_RDONLY, 0777) + f = posix.fdopen(fd, "r") + f.close() + finally: + __builtins__.file = _file + def test_getcwd(self): assert isinstance(self.posix.getcwd(), str) assert isinstance(self.posix.getcwdu(), unicode) From arigo at codespeak.net Fri May 21 15:08:13 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 May 2010 15:08:13 +0200 (CEST) Subject: [pypy-svn] r74626 - pypy/trunk/pypy/config Message-ID: <20100521130813.49C2C282C0D@codespeak.net> Author: arigo Date: Fri May 21 15:08:11 2010 New Revision: 74626 Modified: pypy/trunk/pypy/config/translationoption.py Log: WildChild reports that --gcremovetypeptr is incompatible with --stackless. Modified: pypy/trunk/pypy/config/translationoption.py ============================================================================== --- pypy/trunk/pypy/config/translationoption.py (original) +++ pypy/trunk/pypy/config/translationoption.py Fri May 21 15:08:11 2010 @@ -23,7 +23,8 @@ "translation", "Translation Options", [ BoolOption("stackless", "enable stackless features during compilation", default=False, cmdline="--stackless", - requires=[("translation.type_system", "lltype")]), + requires=[("translation.type_system", "lltype"), + ("translation.gcremovetypeptr", False)]), # XXX? ChoiceOption("type_system", "Type system to use when RTyping", ["lltype", "ootype"], cmdline=None, default="lltype", requires={ From wildchild at codespeak.net Fri May 21 15:19:03 2010 From: wildchild at codespeak.net (wildchild at codespeak.net) Date: Fri, 21 May 2010 15:19:03 +0200 (CEST) Subject: [pypy-svn] r74627 - pypy/branch/build-problems-x86_64-stackless Message-ID: <20100521131903.2DDA3282C0D@codespeak.net> Author: wildchild Date: Fri May 21 15:19:01 2010 New Revision: 74627 Removed: pypy/branch/build-problems-x86_64-stackless/ Log: --stackless problems on x86_64 solved by r74626. From arigo at codespeak.net Fri May 21 15:31:36 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 May 2010 15:31:36 +0200 (CEST) Subject: [pypy-svn] r74628 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100521133136.9A3B4282C0D@codespeak.net> Author: arigo Date: Fri May 21 15:31:34 2010 New Revision: 74628 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py Log: Allow -live- operations to have a label too. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py Fri May 21 15:31:34 2010 @@ -25,6 +25,10 @@ alive = set() must_continue = False + def follow_label(lbl): + alive_at_point = label2alive.get(lbl.name, ()) + alive.update(alive_at_point) + for i in range(len(ssarepr.insns)-1, -1, -1): insn = ssarepr.insns[i] @@ -37,10 +41,14 @@ continue if insn[0] == '-live-': + labels = [] for x in insn[1:]: if isinstance(x, Register): alive.add(x) - ssarepr.insns[i] = insn[:1] + tuple(alive) + elif isinstance(x, TLabel): + follow_label(x) + labels.append(x) + ssarepr.insns[i] = insn[:1] + tuple(alive) + tuple(labels) continue if insn[0] == '---': @@ -63,11 +71,9 @@ if isinstance(y, Register): alive.add(y) elif isinstance(x, TLabel): - alive_at_point = label2alive.get(x.name, ()) - alive.update(alive_at_point) + follow_label(x) elif isinstance(x, SwitchDictDescr): for key, label in x._labels: - alive_at_point = label2alive.get(label.name, ()) - alive.update(alive_at_point) + follow_label(label) return must_continue Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py Fri May 21 15:31:34 2010 @@ -217,3 +217,18 @@ bar %i3, %i2 -> %i5 -live- %i6 """) + + def test_live_with_label(self): + self.liveness_test(""" + -live- L1 + foo %i0 + --- + L1: + bar %i1 + """, """ + -live- %i0, %i1, L1 + foo %i0 + --- + L1: + bar %i1 + """) From arigo at codespeak.net Fri May 21 15:46:59 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 May 2010 15:46:59 +0200 (CEST) Subject: [pypy-svn] r74629 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter codewriter/test metainterp Message-ID: <20100521134659.2FF42282C0D@codespeak.net> Author: arigo Date: Fri May 21 15:46:57 2010 New Revision: 74629 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py Log: Finish the refactoring of codewriter to the new format of -live- operations. Now starting to work on pyjitpl... Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Fri May 21 15:46:57 2010 @@ -45,7 +45,6 @@ self.switchdictdescrs = [] self.count_regs = dict.fromkeys(KINDS, 0) self.liveness = {} - self.nextlive = None self.startpoints = set() self.alllabels = set() @@ -88,21 +87,18 @@ return False def write_insn(self, insn): + if insn[0] == '---': + return if isinstance(insn[0], Label): self.label_positions[insn[0].name] = len(self.code) return if insn[0] == '-live-': - assert self.nextlive is None - self.nextlive = ( - self.get_liveness_info(insn, 'int'), - self.get_liveness_info(insn, 'ref'), - self.get_liveness_info(insn, 'float')) - return - if insn[0] == 'keepalive': - # The 'keepalive' instruction is useful when doing register - # allocation, to ensure that the register holding the value is - # not overwritten too early. But it does not need to be - # present at all in the final code. + key = len(self.code) + live_i, live_r, live_f = self.liveness.get(key, ("", "", "")) + live_i = self.get_liveness_info(live_i, insn[1:], 'int') + live_r = self.get_liveness_info(live_r, insn[1:], 'ref') + live_f = self.get_liveness_info(live_f, insn[1:], 'float') + self.liveness[key] = live_i, live_r, live_f return startposition = len(self.code) self.code.append("temporary placeholder") @@ -154,24 +150,29 @@ argcodes.append('d') elif isinstance(x, IndirectCallTargets): self.indirectcalltargets.update(x.lst) + elif x == '->': + assert '>' not in argcodes + argcodes.append('>') else: raise NotImplementedError(x) # opname = insn[0] - if opname.startswith('G_'): opname = opname[2:] + assert '>' not in argcodes or argcodes.index('>') == len(argcodes) - 2 key = opname + '/' + ''.join(argcodes) num = self.insns.setdefault(key, len(self.insns)) self.code[startposition] = chr(num) self.startpoints.add(startposition) - # - if self.nextlive is not None: - self.liveness[len(self.code)] = self.nextlive - self.nextlive = None - - def get_liveness_info(self, insn, kind): - lives = [chr(reg.index) for reg in insn[1:] if reg.kind == kind] - lives.sort() - return ''.join(lives) + + def get_liveness_info(self, prevstring, args, kind): + """Return a string whose characters are register numbers. + We sort the numbers, too, to increase the chances of duplicate + strings (which are collapsed into a single string during translation). + """ + lives = set(prevstring) # set of characters + for reg in args: + if isinstance(reg, Register) and reg.kind == kind: + lives.add(chr(reg.index)) + return ''.join(sorted(lives)) def fix_labels(self): for name, pos in self.tlabel_positions: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py Fri May 21 15:46:57 2010 @@ -38,20 +38,20 @@ # that we want in the JitCode, but still as a control flow graph transform_graph(graph, self.cpu, self.callcontrol, portal) # - # step 2a: perform register allocation on it + # step 2: perform register allocation on it regallocs = {} for kind in KINDS: regallocs[kind] = perform_register_allocation(graph, kind) # - # step 2b: compute the liveness around certain operations - compute_liveness(graph) - # # step 3: flatten the graph to produce human-readable "assembler", # which means mostly producing a linear list of operations and # inserting jumps or conditional jumps. This is a list of tuples # of the shape ("opname", arg1, ..., argN) or (Label(...),). ssarepr = flatten_graph(graph, regallocs) # + # step 3b: compute the liveness around certain operations + compute_liveness(ssarepr) + # # print the resulting assembler self.print_ssa_repr(ssarepr, portal, verbose) # Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Fri May 21 15:46:57 2010 @@ -168,14 +168,18 @@ elif block.exitswitch is c_last_exception: # An exception block. See test_exc_exitswitch in test_flatten.py # for an example of what kind of code this makes. - lastopname = block.operations[-1].opname - assert lastopname != '-live-' + index = -1 + while True: + lastopname = block.operations[index].opname + if lastopname != '-live-': + break + index -= 1 assert block.exits[0].exitcase is None # is this always True? # if not self._include_all_exc_links: - if (len(block.operations) == 1 or - block.operations[-2].opname != '-live-'): - # cannot actually raise + if index == -1: + # cannot raise: the last instruction is not + # actually a '-live-' self.make_link(block.exits[0]) return # @@ -212,8 +216,7 @@ opname = 'goto_if_not' if isinstance(block.exitswitch, tuple): # special case produced by jtransform.optimize_goto_if_not() - if block.exitswitch[0] != 'int_is_true': - opname = 'goto_if_not_' + block.exitswitch[0] + opname = 'goto_if_not_' + block.exitswitch[0] opargs = block.exitswitch[1:] else: assert block.exitswitch.concretetype == lltype.Bool @@ -221,6 +224,7 @@ # lst = self.flatten_list(opargs) + [TLabel(linkfalse)] self.emitline(opname, *lst) + self.emitline('-live-', TLabel(linkfalse)) # true path: self.make_link(linktrue) # false path: @@ -236,6 +240,7 @@ else: self.emitline("---") # + self.emitline('-live-') switches = [link for link in block.exits if link.exitcase != 'default'] switches.sort(key=lambda link: link.llexitcase) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py Fri May 21 15:46:57 2010 @@ -50,7 +50,7 @@ # output = StringIO() insns = ssarepr.insns - if insns[-1] == ('---',): + if insns and insns[-1] == ('---',): insns = insns[:-1] for asm in insns: if isinstance(asm[0], Label): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Fri May 21 15:46:57 2010 @@ -580,6 +580,7 @@ ('cast_float_to_uint', 'cast_float_to_int'), ('int_add_nonneg_ovf', 'int_add_ovf'), + ('keepalive', '-live-'), ('char_lt', 'int_lt'), ('char_le', 'int_le'), Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py Fri May 21 15:46:57 2010 @@ -12,14 +12,14 @@ ssarepr = SSARepr("test") i0, i1, i2 = Register('int', 0), Register('int', 1), Register('int', 2) ssarepr.insns = [ - ('int_add', i0, i1, i2), + ('int_add', i0, i1, '->', i2), ('int_return', i2), ] assembler = Assembler() jitcode = assembler.assemble(ssarepr) assert jitcode.code == ("\x00\x00\x01\x02" "\x01\x02") - assert assembler.insns == {'int_add/iii': 0, + assert assembler.insns == {'int_add/ii>i': 0, 'int_return/i': 1} assert jitcode.num_regs_i() == 3 assert jitcode.num_regs_r() == 0 @@ -94,8 +94,8 @@ ssarepr.insns = [ (Label('L1'),), ('goto_if_not_int_gt', i0, Constant(4, lltype.Signed), TLabel('L2')), - ('int_add', i1, i0, i1), - ('int_sub', i0, Constant(1, lltype.Signed), i0), + ('int_add', i1, i0, '->', i1), + ('int_sub', i0, Constant(1, lltype.Signed), '->', i0), ('goto', TLabel('L1')), (Label('L2'),), ('int_return', i1), @@ -108,8 +108,8 @@ "\x03\x00\x00" "\x04\x17") assert assembler.insns == {'goto_if_not_int_gt/icL': 0, - 'int_add/iii': 1, - 'int_sub/ici': 2, + 'int_add/ii>i': 1, + 'int_sub/ic>i': 2, 'goto/L': 3, 'int_return/i': 4} @@ -149,13 +149,6 @@ assembler.assemble(ssarepr) assert assembler.indirectcalltargets == set(lst1).union(lst2) -def test_assemble_keepalive(): - ssarepr = SSARepr("test") - ssarepr.insns = [('keepalive', Register('ref', 2))] - assembler = Assembler() - jitcode = assembler.assemble(ssarepr) - assert jitcode.code == "" # the keepalive is removed - def test_num_regs(): assembler = Assembler() ssarepr = SSARepr("test") @@ -176,37 +169,20 @@ def test_liveness(): ssarepr = SSARepr("test") i0, i1, i2 = Register('int', 0), Register('int', 1), Register('int', 2) - i3, i4, i5 = Register('int', 3), Register('int', 4), Register('int', 5) ssarepr.insns = [ - ('-live-', i0), - ('G_int_add', i0, Constant(10, lltype.Signed), i1), + ('int_add', i0, Constant(10, lltype.Signed), '->', i1), ('-live-', i0, i1), - ('G_int_add', i0, Constant(3, lltype.Signed), i2), - ('-live-', i0), - ('G_int_mul', i1, i2, i3), - ('-live-', i3), - ('G_int_add', i0, Constant(6, lltype.Signed), i4), - ('-live-',), - ('G_int_mul', i3, i4, i5), - ('int_return', i5), + ('-live-', i1, i2), + ('int_add', i0, Constant(3, lltype.Signed), '->', i2), + ('-live-', i2), ] assembler = Assembler() jitcode = assembler.assemble(ssarepr) assert jitcode.code == ("\x00\x00\x0A\x01" # ends at 4 - "\x00\x00\x03\x02" # ends at 8 - "\x01\x01\x02\x03" # ends at 12 - "\x00\x00\x06\x04" # ends at 16 - "\x01\x03\x04\x05" # ends at 20 - "\x02\x05") - assert assembler.insns == {'int_add/ici': 0, - 'int_mul/iii': 1, - 'int_return/i': 2} - py.test.raises(MissingLiveness, jitcode._live_vars, 0) - py.test.raises(MissingLiveness, jitcode._live_vars, 3) - py.test.raises(MissingLiveness, jitcode._live_vars, 5) - py.test.raises(MissingLiveness, jitcode._live_vars, 24) - assert jitcode._live_vars(4) == '%i0' - assert jitcode._live_vars(8) == '%i0 %i1' - assert jitcode._live_vars(12) == '%i0' - assert jitcode._live_vars(16) == '%i3' - assert jitcode._live_vars(20) == '' + "\x00\x00\x03\x02") # ends at 8 + assert assembler.insns == {'int_add/ic>i': 0} + for i in range(8): + if i != 4: + py.test.raises(MissingLiveness, jitcode._live_vars, i) + assert jitcode._live_vars(4) == '%i0 %i1 %i2' + assert jitcode._live_vars(8) == '%i2' Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py Fri May 21 15:46:57 2010 @@ -36,8 +36,8 @@ "\x03\x00\x00" "\x04\x01") assert cw.assembler.insns == {'goto_if_not_int_gt/icL': 0, - 'int_add/iii': 1, - 'int_sub/ici': 2, + 'int_add/ii>i': 1, + 'int_sub/ic>i': 2, 'goto/L': 3, 'int_return/i': 4} assert jitcode.num_regs_i() == 2 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Fri May 21 15:46:57 2010 @@ -95,18 +95,18 @@ if transform: from pypy.jit.codewriter.jtransform import transform_graph transform_graph(graphs[0], FakeCPU(), FakeCallControl()) - if liveness: - from pypy.jit.codewriter.liveness import compute_liveness - compute_liveness(graphs[0]) ssarepr = flatten_graph(graphs[0], fake_regallocs(), _include_all_exc_links=not transform) + if liveness: + from pypy.jit.codewriter.liveness import compute_liveness + compute_liveness(ssarepr) assert_format(ssarepr, expected) def test_simple(self): def f(n): return n + 10 self.encoding_test(f, [5], """ - int_add %i0, $10, %i1 + int_add %i0, $10 -> %i1 int_return %i1 """) @@ -117,18 +117,20 @@ a -= 1 return b self.encoding_test(f, [5, 6], """ - int_copy %i0, %i2 - int_copy %i1, %i3 + int_copy %i0 -> %i2 + int_copy %i1 -> %i3 L1: - int_gt %i2, $0, %i4 + int_gt %i2, $0 -> %i4 goto_if_not %i4, L2 - int_copy %i2, %i5 - int_copy %i3, %i6 - int_add %i6, %i5, %i7 - int_sub %i5, $1, %i8 - int_copy %i8, %i2 - int_copy %i7, %i3 + -live- L2 + int_copy %i2 -> %i5 + int_copy %i3 -> %i6 + int_add %i6, %i5 -> %i7 + int_sub %i5, $1 -> %i8 + int_copy %i8 -> %i2 + int_copy %i7 -> %i3 goto L1 + --- L2: int_return %i3 """) @@ -140,17 +142,19 @@ a -= 1 return b self.encoding_test(f, [5, 6], """ - int_copy %i0, %i2 - int_copy %i1, %i3 + int_copy %i0 -> %i2 + int_copy %i1 -> %i3 L1: goto_if_not_int_gt %i2, $0, L2 - int_copy %i2, %i4 - int_copy %i3, %i5 - int_add %i5, %i4, %i6 - int_sub %i4, $1, %i7 - int_copy %i7, %i2 - int_copy %i6, %i3 + -live- L2 + int_copy %i2 -> %i4 + int_copy %i3 -> %i5 + int_add %i5, %i4 -> %i6 + int_sub %i4, $1 -> %i7 + int_copy %i7 -> %i2 + int_copy %i6 -> %i3 goto L1 + --- L2: int_return %i3 """, transform=True) @@ -159,10 +163,10 @@ def f(i, f): return (i*5) + (f*0.25) self.encoding_test(f, [4, 7.5], """ - int_mul %i0, $5, %i1 - float_mul %f0, $0.25, %f1 - cast_int_to_float %i1, %f2 - float_add %f2, %f1, %f3 + int_mul %i0, $5 -> %i1 + float_mul %f0, $0.25 -> %f1 + cast_int_to_float %i1 -> %f2 + float_add %f2, %f1 -> %f3 float_return %f3 """) @@ -180,7 +184,7 @@ flattener = GraphFlattener(None, fake_regallocs()) flattener.serialize_op(op) assert_format(flattener.ssarepr, """ - residual_call_ir_f $12345, I[%i0, %i1], R[%r0, %r1], %f0 + residual_call_ir_f $12345, I[%i0, %i1], R[%r0, %r1] -> %f0 """) def test_same_as_removal(self): @@ -188,7 +192,7 @@ b = chr(a) return ord(b) + a self.encoding_test(f, [65], """ - int_add %i0, %i0, %i1 + int_add %i0, %i0 -> %i1 int_return %i1 """, transform=True) @@ -210,15 +214,19 @@ elif n == 7: return 1212 else: return 42 self.encoding_test(f, [65], """ + -live- int_guard_value %i0 goto_if_not_int_eq %i0, $-5, L1 int_return $12 + --- L1: goto_if_not_int_eq %i0, $2, L2 int_return $51 + --- L2: goto_if_not_int_eq %i0, $7, L3 int_return $1212 + --- L3: int_return $42 """) @@ -233,18 +241,25 @@ elif x == 6: return 54 return -1 self.encoding_test(f, [65], """ + -live- switch %i0, int_return $-1 + --- L1: int_return $61 + --- L2: int_return $511 + --- L3: int_return $-22 + --- L4: int_return $81 + --- L5: int_return $17 + --- L6: int_return $54 """) @@ -267,12 +282,15 @@ direct_call $<* fn g>, %i0 catch_exception L1 int_return $3 + --- L1: goto_if_exception_mismatch $<* struct object_vtable>, L2 int_return $1 + --- L2: goto_if_exception_mismatch $<* struct object_vtable>, L3 int_return $2 + --- L3: reraise """) @@ -295,15 +313,18 @@ return 4 self.encoding_test(f, [65], """ - G_residual_call_ir_v $<* fn g>, , I[%i0], R[] + residual_call_ir_v $<* fn g>, , I[%i0], R[] + -live- catch_exception L1 int_return $4 + --- L1: goto_if_exception_mismatch $<* struct object_vtable>, L2 - last_exc_value %r0 - ref_copy %r0, %r1 - getfield_gc_i %r1, , %i1 + last_exc_value -> %r0 + ref_copy %r0 -> %r1 + getfield_gc_i %r1, -> %i1 int_return %i1 + --- L2: int_return $3 """, transform=True) @@ -332,6 +353,7 @@ direct_call $<* fn g>, %i0 catch_exception L1 void_return + --- L1: raise $<* struct object> """) @@ -340,11 +362,13 @@ def f(i): return not i - # note that 'goto_if_not_int_is_true' is actually the same thing - # as just 'goto_if_not'. + # note that 'goto_if_not_int_is_true' is not the same thing + # as just 'goto_if_not', because the last one expects a boolean self.encoding_test(f, [7], """ - goto_if_not %i0, L1 + goto_if_not_int_is_true %i0, L1 + -live- L1 int_return $False + --- L1: int_return $True """, transform=True) @@ -360,15 +384,19 @@ except ZeroDivisionError: return -42 self.encoding_test(f, [7, 2], """ - G_residual_call_ir_i $<* fn int_floordiv_ovf_zer>, , I[%i0, %i1], R[], %i2 + residual_call_ir_i $<* fn int_floordiv_ovf_zer>, , I[%i0, %i1], R[] -> %i2 + -live- catch_exception L1 int_return %i2 + --- L1: goto_if_exception_mismatch $<* struct object_vtable>, L2 int_return $42 + --- L2: goto_if_exception_mismatch $<* struct object_vtable>, L3 int_return $-42 + --- L3: reraise """, transform=True) @@ -383,16 +411,34 @@ return 42 # XXX so far, this really produces a int_mod_ovf_zer... self.encoding_test(f, [7, 2], """ - G_residual_call_ir_i $<* fn int_mod_ovf_zer>, , I[%i0, %i1], R[], %i2 + residual_call_ir_i $<* fn int_mod_ovf_zer>, , I[%i0, %i1], R[] -> %i2 + -live- catch_exception L1 int_return %i2 + --- L1: goto_if_exception_mismatch $<* struct object_vtable>, L2 int_return $42 + --- L2: reraise """, transform=True) + def test_simple_branch(self): + def f(n, m1, m2): + if n: + return m1 + else: + return m2 + self.encoding_test(f, [4, 5, 6], """ + goto_if_not_int_is_true %i0, L1 + -live- %i1, %i2, L1 + int_return %i1 + --- + L1: + int_return %i2 + """, transform=True, liveness=True) + def test_int_add_ovf(self): def f(i, j): try: @@ -400,10 +446,11 @@ except OverflowError: return 42 self.encoding_test(f, [7, 2], """ - -live- - G_int_add_ovf %i0, %i1, %i2 + int_add_ovf %i0, %i1 -> %i2 + -live- %i2 catch_exception L1 int_return %i2 + --- L1: int_return $42 """, transform=True, liveness=True) @@ -418,13 +465,14 @@ except Exception: return 42 + j self.encoding_test(f, [7, 2], """ - -live- %i1 - G_residual_call_ir_i $<* fn g>, , I[%i0, %i1], R[], %i2 + residual_call_ir_i $<* fn g>, , I[%i0, %i1], R[] -> %i2 + -live- %i1, %i2 catch_exception L1 int_return %i2 + --- L1: - int_copy %i1, %i3 - int_add %i3, $42, %i4 + int_copy %i1 -> %i3 + int_add %i3, $42 -> %i4 int_return %i4 """, transform=True, liveness=True) @@ -438,7 +486,7 @@ except Exception: return 42 + j self.encoding_test(f, [7, 2], """ - residual_call_ir_i $<* fn cannot_raise>, , I[%i0, %i1], R[], %i2 + residual_call_ir_i $<* fn cannot_raise>, , I[%i0, %i1], R[] -> %i2 int_return %i2 """, transform=True, liveness=True) @@ -458,11 +506,12 @@ myjitdriver.jit_merge_point(x=x, y=y) myjitdriver.can_enter_jit(x=y, y=x) self.encoding_test(f, [4, 5], """ - G_int_guard_value %i0 + int_guard_value %i0 + -live- %i0, %i1 jit_merge_point I[%i0], R[], F[], I[%i1], R[], F[] can_enter_jit void_return - """, transform=True) + """, transform=True, liveness=True) def test_keepalive(self): S = lltype.GcStruct('S') @@ -475,9 +524,20 @@ keepalive_until_here(q) return x self.encoding_test(f, [5], """ - G_residual_call_r_r $<* fn g>, , R[], %r0 - G_residual_call_r_r $<* fn g>, , R[], %r1 - keepalive %r0 - keepalive %r1 + residual_call_r_r $<* fn g>, , R[] -> %r0 + -live- + residual_call_r_r $<* fn g>, , R[] -> %r1 + -live- + -live- %r0 + -live- %r1 int_return %i0 """, transform=True) + self.encoding_test(f, [5], """ + residual_call_r_r $<* fn g>, , R[] -> %r0 + -live- %i0, %r0 + residual_call_r_r $<* fn g>, , R[] -> %r1 + -live- %i0, %r0, %r1 + -live- %i0, %r0, %r1 + -live- %i0, %r1 + int_return %i0 + """, transform=True, liveness=True) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Fri May 21 15:46:57 2010 @@ -59,11 +59,13 @@ graph = self.make_graphs(f, [5, 6])[0] self.check_assembler(graph, """ L1: - int_gt %i0, $0, %i2 + int_gt %i0, $0 -> %i2 goto_if_not %i2, L2 - int_add %i1, %i0, %i1 - int_sub %i0, $1, %i0 + -live- L2 + int_add %i1, %i0 -> %i1 + int_sub %i0, $1 -> %i0 goto L1 + --- L2: int_return %i1 """) @@ -76,12 +78,14 @@ graph = self.make_graphs(f, [5, 6])[0] self.check_assembler(graph, """ L1: - int_gt %i0, $0, %i2 + int_gt %i0, $0 -> %i2 goto_if_not %i2, L2 + -live- L2 int_push %i1 - int_copy %i0, %i1 - int_pop %i0 + int_copy %i0 -> %i1 + int_pop -> %i0 goto L1 + --- L2: int_return %i1 """) @@ -94,11 +98,13 @@ graph = self.make_graphs(f, [5, 6])[0] self.check_assembler(graph, """ L1: - int_gt %i0, $0, %i0 + int_gt %i0, $0 -> %i0 goto_if_not %i0, L2 - int_copy %i1, %i0 - int_copy $2, %i1 + -live- L2 + int_copy %i1 -> %i0 + int_copy $2 -> %i1 goto L1 + --- L2: int_return %i1 """) @@ -111,13 +117,15 @@ graph = self.make_graphs(f, [5, 6, 7])[0] self.check_assembler(graph, """ L1: - int_gt %i0, $0, %i3 + int_gt %i0, $0 -> %i3 goto_if_not %i3, L2 + -live- L2 int_push %i1 - int_copy %i2, %i1 - int_copy %i0, %i2 - int_pop %i0 + int_copy %i2 -> %i1 + int_copy %i0 -> %i2 + int_pop -> %i0 goto L1 + --- L2: int_return %i1 """) @@ -130,10 +138,12 @@ graph = self.make_graphs(f, [5, 6, 7])[0] self.check_assembler(graph, """ L1: - int_gt %i0, $0, %i3 + int_gt %i0, $0 -> %i3 goto_if_not %i3, L2 - int_copy %i2, %i1 + -live- L2 + int_copy %i2 -> %i1 goto L1 + --- L2: int_return %i1 """) @@ -152,8 +162,8 @@ block.closeblock(Link([v3], graph.returnblock)) # self.check_assembler(graph, """ - int_add %i0, $1, %i1 - rescall I[%i0, %i1], %i0 + int_add %i0, $1 -> %i1 + rescall I[%i0, %i1] -> %i0 int_return %i0 """) @@ -164,7 +174,8 @@ v4 = Variable(); v4.concretetype = rclass.CLASSTYPE block = Block([]) block.operations = [ - SpaceOperation('G_res_call', [], v1), + SpaceOperation('res_call', [], v1), + SpaceOperation('-live-', [], None), ] graph = FunctionGraph('f', block, v4) exclink = Link([v2], graph.returnblock) @@ -176,13 +187,16 @@ exclink) # self.check_assembler(graph, """ - G_res_call %i0 + res_call -> %i0 + -live- catch_exception L1 int_return %i0 + --- L1: goto_if_exception_mismatch $123, L2 - last_exception %i0 + last_exception -> %i0 int_return %i0 + --- L2: reraise """) @@ -203,8 +217,8 @@ block.closeblock(Link([v3], graph.returnblock)) # self.check_assembler(graph, """ - int_add %i0, $1, %i1 - rescall I[%i0, %i1], %i2 - rescall I[%i0, %i1], %i0 + int_add %i0, $1 -> %i1 + rescall I[%i0, %i1] -> %i2 + rescall I[%i0, %i1] -> %i0 int_return %i0 """) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Fri May 21 15:46:57 2010 @@ -195,8 +195,9 @@ if resulttype == 'i': # argcode should be 'i' too - assert argcodes[next_argcode] == 'i' - next_argcode = next_argcode + 1 + assert argcodes[next_argcode] == '>' + assert argcodes[next_argcode+1] == 'i' + next_argcode = next_argcode + 2 if lltype.typeOf(result) is lltype.Bool: result = int(result) assert lltype.typeOf(result) is lltype.Signed @@ -204,15 +205,17 @@ position += 1 elif resulttype == 'r': # argcode should be 'r' too - assert argcodes[next_argcode] == 'r' - next_argcode = next_argcode + 1 + assert argcodes[next_argcode] == '>' + assert argcodes[next_argcode+1] == 'r' + next_argcode = next_argcode + 2 assert lltype.typeOf(result) == llmemory.GCREF self.registers_r[ord(code[position])] = result position += 1 elif resulttype == 'f': # argcode should be 'f' too - assert argcodes[next_argcode] == 'f' - next_argcode = next_argcode + 1 + assert argcodes[next_argcode] == '>' + assert argcodes[next_argcode+1] == 'f' + next_argcode = next_argcode + 2 assert lltype.typeOf(result) is lltype.Float self.registers_f[ord(code[position])] = result position += 1 @@ -979,16 +982,34 @@ def _prepare_resume_from_failure(self, opnum): from pypy.jit.metainterp.resoperation import rop - if opnum == rop.GUARD_TRUE: # a goto_if_not_xxx that jumps only now + # + if opnum == rop.GUARD_TRUE: + # Produced directly by some goto_if_not_xxx() opcode that did not + # jump, but which must now jump. The pc is just after the opcode. self.position = self.jitcode.follow_jump(self.position) - elif opnum == rop.GUARD_FALSE: # a goto_if_not that stops jumping + # + elif opnum == rop.GUARD_FALSE: + # Produced directly by some goto_if_not_xxx() opcode that jumped, + # but which must no longer jump. The pc is just after the opcode. pass - elif opnum == rop.GUARD_NO_EXCEPTION or opnum == rop.GUARD_EXCEPTION: + # + elif opnum == rop.GUARD_VALUE or opnum == rop.GUARD_CLASS: + # Produced by guard_class(), xxx_guard_value(), or a few other + # opcodes like switch(). The pc is at the start of the opcode + # (so it will be redone). pass - elif opnum == rop.GUARD_CLASS or opnum == rop.GUARD_VALUE: + # + elif opnum == rop.GUARD_NONNULL or opnum == rop.GUARD_ISNULL: + xxx + elif (opnum == rop.GUARD_NO_EXCEPTION or + opnum == rop.GUARD_EXCEPTION or + opnum == rop.GUARD_NOT_FORCED or + opnum == rop.GUARD_NO_OVERFLOW or + opnum == rop.GUARD_OVERFLOW): pass else: - raise NotImplementedError(opnum) + from pypy.jit.metainterp.resoperation import opname + raise NotImplementedError(opname[opnum]) # connect the return of values from the called frame to the # 'xxx_call_yyy' instructions from the caller frame Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py Fri May 21 15:46:57 2010 @@ -366,6 +366,8 @@ except lltype.UninitializedMemoryAccess: return '' +CONST_NULL = ConstPtr(ConstPtr.value) + class ConstObj(Const): type = REF value = ootype.NULL Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Fri May 21 15:46:57 2010 @@ -182,7 +182,6 @@ for _opimpl in ['int_is_true', 'int_is_zero', 'int_neg', 'int_invert', 'cast_ptr_to_int', 'cast_float_to_int', 'cast_int_to_float', 'float_neg', 'float_abs', - 'ptr_iszero', 'ptr_nonzero', ]: exec py.code.Source(''' @arguments("box") @@ -191,6 +190,14 @@ ''' % (_opimpl, _opimpl.upper())).compile() @arguments("box") + def opimpl_ptr_nonzero(self, box): + return self.execute(rop.PTR_NE, box, history.CONST_NULL) + + @arguments("box") + def opimpl_ptr_iszero(self, box): + return self.execute(rop.PTR_EQ, box, history.CONST_NULL) + + @arguments("box") def _opimpl_any_return(self, box): self.metainterp.finishframe(box) @@ -252,13 +259,15 @@ if not switchcase: self.pc = target - for _opimpl in ['int_is_zero', 'ptr_iszero', 'ptr_nonzero']: - exec py.code.Source(''' - @arguments("box", "label") - def opimpl_goto_if_not_%s(self, box, target): - condbox = self.execute(rop.%s, box) - self.opimpl_goto_if_not(condbox, target) - ''' % (_opimpl, _opimpl.upper())).compile() + @arguments("box", "label") + def opimpl_goto_if_not_int_is_true(self, box, target): + condbox = self.execute(rop.INT_IS_TRUE, box) + self.opimpl_goto_if_not(condbox, target) + + @arguments("box", "label") + def opimpl_goto_if_not_int_is_zero(self, box, target): + condbox = self.execute(rop.INT_IS_ZERO, box) + self.opimpl_goto_if_not(condbox, target) for _opimpl in ['int_lt', 'int_le', 'int_eq', 'int_ne', 'int_gt', 'int_ge', 'ptr_eq', 'ptr_ne']: @@ -268,7 +277,29 @@ condbox = self.execute(rop.%s, b1, b2) self.opimpl_goto_if_not(condbox, target) ''' % (_opimpl, _opimpl.upper())).compile() - + + @arguments("box", "label") + def opimpl_goto_if_not_ptr_nonzero(self, box, target): + value = box.nonnull() + if value: + opnum = rop.GUARD_NONNULL + else: + opnum = rop.GUARD_ISNULL + self.generate_guard(opnum, box) + if not value: + self.pc = target + + @arguments("box", "label") + def opimpl_goto_if_not_ptr_iszero(self, box, target): + value = box.nonnull() + if value: + opnum = rop.GUARD_NONNULL + else: + opnum = rop.GUARD_ISNULL + self.generate_guard(opnum, box) + if value: + self.pc = target + ## def follow_jump(self): ## _op_goto_if_not = self.metainterp.staticdata._op_goto_if_not ## assert ord(self.bytecode[self.pc]) == _op_goto_if_not @@ -1003,6 +1034,9 @@ self.indirectcalltargets = list(indirectcalltargets) def finish_setup(self, codewriter, optimizer=None): + from pypy.jit.metainterp.blackhole import BlackholeInterpBuilder + self.blackholeinterpbuilder = BlackholeInterpBuilder(codewriter, self) + # asm = codewriter.assembler self.setup_insns(asm.insns) self.setup_descrs(asm.descrs) @@ -1012,9 +1046,6 @@ RESULT = codewriter.portal_graph.getreturnvar().concretetype self.result_type = history.getkind(RESULT) # - from pypy.jit.metainterp.blackhole import BlackholeInterpBuilder - self.blackholeinterpbuilder = BlackholeInterpBuilder(codewriter, self) - # warmrunnerdesc = self.warmrunnerdesc if warmrunnerdesc is not None: self.num_green_args = warmrunnerdesc.num_green_args @@ -1514,6 +1545,10 @@ frame.pc = frame.jitcode.follow_jump(frame.pc) elif opnum == rop.GUARD_FALSE: # a goto_if_not that stops jumping pass + elif opnum == rop.GUARD_VALUE or opnum == rop.GUARD_CLASS: + pass # the pc is already set to the *start* of the opcode + elif opnum == rop.GUARD_NONNULL or opnum == rop.GUARD_ISNULL: + xxx #self.framestack[-1].ignore_next_guard_nullness(opnum) elif (opnum == rop.GUARD_NO_EXCEPTION or opnum == rop.GUARD_EXCEPTION or opnum == rop.GUARD_NOT_FORCED): exception = self.cpu.grab_exc_value() @@ -1523,13 +1558,13 @@ self.execute_did_not_raise() self.handle_possible_exception() elif opnum == rop.GUARD_NO_OVERFLOW: # an overflow now detected - xxx #self.raise_overflow_error() - elif opnum == rop.GUARD_NONNULL or opnum == rop.GUARD_ISNULL: - xxx #self.framestack[-1].ignore_next_guard_nullness(opnum) - elif opnum == rop.GUARD_CLASS or opnum == rop.GUARD_VALUE: + self.execute_raised(OverflowError(), constant=True) + self.finishframe_exception() + elif opnum == rop.GUARD_OVERFLOW: # no longer overflowing pass else: - raise NotImplementedError(opnum) + from pypy.jit.metainterp.resoperation import opname + raise NotImplementedError(opname[opnum]) def compile(self, original_boxes, live_arg_boxes, start): num_green_args = self.staticdata.num_green_args Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py Fri May 21 15:46:57 2010 @@ -183,8 +183,6 @@ # 'SAME_AS/1', # gets a Const or a Box, turns it into another Box # - 'PTR_ISZERO/1b', - 'PTR_NONZERO/1b', 'PTR_EQ/2b', 'PTR_NE/2b', # From arigo at codespeak.net Fri May 21 16:01:52 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 May 2010 16:01:52 +0200 (CEST) Subject: [pypy-svn] r74630 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter codewriter/test metainterp metainterp/test Message-ID: <20100521140152.9B900282C0D@codespeak.net> Author: arigo Date: Fri May 21 16:01:50 2010 New Revision: 74630 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Log: Progress. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Fri May 21 16:01:50 2010 @@ -298,8 +298,8 @@ jitcode = self.callcontrol.get_jitcode(targetgraph, called_from=self.graph) lst.append(jitcode) - op0 = SpaceOperation('int_guard_value', [op.args[0]], None) - op1 = SpaceOperation('-live-', [op.args[0]], None) + op0 = SpaceOperation('-live-', [], None) + op1 = SpaceOperation('int_guard_value', [op.args[0]], None) op2 = self.handle_residual_call(op, [IndirectCallTargets(lst)]) result = [op0, op1] if isinstance(op2, list): @@ -345,8 +345,8 @@ if hints.get('promote') and op.args[0].concretetype is not lltype.Void: assert op.args[0].concretetype != lltype.Ptr(rstr.STR) kind = getkind(op.args[0].concretetype) - op0 = SpaceOperation('%s_guard_value' % kind, [op.args[0]], None) - op1 = SpaceOperation('-live-', [op.args[0]], None) + op0 = SpaceOperation('-live-', [], None) + op1 = SpaceOperation('%s_guard_value' % kind, [op.args[0]], None) # the special return value None forces op.result to be considered # equal to op.args[0] return [op0, op1, None] @@ -463,8 +463,8 @@ op.args[0].concretetype.TO._hints.get('typeptr')) def handle_getfield_typeptr(self, op): - op0 = SpaceOperation('guard_class', [op.args[0]], op.result) - op1 = SpaceOperation('-live-', [op.args[0]], None) + op0 = SpaceOperation('-live-', [], None) + op1 = SpaceOperation('guard_class', [op.args[0]], op.result) return [op0, op1] def rewrite_op_malloc(self, op): @@ -657,9 +657,9 @@ for v in op.args[2:2+num_green_args]: if isinstance(v, Variable) and v.concretetype is not lltype.Void: kind = getkind(v.concretetype) + ops.append(SpaceOperation('-live-', [], None)) ops.append(SpaceOperation('%s_guard_value' % kind, [v], None)) - ops.append(SpaceOperation('-live-', [v], None)) args = (self.make_three_lists(op.args[2:2+num_green_args]) + self.make_three_lists(op.args[2+num_green_args:])) ops.append(SpaceOperation('jit_merge_point', args, None)) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Fri May 21 16:01:50 2010 @@ -506,8 +506,8 @@ myjitdriver.jit_merge_point(x=x, y=y) myjitdriver.can_enter_jit(x=y, y=x) self.encoding_test(f, [4, 5], """ - int_guard_value %i0 -live- %i0, %i1 + int_guard_value %i0 jit_merge_point I[%i0], R[], F[], I[%i1], R[], F[] can_enter_jit void_return Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Fri May 21 16:01:50 2010 @@ -303,11 +303,11 @@ tr.graph = 'someinitialgraph' oplist = tr.rewrite_operation(op) op0gv, op1gv, op0, op1 = oplist - assert op0gv.opname == 'int_guard_value' - assert op0gv.args == [op.args[0]] - assert op0gv.result is None - assert op1gv.opname == '-live-' + assert op0gv.opname == '-live-' + assert op0gv.args == [] + assert op1gv.opname == 'int_guard_value' assert op1gv.args == [op.args[0]] + assert op1gv.result is None # reskind = getkind(restype)[0] assert op0.opname == 'residual_call_%s_%s' % (expectedkind, reskind) @@ -364,11 +364,11 @@ op = SpaceOperation('getfield', [v_parent, c_name], v_result) oplist = Transformer(FakeCPU()).rewrite_operation(op) op0, op1 = oplist - assert op0.opname == 'guard_class' - assert op0.args == [v_parent] - assert op0.result == v_result - assert op1.opname == '-live-' + assert op0.opname == '-live-' + assert op0.args == [] + assert op1.opname == 'guard_class' assert op1.args == [v_parent] + assert op1.result == v_result def test_setfield(): # XXX a more compact encoding would be possible; see test_getfield() @@ -615,11 +615,11 @@ v2) oplist = Transformer().rewrite_operation(op) op0, op1, op2 = oplist - assert op0.opname == 'int_guard_value' - assert op0.args == [v1] - assert op0.result is None - assert op1.opname == '-live-' + assert op0.opname == '-live-' + assert op0.args == [] + assert op1.opname == 'int_guard_value' assert op1.args == [v1] + assert op1.result is None assert op2 is None def test_promote_2(): @@ -635,9 +635,9 @@ block.closeblock(Link([v2], returnblock)) Transformer().optimize_block(block) assert len(block.operations) == 2 - assert block.operations[0].opname == 'int_guard_value' - assert block.operations[0].args == [v1] - assert block.operations[0].result is None - assert block.operations[1].opname == '-live-' + assert block.operations[0].opname == '-live-' + assert block.operations[0].args == [] + assert block.operations[1].opname == 'int_guard_value' assert block.operations[1].args == [v1] + assert block.operations[1].result is None assert block.exits[0].args == [v1] Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Fri May 21 16:01:50 2010 @@ -196,7 +196,7 @@ if resulttype == 'i': # argcode should be 'i' too assert argcodes[next_argcode] == '>' - assert argcodes[next_argcode+1] == 'i' + assert argcodes[next_argcode + 1] == 'i' next_argcode = next_argcode + 2 if lltype.typeOf(result) is lltype.Bool: result = int(result) @@ -206,7 +206,7 @@ elif resulttype == 'r': # argcode should be 'r' too assert argcodes[next_argcode] == '>' - assert argcodes[next_argcode+1] == 'r' + assert argcodes[next_argcode + 1] == 'r' next_argcode = next_argcode + 2 assert lltype.typeOf(result) == llmemory.GCREF self.registers_r[ord(code[position])] = result @@ -214,7 +214,7 @@ elif resulttype == 'f': # argcode should be 'f' too assert argcodes[next_argcode] == '>' - assert argcodes[next_argcode+1] == 'f' + assert argcodes[next_argcode + 1] == 'f' next_argcode = next_argcode + 2 assert lltype.typeOf(result) is lltype.Float self.registers_f[ord(code[position])] = result @@ -307,12 +307,10 @@ e = get_llexception(self.cpu, e) self.handle_exception_in_frame(e) - def get_result_i(self): - assert self._return_type == 'i' + def get_tmpreg_i(self): return self.tmpreg_i - def get_result_r(self): - assert self._return_type == 'r' + def get_tmpreg_r(self): result = self.tmpreg_r if we_are_translated(): self.tmpreg_r = NULL @@ -320,19 +318,30 @@ del self.tmpreg_r return result - def get_result_f(self): - assert self._return_type == 'f' + def get_tmpreg_f(self): return self.tmpreg_f - def get_result_v(self): + def final_result_i(self): + assert self._return_type == 'i' + return self.get_tmpreg_i() + + def final_result_r(self): + assert self._return_type == 'r' + return self.get_tmpreg_r() + + def final_result_f(self): + assert self._return_type == 'f' + return self.get_tmpreg_f() + + def final_result_v(self): assert self._return_type == 'v' - def _get_result_anytype(self): + def _final_result_anytype(self): "NOT_RPYTHON" - if self._return_type == 'i': return self.get_result_i() - if self._return_type == 'r': return self.get_result_r() - if self._return_type == 'f': return self.get_result_f() - if self._return_type == 'v': return None + if self._return_type == 'i': return self.final_result_i() + if self._return_type == 'r': return self.final_result_r() + if self._return_type == 'f': return self.final_result_f() + if self._return_type == 'v': return self.final_result_v() raise ValueError(self._return_type) def cleanup_registers(self): @@ -531,13 +540,13 @@ @arguments("self", returns="i") def bhimpl_int_pop(self): - return self.get_result_i() + return self.get_tmpreg_i() @arguments("self", returns="r") def bhimpl_ref_pop(self): - return self.get_result_r() + return self.get_tmpreg_r() @arguments("self", returns="f") def bhimpl_float_pop(self): - return self.get_result_f() + return self.get_tmpreg_f() # ---------- # float operations @@ -669,6 +678,8 @@ else: return target + bhimpl_goto_if_not_int_is_true = bhimpl_goto_if_not + @arguments("i", "L", "pc", returns="L") def bhimpl_goto_if_not_int_is_zero(a, target, pc): if not a: @@ -971,11 +982,11 @@ self._done_with_this_frame() kind = self._return_type if kind == 'i': - caller._setup_return_value_i(self.get_result_i()) + caller._setup_return_value_i(self.final_result_i()) elif kind == 'r': - caller._setup_return_value_r(self.get_result_r()) + caller._setup_return_value_r(self.final_result_r()) elif kind == 'f': - caller._setup_return_value_f(self.get_result_f()) + caller._setup_return_value_f(self.final_result_f()) else: assert kind == 'v' return NULL @@ -1028,14 +1039,14 @@ # normally (in general we get a ContinueRunningNormally exception). sd = self.builder.metainterp_sd if sd.result_type == 'void': - self.get_result_v() + self.final_result_v() raise sd.DoneWithThisFrameVoid() elif sd.result_type == 'int': - raise sd.DoneWithThisFrameInt(self.get_result_i()) + raise sd.DoneWithThisFrameInt(self.final_result_i()) elif sd.result_type == 'ref': - raise sd.DoneWithThisFrameRef(self.get_result_r()) + raise sd.DoneWithThisFrameRef(self.final_result_r()) elif sd.result_type == 'float': - raise sd.DoneWithThisFrameFloat(self.get_result_f()) + raise sd.DoneWithThisFrameFloat(self.final_result_f()) else: assert False Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Fri May 21 16:01:50 2010 @@ -894,11 +894,12 @@ virtualizable_boxes = None if metainterp.staticdata.virtualizable_info is not None: virtualizable_boxes = metainterp.virtualizable_boxes - if resumepc < 0: - resumepc = self.pc + saved_pc = self.pc + if resumepc >= 0: + self.pc = resumepc resume.capture_resumedata(metainterp.framestack, virtualizable_boxes, - metainterp.virtualref_boxes, resumedescr, - resumepc) + metainterp.virtualref_boxes, resumedescr) + self.pc = saved_pc self.metainterp.staticdata.profiler.count_ops(opnum, GUARDS) # count metainterp.attach_debug_info(guard_op) @@ -2030,8 +2031,10 @@ args += (value,) # num_return_args = len(argcodes) - next_argcode - assert num_return_args == 0 or num_return_args == 1 - self.pc = position + num_return_args + assert num_return_args == 0 or num_return_args == 2 + if num_return_args: + position += 1 + self.pc = position # if not we_are_translated(): print '\tpyjitpl: %s(%s)' % (name, ', '.join(map(repr, args))), @@ -2045,7 +2048,8 @@ assert resultbox is None else: print '-> %r' % (resultbox,) - result_argcode = argcodes[next_argcode] + assert argcodes[next_argcode] == '>' + result_argcode = argcodes[next_argcode + 1] assert resultbox.type == {'i': history.INT, 'r': history.REF, 'f': history.FLOAT}[result_argcode] Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py Fri May 21 16:01:50 2010 @@ -45,12 +45,12 @@ back.get_list_of_active_boxes()) def capture_resumedata(framestack, virtualizable_boxes, virtualref_boxes, - storage, pc): + storage): n = len(framestack)-1 top = framestack[n] _ensure_parent_resumedata(framestack, n) frame_info_list = FrameInfo(top.parent_resumedata_frame_info_list, - top.jitcode, pc) + top.jitcode, top.pc) storage.rd_frame_info_list = frame_info_list snapshot = Snapshot(top.parent_resumedata_snapshot, top.get_list_of_active_boxes()) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Fri May 21 16:01:50 2010 @@ -45,7 +45,7 @@ raise TypeError(T) blackholeinterp.setposition(cw.mainjitcode, 0) blackholeinterp.run() - return blackholeinterp._get_result_anytype() + return blackholeinterp._final_result_anytype() def _run_with_pyjitpl(cw, args, testself): from pypy.jit.metainterp import simple_optimize From afa at codespeak.net Fri May 21 16:16:22 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 21 May 2010 16:16:22 +0200 (CEST) Subject: [pypy-svn] r74631 - pypy/trunk/pypy/module/cpyext Message-ID: <20100521141622.E1B53282C0D@codespeak.net> Author: afa Date: Fri May 21 16:16:21 2010 New Revision: 74631 Modified: pypy/trunk/pypy/module/cpyext/stubsactive.py Log: Fix signature of PyFile_AsFile, allows compilation with C++ Modified: pypy/trunk/pypy/module/cpyext/stubsactive.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/stubsactive.py (original) +++ pypy/trunk/pypy/module/cpyext/stubsactive.py Fri May 21 16:16:21 2010 @@ -1,11 +1,10 @@ from pypy.module.cpyext.pyobject import PyObject from pypy.module.cpyext.api import cpython_api, Py_ssize_t, CANNOT_FAIL, CConfig +from pypy.module.cpyext.object import FILEP from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.pystate import PyThreadState, PyInterpreterState -FILE = rffi.VOIDP_real.TO -FILEP = lltype.Ptr(FILE) @cpython_api([PyObject], FILEP, error=CANNOT_FAIL) def PyFile_AsFile(space, p): """Return the file object associated with p as a FILE*. From arigo at codespeak.net Fri May 21 16:29:35 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 May 2010 16:29:35 +0200 (CEST) Subject: [pypy-svn] r74632 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter codewriter/test metainterp Message-ID: <20100521142935.74CAE282C0D@codespeak.net> Author: arigo Date: Fri May 21 16:29:33 2010 New Revision: 74632 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitcode.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py Log: Progress. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py Fri May 21 16:29:33 2010 @@ -51,32 +51,21 @@ registers_i, registers_r, registers_f): # 'pc' gives a position in this bytecode. This invokes # 'callback' for each variable that is live across the - # instruction which ends at 'pc'. (It excludes the arguments - # of that instruction which are no longer used afterwards, and - # excludes the return value of that instruction.) More precisely, - # this invokes 'callback(arg, box, index)' where 'box' comes from one - # of the three lists of registers and 'index' is 0, 1, 2... - # If the callback returns a box, then it is stored back. + # instruction boundary at 'pc'. More precisely, + # this invokes 'callback(arg, box)' where 'box' comes from one + # of the three lists of registers. if not we_are_translated() and pc not in self.liveness: self._missing_liveness(pc) live_i, live_r, live_f = self.liveness[pc] # XXX compactify!! - index = 0 for c in live_i: - newbox = callback(arg, registers_i[ord(c)], index) - index += 1 - if newbox is not None: - registers_i[ord(c)] = newbox + x = callback(arg, registers_i[ord(c)]) + assert x is None for c in live_r: - newbox = callback(arg, registers_r[ord(c)], index) - index += 1 - if newbox is not None: - registers_r[ord(c)] = newbox + x = callback(arg, registers_r[ord(c)]) + assert x is None for c in live_f: - newbox = callback(arg, registers_f[ord(c)], index) - index += 1 - if newbox is not None: - registers_f[ord(c)] = newbox - return index + x = callback(arg, registers_f[ord(c)]) + assert x is None enumerate_live_vars._annspecialcase_ = 'specialize:arg(2)' def _live_vars(self, pc): @@ -86,10 +75,8 @@ self.kind = kind def __getitem__(self, index): return '%%%s%d' % (self.kind, index) - def callback(lst, reg, index): - lst.append(reg) lst = [] - self.enumerate_live_vars(pc, callback, lst, + self.enumerate_live_vars(pc, list.append, lst, Names('i'), Names('r'), Names('f')) lst.sort() return ' '.join(lst) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py Fri May 21 16:29:33 2010 @@ -86,4 +86,4 @@ blackholeinterp.setarg_i(0, 6) blackholeinterp.setarg_i(1, 100) blackholeinterp.run() - assert blackholeinterp.get_result_i() == 100+6+5+4+3 + assert blackholeinterp.final_result_i() == 100+6+5+4+3 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitcode.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitcode.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitcode.py Fri May 21 16:29:33 2010 @@ -23,13 +23,12 @@ assert not j.has_liveness_info(4) # seen = [] - def callback(arg, value, index): + def callback(arg, value): assert arg == "foo" - seen.append((value, index)) + seen.append(value) # - total = j.enumerate_live_vars(5, callback, "foo", - {ord(" "): "i10", ord("A"): "i20"}, - {ord("b"): "r30"}, - {ord("C"): "f40", ord("D"): "f50"}) - assert total == 5 - assert seen == [("i10", 0), ("i20", 1), ("r30", 2), ("f40", 3), ("f50", 4)] + j.enumerate_live_vars(5, callback, "foo", + {ord(" "): "i10", ord("A"): "i20"}, + {ord("b"): "r30"}, + {ord("C"): "f40", ord("D"): "f50"}) + assert seen == ["i10", "i20", "r30", "f40", "f50"] Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Fri May 21 16:29:33 2010 @@ -102,24 +102,43 @@ outvalue[startindex+i] = reg prepare_list_of_boxes._annspecialcase_ = 'specialize:arg(4)' - def get_list_of_active_boxes(self): + def get_list_of_active_boxes(self, in_a_call): + if in_a_call: + # If we are not the topmost frame, self._result_argcode contains + # the type of the result of the call instruction in the bytecode. + # We use it to clear the box that will hold the result: this box + # is not defined yet. + argcode = self._result_argcode + index = ord(self.bytecode[self.pc - 1]) + if argcode == 'i': self.registers_i[index] = None + elif argcode == 'r': self.registers_r[index] = None + elif argcode == 'f': self.registers_f[index] = None + # + self._tmp_count = 0 count = self.jitcode.enumerate_live_vars( - self.pc, MIFrame._count_boxes, None, + self.pc, MIFrame._count_boxes, self, self.registers_i, self.registers_r, self.registers_f) - env = [None] * count + # + self._tmp_env = [None] * self._tmp_count + self._tmp_count = 0 self.jitcode.enumerate_live_vars( - self.pc, MIFrame._store_in_env, env, + self.pc, MIFrame._store_in_env, self, self.registers_i, self.registers_r, self.registers_f) + # + env = self._tmp_env + self._tmp_env = None make_sure_not_resized(env) return env - @staticmethod - def _count_boxes(_, box, index): - pass # just used to count how many boxes there are - - @staticmethod - def _store_in_env(env, box, index): - env[index] = box + def _count_boxes(self, box): + if box is not None: # just used to count how many boxes there are + self._tmp_count += 1 + + def _store_in_env(self, box): + if box is not None: + index = self._tmp_count + self._tmp_env[index] = box + self._tmp_count = index + 1 def replace_active_box_in_frame(self, oldbox, newbox): if isinstance(oldbox, history.BoxInt): @@ -2033,7 +2052,14 @@ num_return_args = len(argcodes) - next_argcode assert num_return_args == 0 or num_return_args == 2 if num_return_args: + # Save the type of the resulting box. This is needed if the + # operation is an inlined call and we need to get the list of + # all alive boxes in it, to know that the result box was not + # written yet. + self._result_argcode = argcodes[next_argcode + 1] position += 1 + else: + self._result_argcode = 'v' self.pc = position # if not we_are_translated(): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py Fri May 21 16:29:33 2010 @@ -42,7 +42,7 @@ back.pc) target.parent_resumedata_snapshot = Snapshot( back.parent_resumedata_snapshot, - back.get_list_of_active_boxes()) + back.get_list_of_active_boxes(True)) def capture_resumedata(framestack, virtualizable_boxes, virtualref_boxes, storage): @@ -53,7 +53,7 @@ top.jitcode, top.pc) storage.rd_frame_info_list = frame_info_list snapshot = Snapshot(top.parent_resumedata_snapshot, - top.get_list_of_active_boxes()) + top.get_list_of_active_boxes(False)) snapshot = Snapshot(snapshot, virtualref_boxes[:]) # xxx for now if virtualizable_boxes is not None: snapshot = Snapshot(snapshot, virtualizable_boxes[:]) # xxx for now From hpk at codespeak.net Fri May 21 16:43:25 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 21 May 2010 16:43:25 +0200 (CEST) Subject: [pypy-svn] r74633 - in pypy/branch/py131: py/_code py/_plugin pypy pypy/interpreter pypy/tool/pytest pypy/tool/pytest/test Message-ID: <20100521144325.03255282C0D@codespeak.net> Author: hpk Date: Fri May 21 16:43:24 2010 New Revision: 74633 Modified: pypy/branch/py131/py/_code/source.py pypy/branch/py131/py/_plugin/pytest_pytester.py pypy/branch/py131/pypy/conftest.py pypy/branch/py131/pypy/interpreter/gateway.py pypy/branch/py131/pypy/tool/pytest/appsupport.py pypy/branch/py131/pypy/tool/pytest/test/test_appsupport.py Log: fix applevel function reporting problems, see also http://bitbucket.org/hpk42/py-trunk/issue/94/pytest-can-stop-with-no-output this commit contains: - update from py-trunk 1.3.1a1 series - improvements and fixes to pypy's presentation of tracebacks involving applevel functions - one can now see a more meaningful filename and the function source is printed correctly Modified: pypy/branch/py131/py/_code/source.py ============================================================================== --- pypy/branch/py131/py/_code/source.py (original) +++ pypy/branch/py131/py/_code/source.py Fri May 21 16:43:24 2010 @@ -26,6 +26,8 @@ partlines = [] if isinstance(part, Source): partlines = part.lines + elif isinstance(part, (tuple, list)): + partlines = [x.rstrip("\n") for x in part] elif isinstance(part, py.builtin._basestring): partlines = part.split('\n') if rstrip: @@ -172,7 +174,9 @@ try: #compile(source+'\n', "x", "exec") syntax_checker(source+'\n') - except SyntaxError: + except KeyboardInterrupt: + raise + except Exception: return False else: return True Modified: pypy/branch/py131/py/_plugin/pytest_pytester.py ============================================================================== --- pypy/branch/py131/py/_plugin/pytest_pytester.py (original) +++ pypy/branch/py131/py/_plugin/pytest_pytester.py Fri May 21 16:43:24 2010 @@ -110,7 +110,7 @@ def _makefile(self, ext, args, kwargs): items = list(kwargs.items()) if args: - source = "\n".join(map(str, args)) + source = "\n".join(map(str, args)) + "\n" basename = self.request.function.__name__ items.insert(0, (basename, source)) ret = None @@ -294,8 +294,10 @@ ret = popen.wait() f1.close() f2.close() - out = p1.read("rb").decode("utf-8").splitlines() - err = p2.read("rb").decode("utf-8").splitlines() + out = p1.read("rb") + out = getdecoded(out).splitlines() + err = p2.read("rb") + err = getdecoded(err).splitlines() def dump_lines(lines, fp): try: for line in lines: @@ -360,6 +362,13 @@ child.timeout = expect_timeout return child +def getdecoded(out): + try: + return out.decode("utf-8") + except UnicodeDecodeError: + return "INTERNAL not-utf8-decodeable, truncated string:\n%s" % ( + py.io.saferepr(out),) + class PseudoPlugin: def __init__(self, vars): self.__dict__.update(vars) Modified: pypy/branch/py131/pypy/conftest.py ============================================================================== --- pypy/branch/py131/pypy/conftest.py (original) +++ pypy/branch/py131/pypy/conftest.py Fri May 21 16:43:24 2010 @@ -383,10 +383,15 @@ if option.runappdirect: return target() space = gettestobjspace() - func = app2interp_temp(target) + filename = self._getdynfilename(target) + func = app2interp_temp(target, filename=filename) print "executing", func self.execute_appex(space, func, space) + def _getdynfilename(self, func): + code = getattr(func, 'im_func', func).func_code + return "[%s:%s]" % (code.co_filename, code.co_firstlineno) + class AppTestMethod(AppTestFunction): def setup(self): @@ -410,7 +415,8 @@ if option.runappdirect: return target() space = target.im_self.space - func = app2interp_temp(target.im_func) + filename = self._getdynfilename(target) + func = app2interp_temp(target.im_func, filename=filename) w_instance = self.parent.w_instance self.execute_appex(space, func, space, w_instance) Modified: pypy/branch/py131/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/py131/pypy/interpreter/gateway.py (original) +++ pypy/branch/py131/pypy/interpreter/gateway.py Fri May 21 16:43:24 2010 @@ -821,9 +821,6 @@ # and now for something completely different ... # -class MyStr(str): - pass - class ApplevelClass: """NOT_RPYTHON A container for app-level source code that should be executed @@ -837,8 +834,7 @@ # HAAACK (but a good one) if filename is None: f = sys._getframe(1) - filename = MyStr('<%s:%d>' % (f.f_code.co_filename, f.f_lineno)) - filename.__source__ = py.code.Source(source) + filename = '<%s:%d>' % (f.f_code.co_filename, f.f_lineno) self.filename = filename self.source = str(py.code.Source(source).deindent()) self.modname = modname @@ -848,6 +844,9 @@ self.can_use_geninterp = False else: self.can_use_geninterp = True + # make source code available for tracebacks + lines = [x + "\n" for x in source.split("\n")] + py.std.linecache.cache[filename] = (1, None, lines, filename) def __repr__(self): return "" % (self.filename, self.can_use_geninterp) @@ -1081,14 +1080,14 @@ # ____________________________________________________________ -def appdef(source, applevel=ApplevelClass): +def appdef(source, applevel=ApplevelClass, filename=None): """ NOT_RPYTHON: build an app-level helper function, like for example: myfunc = appdef('''myfunc(x, y): return x+y ''') """ if not isinstance(source, str): - source = str(py.code.Source(source).strip()) + source = py.std.inspect.getsource(source).lstrip() while source.startswith('@py.test.mark.'): # these decorators are known to return the same function # object, we may ignore them @@ -1100,7 +1099,11 @@ assert p >= 0 funcname = source[:p].strip() source = source[p:] - return applevel("def %s%s\n" % (funcname, source)).interphook(funcname) + assert source.strip() + funcsource = "def %s%s\n" % (funcname, source) + #for debugging of wrong source code: py.std.parser.suite(funcsource) + a = applevel(funcsource, filename=filename) + return a.interphook(funcname) applevel = ApplevelClass # backward compatibility app2interp = appdef # backward compatibility @@ -1118,6 +1121,6 @@ return PyPyCacheDir.build_applevelinterp_dict(self, space) # app2interp_temp is used for testing mainly -def app2interp_temp(func, applevel_temp=applevel_temp): +def app2interp_temp(func, applevel_temp=applevel_temp, filename=None): """ NOT_RPYTHON """ - return appdef(func, applevel_temp) + return appdef(func, applevel_temp, filename=filename) Modified: pypy/branch/py131/pypy/tool/pytest/appsupport.py ============================================================================== --- pypy/branch/py131/pypy/tool/pytest/appsupport.py (original) +++ pypy/branch/py131/pypy/tool/pytest/appsupport.py Fri May 21 16:43:24 2010 @@ -11,7 +11,8 @@ self.raw = pycode self.w_file = space.getattr(pycode, space.wrap('co_filename')) self.name = space.getattr(pycode, space.wrap('co_name')) - self.firstlineno = space.unwrap(space.getattr(pycode, space.wrap('co_firstlineno'))) + self.firstlineno = space.unwrap( + space.getattr(pycode, space.wrap('co_firstlineno'))) - 1 #try: # self.path = space.unwrap(space.getattr(self.w_file, space.wrap('__path__'))) #except OperationError: @@ -20,13 +21,14 @@ self.space = space def fullsource(self): + filename = self.space.str_w(self.w_file) + source = py.code.Source(py.std.linecache.getlines(filename)) + if source.lines: + return source try: - return self.space.str_w(self.w_file).__source__ - except AttributeError: - try: - return py.code.Source(self.path.read(mode="rU")) - except py.error.Error: - return None + return py.code.Source(self.path.read(mode="rU")) + except py.error.Error: + return None fullsource = property(fullsource, None, None, "Full source of AppCode") def getargs(self): @@ -211,12 +213,19 @@ source = py.code.Source(expr) frame = space.getexecutioncontext().gettopframe() w_locals = frame.getdictscope() + pycode = frame.pycode + filename = "<%s:%s>" %(pycode.co_filename, frame.f_lineno) + lines = [x + "\n" for x in expr.split("\n")] + py.std.linecache.cache[filename] = (1, None, lines, filename) w_locals = space.call_method(w_locals, 'copy') for key, w_value in kwds_w.items(): space.setitem(w_locals, space.wrap(key), w_value) + #filename = __file__ + #if filename.endswith("pyc"): + # filename = filename[:-1] try: space.exec_(str(source), frame.w_globals, w_locals, - filename=__file__) + filename=filename) except OperationError, e: if e.match(space, w_ExpectedException): return _exc_info(space, e) Modified: pypy/branch/py131/pypy/tool/pytest/test/test_appsupport.py ============================================================================== --- pypy/branch/py131/pypy/tool/pytest/test/test_appsupport.py (original) +++ pypy/branch/py131/pypy/tool/pytest/test/test_appsupport.py Fri May 21 16:43:24 2010 @@ -70,6 +70,49 @@ assert result.ret == 0 result.stdout.fnmatch_lines(["*2 passed*"]) +def test_applevel_raises_simple_display(testdir): + setpypyconftest(testdir) + p = testdir.makepyfile(""" + def app_test_raises(): + raises(ValueError, x) + class AppTestRaises: + def test_func(self): + raises (ValueError, x) + # + """) + result = testdir.runpytest(p, "-s") + assert result.ret == 1 + result.stdout.fnmatch_lines([ + "*E*application-level*NameError*x*not defined", + "*test_func(self)*", + ">*raises*ValueError*", + "*E*application-level*NameError*x*not defined", + "*test_applevel_raises_simple_display*", + ]) + result = testdir.runpytest(p) # this time we may run the pyc file + assert result.ret == 1 + result.stdout.fnmatch_lines([ + "*E*application-level*NameError*x*not defined", + ]) + +def test_applevel_raises_display(testdir): + setpypyconftest(testdir) + p = testdir.makepyfile(""" + def app_test_raises(): + raises(ValueError, "x") + pass + """) + result = testdir.runpytest(p, "-s") + assert result.ret == 1 + result.stdout.fnmatch_lines([ + "*E*application-level*NameError*x*not defined", + ]) + result = testdir.runpytest(p) # this time we may run the pyc file + assert result.ret == 1 + result.stdout.fnmatch_lines([ + "*E*application-level*NameError*x*not defined", + ]) + def app_test_raises(): info = raises(TypeError, id) assert info.type is TypeError From arigo at codespeak.net Fri May 21 16:56:30 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 May 2010 16:56:30 +0200 (CEST) Subject: [pypy-svn] r74634 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter codewriter/test metainterp Message-ID: <20100521145630.284E2282C0D@codespeak.net> Author: arigo Date: Fri May 21 16:56:28 2010 New Revision: 74634 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: Still progress. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Fri May 21 16:56:28 2010 @@ -214,17 +214,24 @@ if linkfalse.llexitcase == True: linkfalse, linktrue = linktrue, linkfalse opname = 'goto_if_not' + livebefore = False if isinstance(block.exitswitch, tuple): # special case produced by jtransform.optimize_goto_if_not() opname = 'goto_if_not_' + block.exitswitch[0] opargs = block.exitswitch[1:] + if opargs[-1] == '-live-before': + livebefore = True + opargs = opargs[:-1] else: assert block.exitswitch.concretetype == lltype.Bool opargs = [block.exitswitch] # lst = self.flatten_list(opargs) + [TLabel(linkfalse)] + if livebefore: + self.emitline('-live-') self.emitline(opname, *lst) - self.emitline('-live-', TLabel(linkfalse)) + if not livebefore: + self.emitline('-live-', TLabel(linkfalse)) # true path: self.make_link(linktrue) # false path: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Fri May 21 16:56:28 2010 @@ -136,6 +136,8 @@ # ok! optimize this case block.operations.remove(op) block.exitswitch = (op.opname,) + tuple(op.args) + if op.opname in ('ptr_iszero', 'ptr_nonzero'): + block.exitswitch += ('-live-before',) # if the variable escape to the next block along a link, # replace it with a constant, because we know its value for link in block.exits: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Fri May 21 16:56:28 2010 @@ -541,3 +541,18 @@ -live- %i0, %r1 int_return %i0 """, transform=True, liveness=True) + + def test_ptr_nonzero(self): + def f(p): + if p: + return 12 + return 34 + S = lltype.GcStruct('S') + self.encoding_test(f, [lltype.malloc(S)], """ + -live- %r0 + goto_if_not_ptr_nonzero %r0, L1 + int_return $12 + --- + L1: + int_return $34 + """, transform=True, liveness=True) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Fri May 21 16:56:28 2010 @@ -142,7 +142,7 @@ res = Transformer().optimize_goto_if_not(block) assert res == True assert block.operations == [] - assert block.exitswitch == (opname, v1) + assert block.exitswitch == (opname, v1, '-live-before') assert block.exits == exits def test_symmetric(): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Fri May 21 16:56:28 2010 @@ -1010,8 +1010,13 @@ # (so it will be redone). pass # - elif opnum == rop.GUARD_NONNULL or opnum == rop.GUARD_ISNULL: - xxx + elif (opnum == rop.GUARD_NONNULL or + opnum == rop.GUARD_ISNULL or + opnum == rop.GUARD_NONNULL_CLASS): + # Produced by goto_if_not_ptr_{non,is}zero(). The pc is at the + # start of the opcode (so it will be redone). This is needed + # because of GUARD_NONNULL_CLASS. + pass elif (opnum == rop.GUARD_NO_EXCEPTION or opnum == rop.GUARD_EXCEPTION or opnum == rop.GUARD_NOT_FORCED or Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Fri May 21 16:56:28 2010 @@ -66,6 +66,7 @@ self.copy_constants(self.registers_i, jitcode.constants_i, ConstInt) self.copy_constants(self.registers_r, jitcode.constants_r, ConstPtr) self.copy_constants(self.registers_f, jitcode.constants_f, ConstFloat) + self._result_argcode = 'v' def copy_constants(self, registers, constants, ConstClass): """Copy jitcode.constants[0] to registers[255], @@ -113,6 +114,7 @@ if argcode == 'i': self.registers_i[index] = None elif argcode == 'r': self.registers_r[index] = None elif argcode == 'f': self.registers_f[index] = None + self._result_argcode = 'v' # done # self._tmp_count = 0 count = self.jitcode.enumerate_live_vars( @@ -297,25 +299,25 @@ self.opimpl_goto_if_not(condbox, target) ''' % (_opimpl, _opimpl.upper())).compile() - @arguments("box", "label") - def opimpl_goto_if_not_ptr_nonzero(self, box, target): + @arguments("orgpc", "box", "label") + def opimpl_goto_if_not_ptr_nonzero(self, orgpc, box, target): value = box.nonnull() if value: opnum = rop.GUARD_NONNULL else: opnum = rop.GUARD_ISNULL - self.generate_guard(opnum, box) + self.generate_guard(opnum, box, resumepc=orgpc) if not value: self.pc = target - @arguments("box", "label") - def opimpl_goto_if_not_ptr_iszero(self, box, target): + @arguments("orgpc", "box", "label") + def opimpl_goto_if_not_ptr_iszero(self, orgpc, box, target): value = box.nonnull() if value: opnum = rop.GUARD_NONNULL else: opnum = rop.GUARD_ISNULL - self.generate_guard(opnum, box) + self.generate_guard(opnum, box, resumepc=orgpc) if value: self.pc = target @@ -1567,8 +1569,10 @@ pass elif opnum == rop.GUARD_VALUE or opnum == rop.GUARD_CLASS: pass # the pc is already set to the *start* of the opcode - elif opnum == rop.GUARD_NONNULL or opnum == rop.GUARD_ISNULL: - xxx #self.framestack[-1].ignore_next_guard_nullness(opnum) + elif (opnum == rop.GUARD_NONNULL or + opnum == rop.GUARD_ISNULL or + opnum == rop.GUARD_NONNULL_CLASS): + pass # the pc is already set to the *start* of the opcode elif (opnum == rop.GUARD_NO_EXCEPTION or opnum == rop.GUARD_EXCEPTION or opnum == rop.GUARD_NOT_FORCED): exception = self.cpu.grab_exc_value() From afa at codespeak.net Fri May 21 17:01:16 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 21 May 2010 17:01:16 +0200 (CEST) Subject: [pypy-svn] r74635 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100521150116.E4427282C0D@codespeak.net> Author: afa Date: Fri May 21 17:01:15 2010 New Revision: 74635 Modified: pypy/trunk/pypy/module/cpyext/eval.py pypy/trunk/pypy/module/cpyext/test/test_eval.py Log: Implement PyEval_GetGlobals(), PyEval_GetLocals() Modified: pypy/trunk/pypy/module/cpyext/eval.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/eval.py (original) +++ pypy/trunk/pypy/module/cpyext/eval.py Fri May 21 17:01:15 2010 @@ -22,6 +22,24 @@ w_builtins = space.builtin.getdict() return borrow_from(None, w_builtins) + at cpython_api([], PyObject, error=CANNOT_FAIL) +def PyEval_GetLocals(space): + """Return a dictionary of the local variables in the current execution + frame, or NULL if no frame is currently executing.""" + caller = space.getexecutioncontext().gettopframe_nohidden() + if caller is None: + return None + return borrow_from(None, caller.getdictscope()) + + at cpython_api([], PyObject, error=CANNOT_FAIL) +def PyEval_GetGlobals(space): + """Return a dictionary of the global variables in the current execution + frame, or NULL if no frame is currently executing.""" + caller = space.getexecutioncontext().gettopframe_nohidden() + if caller is None: + return None + return borrow_from(None, caller.w_globals) + @cpython_api([PyObject, PyObject], PyObject) def PyObject_CallObject(space, w_obj, w_arg): """ Modified: pypy/trunk/pypy/module/cpyext/test/test_eval.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_eval.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_eval.py Fri May 21 17:01:15 2010 @@ -2,6 +2,7 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.eval import Py_single_input, Py_file_input, Py_eval_input +from pypy.interpreter.gateway import interp2app class TestEval(BaseApiTest): def test_eval(self, space, api): @@ -79,10 +80,8 @@ def test_getbuiltins(self, space, api): assert api.PyEval_GetBuiltins() is space.builtin.w_dict - from pypy.interpreter.gateway import ObjSpace, interp2app def cpybuiltins(space): return api.PyEval_GetBuiltins() - cpybuiltins.unwrap_spec = [ObjSpace] w_cpybuiltins = space.wrap(interp2app(cpybuiltins)) w_result = space.appexec([w_cpybuiltins], """(cpybuiltins): @@ -96,6 +95,24 @@ """) assert space.int_w(space.len(w_result)) == 1 + def test_getglobals(self, space, api): + assert api.PyEval_GetLocals() is None + assert api.PyEval_GetGlobals() is None + + def cpyvars(space): + return space.newtuple([api.PyEval_GetGlobals(), + api.PyEval_GetLocals()]) + w_cpyvars = space.wrap(interp2app(cpyvars)) + + w_result = space.appexec([w_cpyvars], """(cpyvars): + x = 1 + return cpyvars() + \ny = 2 + """) + globals, locals = space.unwrap(w_result) + assert sorted(locals) == ['cpyvars', 'x'] + assert sorted(globals) == ['__builtins__', 'anonymous', 'y'] + class AppTestCall(AppTestCpythonExtensionBase): def test_CallFunction(self): From arigo at codespeak.net Fri May 21 17:40:42 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 May 2010 17:40:42 +0200 (CEST) Subject: [pypy-svn] r74636 - in pypy/branch/blackhole-improvement/pypy: jit/metainterp rpython/lltypesystem Message-ID: <20100521154042.ED8D1282C00@codespeak.net> Author: arigo Date: Fri May 21 17:40:41 2010 New Revision: 74636 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py Log: Fix a bug. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Fri May 21 17:40:41 2010 @@ -43,10 +43,6 @@ class MIFrame(object): - # for resume.py operation - parent_resumedata_snapshot = None - parent_resumedata_frame_info_list = None - env = property(lambda: xxx, lambda: xxx) # temporary: no read/write! def __init__(self, metainterp): @@ -67,6 +63,9 @@ self.copy_constants(self.registers_r, jitcode.constants_r, ConstPtr) self.copy_constants(self.registers_f, jitcode.constants_f, ConstFloat) self._result_argcode = 'v' + # for resume.py operation + self.parent_resumedata_snapshot = None + self.parent_resumedata_frame_info_list = None def copy_constants(self, registers, constants, ConstClass): """Copy jitcode.constants[0] to registers[255], @@ -1548,6 +1547,7 @@ # raises in case it works -- which is the common case self.compile(original_boxes, live_arg_boxes, start) # creation of the loop was cancelled! + self.staticdata.log('cancelled, going on...') # Otherwise, no loop found so far, so continue tracing. start = len(self.history.operations) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py Fri May 21 17:40:41 2010 @@ -757,7 +757,7 @@ if tag == TAGCONST: if tagged_eq(tagged, NULLREF): return self.cpu.ts.NULLREF - return self.consts[num].getptr_base() + return self.consts[num].getref_base() elif tag == TAGVIRTUAL: virtuals = self.virtuals assert virtuals is not None Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py Fri May 21 17:40:41 2010 @@ -504,10 +504,12 @@ return model.SomeInteger() def lltype(self): return lltype.Signed - def __cmp__(self, other): - if isinstance(other, AddressAsInt): - return cmp(self.adr, other.adr) - return Symbolic.__cmp__(self, other) + def __eq__(self, other): + return self.adr == cast_int_to_adr(other) + def __ne__(self, other): + return self.adr != cast_int_to_adr(other) + def __nonzero__(self): + return bool(self.adr) def __repr__(self): try: return '' % (self.adr.ptr,) From jcreigh at codespeak.net Fri May 21 17:43:22 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Fri, 21 May 2010 17:43:22 +0200 (CEST) Subject: [pypy-svn] r74637 - in pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86: . test tool Message-ID: <20100521154322.9966B282C00@codespeak.net> Author: jcreigh Date: Fri May 21 17:43:20 2010 New Revision: 74637 Added: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py - copied, changed from r74549, pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/regloc.py Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/codebuf.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/jump.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regalloc.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_assembler.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/tool/instruction_encoding.sh Log: Commiting what I have before attempting a minor refactor. (A few tests even pass!) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Fri May 21 17:43:20 2010 @@ -10,9 +10,15 @@ from pypy.jit.backend.x86.regalloc import RegAlloc, WORD,\ X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs, FRAME_FIXED_SIZE,\ FORCE_INDEX_OFS + +from pypy.jit.backend.x86.regloc import (eax, ecx, edx, ebx, + esp, ebp, esi, edi, + xmm0, xmm1, xmm2, xmm3, + xmm4, xmm5, xmm6, xmm7, + RegLoc, StackLoc, ImmedLoc, imm) + from pypy.rlib.objectmodel import we_are_translated, specialize -from pypy.jit.backend.x86 import codebuf -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86 import rx86, regloc, codebuf from pypy.jit.metainterp.resoperation import rop from pypy.jit.backend.x86.support import values_array from pypy.rlib.debug import debug_print @@ -62,7 +68,7 @@ def make_new_mc(self): new_mc = self._instantiate_mc() debug_print('[new machine code block at', new_mc.tell(), ']') - self._mc.JMP(rel32(new_mc.tell())) + self._mc.JMP_l(new_mc.tell()) if self.function_name is not None: self.end_function(done=False) @@ -89,6 +95,9 @@ method.func_name = name return method +for _name in rx86.all_instructions + regloc.all_extra_instructions: + setattr(MachineCodeBlockWrapper, _name, _new_method(_name)) + for name in dir(codebuf.MachineCodeBlock): if name.upper() == name or name == "writechr": setattr(MachineCodeBlockWrapper, name, _new_method(name)) @@ -183,8 +192,8 @@ addr[5] = 2147483647 # / for abs addr[6] = 0 # addr[7] = 0 # - self.loc_float_const_neg = heap64(float_constants) - self.loc_float_const_abs = heap64(float_constants + 16) + self.loc_float_const_neg = float_constants + self.loc_float_const_abs = float_constants + 16 def _build_malloc_fixedsize_slowpath(self): mc = self.mc2._mc @@ -196,7 +205,7 @@ mc.SUB(edx, eax) # compute the size we want mc.MOV(mem(esp, 4), edx) # save it as the new argument addr = self.cpu.gc_ll_descr.get_malloc_fixedsize_slowpath_addr() - mc.JMP(rel32(addr)) # tail call to the real malloc + mc.JMP_l(addr) # tail call to the real malloc # ---------- second helper for the slow path of malloc ---------- self.malloc_fixedsize_slowpath2 = mc.tell() if self.cpu.supports_floats: # restore the XMM registers @@ -295,7 +304,7 @@ def patch_jump_for_descr(self, faildescr, adr_new_target): adr_jump_offset = faildescr._x86_adr_jump_offset mc = codebuf.InMemoryCodeBuilder(adr_jump_offset, adr_jump_offset + 4) - mc.write(packimm32(adr_new_target - adr_jump_offset - 4)) + mc.writeimm32(adr_new_target - adr_jump_offset - 4) mc.valgrind_invalidated() mc.done() @@ -318,7 +327,7 @@ def _patchable_stackadjust(self): # stack adjustment LEA - self.mc.LEA(esp, fixedsize_ebp_ofs(0)) + self.mc.LEA32_rb(esp.value, 0) return self.mc.tell() - 4 def _patch_stackadjust(self, adr_lea, reserved_depth): @@ -330,12 +339,12 @@ words = (FRAME_FIXED_SIZE - 1) + reserved_depth # align, e.g. for Mac OS X aligned_words = align_stack_words(words+2)-2 # 2 = EIP+EBP - mc.write(packimm32(-WORD * aligned_words)) + mc.writeimm32(-WORD * aligned_words) mc.done() def _call_header(self): self.mc.PUSH(ebp) - self.mc.MOV(ebp, esp) + self.mc.MOV_rr(ebp.value, esp.value) self.mc.PUSH(ebx) self.mc.PUSH(esi) self.mc.PUSH(edi) @@ -354,23 +363,26 @@ self._patch_stackadjust(adr_stackadjust, stackdepth) for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] - if isinstance(loc, REG): - self.mc.MOV(loc, mem(ebp, (2 + i) * WORD)) + if isinstance(loc, RegLoc): + assert not loc.is_xmm + self.mc.MOV_rb(loc.value, (2 + i) * WORD) loc = floatlocs[i] - if isinstance(loc, XMMREG): - self.mc.MOVSD(loc, mem64(ebp, (1 + i) * 2 * WORD)) + if isinstance(loc, RegLoc): + assert loc.is_xmm + self.mc.MOVSD_rb(loc.value, (1 + i) * 2 * WORD) tmp = eax xmmtmp = xmm0 for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] - if loc is not None and not isinstance(loc, REG): - self.mc.MOV(tmp, mem(ebp, (2 + i) * WORD)) + if loc is not None and not isinstance(loc, RegLoc): + self.mc.MOV_rb(tmp.value, (2 + i) * WORD) self.mc.MOV(loc, tmp) loc = floatlocs[i] - if loc is not None and not isinstance(loc, XMMREG): - self.mc.MOVSD(xmmtmp, mem64(ebp, (1 + i) * 2 * WORD)) - self.mc.MOVSD(loc, xmmtmp) - self.mc.JMP(rel32(jmpadr)) + if loc is not None and not isinstance(loc, RegLoc): + self.mc.MOVSD_rb(xmmtmp.value, (1 + i) * 2 * WORD) + assert isinstance(loc, StackLoc) + self.mc.MOVSD_br(loc.value, xmmtmp.value) + self.mc.JMP_l(jmpadr) return adr_stackadjust def _assemble_bootstrap_code(self, inputargs, arglocs): @@ -382,7 +394,7 @@ loc = nonfloatlocs[i] if loc is None: continue - if isinstance(loc, REG): + if isinstance(loc, RegLoc): target = loc else: target = tmp @@ -391,22 +403,24 @@ # reading them self.mc.XOR(target, target) adr = self.fail_boxes_ptr.get_addr_for_num(i) - self.mc.XCHG(target, heap(adr)) + self.mc.XCHG_rj(target.value, adr) else: adr = self.fail_boxes_int.get_addr_for_num(i) - self.mc.MOV(target, heap(adr)) + self.mc.MOV_rj(target.value, adr) if target is not loc: - self.mc.MOV(loc, target) + assert isinstance(loc, StackLoc) + self.mc.MOV_br(loc.value, target.value) for i in range(len(floatlocs)): loc = floatlocs[i] if loc is None: continue adr = self.fail_boxes_float.get_addr_for_num(i) - if isinstance(loc, REG): - self.mc.MOVSD(loc, heap64(adr)) + if isinstance(loc, RegLoc): + self.mc.MOVSD_rj(loc.value, adr) else: - self.mc.MOVSD(xmmtmp, heap64(adr)) - self.mc.MOVSD(loc, xmmtmp) + self.mc.MOVSD_rj(xmmtmp.value, adr) + assert isinstance(loc, StackLoc) + self.mc.MOVSD_br(loc.value, xmmtmp.value) return adr_stackadjust def dump(self, text): @@ -439,7 +453,7 @@ # ------------------------------------------------------------ def mov(self, from_loc, to_loc): - if isinstance(from_loc, XMMREG) or isinstance(to_loc, XMMREG): + if (isinstance(from_loc, RegLoc) and from_loc.is_xmm) or (isinstance(to_loc, RegLoc) and to_loc.is_xmm): self.mc.MOVSD(to_loc, from_loc) else: self.mc.MOV(to_loc, from_loc) @@ -447,10 +461,10 @@ regalloc_mov = mov # legacy interface def regalloc_push(self, loc): - if isinstance(loc, XMMREG): - self.mc.SUB(esp, imm(2*WORD)) - self.mc.MOVSD(mem64(esp, 0), loc) - elif isinstance(loc, MODRM64): + if isinstance(loc, RegLoc) and loc.is_xmm: + self.mc.SUB_ri(esp.value, 2*WORD) + self.mc.MOVSD_sr(0, loc.value) + elif isinstance(loc, StackLoc) and loc.width == 8: # XXX evil trick self.mc.PUSH(mem(ebp, get_ebp_ofs(loc.position))) self.mc.PUSH(mem(ebp, get_ebp_ofs(loc.position + 1))) @@ -543,19 +557,15 @@ if isinstance(op.args[0], Const): self.mc.CMP(arglocs[1], arglocs[0]) if guard_opnum == rop.GUARD_FALSE: - name = 'J' + rev_cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(addr, rev_cond) else: - name = 'J' + false_rev_cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(addr, false_rev_cond) else: self.mc.CMP(arglocs[0], arglocs[1]) if guard_opnum == rop.GUARD_FALSE: - name = 'J' + cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(addr, cond) else: - name = 'J' + false_cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(addr, false_cond) return genop_cmp_guard def _cmpop_guard_float(cond, false_cond, need_jp): @@ -575,9 +585,8 @@ mc = self.mc._mc mc.JP(rel8(2)) getattr(mc, 'J' + cond)(rel8(5)) - return self.implement_guard(addr, mc.JMP) - name = 'J' + false_cond - return self.implement_guard(addr, getattr(self.mc, name)) + return self.implement_guard(addr) + return self.implement_guard(addr, false_cond) return genop_cmp_guard_float @specialize.arg(5) @@ -589,29 +598,30 @@ n = len(arglocs) for i in range(start, n): loc = arglocs[i] - if isinstance(loc, REG): - if isinstance(loc, XMMREG): - mc.MOVSD(mem64(esp, p), loc) + if isinstance(loc, RegLoc): + if loc.is_xmm: + mc.MOVSD_mr((esp.value, p), loc.value) else: - mc.MOV(mem(esp, p), loc) + mc.MOV_mr((esp.value, p), loc.value) p += round_up_to_4(loc.width) p = 0 for i in range(start, n): loc = arglocs[i] - if not isinstance(loc, REG): - if isinstance(loc, MODRM64): + if not isinstance(loc, RegLoc): + # if isinstance(loc, MODRM64): + if False: mc.MOVSD(xmm0, loc) mc.MOVSD(mem64(esp, p), xmm0) else: mc.MOV(tmp, loc) - mc.MOV(mem(esp, p), tmp) + mc.MOV_sr(p, tmp.value) p += round_up_to_4(loc.width) self._regalloc.reserve_param(p//WORD) - mc.CALL(x) + mc.CALL_l(x) self.mark_gc_roots() def call(self, addr, args, res): - self._emit_call(rel32(addr), args) + self._emit_call(addr, args) assert res is eax genop_int_neg = _unaryop("NEG") @@ -679,7 +689,7 @@ else: mc.JP(rel8(2)) mc.JE(rel8(5)) - return self.implement_guard(addr, mc.JMP) + return self.implement_guard(addr) def genop_float_neg(self, op, arglocs, resloc): # Following what gcc does: res = x ^ 0x8000000000000000 @@ -702,7 +712,7 @@ else: mc.JP(rel8(2)) mc.JZ(rel8(5)) - return self.implement_guard(addr, mc.JMP) + return self.implement_guard(addr) def genop_float_is_true(self, op, arglocs, resloc): loc0, loc1 = arglocs @@ -741,28 +751,28 @@ def genop_guard_int_is_true(self, op, guard_op, addr, arglocs, resloc): guard_opnum = guard_op.opnum - self.mc.CMP(arglocs[0], imm8(0)) + self.mc.CMP(arglocs[0], imm(0)) if guard_opnum == rop.GUARD_TRUE: - return self.implement_guard(addr, self.mc.JZ) + return self.implement_guard(addr, 'Z') else: - return self.implement_guard(addr, self.mc.JNZ) + return self.implement_guard(addr, 'NZ') def genop_int_is_true(self, op, arglocs, resloc): - self.mc.CMP(arglocs[0], imm8(0)) + self.mc.CMP(arglocs[0], imm(0)) rl = resloc.lowest8bits() self.mc.SETNE(rl) self.mc.MOVZX(resloc, rl) def genop_guard_bool_not(self, op, guard_op, addr, arglocs, resloc): guard_opnum = guard_op.opnum - self.mc.CMP(arglocs[0], imm8(0)) + self.mc.CMP(arglocs[0], imm(0)) if guard_opnum == rop.GUARD_TRUE: - return self.implement_guard(addr, self.mc.JNZ) + return self.implement_guard(addr, 'NZ') else: - return self.implement_guard(addr, self.mc.JZ) + return self.implement_guard(addr, 'Z') def genop_bool_not(self, op, arglocs, resloc): - self.mc.XOR(arglocs[0], imm8(1)) + self.mc.XOR(arglocs[0], imm(1)) def genop_same_as(self, op, arglocs, resloc): self.mov(arglocs[0], resloc) @@ -771,18 +781,18 @@ def genop_int_mod(self, op, arglocs, resloc): self.mc.CDQ() - self.mc.IDIV(ecx) + self.mc.IDIV_r(ecx.value) genop_int_floordiv = genop_int_mod def genop_uint_floordiv(self, op, arglocs, resloc): self.mc.XOR(edx, edx) - self.mc.DIV(ecx) + self.mc.DIV_r(ecx.value) def genop_new_with_vtable(self, op, arglocs, result_loc): assert result_loc is eax loc_vtable = arglocs[-1] - assert isinstance(loc_vtable, IMM32) + assert isinstance(loc_vtable, ImmedLoc) arglocs = arglocs[:-1] self.call(self.malloc_func_addr, arglocs, eax) # xxx ignore NULL returns for now @@ -790,7 +800,9 @@ def set_vtable(self, loc, loc_vtable): if self.cpu.vtable_offset is not None: - self.mc.MOV(mem(loc, self.cpu.vtable_offset), loc_vtable) + assert isinstance(loc, RegLoc) + assert isinstance(loc_vtable, ImmedLoc) + self.mc.MOV_mi((loc.value, self.cpu.vtable_offset), loc_vtable.value) # XXX genop_new is abused for all varsized mallocs with Boehm, for now # (instead of genop_new_array, genop_newstr, genop_newunicode) @@ -812,18 +824,25 @@ def genop_getfield_gc(self, op, arglocs, resloc): base_loc, ofs_loc, size_loc = arglocs - assert isinstance(size_loc, IMM32) + assert isinstance(size_loc, ImmedLoc) + assert isinstance(resloc, RegLoc) size = size_loc.value + + # FIXME: Should be done in one instruction + self.mc.PUSH(eax) + self.mc.MOV(eax, base_loc) + self.mc.ADD(eax, ofs_loc) if size == 1: - self.mc.MOVZX(resloc, addr8_add(base_loc, ofs_loc)) + self.mc.MOVZX8_rm(resloc.value, (eax.value, 0)) elif size == 2: - self.mc.MOVZX(resloc, addr_add(base_loc, ofs_loc)) + self.mc.MOVZX16_rm(resloc.value, (eax.value, 0)) elif size == WORD: - self.mc.MOV(resloc, addr_add(base_loc, ofs_loc)) + self.mc.MOV_rm(resloc.value, (eax.value, 0)) elif size == 8: - self.mc.MOVSD(resloc, addr64_add(base_loc, ofs_loc)) + self.mc.MOVSD_rm(resloc.value, (eax.value, 0)) else: raise NotImplementedError("getfield size = %d" % size) + self.mc.POP(eax) genop_getfield_raw = genop_getfield_gc genop_getfield_raw_pure = genop_getfield_gc @@ -831,8 +850,8 @@ def genop_getarrayitem_gc(self, op, arglocs, resloc): base_loc, ofs_loc, scale, ofs = arglocs - assert isinstance(ofs, IMM32) - assert isinstance(scale, IMM32) + assert isinstance(ofs, ImmedLoc) + assert isinstance(scale, ImmedLoc) if op.result.type == FLOAT: self.mc.MOVSD(resloc, addr64_add(base_loc, ofs_loc, ofs.value, scale.value)) @@ -851,24 +870,34 @@ def genop_discard_setfield_gc(self, op, arglocs): base_loc, ofs_loc, size_loc, value_loc = arglocs - assert isinstance(size_loc, IMM32) + assert isinstance(size_loc, ImmedLoc) size = size_loc.value + # FIXME: Should be done in one instruction + self.mc.PUSH(eax) + self.mc.PUSH(ebx) + self.mc.MOV(eax, base_loc) + self.mc.ADD(eax, ofs_loc) + self.mc.MOV(ebx, value_loc) if size == WORD * 2: - self.mc.MOVSD(addr64_add(base_loc, ofs_loc), value_loc) + self.mc.MOVSD_mr((eax.value, 0), ebx.value) elif size == WORD: - self.mc.MOV(addr_add(base_loc, ofs_loc), value_loc) + self.mc.MOV_mr((eax.value, 0), ebx.value) elif size == 2: - self.mc.MOV16(addr_add(base_loc, ofs_loc), value_loc) + # XXX: Select 16-bit operand mode in a non-ugly way + self.mc.writechar('\x66') + self.mc.MOV_mr((eax.value, 0), ebx.value) elif size == 1: - self.mc.MOV(addr8_add(base_loc, ofs_loc), value_loc.lowest8bits()) + self.mc.MOV8_mr((eax.value, 0), ebx.lowest8bits()) else: print "[asmgen]setfield addr size %d" % size raise NotImplementedError("Addr size %d" % size) + self.mc.POP(ebx) + self.mc.POP(eax) def genop_discard_setarrayitem_gc(self, op, arglocs): base_loc, ofs_loc, value_loc, scale_loc, baseofs = arglocs - assert isinstance(baseofs, IMM32) - assert isinstance(scale_loc, IMM32) + assert isinstance(baseofs, ImmedLoc) + assert isinstance(scale_loc, ImmedLoc) if op.args[2].type == FLOAT: self.mc.MOVSD(addr64_add(base_loc, ofs_loc, baseofs.value, scale_loc.value), value_loc) @@ -918,7 +947,7 @@ def genop_arraylen_gc(self, op, arglocs, resloc): base_loc, ofs_loc = arglocs - assert isinstance(ofs_loc, IMM32) + assert isinstance(ofs_loc, ImmedLoc) self.mc.MOV(resloc, addr_add_const(base_loc, ofs_loc.value)) def genop_strgetitem(self, op, arglocs, resloc): @@ -942,13 +971,13 @@ def genop_guard_guard_true(self, ign_1, guard_op, addr, locs, ign_2): loc = locs[0] self.mc.TEST(loc, loc) - return self.implement_guard(addr, self.mc.JZ) + return self.implement_guard(addr, 'Z') genop_guard_guard_nonnull = genop_guard_guard_true def genop_guard_guard_no_exception(self, ign_1, guard_op, addr, locs, ign_2): self.mc.CMP(heap(self.cpu.pos_exception()), imm(0)) - return self.implement_guard(addr, self.mc.JNZ) + return self.implement_guard(addr, 'NZ') def genop_guard_guard_exception(self, ign_1, guard_op, addr, locs, resloc): @@ -956,7 +985,7 @@ loc1 = locs[1] self.mc.MOV(loc1, heap(self.cpu.pos_exception())) self.mc.CMP(loc1, loc) - addr = self.implement_guard(addr, self.mc.JNE) + addr = self.implement_guard(addr, 'NE') if resloc is not None: self.mc.MOV(resloc, heap(self.cpu.pos_exc_value())) self.mc.MOV(heap(self.cpu.pos_exception()), imm(0)) @@ -966,9 +995,9 @@ def _gen_guard_overflow(self, guard_op, addr): guard_opnum = guard_op.opnum if guard_opnum == rop.GUARD_NO_OVERFLOW: - return self.implement_guard(addr, self.mc.JO) + return self.implement_guard(addr, 'O') elif guard_opnum == rop.GUARD_OVERFLOW: - return self.implement_guard(addr, self.mc.JNO) + return self.implement_guard(addr, 'NO') else: print "int_xxx_ovf followed by", guard_op.getopname() raise AssertionError @@ -988,7 +1017,7 @@ def genop_guard_guard_false(self, ign_1, guard_op, addr, locs, ign_2): loc = locs[0] self.mc.TEST(loc, loc) - return self.implement_guard(addr, self.mc.JNZ) + return self.implement_guard(addr, 'NZ') genop_guard_guard_isnull = genop_guard_guard_false def genop_guard_guard_value(self, ign_1, guard_op, addr, locs, ign_2): @@ -997,7 +1026,7 @@ self.mc.UCOMISD(locs[0], locs[1]) else: self.mc.CMP(locs[0], locs[1]) - return self.implement_guard(addr, self.mc.JNE) + return self.implement_guard(addr, 'NE') def _cmp_guard_class(self, mc, locs): offset = self.cpu.vtable_offset @@ -1011,7 +1040,7 @@ # - multiply by 4 and use it as an offset in type_info_group # - add 16 bytes, to go past the TYPE_INFO structure loc = locs[1] - assert isinstance(loc, IMM32) + assert isinstance(loc, ImmedLoc) classptr = loc.value # here, we have to go back from 'classptr' to the value expected # from reading the 16 bits in the object header @@ -1026,7 +1055,7 @@ mc = self._start_block() self._cmp_guard_class(mc, locs) self._stop_block() - return self.implement_guard(addr, self.mc.JNE) + return self.implement_guard(addr, 'NE') def genop_guard_guard_nonnull_class(self, ign_1, guard_op, addr, locs, ign_2): @@ -1041,7 +1070,7 @@ mc.overwrite(jb_location-1, [chr(offset)]) self._stop_block() # - return self.implement_guard(addr, self.mc.JNE) + return self.implement_guard(addr, 'NE') def implement_guard_recovery(self, guard_opnum, faildescr, failargs, fail_locs): @@ -1068,12 +1097,12 @@ if box is not None and box.type == FLOAT: withfloats = True break - mc.CALL(rel32(self.failure_recovery_code[exc + 2 * withfloats])) + mc.CALL_l(self.failure_recovery_code[exc + 2 * withfloats]) # write tight data that describes the failure recovery faildescr._x86_failure_recovery_bytecode = mc.tell() self.write_failure_recovery_description(mc, failargs, fail_locs) # write the fail_index too - mc.write(packimm32(fail_index)) + mc.writeimm32(fail_index) # for testing the decoding, write a final byte 0xCC if not we_are_translated(): mc.writechr(0xCC) @@ -1102,11 +1131,11 @@ else: raise AssertionError("bogus kind") loc = locs[i] - if isinstance(loc, MODRM): + if isinstance(loc, StackLoc): n = self.CODE_FROMSTACK//4 + loc.position else: - assert isinstance(loc, REG) - n = loc.op + assert isinstance(loc, RegLoc) + n = loc.value n = kind + 4*n while n > 0x7F: mc.writechr((n & 0x7F) | 0x80) @@ -1159,9 +1188,9 @@ kind = code & 3 code >>= 2 if kind == self.DESCR_FLOAT: - loc = xmm_registers[code] + loc = regloc.XMMREGLOCS[code] else: - loc = registers[code] + loc = regloc.REGLOCS[code] arglocs.append(loc) return arglocs[:] @@ -1201,7 +1230,7 @@ @rgc.no_collect def grab_frame_values(self, bytecode, frame_addr, allregisters): # no malloc allowed here!! - self.fail_ebp = allregisters[16 + ebp.op] + self.fail_ebp = allregisters[16 + ebp.value] num = 0 value_hi = 0 while 1: @@ -1269,7 +1298,7 @@ # original value of the registers, optionally the original # value of XMM registers, and finally a reference to the # recovery bytecode. See _build_failure_recovery() for details. - stack_at_ebp = registers[ebp.op] + stack_at_ebp = registers[ebp.value] bytecode = rffi.cast(rffi.UCHARP, registers[8]) allregisters = rffi.ptradd(registers, -16) return self.grab_frame_values(bytecode, stack_at_ebp, allregisters) @@ -1297,11 +1326,11 @@ mc.PUSH(edx) mc.PUSH(ecx) mc.PUSH(eax) - mc.MOV(esi, esp) + mc.MOV_rr(esi.value, esp.value) if withfloats: - mc.SUB(esp, imm(8*8)) + mc.SUB_ri(esp.value, 8*8) for i in range(8): - mc.MOVSD(mem64(esp, 8*i), xmm_registers[i]) + mc.MOVSD_sr(8*i, i) # we call a provided function that will # - call our on_leave_jitted_hook which will mark @@ -1309,7 +1338,7 @@ # avoid unwarranted freeing # - optionally save exception depending on the flag addr = self.cpu.get_on_leave_jitted_int(save_exception=exc) - mc.CALL(rel32(addr)) + mc.CALL_l(addr) # the following call saves all values from the stack and from # registers to the right 'fail_boxes_' location. @@ -1320,13 +1349,13 @@ # generate_quick_failure(). XXX misaligned stack in the call, but # it's ok because failure_recovery_func is not calling anything more mc.PUSH(esi) - mc.CALL(rel32(failure_recovery_func)) + mc.CALL_l(failure_recovery_func) # returns in eax the fail_index # now we return from the complete frame, which starts from # _assemble_bootstrap_code(). The LEA below throws away most # of the frame, including all the PUSHes that we did just above. - mc.LEA(esp, addr_add(ebp, imm(-3 * WORD))) + mc.LEA_rb(esp.value, -3 * WORD) mc.POP(edi) # [ebp-12] mc.POP(esi) # [ebp-8] mc.POP(ebx) # [ebp-4] @@ -1339,30 +1368,31 @@ mc = self.mc for i in range(len(locs)): loc = locs[i] - if isinstance(loc, REG): + if isinstance(loc, RegLoc): if loc.width == 8: adr = self.fail_boxes_float.get_addr_for_num(i) - mc.MOVSD(heap64(adr), loc) + mc.MOVSD_jr(adr, loc.value) else: if locs_are_ref[i]: adr = self.fail_boxes_ptr.get_addr_for_num(i) else: adr = self.fail_boxes_int.get_addr_for_num(i) - mc.MOV(heap(adr), loc) + mc.MOV_jr(adr, loc.value) for i in range(len(locs)): loc = locs[i] - if not isinstance(loc, REG): + if not isinstance(loc, RegLoc): if loc.width == 8: - mc.MOVSD(xmm0, loc) + assert isinstance(loc, StackLoc) + mc.MOVSD_rb(xmm0.value, loc.value) adr = self.fail_boxes_float.get_addr_for_num(i) - mc.MOVSD(heap64(adr), xmm0) + mc.MOVSD_jr(adr, xmm0.value) else: if locs_are_ref[i]: adr = self.fail_boxes_ptr.get_addr_for_num(i) else: adr = self.fail_boxes_int.get_addr_for_num(i) mc.MOV(eax, loc) - mc.MOV(heap(adr), eax) + mc.MOV_jr(adr, eax.value) # we call a provided function that will # - call our on_leave_jitted_hook which will mark @@ -1370,9 +1400,9 @@ # avoid unwarranted freeing # - optionally save exception depending on the flag addr = self.cpu.get_on_leave_jitted_int(save_exception=exc) - mc.CALL(rel32(addr)) + mc.CALL_l(addr) - mc.LEA(esp, addr_add(ebp, imm(-3 * WORD))) + mc.LEA_rb(esp.value, -3 * WORD) mc.MOV(eax, imm(fail_index)) mc.POP(edi) # [ebp-12] mc.POP(esi) # [ebp-8] @@ -1380,14 +1410,19 @@ mc.POP(ebp) # [ebp] mc.RET() + # FIXME: I changed the third argument to this method, but I don't know + # what to do with @specialize @specialize.arg(2) - def implement_guard(self, addr, emit_jump): - emit_jump(rel32(addr)) + def implement_guard(self, addr, condition=None): + if condition: + self.mc.J_il(rx86.Conditions[condition], addr) + else: + self.mc.JMP_l(addr) return self.mc.tell() - 4 def genop_call(self, op, arglocs, resloc): sizeloc = arglocs[0] - assert isinstance(sizeloc, IMM32) + assert isinstance(sizeloc, ImmedLoc) size = sizeloc.value if isinstance(op.args[0], Const): @@ -1417,7 +1452,7 @@ self.mc.MOV(mem(ebp, FORCE_INDEX_OFS), imm(fail_index)) self.genop_call(op, arglocs, result_loc) self.mc.CMP(mem(ebp, FORCE_INDEX_OFS), imm(0)) - return self.implement_guard(addr, self.mc.JL) + return self.implement_guard(addr, 'L') def genop_guard_call_assembler(self, op, guard_op, addr, arglocs, result_loc): @@ -1450,7 +1485,7 @@ else: assert result_loc is eax or result_loc is None self.mc.CMP(mem(ebp, FORCE_INDEX_OFS), imm(0)) - return self.implement_guard(addr, self.mc.JL) + return self.implement_guard(addr, 'L') def genop_discard_cond_call_gc_wb(self, op, arglocs): # use 'mc._mc' directly instead of 'mc', to avoid @@ -1472,7 +1507,7 @@ # misaligned stack in the call, but it's ok because the write barrier # is not going to call anything more. Also, this assumes that the # write barrier does not touch the xmm registers. - mc.CALL(rel32(descr.get_write_barrier_fn(self.cpu))) + mc.CALL_l(descr.get_write_barrier_fn(self.cpu)) for i in range(len(arglocs)): loc = arglocs[i] assert isinstance(loc, REG) @@ -1484,7 +1519,9 @@ self._stop_block() def genop_force_token(self, op, arglocs, resloc): - self.mc.LEA(resloc, mem(ebp, FORCE_INDEX_OFS)) + # RegAlloc.consider_force_token ensures this: + assert isinstance(resloc, RegLoc) + self.mc.LEA_rb(resloc.value, FORCE_INDEX_OFS) def not_implemented_op_discard(self, op, arglocs): msg = "not implemented operation: %s" % op.getopname() @@ -1512,7 +1549,7 @@ return loop_token._x86_arglocs def closing_jump(self, loop_token): - self.mc.JMP(rel32(loop_token._x86_loop_code)) + self.mc.JMP_l(loop_token._x86_loop_code) def malloc_cond_fixedsize(self, nursery_free_adr, nursery_top_adr, size, tid): @@ -1538,10 +1575,10 @@ # reserve room for the argument to the real malloc and the # 8 saved XMM regs self._regalloc.reserve_param(1+16) - mc.CALL(rel32(slowpath_addr1)) + mc.CALL_l(slowpath_addr1) self.mark_gc_roots() slowpath_addr2 = self.malloc_fixedsize_slowpath2 - mc.CALL(rel32(slowpath_addr2)) + mc.CALL_l(slowpath_addr2) offset = mc.get_relative_pos() - jmp_adr assert 0 < offset <= 127 @@ -1568,31 +1605,6 @@ num = getattr(rop, opname.upper()) genop_list[num] = value -def new_addr_add(heap, mem, memsib): - def addr_add(reg_or_imm1, reg_or_imm2, offset=0, scale=0): - if isinstance(reg_or_imm1, IMM32): - if isinstance(reg_or_imm2, IMM32): - return heap(reg_or_imm1.value + offset + - (reg_or_imm2.value << scale)) - else: - return memsib(None, reg_or_imm2, scale, reg_or_imm1.value + offset) - else: - if isinstance(reg_or_imm2, IMM32): - return mem(reg_or_imm1, offset + (reg_or_imm2.value << scale)) - else: - return memsib(reg_or_imm1, reg_or_imm2, scale, offset) - return addr_add - -addr8_add = new_addr_add(heap8, mem8, memSIB8) -addr_add = new_addr_add(heap, mem, memSIB) -addr64_add = new_addr_add(heap64, mem64, memSIB64) - -def addr_add_const(reg_or_imm1, offset): - if isinstance(reg_or_imm1, IMM32): - return heap(reg_or_imm1.value + offset) - else: - return mem(reg_or_imm1, offset) - def round_up_to_4(size): if size < 4: return 4 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/codebuf.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/codebuf.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/codebuf.py Fri May 21 17:43:20 2010 @@ -2,12 +2,13 @@ import os, sys from pypy.rpython.lltypesystem import lltype, rffi from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.jit.backend.x86.ri386 import I386CodeBuilder +from pypy.jit.backend.x86.rx86 import X86_32_CodeBuilder +from pypy.jit.backend.x86.regloc import LocationCodeBuilder from pypy.rlib.rmmap import PTR, alloc, free from pypy.rlib.debug import make_sure_not_resized -class InMemoryCodeBuilder(I386CodeBuilder): +class InMemoryCodeBuilder(X86_32_CodeBuilder, LocationCodeBuilder): _last_dump_start = 0 def __init__(self, start, end): @@ -31,13 +32,15 @@ def write(self, listofchars): self._pos = self.overwrite(self._pos, listofchars) - def writechr(self, n): - # purely for performance: don't make the one-element list [chr(n)] + def writechar(self, char): pos = self._pos assert pos + 1 <= self._size - self._data[pos] = chr(n) + self._data[pos] = char self._pos = pos + 1 + def writechr(self, n): + self.writechar(chr(n)) + def get_relative_pos(self): return self._pos Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/jump.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/jump.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/jump.py Fri May 21 17:43:20 2010 @@ -1,23 +1,6 @@ import sys from pypy.tool.pairtype import extendabletype -from pypy.jit.backend.x86.ri386 import * - -class __extend__(OPERAND): - __metaclass__ = extendabletype - def _getregkey(self): - raise AssertionError("should only happen to registers and frame " - "positions") - -class __extend__(REG): - __metaclass__ = extendabletype - def _getregkey(self): - return ~self.op - -class __extend__(MODRM): - __metaclass__ = extendabletype - def _getregkey(self): - return self.position - +from pypy.jit.backend.x86.regloc import ImmedLoc, StackLoc def remap_frame_layout(assembler, src_locations, dst_locations, tmpreg): pending_dests = len(dst_locations) @@ -27,7 +10,7 @@ srccount[dst._getregkey()] = 0 for i in range(len(dst_locations)): src = src_locations[i] - if isinstance(src, IMM32): + if isinstance(src, ImmedLoc): continue key = src._getregkey() if key in srccount: @@ -46,7 +29,7 @@ srccount[key] = -1 # means "it's done" pending_dests -= 1 src = src_locations[i] - if not isinstance(src, IMM32): + if not isinstance(src, ImmedLoc): key = src._getregkey() if key in srccount: srccount[key] -= 1 @@ -80,7 +63,7 @@ assert pending_dests == 0 def _move(assembler, src, dst, tmpreg): - if isinstance(dst, MODRM) and isinstance(src, MODRM): + if isinstance(dst, StackLoc) and isinstance(src, StackLoc): assembler.regalloc_mov(src, tmpreg) src = tmpreg assembler.regalloc_mov(src, dst) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regalloc.py Fri May 21 17:43:20 2010 @@ -5,7 +5,7 @@ from pypy.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr, ResOperation, ConstAddr, BoxPtr, LoopToken, INT, REF, FLOAT) -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.regloc import * from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi, rstr from pypy.rlib.objectmodel import we_are_translated from pypy.rlib import rgc @@ -108,14 +108,12 @@ @staticmethod def frame_pos(i, size): if size == 1: - res = mem(ebp, get_ebp_ofs(i)) + return StackLoc(i, get_ebp_ofs(i), size) elif size == 2: - res = mem64(ebp, get_ebp_ofs(i + 1)) + return StackLoc(i, get_ebp_ofs(i+1), size) else: print "Unimplemented size %d" % i raise NotImplementedError("unimplemented size %d" % i) - res.position = i - return res class RegAlloc(object): exc = False @@ -253,13 +251,13 @@ arg = inputargs[i] i += 1 if arg.type == FLOAT: - if isinstance(loc, REG): + if isinstance(loc, RegLoc): self.xrm.reg_bindings[arg] = loc used[loc] = None else: self.fm.frame_bindings[arg] = loc else: - if isinstance(loc, REG): + if isinstance(loc, RegLoc): self.rm.reg_bindings[arg] = loc used[loc] = None else: @@ -819,7 +817,7 @@ def consider_setfield_gc(self, op): ofs_loc, size_loc, ptr = self._unpack_fielddescr(op.descr) - assert isinstance(size_loc, IMM32) + assert isinstance(size_loc, ImmedLoc) if size_loc.value == 1: need_lower_byte = True else: Copied: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py (from r74549, pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/regloc.py) ============================================================================== --- pypy/branch/remove-ri386-multimethod-2/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Fri May 21 17:43:20 2010 @@ -1,4 +1,4 @@ -from pypy.jit.metainterp.history import AbstractValue +from pypy.jit.metainterp.history import AbstractValue, ConstInt from pypy.jit.backend.x86 import rx86 # @@ -14,27 +14,98 @@ def _getregkey(self): return self.value + class StackLoc(AssemblerLocation): _immutable_ = True - def __init__(self, position, ebp_offset): + def __init__(self, position, ebp_offset, num_words): assert ebp_offset < 0 # so no confusion with RegLoc.value self.position = position self.value = ebp_offset + # XXX: Word size hardcoded + self.width = num_words * 4 def __repr__(self): return '%d(%%ebp)' % (self.value,) + def location_code(self): + return 'b' + + # FIXME: This definition of assembler sufficient? + def assembler(self): + return repr(self) + class RegLoc(AssemblerLocation): _immutable_ = True def __init__(self, regnum, is_xmm): assert regnum >= 0 self.value = regnum self.is_xmm = is_xmm + # XXX: Word size + if self.is_xmm: + self.width = 8 + else: + self.width = 4 def __repr__(self): if self.is_xmm: return rx86.R.xmmnames[self.value] else: return rx86.R.names[self.value] + def lowest8bits(self): + # XXX: Only handling i386 al, cl, dl, bl for now + assert self.value < 4 + return self.value + + def location_code(self): + return 'r' + + # FIXME: This definition of assembler sufficient? + def assembler(self): + return '%' + repr(self) + +class ImmedLoc(AssemblerLocation): + _immutable_ = True + # XXX: word size hardcoded. And does this even make sense for an immediate? + width = 4 + def __init__(self, value): + self.value = value + + def location_code(self): + return 'i' + + def getint(self): + return self.value + +class AddressLoc(AssemblerLocation): + _immutable_ = True + + # The address is base_loc + (scaled_loc << scale) + static_offset + def __init__(self, base_loc, scaled_loc, scale, static_offset): + assert 0 <= scale < 4 + assert isinstance(base_loc, ImmedLoc) or isinstance(base_loc, RegLoc) + assert isinstance(scaled_loc, ImmedLoc) or isinstance(scaled_loc, RegLoc) + + if isinstance(base_loc, ImmedLoc): + if isinstance(scaled_loc, ImmedLoc): + self.location_code = 'j' + self.value = base_loc.value + (scaled_loc.value << scale) + static_offset + else: + # FIXME + raise AssertionError("Don't know how to handle this case yet") + else: + if isinstance(scaled_loc, ImmedLoc): + # FIXME: What if base_loc is ebp or esp? + self.location_code = 'm' + self.value = (base_loc.value, (scaled_loc.value << scale) + static_offset) + else: + self.location_code = 'a' + self.value = (base_loc.value, scaled_loc.value, scale, static_offset) + + def location_code(self): + return self.location_code + + def value(self): + return self.value + REGLOCS = [RegLoc(i, is_xmm=False) for i in range(8)] XMMREGLOCS = [RegLoc(i, is_xmm=True) for i in range(8)] eax, ecx, edx, ebx, esp, ebp, esi, edi = REGLOCS @@ -46,14 +117,10 @@ def _binaryop(name): def INSN(self, loc1, loc2): - assert isinstance(loc1, RegLoc) - val1 = loc1.value - if isinstance(loc2, RegLoc): - getattr(self, name + '_rr')(val1, loc2.value) - elif isinstance(loc2, StackLoc): - getattr(self, name + '_rb')(val1, loc2.value) - else: - getattr(self, name + '_ri')(val1, loc2.getint()) + code1 = loc1.location_code() + code2 = loc2.location_code() + # XXX: We just hope that the right method exists + getattr(self, name + '_' + code1 + code2)(loc1.value, loc2.value) return INSN ADD = _binaryop('ADD') @@ -61,6 +128,9 @@ AND = _binaryop('AND') SUB = _binaryop('SUB') XOR = _binaryop('XOR') + MOV = _binaryop('MOV') + MOVSD = _binaryop('MOVSD') + IMUL = _binaryop('IMUL') def PUSH(self, loc): assert isinstance(loc, RegLoc) @@ -96,6 +166,12 @@ assert isinstance(loc0, StackLoc) self.CMP_bi(loc0.value, loc1.getint()) +def imm(x): + # XXX: ri386 migration shim + if isinstance(x, ConstInt): + return ImmedLoc(x.getint()) + else: + return ImmedLoc(x) all_extra_instructions = [name for name in LocationCodeBuilder.__dict__ if name[0].isupper()] Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py Fri May 21 17:43:20 2010 @@ -339,6 +339,8 @@ MOV_rr = insn(rex_w, '\x89', register(2,8), register(1), '\xC0') MOV_br = insn(rex_w, '\x89', register(2,8), stack_bp(1)) MOV_rb = insn(rex_w, '\x8B', register(1,8), stack_bp(2)) + MOV_sr = insn(rex_w, '\x89', register(2,8), stack_sp(1)) + MOV_rs = insn(rex_w, '\x8B', register(1,8), stack_sp(2)) # "MOV reg1, [reg2+offset]" and the opposite direction MOV_rm = insn(rex_w, '\x8B', register(1,8), mem_reg_plus_const(2)) @@ -356,6 +358,11 @@ MOV_rj = insn(rex_w, '\x8B', register(1,8), '\x05', immediate(2)) MOV_jr = insn(rex_w, '\x89', register(2,8), '\x05', immediate(1)) + MOV8_mr = insn(rex_w, '\x88', register(2, 8), mem_reg_plus_const(1)) + + MOVZX8_rm = insn(rex_w, '\x0F\xB6', register(1,8), mem_reg_plus_const(2)) + MOVZX16_rm = insn(rex_w, '\x0F\xB7', register(1,8), mem_reg_plus_const(2)) + # ------------------------------ Arithmetic ------------------------------ ADD_ri, ADD_rr, ADD_rb, _, _ = common_modes(0) @@ -365,6 +372,17 @@ XOR_ri, XOR_rr, XOR_rb, _, _ = common_modes(6) CMP_ri, CMP_rr, CMP_rb, CMP_bi, CMP_br = common_modes(7) + DIV_r = insn(rex_w, '\xF7', register(1), '\xF0') + IDIV_r = insn(rex_w, '\xF7', register(1), '\xF8') + + IMUL_rr = insn(rex_w, '\x0F\xAF', register(1, 8), register(2), '\xC0') + IMUL_rb = insn(rex_w, '\x0F\xAF', register(1, 8), stack_bp(2)) + # XXX: There are more efficient encodings of small immediates + IMUL_rri = insn(rex_w, '\x69', register(1, 8), register(2), '\xC0', immediate(3)) + + def IMUL_ri(self, reg, immed): + return self.IMUL_rri(reg, reg, immed) + # ------------------------------ Misc stuff ------------------------------ NOP = insn('\x90') @@ -381,11 +399,14 @@ CALL_b = insn('\xFF', orbyte(2<<3), stack_bp(1)) XCHG_rm = insn(rex_w, '\x87', register(1,8), mem_reg_plus_const(2)) + XCHG_rj = insn(rex_w, '\x87', register(1,8), '\x05', immediate(2)) JMP_l = insn('\xE9', relative(1)) J_il = insn('\x0F', immediate(1,'o'), '\x80', relative(2)) SET_ir = insn('\x0F', immediate(1,'o'),'\x90', register(2), '\xC0') + CDQ = insn(rex_w, '\x99') + # ------------------------------ SSE2 ------------------------------ MOVSD_rr = xmminsn('\xF2', rex_nw, '\x0F\x10', register(1,8), register(2), @@ -399,6 +420,9 @@ MOVSD_mr = xmminsn('\xF2', rex_nw, '\x0F\x11', register(2,8), mem_reg_plus_const(1)) + MOVSD_rj = xmminsn('\xF2', rex_nw, '\x0F\x10', register(1, 8), '\x05', immediate(2)) + MOVSD_jr = xmminsn('\xF2', rex_nw, '\x0F\x11', register(2, 8), '\x05', immediate(1)) + # ------------------------------------------------------------ Conditions = { @@ -463,6 +487,12 @@ py.test.skip("MOV_rj unsupported") def MOV_jr(self, mem_immed, reg): py.test.skip("MOV_jr unsupported") + def XCHG_rj(self, reg, mem_immed): + py.test.skip("XCGH_rj unsupported") + def MOVSD_rj(self, xmm_reg, mem_immed): + py.test.skip("MOVSD_rj unsupported") + def MOVSD_jr(self, xmm_reg, mem_immed): + py.test.skip("MOVSD_jr unsupported") # ____________________________________________________________ Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_assembler.py Fri May 21 17:43:20 2010 @@ -1,4 +1,4 @@ -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.regloc import * from pypy.jit.backend.x86.assembler import Assembler386, MachineCodeBlockWrapper from pypy.jit.backend.x86.regalloc import X86FrameManager, get_ebp_ofs from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat @@ -21,7 +21,7 @@ return self.base_address + len(self.content) def get_relative_pos(self): return len(self.content) - def JMP(self, *args): + def JMP_l(self, *args): self.content.append(("JMP", args)) def done(self): pass @@ -54,9 +54,9 @@ Assembler386.DESCR_FLOAT + 4*(8+110), Assembler386.CODE_HOLE, Assembler386.CODE_HOLE, - Assembler386.DESCR_INT + 4*ebx.op, - Assembler386.DESCR_REF + 4*esi.op, - Assembler386.DESCR_FLOAT + 4*xmm2.op] + Assembler386.DESCR_INT + 4*ebx.value, + Assembler386.DESCR_REF + 4*esi.value, + Assembler386.DESCR_FLOAT + 4*xmm2.value] double_byte_nums = [] for num in nums[3:6]: double_byte_nums.append((num & 0x7F) | 0x80) @@ -181,9 +181,9 @@ value, lo, hi = get_random_float() expected_floats[i] = value kind = Assembler386.DESCR_FLOAT - if isinstance(loc, REG): - xmmregisters[2*loc.op] = lo - xmmregisters[2*loc.op+1] = hi + if isinstance(loc, RegLoc): + xmmregisters[2*loc.value] = lo + xmmregisters[2*loc.value+1] = hi else: write_in_stack(loc, hi) write_in_stack(loc+1, lo) @@ -199,13 +199,13 @@ value = rffi.cast(rffi.LONG, value) else: assert 0, kind - if isinstance(loc, REG): - registers[loc.op] = value + if isinstance(loc, RegLoc): + registers[loc.value] = value else: write_in_stack(loc, value) - if isinstance(loc, REG): - num = kind + 4*loc.op + if isinstance(loc, RegLoc): + num = kind + 4*loc.value else: num = kind + 4*(8+loc) while num >= 0x80: @@ -225,7 +225,7 @@ assert 0 <= descr_bytecode[i] <= 255 descr_bytes[i] = rffi.cast(rffi.UCHAR, descr_bytecode[i]) registers[8] = rffi.cast(rffi.LONG, descr_bytes) - registers[ebp.op] = rffi.cast(rffi.LONG, stack) + 4*stacklen + registers[ebp.value] = rffi.cast(rffi.LONG, stack) + 4*stacklen # run! assembler = Assembler386(FakeCPU()) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py Fri May 21 17:43:20 2010 @@ -111,6 +111,36 @@ s.SET_ir(5, 2) assert s.getvalue() == '\x0F\x95\xC2' +def test_xchg_rj(): + s = CodeBuilder32() + s.XCHG_rj(edx, 0x01234567) + assert s.getvalue() == '\x87\x15\x67\x45\x23\x01' + +def test_movsd_rj(): + s = CodeBuilder32() + s.MOVSD_rj(xmm2, 0x01234567) + assert s.getvalue() == '\xF2\x0F\x10\x15\x67\x45\x23\x01' + +def test_movzx8_rm(): + s = CodeBuilder32() + s.MOVZX8_rm(ecx, (eax, 16)) + assert s.getvalue() == '\x0F\xB6\x48\x10' + +def test_movzx16_rm(): + s = CodeBuilder32() + s.MOVZX16_rm(ecx, (eax, 16)) + assert s.getvalue() == '\x0F\xB7\x48\x10' + +def test_div(): + s = CodeBuilder32() + s.DIV_r(ecx) + assert s.getvalue() == '\xF7\xF1' + +def test_imul_rri(): + s = CodeBuilder32() + # Multiply ecx by 0x01234567 and store the result in ebx + s.IMUL_rri(ebx, ecx, 0x01234567) + assert s.getvalue() == '\x69\xD9\x67\x45\x23\x01' class CodeBuilder64(CodeBuilderMixin, X86_64_CodeBuilder): pass Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/tool/instruction_encoding.sh ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/tool/instruction_encoding.sh (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/tool/instruction_encoding.sh Fri May 21 17:43:20 2010 @@ -3,10 +3,16 @@ # Tool to quickly see how the GNU assembler encodes an instruction # (AT&T syntax only for now) +# Provide readline if available +if which rlwrap > /dev/null && [ "$INSIDE_RLWRAP" = "" ]; then + export INSIDE_RLWRAP=1 + exec rlwrap "$0" +fi + while :; do echo -n '? ' read instruction echo "$instruction" | as - objdump --disassemble ./a.out | tail -n +8 + objdump --disassemble ./a.out | grep '^ *[0-9a-f]\+:' rm -f ./a.out done From arigo at codespeak.net Fri May 21 17:52:12 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 May 2010 17:52:12 +0200 (CEST) Subject: [pypy-svn] r74638 - in pypy/branch/blackhole-improvement/pypy/jit/metainterp: . test Message-ID: <20100521155212.B9319282C00@codespeak.net> Author: arigo Date: Fri May 21 17:52:11 2010 New Revision: 74638 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py Log: Fix the test. Remove some tests that are now useless. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Fri May 21 17:52:11 2010 @@ -83,7 +83,8 @@ # def dispatch_loop(self, code, position): while True: - if not we_are_translated(): + if (not we_are_translated() + and self.jitcode._startpoints is not None): assert position in self.jitcode._startpoints, ( "the current position %d is in the middle of " "an instruction!" % position) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py Fri May 21 17:52:11 2010 @@ -32,52 +32,56 @@ jitcode.setup("\x00\x00\x01\x02" "\x01\x02", []) - blackholeinterp = getblackholeinterp({'int_add/iii': 0, + blackholeinterp = getblackholeinterp({'int_add/ii>i': 0, 'int_return/i': 1}) + blackholeinterp.setposition(jitcode, 0) blackholeinterp.setarg_i(0, 40) blackholeinterp.setarg_i(1, 2) - blackholeinterp.run(jitcode, 0) - assert blackholeinterp.get_result_i() == 42 + blackholeinterp.run() + assert blackholeinterp.final_result_i() == 42 def test_simple_const(): jitcode = JitCode("test") jitcode.setup("\x00\x30\x01\x02" "\x01\x02", []) - blackholeinterp = getblackholeinterp({'int_sub/cii': 0, + blackholeinterp = getblackholeinterp({'int_sub/ci>i': 0, 'int_return/i': 1}) + blackholeinterp.setposition(jitcode, 0) blackholeinterp.setarg_i(1, 6) - blackholeinterp.run(jitcode, 0) - assert blackholeinterp.get_result_i() == 42 + blackholeinterp.run() + assert blackholeinterp.final_result_i() == 42 def test_simple_bigconst(): jitcode = JitCode("test") jitcode.setup("\x00\xFD\x01\x02" "\x01\x02", [666, 666, 10042, 666]) - blackholeinterp = getblackholeinterp({'int_sub/iii': 0, + blackholeinterp = getblackholeinterp({'int_sub/ii>i': 0, 'int_return/i': 1}) + blackholeinterp.setposition(jitcode, 0) blackholeinterp.setarg_i(1, 10000) - blackholeinterp.run(jitcode, 0) - assert blackholeinterp.get_result_i() == 42 + blackholeinterp.run() + assert blackholeinterp.final_result_i() == 42 def test_simple_loop(): jitcode = JitCode("test") - jitcode.setup("\x00\x10\x00\x16\x02" # L1: goto_if_not_int_gt L2, %i0, 2 + jitcode.setup("\x00\x16\x02\x10\x00" # L1: goto_if_not_int_gt %i0, 2, L2 "\x01\x17\x16\x17" # int_add %i1, %i0, %i1 "\x02\x16\x01\x16" # int_sub %i0, $1, %i0 "\x03\x00\x00" # goto L1 "\x04\x17", # L2: int_return %i1 []) - blackholeinterp = getblackholeinterp({'goto_if_not_int_gt/Lic': 0, - 'int_add/iii': 1, - 'int_sub/ici': 2, + blackholeinterp = getblackholeinterp({'goto_if_not_int_gt/icL': 0, + 'int_add/ii>i': 1, + 'int_sub/ic>i': 2, 'goto/L': 3, 'int_return/i': 4}) + blackholeinterp.setposition(jitcode, 0) blackholeinterp.setarg_i(0x16, 6) # %i0 blackholeinterp.setarg_i(0x17, 100) # %i1 - blackholeinterp.run(jitcode, 0) - assert blackholeinterp.get_result_i() == 100+6+5+4+3 + blackholeinterp.run() + assert blackholeinterp.final_result_i() == 100+6+5+4+3 def test_simple_exception(): jitcode = JitCode("test") @@ -88,101 +92,17 @@ "\x03\x2A", # L1: int_return $42 [321]) # <-- address of the function g blackholeinterp = getblackholeinterp({'catch_exception/L': 0, - 'residual_call_ir_i/idIRi': 1, + 'residual_call_ir_i/idIR>i': 1, 'int_return/i': 2, 'int_return/c': 3}, [""]) # + blackholeinterp.setposition(jitcode, 0) blackholeinterp.setarg_i(0x9, 100) - blackholeinterp.run(jitcode, 0) - assert blackholeinterp.get_result_i() == 200 + blackholeinterp.run() + assert blackholeinterp.final_result_i() == 200 # + blackholeinterp.setposition(jitcode, 0) blackholeinterp.setarg_i(0x9, -100) - blackholeinterp.run(jitcode, 0) - assert blackholeinterp.get_result_i() == 42 - -# ____________________________________________________________ - -class BlackholeTests(object): - - def meta_interp(self, *args): - def counting_init(frame, metainterp, jitcode, greenkey=None): - previnit(frame, metainterp, jitcode, greenkey) - self.seen_frames.append(jitcode.name) - # - from pypy.jit.metainterp import pyjitpl - previnit = pyjitpl.MIFrame.__init__.im_func - try: - self.seen_frames = [] - pyjitpl.MIFrame.__init__ = counting_init - return super(BlackholeTests, self).meta_interp(*args) - finally: - pyjitpl.MIFrame.__init__ = previnit - - def test_calls_not_followed(self): - myjitdriver = JitDriver(greens = [], reds = ['n']) - def h(): - return 42 - def g(): - return h() - def f(n): - while n > 0: - myjitdriver.can_enter_jit(n=n) - myjitdriver.jit_merge_point(n=n) - n -= 1 - return g() - res = self.meta_interp(f, [7]) - assert res == 42 - assert self.seen_frames == ['f', 'f'] - - def test_indirect_calls_not_followed(self): - myjitdriver = JitDriver(greens = [], reds = ['n']) - def h(): - return 42 - def g(): - return h() - def f(n): - while n > 0: - myjitdriver.can_enter_jit(n=n) - myjitdriver.jit_merge_point(n=n) - n -= 1 - if n < 0: - call = h - else: - call = g - return call() - res = self.meta_interp(f, [7]) - assert res == 42 - assert self.seen_frames == ['f', 'f'] - - def test_oosends_not_followed(self): - myjitdriver = JitDriver(greens = [], reds = ['n']) - class A: - def meth(self): - return 42 - class B(A): - def meth(self): - return 45 - class C(A): - def meth(self): - return 64 - def f(n): - while n > 0: - myjitdriver.can_enter_jit(n=n) - myjitdriver.jit_merge_point(n=n) - n -= 1 - if n < 0: - x = B() - else: - x = C() - return x.meth() - res = self.meta_interp(f, [7]) - assert res == 64 - assert self.seen_frames == ['f', 'f'] - - -class TestLLtype(BlackholeTests, LLJitMixin): - pass - -class TestOOtype(BlackholeTests, OOJitMixin): - pass + blackholeinterp.run() + assert blackholeinterp.final_result_i() == 42 From afa at codespeak.net Fri May 21 17:53:20 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 21 May 2010 17:53:20 +0200 (CEST) Subject: [pypy-svn] r74639 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100521155320.21726282C00@codespeak.net> Author: afa Date: Fri May 21 17:53:18 2010 New Revision: 74639 Modified: pypy/trunk/pypy/module/cpyext/object.py pypy/trunk/pypy/module/cpyext/test/test_object.py Log: Add PyObject_SetItem, PyObject_DelItem, PyObject_DelAttrString Modified: pypy/trunk/pypy/module/cpyext/object.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/object.py (original) +++ pypy/trunk/pypy/module/cpyext/object.py Fri May 21 17:53:18 2010 @@ -128,6 +128,14 @@ space.delattr(w_obj, w_name) return 0 + at cpython_api([PyObject, CONST_STRING], rffi.INT_real, error=-1) +def PyObject_DelAttrString(space, w_obj, name_ptr): + """Delete attribute named attr_name, for object o. Returns -1 on failure. + This is the equivalent of the Python statement del o.attr_name.""" + w_name = space.wrap(rffi.charp2str(name_ptr)) + space.delattr(w_obj, w_name) + return 0 + @cpython_api([PyObject], lltype.Void) def PyObject_ClearWeakRefs(space, w_object): w_object.clear_all_weakrefs() @@ -148,6 +156,20 @@ This is the equivalent of the Python expression o[key].""" return space.getitem(w_obj, w_key) + at cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) +def PyObject_SetItem(space, w_obj, w_key, w_value): + """Map the object key to the value v. Returns -1 on failure. This is the + equivalent of the Python statement o[key] = v.""" + space.setitem(w_obj, w_key, w_value) + return 0 + + at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) +def PyObject_DelItem(space, w_obj, w_key): + """Delete the mapping for key from o. Returns -1 on failure. This is the + equivalent of the Python statement del o[key].""" + space.delitem(w_obj, w_key) + return 0 + @cpython_api([PyObject, PyTypeObjectPtr], PyObject) def PyObject_Init(space, op, type): """Initialize a newly-allocated object op with its type and initial Modified: pypy/trunk/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_object.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_object.py Fri May 21 17:53:18 2010 @@ -63,6 +63,9 @@ assert not api.PyObject_GetAttrString(space.wrap(""), charp2) assert api.PyErr_Occurred() is space.w_AttributeError api.PyErr_Clear() + assert api.PyObject_DelAttrString(space.wrap(""), charp1) == -1 + assert api.PyErr_Occurred() is space.w_AttributeError + api.PyErr_Clear() rffi.free_charp(charp1) rffi.free_charp(charp2) @@ -78,6 +81,14 @@ space.setitem(w_d, space.wrap("a key!"), space.wrap(72)) assert space.unwrap(api.PyObject_GetItem(w_d, space.wrap("a key!"))) == 72 + assert api.PyObject_SetItem(w_d, space.wrap("key"), space.w_None) == 0 + assert space.getitem(w_d, space.wrap("key")) is space.w_None + + assert api.PyObject_DelItem(w_d, space.wrap("key")) == 0 + assert api.PyObject_GetItem(w_d, space.wrap("key")) is None + assert api.PyErr_Occurred() is space.w_KeyError + api.PyErr_Clear() + def test_size(self, space, api): assert api.PyObject_Size(space.newlist([space.w_None])) == 1 From arigo at codespeak.net Fri May 21 18:09:50 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 May 2010 18:09:50 +0200 (CEST) Subject: [pypy-svn] r74640 - pypy/branch/blackhole-improvement/pypy/jit/codewriter/test Message-ID: <20100521160950.98C66282C00@codespeak.net> Author: arigo Date: Fri May 21 18:09:49 2010 New Revision: 74640 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py Log: Port a test from the old metainterp/test/test_codewriter.py. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py Fri May 21 18:09:49 2010 @@ -1,21 +1,33 @@ import py from pypy.jit.codewriter.codewriter import CodeWriter from pypy.jit.codewriter import support +from pypy.jit.metainterp.history import AbstractDescr from pypy.rpython.lltypesystem import lltype, llmemory -class FakeRTyper: - class annotator: - translator = None - class type_system: - name = 'lltypesystem' - def getcallable(self, graph): - F = lltype.FuncType([], lltype.Signed) - return lltype.functionptr(F, 'bar') +class FakeCallDescr(AbstractDescr): + def __init__(self, FUNC, ARGS, RESULT, effectinfo=None): + self.FUNC = FUNC + self.ARGS = ARGS + self.RESULT = RESULT + self.effectinfo = effectinfo + def get_extra_info(self): + return self.effectinfo + +class FakeFieldDescr(AbstractDescr): + def __init__(self, STRUCT, fieldname): + self.STRUCT = STRUCT + self.fieldname = fieldname + +class FakeSizeDescr(AbstractDescr): + def __init__(self, STRUCT): + self.STRUCT = STRUCT class FakeCPU: - rtyper = FakeRTyper() - def calldescrof(self, FUNC, ARGS, RESULT): - return ('calldescr', FUNC, ARGS, RESULT) + def __init__(self, rtyper): + self.rtyper = rtyper + calldescrof = FakeCallDescr + fielddescrof = FakeFieldDescr + sizeof = FakeSizeDescr class FakePolicy: def look_inside_graph(self, graph): @@ -57,7 +69,7 @@ return ggg(b) - ggg(a) rtyper = support.annotate(fff, [35, 42]) maingraph = rtyper.annotator.translator.graphs[0] - cw = CodeWriter(FakeCPU(), maingraph) + cw = CodeWriter(FakeCPU(rtyper), maingraph) cw.find_all_graphs(FakePolicy()) cw.make_jitcodes(verbose=True) jitcode = cw.mainjitcode @@ -69,7 +81,7 @@ assert jitcode2.name == 'ggg' assert 'ggg' in jitcode.dump() assert lltype.typeOf(jitcode2.fnaddr) == llmemory.Address - assert jitcode2.calldescr[0] == 'calldescr' + assert isinstance(jitcode2.calldescr, FakeCallDescr) def test_integration(): from pypy.jit.metainterp.blackhole import BlackholeInterpBuilder @@ -87,3 +99,31 @@ blackholeinterp.setarg_i(1, 100) blackholeinterp.run() assert blackholeinterp.final_result_i() == 100+6+5+4+3 + +def test_instantiate(): + class A1: id = 651 + class A2(A1): id = 652 + class B1: id = 661 + class B2(B1): id = 662 + def f(n): + if n > 5: + x, y = A1, B1 + else: + x, y = A2, B2 + n += 1 + return x().id + y().id + n + rtyper = support.annotate(f, [35]) + maingraph = rtyper.annotator.translator.graphs[0] + cw = CodeWriter(FakeCPU(rtyper), maingraph) + cw.find_all_graphs(FakePolicy()) + cw.make_jitcodes(verbose=True) + # + assert len(cw.assembler.indirectcalltargets) == 4 + names = [jitcode.name for jitcode in cw.assembler.indirectcalltargets] + for expected in ['A1', 'A2', 'B1', 'B2']: + for name in names: + if name.startswith('instantiate_') and name.endswith(expected): + break + else: + assert 0, "missing instantiate_*_%s in:\n%r" % (expected, + names) From hpk at codespeak.net Fri May 21 18:10:50 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 21 May 2010 18:10:50 +0200 (CEST) Subject: [pypy-svn] r74641 - pypy/branch/py131/py/_plugin Message-ID: <20100521161050.5D8B9282C00@codespeak.net> Author: hpk Date: Fri May 21 18:10:48 2010 New Revision: 74641 Modified: pypy/branch/py131/py/_plugin/pytest_mark.py pypy/branch/py131/py/_plugin/pytest_skipping.py Log: update to py-trunk 1.3.1a1 - fixing skipping/xfailing if specified multiple times Modified: pypy/branch/py131/py/_plugin/pytest_mark.py ============================================================================== --- pypy/branch/py131/py/_plugin/pytest_mark.py (original) +++ pypy/branch/py131/py/_plugin/pytest_mark.py Fri May 21 18:10:48 2010 @@ -34,38 +34,43 @@ .. _`scoped-marking`: -Marking classes or modules +Marking whole classes or modules ---------------------------------------------------- -To mark all methods of a class set a ``pytestmark`` attribute like this:: +If you are programming with Python2.6 you may use ``py.test.mark`` decorators +with classes to apply markers to all its test methods:: + + @py.test.mark.webtest + class TestClass: + def test_startup(self): + ... + +This is equivalent to directly applying the decorator to the +``test_startup`` function. + +To remain compatible with Python2.5 you can instead set a +``pytestmark`` attribute on a TestClass like this:: import py class TestClass: pytestmark = py.test.mark.webtest -You can re-use the same markers that you would use for decorating -a function - in fact this marker decorator will be applied -to all test methods of the class. +or if you need to use multiple markers:: + + import py + + class TestClass: + pytestmark = [py.test.mark.webtest, pytest.mark.slowtest] You can also set a module level marker:: import py pytestmark = py.test.mark.webtest -in which case then the marker decorator will be applied to all functions and +in which case then it will be applied to all functions and methods defined in the module. -The order in which marker functions are called is this:: - - per-function (upon import of module already) - per-class - per-module - -Later called markers may overwrite previous key-value settings. -Positional arguments are all appended to the same 'args' list -of the Marker object. - Using "-k MARKNAME" to select tests ---------------------------------------------------- @@ -105,15 +110,23 @@ """ if passed a single callable argument: decorate it with mark info. otherwise add *args/**kwargs in-place to mark information. """ if args: - if len(args) == 1 and hasattr(args[0], '__call__'): - func = args[0] - holder = getattr(func, self.markname, None) - if holder is None: - holder = MarkInfo(self.markname, self.args, self.kwargs) - setattr(func, self.markname, holder) + func = args[0] + if len(args) == 1 and hasattr(func, '__call__') or \ + hasattr(func, '__bases__'): + if hasattr(func, '__bases__'): + l = func.__dict__.setdefault("pytestmark", []) + if not isinstance(l, list): + func.pytestmark = [l, self] + else: + l.append(self) else: - holder.kwargs.update(self.kwargs) - holder.args.extend(self.args) + holder = getattr(func, self.markname, None) + if holder is None: + holder = MarkInfo(self.markname, self.args, self.kwargs) + setattr(func, self.markname, holder) + else: + holder.kwargs.update(self.kwargs) + holder.args.extend(self.args) return func else: self.args.extend(args) @@ -147,6 +160,10 @@ func = getattr(func, 'im_func', func) # py2 for parent in [x for x in (mod, cls) if x]: marker = getattr(parent.obj, 'pytestmark', None) - if isinstance(marker, MarkDecorator): - marker(func) + if marker is not None: + if not isinstance(marker, list): + marker = [marker] + for mark in marker: + if isinstance(mark, MarkDecorator): + mark(func) return item Modified: pypy/branch/py131/py/_plugin/pytest_skipping.py ============================================================================== --- pypy/branch/py131/py/_plugin/pytest_skipping.py (original) +++ pypy/branch/py131/py/_plugin/pytest_skipping.py Fri May 21 18:10:48 2010 @@ -60,6 +60,19 @@ # The ``pytestmark`` decorator will be applied to each test function. +If your code targets python2.6 or above you can equivalently use +the skipif decorator on classes:: + + @py.test.mark.skipif("sys.platform == 'win32'") + class TestPosixCalls: + + def test_function(self): + # will not be setup or run under 'win32' platform + # + +It is fine in general to apply multiple "skipif" decorators +on a single function - this means that if any of the conditions +apply the function will be skipped. .. _`whole class- or module level`: mark.html#scoped-marking @@ -144,17 +157,20 @@ def istrue(self): if self.holder: d = {'os': py.std.os, 'sys': py.std.sys, 'config': self.item.config} - self.result = True - for expr in self.holder.args: - self.expr = expr - if isinstance(expr, str): - result = cached_eval(self.item.config, expr, d) - else: - result = expr - if not result: - self.result = False + if self.holder.args: + self.result = False + for expr in self.holder.args: self.expr = expr - break + if isinstance(expr, str): + result = cached_eval(self.item.config, expr, d) + else: + result = expr + if result: + self.result = True + self.expr = expr + break + else: + self.result = True return getattr(self, 'result', False) def get(self, attr, default=None): From jcreigh at codespeak.net Fri May 21 18:42:12 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Fri, 21 May 2010 18:42:12 +0200 (CEST) Subject: [pypy-svn] r74642 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86 Message-ID: <20100521164212.672A2282C00@codespeak.net> Author: jcreigh Date: Fri May 21 18:42:10 2010 New Revision: 74642 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Log: fix AddressLoc, use it in a few places Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Fri May 21 18:42:10 2010 @@ -15,7 +15,8 @@ esp, ebp, esi, edi, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, - RegLoc, StackLoc, ImmedLoc, imm) + RegLoc, StackLoc, + ImmedLoc, AddressLoc, imm) from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.jit.backend.x86 import rx86, regloc, codebuf @@ -828,21 +829,17 @@ assert isinstance(resloc, RegLoc) size = size_loc.value - # FIXME: Should be done in one instruction - self.mc.PUSH(eax) - self.mc.MOV(eax, base_loc) - self.mc.ADD(eax, ofs_loc) + source_addr = AddressLoc(base_loc, ofs_loc) if size == 1: - self.mc.MOVZX8_rm(resloc.value, (eax.value, 0)) + self.mc.MOVZX8(resloc, source_addr) elif size == 2: - self.mc.MOVZX16_rm(resloc.value, (eax.value, 0)) + self.mc.MOVZX16(resloc, source_addr) elif size == WORD: - self.mc.MOV_rm(resloc.value, (eax.value, 0)) + self.mc.MOV(resloc, source_addr) elif size == 8: - self.mc.MOVSD_rm(resloc.value, (eax.value, 0)) + self.mc.MOVSD(resloc, source_addr) else: raise NotImplementedError("getfield size = %d" % size) - self.mc.POP(eax) genop_getfield_raw = genop_getfield_gc genop_getfield_raw_pure = genop_getfield_gc @@ -872,42 +869,31 @@ base_loc, ofs_loc, size_loc, value_loc = arglocs assert isinstance(size_loc, ImmedLoc) size = size_loc.value - # FIXME: Should be done in one instruction - self.mc.PUSH(eax) - self.mc.PUSH(ebx) - self.mc.MOV(eax, base_loc) - self.mc.ADD(eax, ofs_loc) - self.mc.MOV(ebx, value_loc) + dest_addr = AddressLoc(base_loc, ofs_loc) if size == WORD * 2: - self.mc.MOVSD_mr((eax.value, 0), ebx.value) + self.mc.MOVSD(dest_addr, value_loc) elif size == WORD: - self.mc.MOV_mr((eax.value, 0), ebx.value) + self.mc.MOV(dest_addr, value_loc) elif size == 2: - # XXX: Select 16-bit operand mode in a non-ugly way - self.mc.writechar('\x66') - self.mc.MOV_mr((eax.value, 0), ebx.value) + self.mc.MOV16(dest_addr, value_loc) elif size == 1: - self.mc.MOV8_mr((eax.value, 0), ebx.lowest8bits()) + self.mc.MOV8(dest_addr, value_loc.lowest8bits()) else: print "[asmgen]setfield addr size %d" % size raise NotImplementedError("Addr size %d" % size) - self.mc.POP(ebx) - self.mc.POP(eax) def genop_discard_setarrayitem_gc(self, op, arglocs): base_loc, ofs_loc, value_loc, scale_loc, baseofs = arglocs assert isinstance(baseofs, ImmedLoc) assert isinstance(scale_loc, ImmedLoc) + dest_addr = AddressLoc(base_loc, ofs_loc, scale_loc.value, baseofs.value) if op.args[2].type == FLOAT: - self.mc.MOVSD(addr64_add(base_loc, ofs_loc, baseofs.value, - scale_loc.value), value_loc) + self.mc.MOVSD(dest_addr, value_loc) else: if scale_loc.value == 2: - self.mc.MOV(addr_add(base_loc, ofs_loc, baseofs.value, - scale_loc.value), value_loc) + self.mc.MOV(dest_addr, value_loc) elif scale_loc.value == 0: - self.mc.MOV(addr8_add(base_loc, ofs_loc, baseofs.value, - scale_loc.value), value_loc.lowest8bits()) + self.mc.MOV8(dest_addr, value_loc.lowest8bits()) else: raise NotImplementedError("scale = %d" % scale_loc.value) @@ -916,17 +902,17 @@ basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) assert itemsize == 1 - self.mc.MOV(addr8_add(base_loc, ofs_loc, basesize), - val_loc.lowest8bits()) + dest_addr = AddressLoc(base_loc, ofs_loc, 0, basesize) + self.mc.MOV8(dest_addr, val_loc.lowest8bits()) def genop_discard_unicodesetitem(self, op, arglocs): base_loc, ofs_loc, val_loc = arglocs basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) if itemsize == 4: - self.mc.MOV(addr_add(base_loc, ofs_loc, basesize, 2), val_loc) + self.mc.MOV(AddressLoc(base_loc, ofs_loc, 2, basesize), val_loc) elif itemsize == 2: - self.mc.MOV16(addr_add(base_loc, ofs_loc, basesize, 1), val_loc) + self.mc.MOV16(AddressLoc(base_loc, ofs_loc, 1, basesize), val_loc) else: assert 0, itemsize Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Fri May 21 18:42:10 2010 @@ -53,7 +53,8 @@ def lowest8bits(self): # XXX: Only handling i386 al, cl, dl, bl for now assert self.value < 4 - return self.value + assert not self.is_xmm + return self def location_code(self): return 'r' @@ -79,14 +80,14 @@ _immutable_ = True # The address is base_loc + (scaled_loc << scale) + static_offset - def __init__(self, base_loc, scaled_loc, scale, static_offset): + def __init__(self, base_loc, scaled_loc, scale=0, static_offset=0): assert 0 <= scale < 4 assert isinstance(base_loc, ImmedLoc) or isinstance(base_loc, RegLoc) assert isinstance(scaled_loc, ImmedLoc) or isinstance(scaled_loc, RegLoc) if isinstance(base_loc, ImmedLoc): if isinstance(scaled_loc, ImmedLoc): - self.location_code = 'j' + self._location_code = 'j' self.value = base_loc.value + (scaled_loc.value << scale) + static_offset else: # FIXME @@ -94,14 +95,14 @@ else: if isinstance(scaled_loc, ImmedLoc): # FIXME: What if base_loc is ebp or esp? - self.location_code = 'm' + self._location_code = 'm' self.value = (base_loc.value, (scaled_loc.value << scale) + static_offset) else: - self.location_code = 'a' + self._location_code = 'a' self.value = (base_loc.value, scaled_loc.value, scale, static_offset) def location_code(self): - return self.location_code + return self._location_code def value(self): return self.value @@ -125,12 +126,20 @@ ADD = _binaryop('ADD') OR = _binaryop('OR') + XOR = _binaryop('XOR') + AND = _binaryop('AND') SUB = _binaryop('SUB') - XOR = _binaryop('XOR') + IMUL = _binaryop('IMUL') + MOV = _binaryop('MOV') + MOV8 = _binaryop('MOV8') MOVSD = _binaryop('MOVSD') - IMUL = _binaryop('IMUL') + + def MOV16(self, dest_loc, src_loc): + # Select 16-bit operand mode + self.writechar('\x66') + self.MOV(dest_loc, src_loc) def PUSH(self, loc): assert isinstance(loc, RegLoc) From arigo at codespeak.net Fri May 21 18:44:13 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 May 2010 18:44:13 +0200 (CEST) Subject: [pypy-svn] r74643 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter codewriter/test metainterp Message-ID: <20100521164413.3B9A4282C00@codespeak.net> Author: arigo Date: Fri May 21 18:44:11 2010 New Revision: 74643 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_call.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py Log: Port a first test about virtualrefs. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py Fri May 21 18:44:11 2010 @@ -15,6 +15,7 @@ class CallControl(object): + virtualref_info = None # optionally set from outside to a VirtualRefInfo() def __init__(self, cpu=None, portal_graph=None): self.cpu = cpu Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py Fri May 21 18:44:11 2010 @@ -68,6 +68,9 @@ self.transform_graph_to_jitcode(graph, jitcode, graph is maingraph, verbose) + def setup_vrefinfo(self, vrefinfo): + self.callcontrol.virtualref_info = vrefinfo + def find_all_graphs(self, policy): return self.callcontrol.find_all_graphs(policy) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Fri May 21 18:44:11 2010 @@ -288,9 +288,18 @@ def handle_builtin_call(self, op): oopspec_name, args = support.decode_builtin_call(op) + if oopspec_name.startswith('virtual_ref'): + return self._handle_virtual_ref_call(op, oopspec_name, args) op1 = self._prepare_builtin_call(op, oopspec_name, args) return self.handle_residual_call(op1) + def _handle_virtual_ref_call(self, op, oopspec_name, args): + vrefinfo = self.callcontrol.virtualref_info + heaptracker.register_known_gctype(self.cpu, + vrefinfo.jit_virtual_ref_vtable, + vrefinfo.JIT_VIRTUAL_REF) + return SpaceOperation(oopspec_name, list(args), op.result) + handle_residual_indirect_call = handle_residual_call def handle_regular_indirect_call(self, op): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_call.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_call.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_call.py Fri May 21 18:44:11 2010 @@ -146,3 +146,8 @@ assert jitcode.name == "foo" pending = list(cc.enum_pending_graphs()) assert pending == [(somegraph, jitcode)] + +# ____________________________________________________________ + +def test_jit_force_virtualizable_effectinfo(): + py.test.skip("XXX add a test for CallControl.getcalldescr() -> EF_xxx") Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Fri May 21 18:44:11 2010 @@ -10,6 +10,7 @@ from pypy.rlib.rarithmetic import ovfcheck from pypy.rlib.jit import dont_look_inside, _we_are_jitted, JitDriver from pypy.rlib.objectmodel import keepalive_until_here +from pypy.rlib import jit class FakeRegAlloc: @@ -23,6 +24,11 @@ self.num_colors += 1 return self.seen[v] +def fake_regallocs(): + return {'int': FakeRegAlloc(), + 'ref': FakeRegAlloc(), + 'float': FakeRegAlloc()} + class FakeDescr(AbstractDescr): def __repr__(self): return '' @@ -38,11 +44,14 @@ _builtin_func_for_spec_cache = FakeDict() class FakeCPU: - rtyper = FakeRTyper() + def __init__(self, rtyper): + self.rtyper = rtyper def calldescrof(self, FUNC, ARGS, RESULT): return FakeDescr() def fielddescrof(self, STRUCT, name): return FakeDescr() + def sizeof(self, STRUCT): + return FakeDescr() class FakeCallControl: _descr_cannot_raise = FakeDescr() @@ -60,10 +69,15 @@ def found_jitdriver(self, jitdriver): assert isinstance(jitdriver, JitDriver) -def fake_regallocs(): - return {'int': FakeRegAlloc(), - 'ref': FakeRegAlloc(), - 'float': FakeRegAlloc()} +class FakeCallControlWithVRefInfo: + class virtualref_info: + JIT_VIRTUAL_REF = lltype.GcStruct('VREF', ('parent', rclass.OBJECT)) + jit_virtual_ref_vtable = lltype.malloc(rclass.OBJECT_VTABLE, + immortal=True) + def guess_call_kind(self, op): + return 'builtin' + +# ____________________________________________________________ def test_reorder_renaming_list(): result = reorder_renaming_list([], []) @@ -83,6 +97,9 @@ def test_repr(): assert repr(Register('int', 13)) == '%i13' +# ____________________________________________________________ + + class TestFlatten: def make_graphs(self, func, values, type_system='lltype'): @@ -90,11 +107,12 @@ return self.rtyper.annotator.translator.graphs def encoding_test(self, func, args, expected, - transform=False, liveness=False): + transform=False, liveness=False, cc=None): graphs = self.make_graphs(func, args) if transform: from pypy.jit.codewriter.jtransform import transform_graph - transform_graph(graphs[0], FakeCPU(), FakeCallControl()) + cc = cc or FakeCallControl() + transform_graph(graphs[0], FakeCPU(self.rtyper), cc) ssarepr = flatten_graph(graphs[0], fake_regallocs(), _include_all_exc_links=not transform) if liveness: @@ -556,3 +574,15 @@ L1: int_return $34 """, transform=True, liveness=True) + + def test_vref_simple(self): + from pypy.jit.codewriter.call import CallControl + class X: + pass + def f(): + return jit.virtual_ref(X()) + self.encoding_test(f, [], """ + new_with_vtable -> %r0 + virtual_ref %r0 -> %r1 + ref_return %r1 + """, transform=True, cc=FakeCallControlWithVRefInfo()) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py Fri May 21 18:44:11 2010 @@ -164,6 +164,7 @@ # from pypy.jit.metainterp.virtualref import VirtualRefInfo self.metainterp_sd.virtualref_info = VirtualRefInfo(self) + self.codewriter.setup_vrefinfo(self.metainterp_sd.virtualref_info) if self.jitdriver.virtualizables: from pypy.jit.metainterp.virtualizable import VirtualizableInfo self.metainterp_sd.virtualizable_info = VirtualizableInfo(self) From arigo at codespeak.net Fri May 21 18:51:26 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 May 2010 18:51:26 +0200 (CEST) Subject: [pypy-svn] r74644 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100521165126.731CC282C00@codespeak.net> Author: arigo Date: Fri May 21 18:51:24 2010 New Revision: 74644 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Log: Next test. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Fri May 21 18:51:24 2010 @@ -293,13 +293,6 @@ op1 = self._prepare_builtin_call(op, oopspec_name, args) return self.handle_residual_call(op1) - def _handle_virtual_ref_call(self, op, oopspec_name, args): - vrefinfo = self.callcontrol.virtualref_info - heaptracker.register_known_gctype(self.cpu, - vrefinfo.jit_virtual_ref_vtable, - vrefinfo.JIT_VIRTUAL_REF) - return SpaceOperation(oopspec_name, list(args), op.result) - handle_residual_indirect_call = handle_residual_call def handle_regular_indirect_call(self, op): @@ -679,6 +672,19 @@ def handle_jit_marker__can_enter_jit(self, op, jitdriver): return SpaceOperation('can_enter_jit', [], None) + # ---------- + # VirtualRefs. + + def _handle_virtual_ref_call(self, op, oopspec_name, args): + vrefinfo = self.callcontrol.virtualref_info + heaptracker.register_known_gctype(self.cpu, + vrefinfo.jit_virtual_ref_vtable, + vrefinfo.JIT_VIRTUAL_REF) + return SpaceOperation(oopspec_name, list(args), op.result) + + def rewrite_op_jit_force_virtual(self, op): + return self._do_builtin_call(op) + # ____________________________________________________________ def _with_prefix(prefix): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Fri May 21 18:51:24 2010 @@ -75,7 +75,13 @@ jit_virtual_ref_vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) def guess_call_kind(self, op): - return 'builtin' + if hasattr(op.args[0].value._obj.graph.func, 'oopspec'): + return 'builtin' + return 'residual' + def getcalldescr(self, op): + return FakeDescr() + def calldescr_canraise(self, calldescr): + return False # ____________________________________________________________ @@ -576,7 +582,6 @@ """, transform=True, liveness=True) def test_vref_simple(self): - from pypy.jit.codewriter.call import CallControl class X: pass def f(): @@ -586,3 +591,19 @@ virtual_ref %r0 -> %r1 ref_return %r1 """, transform=True, cc=FakeCallControlWithVRefInfo()) + + def test_vref_forced(self): + class X: + pass + def f(): + vref = jit.virtual_ref(X()) + return vref() + # The call vref() is a jit_force_virtual operation in the original + # graph. It becomes in the JitCode a residual call to a helper that + # contains itself a jit_force_virtual. + self.encoding_test(f, [], """ + new_with_vtable -> %r0 + virtual_ref %r0 -> %r1 + residual_call_r_r $<* fn _ll_1_jit_force_virtual__objectPtr>, , R[%r1] -> %r2 + ref_return %r2 + """, transform=True, cc=FakeCallControlWithVRefInfo()) From arigo at codespeak.net Fri May 21 18:59:27 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 May 2010 18:59:27 +0200 (CEST) Subject: [pypy-svn] r74645 - pypy/branch/blackhole-improvement/pypy/jit/codewriter/test Message-ID: <20100521165927.E0D48282C00@codespeak.net> Author: arigo Date: Fri May 21 18:59:26 2010 New Revision: 74645 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_call.py Log: Fix a test. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_call.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_call.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_call.py Fri May 21 18:59:26 2010 @@ -1,3 +1,4 @@ +import py from pypy.objspace.flow.model import SpaceOperation, Constant, Variable from pypy.rpython.lltypesystem import lltype from pypy.translator.unsimplify import varoftype @@ -137,8 +138,17 @@ # ____________________________________________________________ def test_get_jitcode(): - from pypy.jit.codewriter.test.test_codewriter import FakeCPU - cc = CallControl(FakeCPU()) + from pypy.jit.codewriter.test.test_flatten import FakeCPU + class FakeRTyper: + class annotator: + translator = None + class type_system: + name = 'lltypesystem' + def getcallable(self, graph): + F = lltype.FuncType([], lltype.Signed) + return lltype.functionptr(F, 'bar') + # + cc = CallControl(FakeCPU(FakeRTyper())) class somegraph: name = "foo" jitcode = cc.get_jitcode(somegraph) From arigo at codespeak.net Fri May 21 19:07:08 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 May 2010 19:07:08 +0200 (CEST) Subject: [pypy-svn] r74646 - pypy/branch/blackhole-improvement/pypy/jit/codewriter/test Message-ID: <20100521170708.4133D282C00@codespeak.net> Author: arigo Date: Fri May 21 19:07:06 2010 New Revision: 74646 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Log: Fix test. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Fri May 21 19:07:06 2010 @@ -40,11 +40,9 @@ c_func = Constant(f, lltype.typeOf(f)) return c_func, lltype.Signed -class FakeRTyper(object): - _builtin_func_for_spec_cache = FakeDict() - class FakeCPU: def __init__(self, rtyper): + rtyper._builtin_func_for_spec_cache = FakeDict() self.rtyper = rtyper def calldescrof(self, FUNC, ARGS, RESULT): return FakeDescr() @@ -75,9 +73,9 @@ jit_virtual_ref_vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) def guess_call_kind(self, op): - if hasattr(op.args[0].value._obj.graph.func, 'oopspec'): - return 'builtin' - return 'residual' + if op.args[0].value._obj._name == 'jit_force_virtual': + return 'residual' + return 'builtin' def getcalldescr(self, op): return FakeDescr() def calldescr_canraise(self, calldescr): @@ -604,6 +602,6 @@ self.encoding_test(f, [], """ new_with_vtable -> %r0 virtual_ref %r0 -> %r1 - residual_call_r_r $<* fn _ll_1_jit_force_virtual__objectPtr>, , R[%r1] -> %r2 + residual_call_r_r $<* fn jit_force_virtual>, , R[%r1] -> %r2 ref_return %r2 """, transform=True, cc=FakeCallControlWithVRefInfo()) From arigo at codespeak.net Fri May 21 19:53:32 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 May 2010 19:53:32 +0200 (CEST) Subject: [pypy-svn] r74647 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100521175332.6C2F4282C0E@codespeak.net> Author: arigo Date: Fri May 21 19:53:29 2010 New Revision: 74647 Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py (contents, props changed) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Log: Start writing the list oopspec operations. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Fri May 21 19:53:29 2010 @@ -5,6 +5,7 @@ from pypy.objspace.flow.model import Block, Link, c_last_exception from pypy.jit.codewriter.flatten import ListOfKind, IndirectCallTargets from pypy.jit.codewriter import support, heaptracker +from pypy.jit.metainterp.typesystem import deref, arrayItem from pypy.rlib import objectmodel from pypy.rlib.jit import _we_are_jitted @@ -288,10 +289,19 @@ def handle_builtin_call(self, op): oopspec_name, args = support.decode_builtin_call(op) - if oopspec_name.startswith('virtual_ref'): - return self._handle_virtual_ref_call(op, oopspec_name, args) - op1 = self._prepare_builtin_call(op, oopspec_name, args) - return self.handle_residual_call(op1) + # dispatch to various implementations depending on the oopspec_name + if oopspec_name.startswith('list.') or oopspec_name == 'newlist': + prepare = self._handle_list_call + elif oopspec_name.startswith('virtual_ref'): + prepare = self._handle_virtual_ref_call + else: + prepare = self.prepare_builtin_call + op1 = prepare(oopspec_name, args, op.result) + # If the resulting op1 is still a direct_call, turn it into a + # residual_call. + if op1 is None or op1.opname == 'direct_call': + op1 = self.handle_residual_call(op1 or op) + return op1 handle_residual_indirect_call = handle_residual_call @@ -312,21 +322,21 @@ result.append(op2) return result - def _prepare_builtin_call(self, op, oopspec_name, args, + def prepare_builtin_call(self, oopspec_name, args, result, extra=None, extrakey=None): argtypes = [v.concretetype for v in args] - resulttype = op.result.concretetype + resulttype = result.concretetype c_func, TP = support.builtin_func_for_spec(self.cpu.rtyper, oopspec_name, argtypes, resulttype, extra, extrakey) - return SpaceOperation('direct_call', [c_func] + args, op.result) + return SpaceOperation('direct_call', [c_func] + args, result) def _do_builtin_call(self, op, oopspec_name=None, args=None, extra=None, extrakey=None): if oopspec_name is None: oopspec_name = op.opname if args is None: args = op.args - op1 = self._prepare_builtin_call(op, oopspec_name, args, - extra, extrakey) + op1 = self.prepare_builtin_call(oopspec_name, args, op.result, + extra, extrakey) return self.rewrite_op_direct_call(op1) # XXX some of the following functions should not become residual calls @@ -673,14 +683,50 @@ return SpaceOperation('can_enter_jit', [], None) # ---------- + # Lists. + + def _handle_list_call(self, oopspec_name, args, result): + if oopspec_name.startswith('new'): + LIST = deref(result.concretetype) + else: + LIST = deref(args[0].concretetype) + resizable = isinstance(LIST, lltype.GcStruct) + assert resizable == (not isinstance(LIST, lltype.GcArray)) + if resizable: + prefix = 'do_resizable_' + else: + prefix = 'do_fixed_' + if self._array_of_voids(LIST): + return None # arrays of voids: not supported + arraydescr = self.cpu.arraydescrof(LIST) + # + try: + meth = getattr(self, prefix + oopspec_name.replace('.', '_')) + except AttributeError: + return None + else: + return meth(LIST, arraydescr, args, result) + + def do_fixed_newlist(self, LIST, arraydescr, args, result): + # normalize number of arguments + if len(args) < 1: + args.append(Constant(0, lltype.Signed)) + if len(args) > 1: + v_default = args[1] + if (not isinstance(v_default, Constant) or + v_default.value != arrayItem(LIST)._defl()): + return None # variable or non-null initial value + return SpaceOperation('new_array', [arraydescr, args[0]], result) + + # ---------- # VirtualRefs. - def _handle_virtual_ref_call(self, op, oopspec_name, args): + def _handle_virtual_ref_call(self, oopspec_name, args, result): vrefinfo = self.callcontrol.virtualref_info heaptracker.register_known_gctype(self.cpu, vrefinfo.jit_virtual_ref_vtable, vrefinfo.JIT_VIRTUAL_REF) - return SpaceOperation(oopspec_name, list(args), op.result) + return SpaceOperation(oopspec_name, list(args), result) def rewrite_op_jit_force_virtual(self, op): return self._do_builtin_call(op) Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py ============================================================================== --- (empty file) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py Fri May 21 19:53:29 2010 @@ -0,0 +1,54 @@ +from pypy.rpython.lltypesystem import lltype +from pypy.translator.unsimplify import varoftype +from pypy.objspace.flow.model import Constant + +from pypy.jit.codewriter.jtransform import Transformer +from pypy.jit.codewriter.flatten import GraphFlattener +from pypy.jit.codewriter.format import assert_format +from pypy.jit.codewriter.test.test_flatten import fake_regallocs +from pypy.jit.metainterp.history import AbstractDescr + +# ____________________________________________________________ + +FIXEDLIST = lltype.Ptr(lltype.GcArray(lltype.Signed)) + +class FakeCPU: + class arraydescrof(AbstractDescr): + def __init__(self, ARRAY): + self.ARRAY = ARRAY + def __repr__(self): + return '' + +def builtin_test(oopspec_name, args, RESTYPE, expected): + v_result = varoftype(RESTYPE) + tr = Transformer(FakeCPU()) + oplist = tr._handle_list_call(oopspec_name, args, v_result) + if expected is None: + assert oplist is None + else: + assert oplist is not None + flattener = GraphFlattener(None, fake_regallocs()) + if not isinstance(oplist, list): + oplist = [oplist] + for op1 in oplist: + flattener.serialize_op(op1) + assert_format(flattener.ssarepr, expected) + +# ____________________________________________________________ + +def test_newlist(): + builtin_test('newlist', [], FIXEDLIST, + """new_array , $0 -> %r0""") + builtin_test('newlist', [Constant(5, lltype.Signed)], FIXEDLIST, + """new_array , $5 -> %r0""") + builtin_test('newlist', [varoftype(lltype.Signed)], FIXEDLIST, + """new_array , %i0 -> %r0""") + builtin_test('newlist', [Constant(5, lltype.Signed), + Constant(0, lltype.Signed)], FIXEDLIST, + """new_array , $5 -> %r0""") + builtin_test('newlist', [Constant(5, lltype.Signed), + Constant(1, lltype.Signed)], FIXEDLIST, + None) + builtin_test('newlist', [Constant(5, lltype.Signed), + varoftype(lltype.Signed)], FIXEDLIST, + None) From wildchild at codespeak.net Fri May 21 20:10:53 2010 From: wildchild at codespeak.net (wildchild at codespeak.net) Date: Fri, 21 May 2010 20:10:53 +0200 (CEST) Subject: [pypy-svn] r74648 - in pypy/trunk/pypy: rlib translator/platform Message-ID: <20100521181053.1A6BD282C00@codespeak.net> Author: wildchild Date: Fri May 21 20:10:51 2010 New Revision: 74648 Modified: pypy/trunk/pypy/rlib/rlocale.py pypy/trunk/pypy/translator/platform/__init__.py pypy/trunk/pypy/translator/platform/darwin.py pypy/trunk/pypy/translator/platform/freebsd7.py Log: Removing a Darwin hardcoded include path. Adding a more "general" way of adding platform related includes/libraries path. For FreeBSD, all ports are always installed in /usr/local, /usr is for the base system. For MacOS /opt/local is the default installation path for MacPorts and /usr/local for Homebrew and self-compiled packages. In a not so far future, user provided CFLAGS and LDFLAGS should be used to provide non-standard paths. Modified: pypy/trunk/pypy/rlib/rlocale.py ============================================================================== --- pypy/trunk/pypy/rlib/rlocale.py (original) +++ pypy/trunk/pypy/rlib/rlocale.py Fri May 21 20:10:51 2010 @@ -17,7 +17,6 @@ class CConfig: includes = ['locale.h', 'limits.h'] - include_dirs = [] if HAVE_LANGINFO: includes += ['langinfo.h'] @@ -25,11 +24,8 @@ includes += ['libintl.h'] if sys.platform == 'win32': includes += ['windows.h'] - if sys.platform == 'darwin': - include_dirs += ['/opt/local/include'] _compilation_info_ = ExternalCompilationInfo( - includes=includes, - include_dirs=include_dirs + includes=includes ) HAVE_BIND_TEXTDOMAIN_CODESET = platform.Has('bind_textdomain_codeset') lconv = platform.Struct("struct lconv", [ Modified: pypy/trunk/pypy/translator/platform/__init__.py ============================================================================== --- pypy/trunk/pypy/translator/platform/__init__.py (original) +++ pypy/trunk/pypy/translator/platform/__init__.py Fri May 21 20:10:51 2010 @@ -113,7 +113,6 @@ log.WARNING(line) def _preprocess_include_dirs(self, include_dirs): - # hook for maemo return include_dirs def _compile_args_from_eci(self, eci, standalone): @@ -125,9 +124,13 @@ extra = self.shared_only cflags = self.cflags + extra return (cflags + list(eci.compile_extra) + args) + + def _preprocess_library_dirs(self, library_dirs): + return library_dirs def _link_args_from_eci(self, eci, standalone): - library_dirs = self._libdirs(eci.library_dirs) + library_dirs = self._preprocess_library_dirs(eci.library_dirs) + library_dirs = self._libdirs(library_dirs) libraries = self._libs(eci.libraries) link_files = self._linkfiles(eci.link_files) return (library_dirs + self.link_flags + Modified: pypy/trunk/pypy/translator/platform/darwin.py ============================================================================== --- pypy/trunk/pypy/translator/platform/darwin.py (original) +++ pypy/trunk/pypy/translator/platform/darwin.py Fri May 21 20:10:51 2010 @@ -20,6 +20,18 @@ def _args_for_shared(self, args): return (self.shared_only + ['-dynamiclib', '-undefined', 'dynamic_lookup'] + args) + + def _preprocess_include_dirs(self, include_dirs): + res_incl_dirs = [dir for dir in include_dirs] + res_incl_dirs.append('/usr/local/include') # Homebrew + res_incl_dirs.append('/opt/local/include') # MacPorts + return res_incl_dirs + + def _preprocess_library_dirs(self, library_dirs): + res_lib_dirs = [dir for dir in library_dirs] + res_lib_dirs.append('/usr/local/lib') # Homebrew + res_lib_dirs.append('/opt/local/lib') # MacPorts + return res_lib_dirs def include_dirs_for_libffi(self): return ['/usr/include/ffi'] Modified: pypy/trunk/pypy/translator/platform/freebsd7.py ============================================================================== --- pypy/trunk/pypy/translator/platform/freebsd7.py (original) +++ pypy/trunk/pypy/translator/platform/freebsd7.py Fri May 21 20:10:51 2010 @@ -15,6 +15,16 @@ def _args_for_shared(self, args): return ['-shared'] + args + def _preprocess_include_dirs(self, include_dirs): + res_incl_dirs = [dir for dir in include_dirs] + res_incl_dirs.append('/usr/local/include') + return res_incl_dirs + + def _preprocess_library_dirs(self, library_dirs): + res_lib_dirs = [dir for dir in library_dirs] + res_lib_dirs.append('/usr/local/lib') + return res_lib_dirs + def include_dirs_for_libffi(self): return ['/usr/local/include'] From wildchild at codespeak.net Fri May 21 20:14:33 2010 From: wildchild at codespeak.net (wildchild at codespeak.net) Date: Fri, 21 May 2010 20:14:33 +0200 (CEST) Subject: [pypy-svn] r74649 - pypy/trunk/pypy/translator/platform Message-ID: <20100521181433.D952F282C00@codespeak.net> Author: wildchild Date: Fri May 21 20:14:32 2010 New Revision: 74649 Modified: pypy/trunk/pypy/translator/platform/freebsd7.py Log: Didn't know a tuple could be converted to a list like this! Modified: pypy/trunk/pypy/translator/platform/freebsd7.py ============================================================================== --- pypy/trunk/pypy/translator/platform/freebsd7.py (original) +++ pypy/trunk/pypy/translator/platform/freebsd7.py Fri May 21 20:14:32 2010 @@ -16,12 +16,12 @@ return ['-shared'] + args def _preprocess_include_dirs(self, include_dirs): - res_incl_dirs = [dir for dir in include_dirs] + res_incl_dirs = list(include_dirs) res_incl_dirs.append('/usr/local/include') return res_incl_dirs def _preprocess_library_dirs(self, library_dirs): - res_lib_dirs = [dir for dir in library_dirs] + res_lib_dirs = list(library_dirs) res_lib_dirs.append('/usr/local/lib') return res_lib_dirs From jcreigh at codespeak.net Fri May 21 21:25:56 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Fri, 21 May 2010 21:25:56 +0200 (CEST) Subject: [pypy-svn] r74650 - in pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86: . test Message-ID: <20100521192556.81616282C0D@codespeak.net> Author: jcreigh Date: Fri May 21 21:25:54 2010 New Revision: 74650 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py Log: fix issues causing rx86 test failures Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py Fri May 21 21:25:54 2010 @@ -377,8 +377,17 @@ IMUL_rr = insn(rex_w, '\x0F\xAF', register(1, 8), register(2), '\xC0') IMUL_rb = insn(rex_w, '\x0F\xAF', register(1, 8), stack_bp(2)) - # XXX: There are more efficient encodings of small immediates - IMUL_rri = insn(rex_w, '\x69', register(1, 8), register(2), '\xC0', immediate(3)) + # 8-bit immediate + _IMUL_rri8 = insn(rex_w, '\x6B', register(1, 8), register(2), '\xC0', immediate(3, 'b')) + # 32-bit immediate + _IMUL_rri32 = insn(rex_w, '\x69', register(1, 8), register(2), '\xC0', immediate(3)) + + def IMUL_rri(self, reg1, reg2, immed): + if single_byte(immed): + self._IMUL_rri8(reg1, reg2, immed) + else: + assert fits_in_32bits(immed) + self._IMUL_rri32(reg1, reg2, immed) def IMUL_ri(self, reg, immed): return self.IMUL_rri(reg, reg, immed) @@ -405,7 +414,8 @@ J_il = insn('\x0F', immediate(1,'o'), '\x80', relative(2)) SET_ir = insn('\x0F', immediate(1,'o'),'\x90', register(2), '\xC0') - CDQ = insn(rex_w, '\x99') + # The 64-bit version of this, CQO, is defined in X86_64_CodeBuilder + CDQ = insn(rex_nw, '\x99') # ------------------------------ SSE2 ------------------------------ @@ -462,6 +472,8 @@ self.writechar(chr((imm >> 48) & 0xFF)) self.writechar(chr((imm >> 56) & 0xFF)) + CQO = insn(rex_w, '\x99') + # MOV_ri from the parent class is not wrong, but here is a better encoding # for the common case where the immediate fits in 32 bits _MOV_ri32 = insn(rex_w, '\xC7', register(1), '\xC0', immediate(2, 'i')) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py Fri May 21 21:25:54 2010 @@ -142,6 +142,10 @@ s.IMUL_rri(ebx, ecx, 0x01234567) assert s.getvalue() == '\x69\xD9\x67\x45\x23\x01' + s = CodeBuilder32() + s.IMUL_rri(ebx, ecx, 0x2A) + assert s.getvalue() == '\x6B\xD9\x2A' + class CodeBuilder64(CodeBuilderMixin, X86_64_CodeBuilder): pass From afa at codespeak.net Fri May 21 21:27:36 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 21 May 2010 21:27:36 +0200 (CEST) Subject: [pypy-svn] r74651 - in pypy/trunk/pypy/module/cpyext: . include test Message-ID: <20100521192736.F2797282C0D@codespeak.net> Author: afa Date: Fri May 21 21:27:35 2010 New Revision: 74651 Modified: pypy/trunk/pypy/module/cpyext/funcobject.py pypy/trunk/pypy/module/cpyext/include/funcobject.h pypy/trunk/pypy/module/cpyext/test/test_funcobject.py Log: Add PyMethod_Self() and PyMethod_GET_SELF() Modified: pypy/trunk/pypy/module/cpyext/funcobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/funcobject.py (original) +++ pypy/trunk/pypy/module/cpyext/funcobject.py Fri May 21 21:27:35 2010 @@ -50,6 +50,13 @@ return borrow_from(w_method, w_method.w_function) @cpython_api([PyObject], PyObject) +def PyMethod_Self(space, w_method): + """Return the instance associated with the method meth if it is bound, + otherwise return NULL.""" + assert isinstance(w_method, Method) + return borrow_from(w_method, w_method.w_instance) + + at cpython_api([PyObject], PyObject) def PyMethod_Class(space, w_method): """Return the class object from which the method meth was created; if this was created from an instance, it will be the class of the instance.""" Modified: pypy/trunk/pypy/module/cpyext/include/funcobject.h ============================================================================== --- pypy/trunk/pypy/module/cpyext/include/funcobject.h (original) +++ pypy/trunk/pypy/module/cpyext/include/funcobject.h Fri May 21 21:27:35 2010 @@ -12,8 +12,9 @@ PyObject *func_name; /* The __name__ attribute, a string object */ } PyFunctionObject; -#define PyMethod_GET_CLASS(obj) PyMethod_Class(obj) -#define PyMethod_GET_FUNCTION(obj) PyMethod_Function(obj) +#define PyMethod_GET_FUNCTION(obj) PyMethod_Function((PyObject*)(obj)) +#define PyMethod_GET_SELF(obj) PyMethod_Self((PyObject*)(obj)) +#define PyMethod_GET_CLASS(obj) PyMethod_Class((PyObject*)(obj)) #ifdef __cplusplus } Modified: pypy/trunk/pypy/module/cpyext/test/test_funcobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_funcobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_funcobject.py Fri May 21 21:27:35 2010 @@ -26,9 +26,12 @@ """) w_function = space.getattr(w_method, space.wrap("im_func")) + w_self = space.getattr(w_method, space.wrap("im_self")) + w_class = space.getattr(w_method, space.wrap("im_class")) + assert space.is_w(api.PyMethod_Function(w_method), w_function) + assert space.is_w(api.PyMethod_Self(w_method), w_self) + assert space.is_w(api.PyMethod_Class(w_method), w_class) - w_class = space.getattr(w_method, space.wrap("im_class")) - w_self = space.getattr(w_method, space.wrap("im_self")) w_method2 = api.PyMethod_New(w_function, w_self, w_class) assert space.eq_w(w_method, w_method2) From afa at codespeak.net Fri May 21 21:41:04 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 21 May 2010 21:41:04 +0200 (CEST) Subject: [pypy-svn] r74652 - pypy/trunk/pypy/module/cpyext Message-ID: <20100521194104.193A6282C00@codespeak.net> Author: afa Date: Fri May 21 21:41:02 2010 New Revision: 74652 Modified: pypy/trunk/pypy/module/cpyext/stubs.py Log: These functions are implemented now Modified: pypy/trunk/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/stubs.py (original) +++ pypy/trunk/pypy/module/cpyext/stubs.py Fri May 21 21:41:02 2010 @@ -2275,19 +2275,6 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject], PyObject) -def PyMethod_Self(space, meth): - """Return the instance associated with the method meth if it is bound, otherwise - return NULL.""" - borrow_from() - raise NotImplementedError - - at cpython_api([PyObject], PyObject) -def PyMethod_GET_SELF(space, meth): - """Macro version of PyMethod_Self() which avoids error checking.""" - borrow_from() - raise NotImplementedError - @cpython_api([], rffi.INT_real, error=CANNOT_FAIL) def PyMethod_ClearFreeList(space, ): """Clear the free list. Return the total number of freed items. @@ -2357,12 +2344,6 @@ """ raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP], rffi.INT_real, error=-1) -def PyObject_DelAttrString(space, o, attr_name): - """Delete attribute named attr_name, for object o. Returns -1 on failure. - This is the equivalent of the Python statement del o.attr_name.""" - raise NotImplementedError - @cpython_api([PyObject, PyObject, rffi.INTP], rffi.INT_real, error=-1) def PyObject_Cmp(space, o1, o2, result): """ @@ -2391,18 +2372,6 @@ """ raise NotImplementedError - at cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) -def PyObject_SetItem(space, o, key, v): - """Map the object key to the value v. Returns -1 on failure. This is the - equivalent of the Python statement o[key] = v.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) -def PyObject_DelItem(space, o, key): - """Delete the mapping for key from o. Returns -1 on failure. This is the - equivalent of the Python statement del o[key].""" - raise NotImplementedError - @cpython_api([PyObject], PyObject) def PyObject_Dir(space, o): """This is equivalent to the Python expression dir(o), returning a (possibly @@ -2412,27 +2381,6 @@ is active then NULL is returned but PyErr_Occurred() will return false.""" raise NotImplementedError - at cpython_api([], PyObject) -def PyEval_GetBuiltins(space, ): - """Return a dictionary of the builtins in the current execution frame, - or the interpreter of the thread state if no frame is currently executing.""" - borrow_from() - raise NotImplementedError - - at cpython_api([], PyObject) -def PyEval_GetLocals(space, ): - """Return a dictionary of the local variables in the current execution frame, - or NULL if no frame is currently executing.""" - borrow_from() - raise NotImplementedError - - at cpython_api([], PyObject) -def PyEval_GetGlobals(space, ): - """Return a dictionary of the global variables in the current execution frame, - or NULL if no frame is currently executing.""" - borrow_from() - raise NotImplementedError - @cpython_api([], PyFrameObject) def PyEval_GetFrame(space, ): """Return the current thread state's frame, which is NULL if no frame is From arigo at codespeak.net Fri May 21 22:22:58 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 May 2010 22:22:58 +0200 (CEST) Subject: [pypy-svn] r74653 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100521202258.7EF43282C00@codespeak.net> Author: arigo Date: Fri May 21 22:22:55 2010 New Revision: 74653 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py Log: Work in progress. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Fri May 21 22:22:55 2010 @@ -1,5 +1,6 @@ import py, sys from pypy.rpython.lltypesystem import lltype, rstr +from pypy.rpython import rlist from pypy.jit.metainterp.history import getkind from pypy.objspace.flow.model import SpaceOperation, Variable, Constant from pypy.objspace.flow.model import Block, Link, c_last_exception @@ -8,6 +9,7 @@ from pypy.jit.metainterp.typesystem import deref, arrayItem from pypy.rlib import objectmodel from pypy.rlib.jit import _we_are_jitted +from pypy.translator.simplify import get_funcobj def transform_graph(graph, cpu=None, callcontrol=None, portal=True): @@ -296,7 +298,7 @@ prepare = self._handle_virtual_ref_call else: prepare = self.prepare_builtin_call - op1 = prepare(oopspec_name, args, op.result) + op1 = prepare(op, oopspec_name, args) # If the resulting op1 is still a direct_call, turn it into a # residual_call. if op1 is None or op1.opname == 'direct_call': @@ -322,20 +324,20 @@ result.append(op2) return result - def prepare_builtin_call(self, oopspec_name, args, result, + def prepare_builtin_call(self, op, oopspec_name, args, extra=None, extrakey=None): argtypes = [v.concretetype for v in args] - resulttype = result.concretetype + resulttype = op.result.concretetype c_func, TP = support.builtin_func_for_spec(self.cpu.rtyper, oopspec_name, argtypes, resulttype, extra, extrakey) - return SpaceOperation('direct_call', [c_func] + args, result) + return SpaceOperation('direct_call', [c_func] + args, op.result) def _do_builtin_call(self, op, oopspec_name=None, args=None, extra=None, extrakey=None): if oopspec_name is None: oopspec_name = op.opname if args is None: args = op.args - op1 = self.prepare_builtin_call(oopspec_name, args, op.result, + op1 = self.prepare_builtin_call(op, oopspec_name, args, extra, extrakey) return self.rewrite_op_direct_call(op1) @@ -685,9 +687,9 @@ # ---------- # Lists. - def _handle_list_call(self, oopspec_name, args, result): + def _handle_list_call(self, op, oopspec_name, args): if oopspec_name.startswith('new'): - LIST = deref(result.concretetype) + LIST = deref(op.result.concretetype) else: LIST = deref(args[0].concretetype) resizable = isinstance(LIST, lltype.GcStruct) @@ -699,34 +701,86 @@ if self._array_of_voids(LIST): return None # arrays of voids: not supported arraydescr = self.cpu.arraydescrof(LIST) + descrs = (arraydescr,) # try: meth = getattr(self, prefix + oopspec_name.replace('.', '_')) except AttributeError: return None else: - return meth(LIST, arraydescr, args, result) + return meth(op, args, *descrs) - def do_fixed_newlist(self, LIST, arraydescr, args, result): + def do_fixed_newlist(self, op, args, arraydescr): # normalize number of arguments if len(args) < 1: args.append(Constant(0, lltype.Signed)) if len(args) > 1: v_default = args[1] + ARRAY = deref(op.result.concretetype) if (not isinstance(v_default, Constant) or - v_default.value != arrayItem(LIST)._defl()): + v_default.value != arrayItem(ARRAY)._defl()): return None # variable or non-null initial value - return SpaceOperation('new_array', [arraydescr, args[0]], result) + return SpaceOperation('new_array', [arraydescr, args[0]], op.result) + + def do_fixed_list_len(self, op, args, arraydescr): + return SpaceOperation('arraylen_gc', [args[0], arraydescr], op.result) + + do_fixed_list_len_foldable = do_fixed_list_len + + def _get_list_nonneg_canraise_flags(self, op): + # xxx break of abstraction: + func = get_funcobj(op.args[0].value)._callable + # base hints on the name of the ll function, which is a bit xxx-ish + # but which is safe for now + assert func.func_name.startswith('ll_') + non_negative = '_nonneg' in func.func_name + fast = '_fast' in func.func_name + if fast: + can_raise = False + non_negative = True + else: + tag = op.args[1].value + assert tag in (rlist.dum_nocheck, rlist.dum_checkidx) + can_raise = tag != rlist.dum_nocheck + return non_negative, can_raise + + def _prepare_list_getset(self, op, descr, args, checkname): + non_negative, can_raise = self._get_list_nonneg_canraise_flags(op) + if can_raise: + return None, None + if non_negative: + return args[1], [] + else: + v_posindex = Variable('posindex') + v_posindex.concretetype = lltype.Signed + op = SpaceOperation(checkname, [args[0], + descr, args[1]], v_posindex) + return v_posindex, [op] + + def do_fixed_list_getitem(self, op, args, arraydescr, pure=False): + v_index, extraop = self._prepare_list_getset(op, arraydescr, args, + 'check_neg_index') + if v_index is None: + return None + extra = getkind(op.result.concretetype)[0] + if pure: + extra = 'pure_' + extra + op = SpaceOperation('getarrayitem_gc_%s' % extra, + [args[0], arraydescr, v_index], op.result) + return extraop + [op] + + def do_fixed_list_getitem_foldable(self, op, args, arraydescr): + return self.do_fixed_list_getitem(op, args, arraydescr, pure=True) # ---------- # VirtualRefs. - def _handle_virtual_ref_call(self, oopspec_name, args, result): + def _handle_virtual_ref_call(self, op, oopspec_name, args): vrefinfo = self.callcontrol.virtualref_info heaptracker.register_known_gctype(self.cpu, vrefinfo.jit_virtual_ref_vtable, vrefinfo.JIT_VIRTUAL_REF) - return SpaceOperation(oopspec_name, list(args), result) + return SpaceOperation(oopspec_name, list(args), op.result) def rewrite_op_jit_force_virtual(self, op): return self._do_builtin_call(op) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py Fri May 21 22:22:55 2010 @@ -1,6 +1,6 @@ from pypy.rpython.lltypesystem import lltype from pypy.translator.unsimplify import varoftype -from pypy.objspace.flow.model import Constant +from pypy.objspace.flow.model import Constant, SpaceOperation from pypy.jit.codewriter.jtransform import Transformer from pypy.jit.codewriter.flatten import GraphFlattener @@ -22,7 +22,17 @@ def builtin_test(oopspec_name, args, RESTYPE, expected): v_result = varoftype(RESTYPE) tr = Transformer(FakeCPU()) - oplist = tr._handle_list_call(oopspec_name, args, v_result) + if '/' in oopspec_name: + oopspec_name, property = oopspec_name.split('/') + def force_flags(op): + if property == 'NONNEG': return True, False + if property == 'NEG': return False, False + if property == 'CANRAISE': return False, True + raise ValueError(property) + tr._get_list_nonneg_canraise_flags = force_flags + op = SpaceOperation('direct_call', [Constant("myfunc")] + args, + v_result) + oplist = tr._handle_list_call(op, oopspec_name, args) if expected is None: assert oplist is None else: @@ -47,8 +57,67 @@ Constant(0, lltype.Signed)], FIXEDLIST, """new_array , $5 -> %r0""") builtin_test('newlist', [Constant(5, lltype.Signed), - Constant(1, lltype.Signed)], FIXEDLIST, - None) + Constant(1, lltype.Signed)], FIXEDLIST, None) builtin_test('newlist', [Constant(5, lltype.Signed), - varoftype(lltype.Signed)], FIXEDLIST, - None) + varoftype(lltype.Signed)], FIXEDLIST, None) + +def test_fixed_ll_arraycopy(): + xxx + +def test_fixed_getitem(): + builtin_test('list.getitem/NONNEG', + [varoftype(FIXEDLIST), varoftype(lltype.Signed)], + lltype.Signed, """ + getarrayitem_gc_i %r0, , %i0 -> %i1 + """) + builtin_test('list.getitem/NEG', + [varoftype(FIXEDLIST), varoftype(lltype.Signed)], + lltype.Signed, """ + check_neg_index %r0, , %i0 -> %i1 + getarrayitem_gc_i %r0, , %i1 -> %i2 + """) + builtin_test('list.getitem/CANRAISE', + [varoftype(FIXEDLIST), varoftype(lltype.Signed)], + lltype.Signed, None) + +def test_fixed_getitem_foldable(): + builtin_test('list.getitem_foldable/NONNEG', + [varoftype(FIXEDLIST), varoftype(lltype.Signed)], + lltype.Signed, """ + getarrayitem_gc_pure_i %r0, , %i0 -> %i1 + """) + builtin_test('list.getitem_foldable/NEG', + [varoftype(FIXEDLIST), varoftype(lltype.Signed)], + lltype.Signed, """ + check_neg_index %r0, , %i0 -> %i1 + getarrayitem_gc_pure_i %r0, , %i1 -> %i2 + """) + builtin_test('list.getitem_foldable/CANRAISE', + [varoftype(FIXEDLIST), varoftype(lltype.Signed)], + lltype.Signed, None) + +def test_fixed_setitem(): + xxx + +def test_fixed_len(): + builtin_test('list.len', [varoftype(FIXEDLIST)], lltype.Signed, + """arraylen_gc %r0, -> %i0""") + +def test_fixed_len_foldable(): + builtin_test('list.len_foldable', [varoftype(FIXEDLIST)], lltype.Signed, + """arraylen_gc %r0, -> %i0""") + +def test_resizable_newlist(): + xxx + +def test_resizable_getitem(): + xxx + +def test_resizable_setitem(): + xxx + +def test_resizable_len(): + xxx + +def test_resizable_unsupportedop(): + builtin_test('list.foobar', [varoftype(VARLIST)], lltype.Signed, None) From arigo at codespeak.net Fri May 21 22:27:03 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 May 2010 22:27:03 +0200 (CEST) Subject: [pypy-svn] r74654 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100521202703.9A685282C00@codespeak.net> Author: arigo Date: Fri May 21 22:27:02 2010 New Revision: 74654 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py Log: list.setitem. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Fri May 21 22:27:02 2010 @@ -772,6 +772,16 @@ def do_fixed_list_getitem_foldable(self, op, args, arraydescr): return self.do_fixed_list_getitem(op, args, arraydescr, pure=True) + def do_fixed_list_setitem(self, op, args, arraydescr): + v_index, extraop = self._prepare_list_getset(op, arraydescr, args, + 'check_neg_index') + if v_index is None: + return None + kind = getkind(op.args[2].concretetype)[0] + op = SpaceOperation('setarrayitem_gc_%s' % kind, + [args[0], arraydescr, v_index, args[2]], None) + return extraop + [op] + # ---------- # VirtualRefs. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py Fri May 21 22:27:02 2010 @@ -97,7 +97,23 @@ lltype.Signed, None) def test_fixed_setitem(): - xxx + builtin_test('list.setitem/NONNEG', [varoftype(FIXEDLIST), + varoftype(lltype.Signed), + varoftype(lltype.Signed)], + lltype.Void, """ + setarrayitem_gc_i %r0, , %i0, %i1 + """) + builtin_test('list.setitem/NEG', [varoftype(FIXEDLIST), + varoftype(lltype.Signed), + varoftype(lltype.Signed)], + lltype.Void, """ + check_neg_index %r0, , %i0 -> %i1 + setarrayitem_gc_i %r0, , %i1, %i2 + """) + builtin_test('list.setitem/CANRAISE', [varoftype(FIXEDLIST), + varoftype(lltype.Signed), + varoftype(lltype.Signed)], + lltype.Void, None) def test_fixed_len(): builtin_test('list.len', [varoftype(FIXEDLIST)], lltype.Signed, From jcreigh at codespeak.net Fri May 21 22:59:16 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Fri, 21 May 2010 22:59:16 +0200 (CEST) Subject: [pypy-svn] r74655 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86 Message-ID: <20100521205916.C5802282C00@codespeak.net> Author: jcreigh Date: Fri May 21 22:59:14 2010 New Revision: 74655 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Log: small fix for _emit_call Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Fri May 21 22:59:14 2010 @@ -601,18 +601,17 @@ loc = arglocs[i] if isinstance(loc, RegLoc): if loc.is_xmm: - mc.MOVSD_mr((esp.value, p), loc.value) + mc.MOVSD_sr(p, loc.value) else: - mc.MOV_mr((esp.value, p), loc.value) + mc.MOV_sr(p, loc.value) p += round_up_to_4(loc.width) p = 0 for i in range(start, n): loc = arglocs[i] if not isinstance(loc, RegLoc): - # if isinstance(loc, MODRM64): - if False: + if loc.width == 8: mc.MOVSD(xmm0, loc) - mc.MOVSD(mem64(esp, p), xmm0) + mc.MOVSD_sr(p, xmm0.value) else: mc.MOV(tmp, loc) mc.MOV_sr(p, tmp.value) From wildchild at codespeak.net Fri May 21 23:11:27 2010 From: wildchild at codespeak.net (wildchild at codespeak.net) Date: Fri, 21 May 2010 23:11:27 +0200 (CEST) Subject: [pypy-svn] r74656 - pypy/trunk/pypy/translator/platform Message-ID: <20100521211127.9835A282C00@codespeak.net> Author: wildchild Date: Fri May 21 23:11:26 2010 New Revision: 74656 Modified: pypy/trunk/pypy/translator/platform/darwin.py Log: Didn't know a tuple could be converted to a list like this! Modified: pypy/trunk/pypy/translator/platform/darwin.py ============================================================================== --- pypy/trunk/pypy/translator/platform/darwin.py (original) +++ pypy/trunk/pypy/translator/platform/darwin.py Fri May 21 23:11:26 2010 @@ -22,13 +22,13 @@ + args) def _preprocess_include_dirs(self, include_dirs): - res_incl_dirs = [dir for dir in include_dirs] + res_incl_dirs = list(include_dirs) res_incl_dirs.append('/usr/local/include') # Homebrew res_incl_dirs.append('/opt/local/include') # MacPorts return res_incl_dirs def _preprocess_library_dirs(self, library_dirs): - res_lib_dirs = [dir for dir in library_dirs] + res_lib_dirs = list(library_dirs) res_lib_dirs.append('/usr/local/lib') # Homebrew res_lib_dirs.append('/opt/local/lib') # MacPorts return res_lib_dirs From afa at codespeak.net Fri May 21 23:31:05 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 21 May 2010 23:31:05 +0200 (CEST) Subject: [pypy-svn] r74657 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100521213105.B6839282C00@codespeak.net> Author: afa Date: Fri May 21 23:31:04 2010 New Revision: 74657 Modified: pypy/trunk/pypy/module/cpyext/api.py pypy/trunk/pypy/module/cpyext/eval.py pypy/trunk/pypy/module/cpyext/object.py pypy/trunk/pypy/module/cpyext/test/test_eval.py Log: Implement PyRun_File() Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Fri May 21 23:31:04 2010 @@ -65,6 +65,16 @@ assert CONST_STRING is not rffi.CCHARP assert CONST_WSTRING is not rffi.CWCHARP +FILEP = rffi.COpaquePtr('FILE') +fopen = rffi.llexternal('fopen', [CONST_STRING, CONST_STRING], FILEP) +fwrite = rffi.llexternal('fwrite', + [rffi.VOIDP, rffi.SIZE_T, rffi.SIZE_T, FILEP], + rffi.SIZE_T) +fread = rffi.llexternal('fread', + [rffi.VOIDP, rffi.SIZE_T, rffi.SIZE_T, FILEP], + rffi.SIZE_T) +feof = rffi.llexternal('feof', [FILEP], rffi.INT) + constant_names = """ Py_TPFLAGS_READY Py_TPFLAGS_READYING METH_COEXIST METH_STATIC METH_CLASS Modified: pypy/trunk/pypy/module/cpyext/eval.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/eval.py (original) +++ pypy/trunk/pypy/module/cpyext/eval.py Fri May 21 23:31:04 2010 @@ -1,7 +1,9 @@ from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING +from pypy.module.cpyext.api import ( + cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, fread, feof) from pypy.module.cpyext.pyobject import PyObject, borrow_from +from pypy.module.__builtin__ import compiling @cpython_api([PyObject, PyObject, PyObject], PyObject) def PyEval_CallObjectWithKeywords(space, w_obj, w_arg, w_kwds): @@ -66,13 +68,8 @@ Py_file_input = 257 Py_eval_input = 258 - at cpython_api([CONST_STRING, rffi.INT_real, PyObject, PyObject], PyObject) -def PyRun_String(space, str, start, w_globals, w_locals): - """This is a simplified interface to PyRun_StringFlags() below, leaving - flags set to NULL.""" - from pypy.module.__builtin__ import compiling - w_source = space.wrap(rffi.charp2str(str)) - filename = "" +def run_string(space, source, filename, start, w_globals, w_locals): + w_source = space.wrap(source) start = rffi.cast(lltype.Signed, start) if start == Py_file_input: mode = 'exec' @@ -86,3 +83,30 @@ w_code = compiling.compile(space, w_source, filename, mode) return compiling.eval(space, w_code, w_globals, w_locals) + at cpython_api([CONST_STRING, rffi.INT_real,PyObject, PyObject], PyObject) +def PyRun_String(space, str, start, w_globals, w_locals): + """This is a simplified interface to PyRun_StringFlags() below, leaving + flags set to NULL.""" + source = rffi.charp2str(str) + filename = "" + return run_string(space, source, filename, start, w_globals, w_locals) + + at cpython_api([FILEP, CONST_STRING, rffi.INT_real, PyObject, PyObject], PyObject) +def PyRun_File(space, fp, filename, start, w_globals, w_locals): + """This is a simplified interface to PyRun_FileExFlags() below, leaving + closeit set to 0 and flags set to NULL.""" + BUF_SIZE = 8192 + source = "" + buf = lltype.malloc(rffi.CCHARP.TO, BUF_SIZE, flavor='raw') + try: + while True: + count = fread(buf, 1, BUF_SIZE, fp) + source += rffi.charpsize2str(buf, count) + if count < BUF_SIZE: + if feof(fp): + break + PyErr_SetFromErrno(PyExc_IOError) + finally: + lltype.free(buf, flavor='raw') + return run_string(space, source, filename, start, w_globals, w_locals) + Modified: pypy/trunk/pypy/module/cpyext/object.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/object.py (original) +++ pypy/trunk/pypy/module/cpyext/object.py Fri May 21 23:31:04 2010 @@ -2,7 +2,7 @@ from pypy.module.cpyext.api import ( cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP, PyVarObject, Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT, - Py_GE, CONST_STRING) + Py_GE, CONST_STRING, FILEP, fwrite) from pypy.module.cpyext.pyobject import ( PyObject, PyObjectP, create_ref, from_ref, Py_IncRef, Py_DecRef, track_reference) @@ -390,11 +390,6 @@ sizep[0] = size return 0 -FILEP = rffi.COpaquePtr('FILE') -fwrite = rffi.llexternal('fwrite', - [rffi.VOIDP, rffi.SIZE_T, rffi.SIZE_T, FILEP], - rffi.SIZE_T) - # Also in include/object.h Py_PRINT_RAW = 1 # No string quotes etc. Modified: pypy/trunk/pypy/module/cpyext/test/test_eval.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_eval.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_eval.py Fri May 21 23:31:04 2010 @@ -1,8 +1,11 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.test.test_api import BaseApiTest -from pypy.module.cpyext.eval import Py_single_input, Py_file_input, Py_eval_input +from pypy.module.cpyext.eval import ( + Py_single_input, Py_file_input, Py_eval_input) +from pypy.module.cpyext.api import fopen from pypy.interpreter.gateway import interp2app +from pypy.tool.udir import udir class TestEval(BaseApiTest): def test_eval(self, space, api): @@ -77,6 +80,17 @@ assert 42 * 43 == space.unwrap( api.PyObject_GetItem(w_globals, space.wrap("a"))) + def test_run_file(self, space, api): + filepath = udir / "cpyext_test_runfile.py" + filepath.write("raise ZeroDivisionError") + fp = fopen(str(filepath), "rb") + filename = rffi.str2charp(str(filepath)) + w_globals = w_locals = space.newdict() + api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals) + rffi.free_charp(filename) + assert api.PyErr_Occurred() is space.w_ZeroDivisionError + api.PyErr_Clear() + def test_getbuiltins(self, space, api): assert api.PyEval_GetBuiltins() is space.builtin.w_dict From afa at codespeak.net Fri May 21 23:36:12 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 21 May 2010 23:36:12 +0200 (CEST) Subject: [pypy-svn] r74658 - pypy/trunk/pypy/module/cpyext Message-ID: <20100521213612.24CAD282C00@codespeak.net> Author: afa Date: Fri May 21 23:36:10 2010 New Revision: 74658 Modified: pypy/trunk/pypy/module/cpyext/eval.py Log: Fix translation Modified: pypy/trunk/pypy/module/cpyext/eval.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/eval.py (original) +++ pypy/trunk/pypy/module/cpyext/eval.py Fri May 21 23:36:10 2010 @@ -3,6 +3,7 @@ from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, fread, feof) from pypy.module.cpyext.pyobject import PyObject, borrow_from +from pypy.module.cpyext.pyerrors import PyErr_SetFromErrno from pypy.module.__builtin__ import compiling @cpython_api([PyObject, PyObject, PyObject], PyObject) From afa at codespeak.net Fri May 21 23:57:20 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 21 May 2010 23:57:20 +0200 (CEST) Subject: [pypy-svn] r74659 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100521215720.1D375282C00@codespeak.net> Author: afa Date: Fri May 21 23:57:17 2010 New Revision: 74659 Modified: pypy/trunk/pypy/module/cpyext/api.py pypy/trunk/pypy/module/cpyext/eval.py pypy/trunk/pypy/module/cpyext/test/test_eval.py Log: really fix this line, with a test this time. Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Fri May 21 23:57:17 2010 @@ -67,6 +67,7 @@ FILEP = rffi.COpaquePtr('FILE') fopen = rffi.llexternal('fopen', [CONST_STRING, CONST_STRING], FILEP) +fclose = rffi.llexternal('fclose', [FILEP], rffi.INT) fwrite = rffi.llexternal('fwrite', [rffi.VOIDP, rffi.SIZE_T, rffi.SIZE_T, FILEP], rffi.SIZE_T) Modified: pypy/trunk/pypy/module/cpyext/eval.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/eval.py (original) +++ pypy/trunk/pypy/module/cpyext/eval.py Fri May 21 23:57:17 2010 @@ -106,7 +106,7 @@ if count < BUF_SIZE: if feof(fp): break - PyErr_SetFromErrno(PyExc_IOError) + PyErr_SetFromErrno(space, space.w_IOError) finally: lltype.free(buf, flavor='raw') return run_string(space, source, filename, start, w_globals, w_locals) Modified: pypy/trunk/pypy/module/cpyext/test/test_eval.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_eval.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_eval.py Fri May 21 23:57:17 2010 @@ -3,7 +3,7 @@ from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.eval import ( Py_single_input, Py_file_input, Py_eval_input) -from pypy.module.cpyext.api import fopen +from pypy.module.cpyext.api import fopen, fclose from pypy.interpreter.gateway import interp2app from pypy.tool.udir import udir @@ -87,10 +87,17 @@ filename = rffi.str2charp(str(filepath)) w_globals = w_locals = space.newdict() api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals) - rffi.free_charp(filename) + fclose(fp) assert api.PyErr_Occurred() is space.w_ZeroDivisionError api.PyErr_Clear() + # retry on closed file + api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals) + assert api.PyErr_Occurred() is space.w_OSError + api.PyErr_Clear() + + rffi.free_charp(filename) + def test_getbuiltins(self, space, api): assert api.PyEval_GetBuiltins() is space.builtin.w_dict From afa at codespeak.net Sat May 22 00:08:19 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 22 May 2010 00:08:19 +0200 (CEST) Subject: [pypy-svn] r74660 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100521220819.C37C0282C00@codespeak.net> Author: afa Date: Sat May 22 00:08:18 2010 New Revision: 74660 Modified: pypy/trunk/pypy/module/cpyext/pyerrors.py pypy/trunk/pypy/module/cpyext/test/test_eval.py Log: Let PyErr_SetFromErrno use the type given as argument... Modified: pypy/trunk/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyerrors.py (original) +++ pypy/trunk/pypy/module/cpyext/pyerrors.py Sat May 22 00:08:18 2010 @@ -1,7 +1,7 @@ import os from pypy.rpython.lltypesystem import rffi, lltype -from pypy.interpreter.error import OperationError, wrap_oserror +from pypy.interpreter.error import OperationError from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING from pypy.module.exceptions.interp_exceptions import W_RuntimeWarning from pypy.module.cpyext.pyobject import ( @@ -116,7 +116,9 @@ Return value: always NULL.""" # XXX Doesn't actually do anything with PyErr_CheckSignals. errno = get_errno() - raise wrap_oserror(space, OSError(errno, "PyErr_SetFromErrno")) + msg = os.strerror(errno) + w_error = space.call_function(w_type, space.wrap(errno), space.wrap(msg)) + raise OperationError(w_type, w_error) @cpython_api([], rffi.INT_real, error=-1) def PyErr_CheckSignals(space): Modified: pypy/trunk/pypy/module/cpyext/test/test_eval.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_eval.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_eval.py Sat May 22 00:08:18 2010 @@ -93,7 +93,7 @@ # retry on closed file api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals) - assert api.PyErr_Occurred() is space.w_OSError + assert api.PyErr_Occurred() is space.w_IOError api.PyErr_Clear() rffi.free_charp(filename) From afa at codespeak.net Sat May 22 00:17:07 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 22 May 2010 00:17:07 +0200 (CEST) Subject: [pypy-svn] r74661 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100521221707.A5436282C00@codespeak.net> Author: afa Date: Sat May 22 00:17:05 2010 New Revision: 74661 Modified: pypy/trunk/pypy/module/cpyext/sequence.py pypy/trunk/pypy/module/cpyext/stubs.py pypy/trunk/pypy/module/cpyext/test/test_sequence.py Log: PySequence_SetSlice, PySequence_DelSlice Modified: pypy/trunk/pypy/module/cpyext/sequence.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/sequence.py (original) +++ pypy/trunk/pypy/module/cpyext/sequence.py Sat May 22 00:17:05 2010 @@ -67,12 +67,23 @@ @cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject) def PySequence_GetSlice(space, w_obj, start, end): """Return the slice of sequence object o between i1 and i2, or NULL on - failure. This is the equivalent of the Python expression o[i1:i2]. - - This function used an int type for i1 and i2. This might - require changes in your code for properly supporting 64-bit systems.""" + failure. This is the equivalent of the Python expression o[i1:i2].""" return space.getslice(w_obj, space.wrap(start), space.wrap(end)) + at cpython_api([PyObject, Py_ssize_t, Py_ssize_t, PyObject], rffi.INT_real, error=-1) +def PySequence_SetSlice(space, w_obj, start, end, w_value): + """Assign the sequence object v to the slice in sequence object o from i1 to + i2. This is the equivalent of the Python statement o[i1:i2] = v.""" + space.setslice(w_obj, space.wrap(start), space.wrap(end), w_value) + return 0 + + at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], rffi.INT_real, error=-1) +def PySequence_DelSlice(space, w_obj, start, end): + """Delete the slice in sequence object o from i1 to i2. Returns -1 on + failure. This is the equivalent of the Python statement del o[i1:i2].""" + space.delslice(w_obj, space.wrap(start), space.wrap(end)) + return 0 + @cpython_api([PyObject, Py_ssize_t], PyObject) def PySequence_GetItem(space, w_obj, i): """Return the ith element of o, or NULL on failure. This is the equivalent of Modified: pypy/trunk/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/stubs.py (original) +++ pypy/trunk/pypy/module/cpyext/stubs.py Sat May 22 00:17:05 2010 @@ -2459,24 +2459,6 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, Py_ssize_t, Py_ssize_t, PyObject], rffi.INT_real, error=-1) -def PySequence_SetSlice(space, o, i1, i2, v): - """Assign the sequence object v to the slice in sequence object o from i1 to - i2. This is the equivalent of the Python statement o[i1:i2] = v. - - This function used an int type for i1 and i2. This might - require changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - - at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], rffi.INT_real, error=-1) -def PySequence_DelSlice(space, o, i1, i2): - """Delete the slice in sequence object o from i1 to i2. Returns -1 on - failure. This is the equivalent of the Python statement del o[i1:i2]. - - This function used an int type for i1 and i2. This might - require changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([PyObject, PyObject], Py_ssize_t, error=-1) def PySequence_Count(space, o, value): """Return the number of occurrences of value in o, that is, return the number Modified: pypy/trunk/pypy/module/cpyext/test/test_sequence.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_sequence.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_sequence.py Sat May 22 00:17:05 2010 @@ -41,9 +41,14 @@ rffi.free_charp(message) def test_get_slice(self, space, api): - w_t = space.wrap((1, 2, 3, 4, 5)) - assert space.unwrap(api.PySequence_GetSlice(w_t, 2, 4)) == (3, 4) - assert space.unwrap(api.PySequence_GetSlice(w_t, 1, -1)) == (2, 3, 4) + w_t = space.wrap([1, 2, 3, 4, 5]) + assert space.unwrap(api.PySequence_GetSlice(w_t, 2, 4)) == [3, 4] + assert space.unwrap(api.PySequence_GetSlice(w_t, 1, -1)) == [2, 3, 4] + + assert api.PySequence_DelSlice(w_t, 1, 4) == 0 + assert space.eq_w(w_t, space.wrap([1, 5])) + assert api.PySequence_SetSlice(w_t, 1, 1, space.wrap((3,))) == 0 + assert space.eq_w(w_t, space.wrap([1, 3, 5])) def test_iter(self, space, api): w_t = space.wrap((1, 2)) From afa at codespeak.net Sat May 22 00:29:50 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 22 May 2010 00:29:50 +0200 (CEST) Subject: [pypy-svn] r74662 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100521222950.77094282C00@codespeak.net> Author: afa Date: Sat May 22 00:29:47 2010 New Revision: 74662 Modified: pypy/trunk/pypy/module/cpyext/eval.py pypy/trunk/pypy/module/cpyext/test/test_eval.py Log: add _PyEval_SliceIndex(), undocumented but still used by Boost.Python... It's the first time I see this function. Modified: pypy/trunk/pypy/module/cpyext/eval.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/eval.py (original) +++ pypy/trunk/pypy/module/cpyext/eval.py Sat May 22 00:29:47 2010 @@ -1,7 +1,7 @@ from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import ( - cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, fread, feof) + cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, fread, feof, Py_ssize_tP) from pypy.module.cpyext.pyobject import PyObject, borrow_from from pypy.module.cpyext.pyerrors import PyErr_SetFromErrno from pypy.module.__builtin__ import compiling @@ -111,3 +111,21 @@ lltype.free(buf, flavor='raw') return run_string(space, source, filename, start, w_globals, w_locals) +# Undocumented function! + at cpython_api([PyObject, Py_ssize_tP], rffi.INT_real, error=0) +def _PyEval_SliceIndex(space, w_obj, pi): + """Extract a slice index from a PyInt or PyLong or an object with the + nb_index slot defined, and store in *pi. + Silently reduce values larger than PY_SSIZE_T_MAX to PY_SSIZE_T_MAX, + and silently boost values less than -PY_SSIZE_T_MAX-1 to -PY_SSIZE_T_MAX-1. + + Return 0 on error, 1 on success. + + Note: If v is NULL, return success without storing into *pi. This + is because_PyEval_SliceIndex() is called by apply_slice(), which can be + called by the SLICE opcode with v and/or w equal to NULL. + """ + if w_obj is not None: + pi[0] = space.getindex_w(w_obj, None) + return 1 + Modified: pypy/trunk/pypy/module/cpyext/test/test_eval.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_eval.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_eval.py Sat May 22 00:29:47 2010 @@ -3,9 +3,10 @@ from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.eval import ( Py_single_input, Py_file_input, Py_eval_input) -from pypy.module.cpyext.api import fopen, fclose +from pypy.module.cpyext.api import fopen, fclose, Py_ssize_tP from pypy.interpreter.gateway import interp2app from pypy.tool.udir import udir +import sys class TestEval(BaseApiTest): def test_eval(self, space, api): @@ -134,6 +135,18 @@ assert sorted(locals) == ['cpyvars', 'x'] assert sorted(globals) == ['__builtins__', 'anonymous', 'y'] + def test_sliceindex(self, space, api): + pi = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw') + assert api._PyEval_SliceIndex(space.w_None, pi) == 0 + api.PyErr_Clear() + + assert api._PyEval_SliceIndex(space.wrap(123), pi) == 1 + assert pi[0] == 123 + + assert api._PyEval_SliceIndex(space.wrap(1 << 66), pi) == 1 + assert pi[0] == sys.maxint + + lltype.free(pi, flavor='raw') class AppTestCall(AppTestCpythonExtensionBase): def test_CallFunction(self): From afa at codespeak.net Sat May 22 00:31:15 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 22 May 2010 00:31:15 +0200 (CEST) Subject: [pypy-svn] r74663 - pypy/trunk/pypy/module/cpyext Message-ID: <20100521223115.C0ABA282C00@codespeak.net> Author: afa Date: Sat May 22 00:31:14 2010 New Revision: 74663 Modified: pypy/trunk/pypy/module/cpyext/eval.py Log: Fix annotation Modified: pypy/trunk/pypy/module/cpyext/eval.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/eval.py (original) +++ pypy/trunk/pypy/module/cpyext/eval.py Sat May 22 00:31:14 2010 @@ -85,10 +85,10 @@ return compiling.eval(space, w_code, w_globals, w_locals) @cpython_api([CONST_STRING, rffi.INT_real,PyObject, PyObject], PyObject) -def PyRun_String(space, str, start, w_globals, w_locals): +def PyRun_String(space, source, start, w_globals, w_locals): """This is a simplified interface to PyRun_StringFlags() below, leaving flags set to NULL.""" - source = rffi.charp2str(str) + source = rffi.charp2str(source) filename = "" return run_string(space, source, filename, start, w_globals, w_locals) @@ -98,6 +98,7 @@ closeit set to 0 and flags set to NULL.""" BUF_SIZE = 8192 source = "" + filename = charp2str(filename) buf = lltype.malloc(rffi.CCHARP.TO, BUF_SIZE, flavor='raw') try: while True: From afa at codespeak.net Sat May 22 00:41:32 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 22 May 2010 00:41:32 +0200 (CEST) Subject: [pypy-svn] r74664 - pypy/trunk/pypy/module/cpyext Message-ID: <20100521224132.4567D282C00@codespeak.net> Author: afa Date: Sat May 22 00:41:29 2010 New Revision: 74664 Modified: pypy/trunk/pypy/module/cpyext/eval.py Log: Typo, it's time to go to bed. Modified: pypy/trunk/pypy/module/cpyext/eval.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/eval.py (original) +++ pypy/trunk/pypy/module/cpyext/eval.py Sat May 22 00:41:29 2010 @@ -98,7 +98,7 @@ closeit set to 0 and flags set to NULL.""" BUF_SIZE = 8192 source = "" - filename = charp2str(filename) + filename = rffi.charp2str(filename) buf = lltype.malloc(rffi.CCHARP.TO, BUF_SIZE, flavor='raw') try: while True: From afa at codespeak.net Sat May 22 00:52:15 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 22 May 2010 00:52:15 +0200 (CEST) Subject: [pypy-svn] r74665 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100521225215.A4069282C00@codespeak.net> Author: afa Date: Sat May 22 00:52:08 2010 New Revision: 74665 Modified: pypy/trunk/pypy/module/cpyext/api.py pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Log: Expose PyModuleType, PyPropertyTyp Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Sat May 22 00:52:08 2010 @@ -18,6 +18,8 @@ from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.gateway import ObjSpace, unwrap_spec from pypy.interpreter.nestedscope import Cell +from pypy.interpreter.module import Module +from pypy.module.__builtin__.descriptor import W_Property from pypy.rlib.entrypoint import entrypoint from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.objectmodel import specialize @@ -323,6 +325,8 @@ 'None': 'space.type(space.w_None)', 'NotImplemented': 'space.type(space.w_NotImplemented)', 'Cell': 'space.gettypeobject(Cell.typedef)', + 'Module': 'space.gettypeobject(Module.typedef)', + 'Property': 'space.gettypeobject(W_Property.typedef)', }.items(): GLOBALS['Py%s_Type#' % (cpyname, )] = ('PyTypeObject*', pypyexpr) Modified: pypy/trunk/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Sat May 22 00:52:08 2010 @@ -646,3 +646,21 @@ h = mod.get_hash() assert h != 0 assert h % 4 == 0 # it's the pointer value + + def test_types(self): + """test the presence of random types""" + + mod = self.import_extension('foo', [ + ('get_names', 'METH_NOARGS', + ''' + /* XXX in tests, the C type is not correct */ + #define NAME(type) ((PyTypeObject*)&type)->tp_name + return Py_BuildValue("sss", + NAME(PyCell_Type), + NAME(PyModule_Type), + NAME(PyProperty_Type) + ); + ''' + ), + ]) + assert mod.get_names() == ('cell', 'module', 'property') From arigo at codespeak.net Sat May 22 10:15:14 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 May 2010 10:15:14 +0200 (CEST) Subject: [pypy-svn] r74666 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100522081514.C4B7B282BAD@codespeak.net> Author: arigo Date: Sat May 22 10:15:11 2010 New Revision: 74666 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py Log: The fixed list part. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Sat May 22 10:15:11 2010 @@ -710,22 +710,6 @@ else: return meth(op, args, *descrs) - def do_fixed_newlist(self, op, args, arraydescr): - # normalize number of arguments - if len(args) < 1: - args.append(Constant(0, lltype.Signed)) - if len(args) > 1: - v_default = args[1] - ARRAY = deref(op.result.concretetype) - if (not isinstance(v_default, Constant) or - v_default.value != arrayItem(ARRAY)._defl()): - return None # variable or non-null initial value - return SpaceOperation('new_array', [arraydescr, args[0]], op.result) - - def do_fixed_list_len(self, op, args, arraydescr): - return SpaceOperation('arraylen_gc', [args[0], arraydescr], op.result) - - do_fixed_list_len_foldable = do_fixed_list_len def _get_list_nonneg_canraise_flags(self, op): # xxx break of abstraction: @@ -757,6 +741,25 @@ descr, args[1]], v_posindex) return v_posindex, [op] + # ---------- fixed lists ---------- + + def do_fixed_newlist(self, op, args, arraydescr): + # normalize number of arguments + if len(args) < 1: + args.append(Constant(0, lltype.Signed)) + if len(args) > 1: + v_default = args[1] + ARRAY = deref(op.result.concretetype) + if (not isinstance(v_default, Constant) or + v_default.value != arrayItem(ARRAY)._defl()): + return None # variable or non-null initial value + return SpaceOperation('new_array', [arraydescr, args[0]], op.result) + + def do_fixed_list_len(self, op, args, arraydescr): + return SpaceOperation('arraylen_gc', [args[0], arraydescr], op.result) + + do_fixed_list_len_foldable = do_fixed_list_len + def do_fixed_list_getitem(self, op, args, arraydescr, pure=False): v_index, extraop = self._prepare_list_getset(op, arraydescr, args, 'check_neg_index') @@ -782,6 +785,16 @@ [args[0], arraydescr, v_index, args[2]], None) return extraop + [op] + def do_fixed_list_ll_arraycopy(self, op, args, arraydescr): + calldescr = self.callcontrol.getcalldescr(op) + return SpaceOperation('arraycopy', + [calldescr, op.args[0]] + args + [arraydescr], + op.result) + + # ---------- resizable lists ---------- + + # xxx + # ---------- # VirtualRefs. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py Sat May 22 10:15:11 2010 @@ -19,9 +19,16 @@ def __repr__(self): return '' +class FakeCallControl: + class getcalldescr(AbstractDescr): + def __init__(self, op): + self.op = op + def __repr__(self): + return '' + def builtin_test(oopspec_name, args, RESTYPE, expected): v_result = varoftype(RESTYPE) - tr = Transformer(FakeCPU()) + tr = Transformer(FakeCPU(), FakeCallControl()) if '/' in oopspec_name: oopspec_name, property = oopspec_name.split('/') def force_flags(op): @@ -30,7 +37,8 @@ if property == 'CANRAISE': return False, True raise ValueError(property) tr._get_list_nonneg_canraise_flags = force_flags - op = SpaceOperation('direct_call', [Constant("myfunc")] + args, + op = SpaceOperation('direct_call', + [Constant("myfunc", lltype.Void)] + args, v_result) oplist = tr._handle_list_call(op, oopspec_name, args) if expected is None: @@ -62,7 +70,15 @@ varoftype(lltype.Signed)], FIXEDLIST, None) def test_fixed_ll_arraycopy(): - xxx + builtin_test('list.ll_arraycopy', + [varoftype(FIXEDLIST), + varoftype(FIXEDLIST), + varoftype(lltype.Signed), + varoftype(lltype.Signed), + varoftype(lltype.Signed)], + lltype.Void, """ + arraycopy , $'myfunc', %r0, %r1, %i0, %i1, %i2, + """) def test_fixed_getitem(): builtin_test('list.getitem/NONNEG', From arigo at codespeak.net Sat May 22 10:40:56 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 May 2010 10:40:56 +0200 (CEST) Subject: [pypy-svn] r74667 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100522084056.B609F282BAD@codespeak.net> Author: arigo Date: Sat May 22 10:40:55 2010 New Revision: 74667 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py Log: Resizable lists. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Sat May 22 10:40:55 2010 @@ -298,10 +298,13 @@ prepare = self._handle_virtual_ref_call else: prepare = self.prepare_builtin_call - op1 = prepare(op, oopspec_name, args) + try: + op1 = prepare(op, oopspec_name, args) + except NotSupported: + op1 = op # If the resulting op1 is still a direct_call, turn it into a # residual_call. - if op1 is None or op1.opname == 'direct_call': + if op1.opname == 'direct_call': op1 = self.handle_residual_call(op1 or op) return op1 @@ -688,6 +691,10 @@ # Lists. def _handle_list_call(self, op, oopspec_name, args): + """Try to transform the call to a list-handling helper. + If no transformation is available, raise NotSupported + (in which case the original call is written as a residual call). + """ if oopspec_name.startswith('new'): LIST = deref(op.result.concretetype) else: @@ -696,20 +703,25 @@ assert resizable == (not isinstance(LIST, lltype.GcArray)) if resizable: prefix = 'do_resizable_' + ARRAY = LIST.items.TO + if self._array_of_voids(ARRAY): + raise NotSupported("resizable lists of voids") + descrs = (self.cpu.arraydescrof(ARRAY), + self.cpu.fielddescrof(LIST, 'length'), + self.cpu.fielddescrof(LIST, 'items'), + self.cpu.sizeof(LIST)) else: prefix = 'do_fixed_' if self._array_of_voids(LIST): - return None # arrays of voids: not supported + raise NotSupported("fixed lists of voids") arraydescr = self.cpu.arraydescrof(LIST) descrs = (arraydescr,) # try: meth = getattr(self, prefix + oopspec_name.replace('.', '_')) except AttributeError: - return None - else: - return meth(op, args, *descrs) - + raise NotSupported(prefix + oopspec_name) + return meth(op, args, *descrs) def _get_list_nonneg_canraise_flags(self, op): # xxx break of abstraction: @@ -731,7 +743,7 @@ def _prepare_list_getset(self, op, descr, args, checkname): non_negative, can_raise = self._get_list_nonneg_canraise_flags(op) if can_raise: - return None, None + raise NotSupported("list operation can raise") if non_negative: return args[1], [] else: @@ -741,19 +753,24 @@ descr, args[1]], v_posindex) return v_posindex, [op] - # ---------- fixed lists ---------- - - def do_fixed_newlist(self, op, args, arraydescr): - # normalize number of arguments - if len(args) < 1: - args.append(Constant(0, lltype.Signed)) + def _get_initial_newlist_length(self, op, args): + # normalize number of arguments to the 'newlist' function if len(args) > 1: - v_default = args[1] + v_default = args[1] # initial value: must be 0 or NULL ARRAY = deref(op.result.concretetype) if (not isinstance(v_default, Constant) or v_default.value != arrayItem(ARRAY)._defl()): - return None # variable or non-null initial value - return SpaceOperation('new_array', [arraydescr, args[0]], op.result) + raise NotSupported("variable or non-null initial value") + if len(args) >= 1: + return args[0] + else: + return Constant(0, lltype.Signed) # length: default to 0 + + # ---------- fixed lists ---------- + + def do_fixed_newlist(self, op, args, arraydescr): + v_length = self._get_initial_newlist_length(op, args) + return SpaceOperation('new_array', [arraydescr, v_length], op.result) def do_fixed_list_len(self, op, args, arraydescr): return SpaceOperation('arraylen_gc', [args[0], arraydescr], op.result) @@ -763,8 +780,6 @@ def do_fixed_list_getitem(self, op, args, arraydescr, pure=False): v_index, extraop = self._prepare_list_getset(op, arraydescr, args, 'check_neg_index') - if v_index is None: - return None extra = getkind(op.result.concretetype)[0] if pure: extra = 'pure_' + extra @@ -778,8 +793,6 @@ def do_fixed_list_setitem(self, op, args, arraydescr): v_index, extraop = self._prepare_list_getset(op, arraydescr, args, 'check_neg_index') - if v_index is None: - return None kind = getkind(op.args[2].concretetype)[0] op = SpaceOperation('setarrayitem_gc_%s' % kind, [args[0], arraydescr, v_index, args[2]], None) @@ -793,7 +806,38 @@ # ---------- resizable lists ---------- - # xxx + def do_resizable_newlist(self, op, args, arraydescr, lengthdescr, + itemsdescr, structdescr): + v_length = self._get_initial_newlist_length(op, args) + return SpaceOperation('newlist', + [structdescr, lengthdescr, itemsdescr, + arraydescr, v_length], + op.result) + + def do_resizable_list_getitem(self, op, args, arraydescr, lengthdescr, + itemsdescr, structdescr): + v_index, extraop = self._prepare_list_getset(op, lengthdescr, args, + 'check_resizable_neg_index') + kind = getkind(op.result.concretetype)[0] + op = SpaceOperation('getlistitem_gc_%s' % kind, + [args[0], itemsdescr, arraydescr, v_index], + op.result) + return extraop + [op] + + def do_resizable_list_setitem(self, op, args, arraydescr, lengthdescr, + itemsdescr, structdescr): + v_index, extraop = self._prepare_list_getset(op, lengthdescr, args, + 'check_resizable_neg_index') + kind = getkind(op.args[2].concretetype)[0] + op = SpaceOperation('setlistitem_gc_%s' % kind, + [args[0], itemsdescr, arraydescr, + v_index, args[2]], None) + return extraop + [op] + + def do_resizable_list_len(self, op, args, arraydescr, lengthdescr, + itemsdescr, structdescr): + return SpaceOperation('getfield_gc_i', + [args[0], lengthdescr], op.result) # ---------- # VirtualRefs. @@ -810,6 +854,9 @@ # ____________________________________________________________ +class NotSupported(Exception): + pass + def _with_prefix(prefix): result = {} for name in dir(Transformer): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py Sat May 22 10:40:55 2010 @@ -2,7 +2,7 @@ from pypy.translator.unsimplify import varoftype from pypy.objspace.flow.model import Constant, SpaceOperation -from pypy.jit.codewriter.jtransform import Transformer +from pypy.jit.codewriter.jtransform import Transformer, NotSupported from pypy.jit.codewriter.flatten import GraphFlattener from pypy.jit.codewriter.format import assert_format from pypy.jit.codewriter.test.test_flatten import fake_regallocs @@ -11,6 +11,10 @@ # ____________________________________________________________ FIXEDLIST = lltype.Ptr(lltype.GcArray(lltype.Signed)) +VARLIST = lltype.Ptr(lltype.GcStruct('VARLIST', + ('length', lltype.Signed), + ('items', FIXEDLIST), + adtmeths={"ITEM": lltype.Signed})) class FakeCPU: class arraydescrof(AbstractDescr): @@ -18,6 +22,17 @@ self.ARRAY = ARRAY def __repr__(self): return '' + class fielddescrof(AbstractDescr): + def __init__(self, STRUCT, fieldname): + self.STRUCT = STRUCT + self.fieldname = fieldname + def __repr__(self): + return '' % self.fieldname + class sizeof(AbstractDescr): + def __init__(self, STRUCT): + self.STRUCT = STRUCT + def __repr__(self): + return '' class FakeCallControl: class getcalldescr(AbstractDescr): @@ -40,10 +55,12 @@ op = SpaceOperation('direct_call', [Constant("myfunc", lltype.Void)] + args, v_result) - oplist = tr._handle_list_call(op, oopspec_name, args) - if expected is None: - assert oplist is None + try: + oplist = tr._handle_list_call(op, oopspec_name, args) + except NotSupported: + assert expected is NotSupported else: + assert expected is not NotSupported assert oplist is not None flattener = GraphFlattener(None, fake_regallocs()) if not isinstance(oplist, list): @@ -53,6 +70,7 @@ assert_format(flattener.ssarepr, expected) # ____________________________________________________________ +# Fixed lists def test_newlist(): builtin_test('newlist', [], FIXEDLIST, @@ -65,9 +83,11 @@ Constant(0, lltype.Signed)], FIXEDLIST, """new_array , $5 -> %r0""") builtin_test('newlist', [Constant(5, lltype.Signed), - Constant(1, lltype.Signed)], FIXEDLIST, None) + Constant(1, lltype.Signed)], FIXEDLIST, + NotSupported) builtin_test('newlist', [Constant(5, lltype.Signed), - varoftype(lltype.Signed)], FIXEDLIST, None) + varoftype(lltype.Signed)], FIXEDLIST, + NotSupported) def test_fixed_ll_arraycopy(): builtin_test('list.ll_arraycopy', @@ -94,7 +114,7 @@ """) builtin_test('list.getitem/CANRAISE', [varoftype(FIXEDLIST), varoftype(lltype.Signed)], - lltype.Signed, None) + lltype.Signed, NotSupported) def test_fixed_getitem_foldable(): builtin_test('list.getitem_foldable/NONNEG', @@ -110,7 +130,7 @@ """) builtin_test('list.getitem_foldable/CANRAISE', [varoftype(FIXEDLIST), varoftype(lltype.Signed)], - lltype.Signed, None) + lltype.Signed, NotSupported) def test_fixed_setitem(): builtin_test('list.setitem/NONNEG', [varoftype(FIXEDLIST), @@ -129,7 +149,7 @@ builtin_test('list.setitem/CANRAISE', [varoftype(FIXEDLIST), varoftype(lltype.Signed), varoftype(lltype.Signed)], - lltype.Void, None) + lltype.Void, NotSupported) def test_fixed_len(): builtin_test('list.len', [varoftype(FIXEDLIST)], lltype.Signed, @@ -139,17 +159,67 @@ builtin_test('list.len_foldable', [varoftype(FIXEDLIST)], lltype.Signed, """arraylen_gc %r0, -> %i0""") +# ____________________________________________________________ +# Resizable lists + def test_resizable_newlist(): - xxx + alldescrs = (", ," + " , ") + builtin_test('newlist', [], VARLIST, + """newlist """+alldescrs+""", $0 -> %r0""") + builtin_test('newlist', [Constant(5, lltype.Signed)], VARLIST, + """newlist """+alldescrs+""", $5 -> %r0""") + builtin_test('newlist', [varoftype(lltype.Signed)], VARLIST, + """newlist """+alldescrs+""", %i0 -> %r0""") + builtin_test('newlist', [Constant(5, lltype.Signed), + Constant(0, lltype.Signed)], VARLIST, + """newlist """+alldescrs+""", $5 -> %r0""") + builtin_test('newlist', [Constant(5, lltype.Signed), + Constant(1, lltype.Signed)], VARLIST, + NotSupported) + builtin_test('newlist', [Constant(5, lltype.Signed), + varoftype(lltype.Signed)], VARLIST, + NotSupported) def test_resizable_getitem(): - xxx + builtin_test('list.getitem/NONNEG', + [varoftype(VARLIST), varoftype(lltype.Signed)], + lltype.Signed, """ + getlistitem_gc_i %r0, , , %i0 -> %i1 + """) + builtin_test('list.getitem/NEG', + [varoftype(VARLIST), varoftype(lltype.Signed)], + lltype.Signed, """ + check_resizable_neg_index %r0, , %i0 -> %i1 + getlistitem_gc_i %r0, , , %i1 -> %i2 + """) + builtin_test('list.getitem/CANRAISE', + [varoftype(VARLIST), varoftype(lltype.Signed)], + lltype.Signed, NotSupported) def test_resizable_setitem(): - xxx + builtin_test('list.setitem/NONNEG', [varoftype(VARLIST), + varoftype(lltype.Signed), + varoftype(lltype.Signed)], + lltype.Void, """ + setlistitem_gc_i %r0, , , %i0, %i1 + """) + builtin_test('list.setitem/NEG', [varoftype(VARLIST), + varoftype(lltype.Signed), + varoftype(lltype.Signed)], + lltype.Void, """ + check_resizable_neg_index %r0, , %i0 -> %i1 + setlistitem_gc_i %r0, , , %i1, %i2 + """) + builtin_test('list.setitem/CANRAISE', [varoftype(VARLIST), + varoftype(lltype.Signed), + varoftype(lltype.Signed)], + lltype.Void, NotSupported) def test_resizable_len(): - xxx + builtin_test('list.len', [varoftype(VARLIST)], lltype.Signed, + """getfield_gc_i %r0, -> %i0""") def test_resizable_unsupportedop(): - builtin_test('list.foobar', [varoftype(VARLIST)], lltype.Signed, None) + builtin_test('list.foobar', [varoftype(VARLIST)], lltype.Signed, + NotSupported) From hpk at codespeak.net Sat May 22 11:00:41 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sat, 22 May 2010 11:00:41 +0200 (CEST) Subject: [pypy-svn] r74668 - pypy/branch/py131/pypy/interpreter/test Message-ID: <20100522090041.A0A29282BD4@codespeak.net> Author: hpk Date: Sat May 22 11:00:40 2010 New Revision: 74668 Modified: pypy/branch/py131/pypy/interpreter/test/test_code.py Log: fix and unify testing for appdirect/cpython-pypy Modified: pypy/branch/py131/pypy/interpreter/test/test_code.py ============================================================================== --- pypy/branch/py131/pypy/interpreter/test/test_code.py (original) +++ pypy/branch/py131/pypy/interpreter/test/test_code.py Sat May 22 11:00:40 2010 @@ -6,11 +6,7 @@ def setup_class(cls): space = gettestobjspace() cls.space = space - if py.test.config.option.runappdirect: - filename = __file__ - else: - filename = gateway.__file__ - + filename = __file__ if filename[-3:] != '.py': filename = filename[:-1] From afa at codespeak.net Sat May 22 11:25:57 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 22 May 2010 11:25:57 +0200 (CEST) Subject: [pypy-svn] r74669 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100522092557.61A2436C53C@codespeak.net> Author: afa Date: Sat May 22 11:25:54 2010 New Revision: 74669 Modified: pypy/trunk/pypy/module/cpyext/api.py pypy/trunk/pypy/module/cpyext/test/test_eval.py Log: Fix segfault in test: don't close the file with fclose(), this frees the FILE* structure. Use os.close() instead. Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Sat May 22 11:25:54 2010 @@ -67,6 +67,7 @@ assert CONST_STRING is not rffi.CCHARP assert CONST_WSTRING is not rffi.CWCHARP +# FILE* interface FILEP = rffi.COpaquePtr('FILE') fopen = rffi.llexternal('fopen', [CONST_STRING, CONST_STRING], FILEP) fclose = rffi.llexternal('fclose', [FILEP], rffi.INT) @@ -77,6 +78,11 @@ [rffi.VOIDP, rffi.SIZE_T, rffi.SIZE_T, FILEP], rffi.SIZE_T) feof = rffi.llexternal('feof', [FILEP], rffi.INT) +if sys.platform == 'win32': + fileno = rffi.llexternal('_fileno', [FILEP], rffi.INT) +else: + fileno = rffi.llexternal('fileno', [FILEP], rffi.INT) + constant_names = """ Py_TPFLAGS_READY Py_TPFLAGS_READYING Modified: pypy/trunk/pypy/module/cpyext/test/test_eval.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_eval.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_eval.py Sat May 22 11:25:54 2010 @@ -3,10 +3,10 @@ from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.eval import ( Py_single_input, Py_file_input, Py_eval_input) -from pypy.module.cpyext.api import fopen, fclose, Py_ssize_tP +from pypy.module.cpyext.api import fopen, fclose, fileno, Py_ssize_tP from pypy.interpreter.gateway import interp2app from pypy.tool.udir import udir -import sys +import sys, os class TestEval(BaseApiTest): def test_eval(self, space, api): @@ -92,8 +92,11 @@ assert api.PyErr_Occurred() is space.w_ZeroDivisionError api.PyErr_Clear() - # retry on closed file + # try again, but with a closed file + fp = fopen(str(filepath), "rb") + os.close(fileno(fp)) api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals) + fclose(fp) assert api.PyErr_Occurred() is space.w_IOError api.PyErr_Clear() From arigo at codespeak.net Sat May 22 12:29:41 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 May 2010 12:29:41 +0200 (CEST) Subject: [pypy-svn] r74670 - in pypy/branch/blackhole-improvement/pypy/jit: backend backend/llgraph backend/x86 codewriter metainterp metainterp/test Message-ID: <20100522102941.06184282BAD@codespeak.net> Author: arigo Date: Sat May 22 12:29:39 2010 New Revision: 74670 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py pypy/branch/blackhole-improvement/pypy/jit/backend/model.py pypy/branch/blackhole-improvement/pypy/jit/backend/x86/assembler.py pypy/branch/blackhole-improvement/pypy/jit/backend/x86/regalloc.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/simple_optimize.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/specnode.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_list.py Log: * Pass metainterp/test/test_list.py. * other misc stuff. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py Sat May 22 12:29:39 2010 @@ -155,16 +155,6 @@ 'force_token' : ((), 'int'), 'call_may_force' : (('int', 'varargs'), 'intorptr'), 'guard_not_forced': ((), None), - 'virtual_ref' : (('ref', 'int'), 'ref'), - 'virtual_ref_finish': (('ref', 'ref'), None), - #'getitem' : (('void', 'ref', 'int'), 'int'), - #'setitem' : (('void', 'ref', 'int', 'int'), None), - #'newlist' : (('void', 'varargs'), 'ref'), - #'append' : (('void', 'ref', 'int'), None), - #'insert' : (('void', 'ref', 'int', 'int'), None), - #'pop' : (('void', 'ref',), 'int'), - #'len' : (('void', 'ref',), 'int'), - #'listnonzero' : (('void', 'ref',), 'int'), } # ____________________________________________________________ @@ -743,7 +733,7 @@ if arraydescr.typeinfo == REF: do_setarrayitem_gc_ptr(array, index, newvalue) elif arraydescr.typeinfo == INT: - do_setarrayitem_gc_int(array, index, newvalue, self.memocast) + do_setarrayitem_gc_int(array, index, newvalue) elif arraydescr.typeinfo == FLOAT: do_setarrayitem_gc_float(array, index, newvalue) else: @@ -865,12 +855,6 @@ if forced: raise GuardFailed - def op_virtual_ref(self, _, virtual, index): - return virtual - - def op_virtual_ref_finish(self, _, vref, virtual): - pass - class OOFrame(Frame): Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py Sat May 22 12:29:39 2010 @@ -286,10 +286,6 @@ # ---------- the backend-dependent operations ---------- -## def do_arraylen_gc(self, arraybox, arraydescr): -## array = arraybox.getref_base() -## return history.BoxInt(llimpl.do_arraylen_gc(arraydescr, array)) - def bh_strlen(self, string): return llimpl.do_strlen(string) @@ -302,36 +298,15 @@ def bh_unicodegetitem(self, string, index): return llimpl.do_unicodegetitem(string, index) -## def do_getarrayitem_gc(self, arraybox, indexbox, arraydescr): -## assert isinstance(arraydescr, Descr) -## array = arraybox.getref_base() -## index = indexbox.getint() -## if arraydescr.typeinfo == REF: -## return history.BoxPtr(llimpl.do_getarrayitem_gc_ptr(array, index)) -## elif arraydescr.typeinfo == INT: -## return history.BoxInt(llimpl.do_getarrayitem_gc_int(array, index, -## self.memo_cast)) -## elif arraydescr.typeinfo == FLOAT: -## return history.BoxFloat(llimpl.do_getarrayitem_gc_float(array, -## index)) -## else: -## raise NotImplementedError - -## def do_getfield_gc(self, structbox, fielddescr): -## assert isinstance(fielddescr, Descr) -## struct = structbox.getref_base() -## if fielddescr.typeinfo == REF: -## return history.BoxPtr(llimpl.do_getfield_gc_ptr(struct, -## fielddescr.ofs)) -## elif fielddescr.typeinfo == INT: -## return history.BoxInt(llimpl.do_getfield_gc_int(struct, -## fielddescr.ofs, -## self.memo_cast)) -## elif fielddescr.typeinfo == FLOAT: -## return history.BoxFloat(llimpl.do_getfield_gc_float(struct, -## fielddescr.ofs)) -## else: -## raise NotImplementedError + def bh_getarrayitem_gc_i(self, arraydescr, array, index): + assert isinstance(arraydescr, Descr) + return llimpl.do_getarrayitem_gc_int(array, index) + def bh_getarrayitem_gc_r(self, arraydescr, array, index): + assert isinstance(arraydescr, Descr) + return llimpl.do_getarrayitem_gc_ptr(array, index) + def bh_getarrayitem_gc_f(self, arraydescr, array, index): + assert isinstance(arraydescr, Descr) + return llimpl.do_getarrayitem_gc_float(array, index) def bh_getfield_gc_i(self, struct, fielddescr): assert isinstance(fielddescr, Descr) @@ -343,24 +318,6 @@ assert isinstance(fielddescr, Descr) return llimpl.do_getfield_gc_float(struct, fielddescr.ofs) -## def do_getfield_raw(self, structbox, fielddescr): -## assert isinstance(fielddescr, Descr) -## struct = self.cast_int_to_adr(structbox.getint()) -## if fielddescr.typeinfo == REF: -## return history.BoxPtr(llimpl.do_getfield_raw_ptr(struct, -## fielddescr.ofs, -## self.memo_cast)) -## elif fielddescr.typeinfo == INT: -## return history.BoxInt(llimpl.do_getfield_raw_int(struct, -## fielddescr.ofs, -## self.memo_cast)) -## elif fielddescr.typeinfo == FLOAT: -## return history.BoxFloat(llimpl.do_getfield_raw_float(struct, -## fielddescr.ofs, -## self.memo_cast)) -## else: -## raise NotImplementedError - def bh_getfield_raw_i(self, struct, fielddescr): assert isinstance(fielddescr, Descr) return llimpl.do_getfield_raw_int(struct, fielddescr.ofs) @@ -371,10 +328,6 @@ assert isinstance(fielddescr, Descr) return llimpl.do_getfield_raw_float(struct, fielddescr.ofs) -## def do_new(self, size): -## assert isinstance(size, Descr) -## return history.BoxPtr(llimpl.do_new(size.ofs)) - def bh_new(self, sizedescr): assert isinstance(sizedescr, Descr) return llimpl.do_new(sizedescr.ofs) @@ -391,51 +344,25 @@ result_adr = llmemory.cast_ptr_to_adr(struct.typeptr) return llmemory.cast_adr_to_int(result_adr) -## def do_new_array(self, countbox, size): -## assert isinstance(size, Descr) -## count = countbox.getint() -## return history.BoxPtr(llimpl.do_new_array(size.ofs, count)) - def bh_new_array(self, arraydescr, length): assert isinstance(arraydescr, Descr) return llimpl.do_new_array(arraydescr.ofs, length) -## def do_setarrayitem_gc(self, arraybox, indexbox, newvaluebox, arraydescr): -## assert isinstance(arraydescr, Descr) -## array = arraybox.getref_base() -## index = indexbox.getint() -## if arraydescr.typeinfo == REF: -## newvalue = newvaluebox.getref_base() -## llimpl.do_setarrayitem_gc_ptr(array, index, newvalue) -## elif arraydescr.typeinfo == INT: -## newvalue = newvaluebox.getint() -## llimpl.do_setarrayitem_gc_int(array, index, newvalue, -## self.memo_cast) -## elif arraydescr.typeinfo == FLOAT: -## newvalue = newvaluebox.getfloat() -## llimpl.do_setarrayitem_gc_float(array, index, newvalue) -## else: -## raise NotImplementedError + def bh_arraylen_gc(self, arraydescr, array): + assert isinstance(arraydescr, Descr) + return llimpl.do_arraylen_gc(arraydescr, array) + + def bh_setarrayitem_gc_i(self, arraydescr, array, index, newvalue): + assert isinstance(arraydescr, Descr) + llimpl.do_setarrayitem_gc_int(array, index, newvalue) def bh_setarrayitem_gc_r(self, arraydescr, array, index, newvalue): assert isinstance(arraydescr, Descr) llimpl.do_setarrayitem_gc_ptr(array, index, newvalue) -## def do_setfield_gc(self, structbox, newvaluebox, fielddescr): -## assert isinstance(fielddescr, Descr) -## struct = structbox.getref_base() -## if fielddescr.typeinfo == REF: -## newvalue = newvaluebox.getref_base() -## llimpl.do_setfield_gc_ptr(struct, fielddescr.ofs, newvalue) -## elif fielddescr.typeinfo == INT: -## newvalue = newvaluebox.getint() -## llimpl.do_setfield_gc_int(struct, fielddescr.ofs, newvalue, -## self.memo_cast) -## elif fielddescr.typeinfo == FLOAT: -## newvalue = newvaluebox.getfloat() -## llimpl.do_setfield_gc_float(struct, fielddescr.ofs, newvalue) -## else: -## raise NotImplementedError + def bh_setarrayitem_gc_f(self, arraydescr, array, index, newvalue): + assert isinstance(arraydescr, Descr) + llimpl.do_setarrayitem_gc_float(array, index, newvalue) def bh_setfield_gc_i(self, struct, fielddescr, newvalue): assert isinstance(fielddescr, Descr) @@ -447,24 +374,6 @@ assert isinstance(fielddescr, Descr) llimpl.do_setfield_gc_float(struct, fielddescr.ofs, newvalue) -## def do_setfield_raw(self, structbox, newvaluebox, fielddescr): -## assert isinstance(fielddescr, Descr) -## struct = self.cast_int_to_adr(structbox.getint()) -## if fielddescr.typeinfo == REF: -## newvalue = newvaluebox.getref_base() -## llimpl.do_setfield_raw_ptr(struct, fielddescr.ofs, newvalue, -## self.memo_cast) -## elif fielddescr.typeinfo == INT: -## newvalue = newvaluebox.getint() -## llimpl.do_setfield_raw_int(struct, fielddescr.ofs, newvalue, -## self.memo_cast) -## elif fielddescr.typeinfo == FLOAT: -## newvalue = newvaluebox.getfloat() -## llimpl.do_setfield_raw_float(struct, fielddescr.ofs, newvalue, -## self.memo_cast) -## else: -## raise NotImplementedError - def bh_setfield_raw_i(self, struct, fielddescr, newvalue): assert isinstance(fielddescr, Descr) llimpl.do_setfield_raw_int(struct, fielddescr.ofs, newvalue) @@ -475,9 +384,6 @@ assert isinstance(fielddescr, Descr) llimpl.do_setfield_raw_float(struct, fielddescr.ofs, newvalue) -## def do_same_as(self, box1): -## return box1.clonebox() - def bh_newstr(self, length): return llimpl.do_newstr(length) @@ -516,10 +422,6 @@ for x in args_f: llimpl.do_call_pushfloat(x) -## def do_cast_ptr_to_int(self, ptrbox): -## return history.BoxInt(llimpl.cast_to_int(ptrbox.getref_base(), -## self.memo_cast)) - def bh_cast_ptr_to_int(self, ptr): return llimpl.cast_to_int(ptr) Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/model.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/model.py Sat May 22 12:29:39 2010 @@ -144,7 +144,14 @@ # lltype specific operations # -------------------------- - + + def bh_getarrayitem_gc_i(self, arraydescr, array, index): + raise NotImplementedError + def bh_getarrayitem_gc_r(self, arraydescr, array, index): + raise NotImplementedError + def bh_getarrayitem_gc_f(self, arraydescr, array, index): + raise NotImplementedError + def bh_getfield_gc_i(self, struct, fielddescr): raise NotImplementedError def bh_getfield_gc_r(self, struct, fielddescr): @@ -166,6 +173,9 @@ def bh_new_array(self, arraydescr, length): raise NotImplementedError + def bh_arraylen_gc(self, arraydescr, array): + raise NotImplementedError + def bh_classof(self, struct): raise NotImplementedError Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/x86/assembler.py Sat May 22 12:29:39 2010 @@ -747,7 +747,6 @@ def genop_same_as(self, op, arglocs, resloc): self.mov(arglocs[0], resloc) genop_cast_ptr_to_int = genop_same_as - genop_virtual_ref = genop_same_as def genop_int_mod(self, op, arglocs, resloc): self.mc.CDQ() Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/x86/regalloc.py Sat May 22 12:29:39 2010 @@ -894,7 +894,6 @@ resloc = self.force_allocate_reg(op.result) self.Perform(op, [argloc], resloc) consider_cast_ptr_to_int = consider_same_as - consider_virtual_ref = consider_same_as def consider_strlen(self, op): base_loc = self.rm.make_sure_var_in_reg(op.args[0], op.args) @@ -953,9 +952,6 @@ def consider_debug_merge_point(self, op): pass - def consider_virtual_ref_finish(self, op): - self.possibly_free_vars(op.args) - def get_mark_gc_roots(self, gcrootmap): shape = gcrootmap.get_basic_shape() for v, val in self.fm.frame_bindings.items(): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Sat May 22 12:29:39 2010 @@ -6,6 +6,7 @@ from pypy.objspace.flow.model import Block, Link, c_last_exception from pypy.jit.codewriter.flatten import ListOfKind, IndirectCallTargets from pypy.jit.codewriter import support, heaptracker +from pypy.jit.codewriter.policy import log from pypy.jit.metainterp.typesystem import deref, arrayItem from pypy.rlib import objectmodel from pypy.rlib.jit import _we_are_jitted @@ -304,7 +305,7 @@ op1 = op # If the resulting op1 is still a direct_call, turn it into a # residual_call. - if op1.opname == 'direct_call': + if isinstance(op1, SpaceOperation) and op1.opname == 'direct_call': op1 = self.handle_residual_call(op1 or op) return op1 @@ -401,7 +402,8 @@ arraydescr = self.cpu.arraydescrof(ARRAY) kind = getkind(op.args[2].concretetype) return SpaceOperation('setarrayitem_gc_%s' % kind[0], - [arraydescr] + op.args, None) + [op.args[0], arraydescr, op.args[1], op.args[2]], + None) def _array_of_voids(self, ARRAY): #if isinstance(ARRAY, ootype.Array): @@ -687,6 +689,11 @@ def handle_jit_marker__can_enter_jit(self, op, jitdriver): return SpaceOperation('can_enter_jit', [], None) + def rewrite_op_debug_assert(self, op): + log.WARNING("found debug_assert in %r; should have be removed" % + (self.graph,)) + return [] + # ---------- # Lists. @@ -793,7 +800,7 @@ def do_fixed_list_setitem(self, op, args, arraydescr): v_index, extraop = self._prepare_list_getset(op, arraydescr, args, 'check_neg_index') - kind = getkind(op.args[2].concretetype)[0] + kind = getkind(args[2].concretetype)[0] op = SpaceOperation('setarrayitem_gc_%s' % kind, [args[0], arraydescr, v_index, args[2]], None) return extraop + [op] Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Sat May 22 12:29:39 2010 @@ -781,6 +781,37 @@ raise CRN(*results) # ---------- + # list operations + + @arguments("r", "d", "i", returns="i") + def bhimpl_check_neg_index(array, arraydescr, index): + xxx + + @arguments("r", "d", "i", returns="i") + def bhimpl_check_resizable_neg_index(list, lengthdescr, index): + xxx + + @arguments() + def bhimpl_getlistitem_gc_i(yyy): + xxx + @arguments() + def bhimpl_getlistitem_gc_r(yyy): + xxx + @arguments() + def bhimpl_getlistitem_gc_f(yyy): + xxx + + @arguments() + def bhimpl_setlistitem_gc_i(yyy): + xxx + @arguments() + def bhimpl_setlistitem_gc_r(yyy): + xxx + @arguments() + def bhimpl_setlistitem_gc_f(yyy): + xxx + + # ---------- # the following operations are directly implemented by the backend @arguments("cpu", "i", "d", "R", returns="i") @@ -862,9 +893,38 @@ @arguments("cpu", "d", "i", returns="r") def bhimpl_new_array(cpu, arraydescr, length): return cpu.bh_new_array(arraydescr, length) - @arguments("cpu", "d", "r", "i", "r") - def bhimpl_setarrayitem_gc_r(cpu, arraydescr, array, index, newvalue): + + @arguments("cpu", "r", "d", "i", returns="i") + def bhimpl_getarrayitem_gc_i(cpu, array, arraydescr, index): + return cpu.bh_getarrayitem_gc_i(arraydescr, array, index) + @arguments("cpu", "r", "d", "i", returns="r") + def bhimpl_getarrayitem_gc_r(cpu, array, arraydescr, index): + return cpu.bh_getarrayitem_gc_r(arraydescr, array, index) + @arguments("cpu", "r", "d", "i", returns="f") + def bhimpl_getarrayitem_gc_f(cpu, array, arraydescr, index): + return cpu.bh_getarrayitem_gc_f(arraydescr, array, index) + + bhimpl_getarrayitem_gc_pure_i = bhimpl_getarrayitem_gc_i + bhimpl_getarrayitem_gc_pure_r = bhimpl_getarrayitem_gc_r + bhimpl_getarrayitem_gc_pure_f = bhimpl_getarrayitem_gc_f + + @arguments("cpu", "r", "d", "i", "i") + def bhimpl_setarrayitem_gc_i(cpu, array, arraydescr, index, newvalue): + cpu.bh_setarrayitem_gc_i(arraydescr, array, index, newvalue) + @arguments("cpu", "r", "d", "i", "r") + def bhimpl_setarrayitem_gc_r(cpu, array, arraydescr, index, newvalue): cpu.bh_setarrayitem_gc_r(arraydescr, array, index, newvalue) + @arguments("cpu", "r", "d", "i", "f") + def bhimpl_setarrayitem_gc_f(cpu, array, arraydescr, index, newvalue): + cpu.bh_setarrayitem_gc_f(arraydescr, array, index, newvalue) + + @arguments("cpu", "r", "d", returns="i") + def bhimpl_arraylen_gc(cpu, array, arraydescr): + return cpu.bh_arraylen_gc(arraydescr, array) + + @arguments("cpu", "d", "i", "r", "r", "i", "i", "i", "d") + def bhimpl_arraycopy(cpu, calldescr, func, x1, x2, x3, x4, x5, arraydescr): + cpu.bh_call_v(func, calldescr, [x3, x4, x5], [x1, x2], None) @arguments("cpu", "r", "d", returns="i") def bhimpl_getfield_gc_i(cpu, struct, fielddescr): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py Sat May 22 12:29:39 2010 @@ -91,11 +91,11 @@ array = arraybox.getref_base() index = indexbox.getint() if arraydescr.is_array_of_pointers(): - return BoxPtr(cpu.bh_setarrayitem_gc_r(arraydescr, array, index)) + return BoxPtr(cpu.bh_getarrayitem_gc_r(arraydescr, array, index)) elif arraydescr.is_array_of_floats(): - return BoxFloat(cpu.bh_setarrayitem_gc_f(arraydescr, array, index)) + return BoxFloat(cpu.bh_getarrayitem_gc_f(arraydescr, array, index)) else: - return BoxInt(cpu.bh_setarrayitem_gc_i(arraydescr, array, index)) + return BoxInt(cpu.bh_getarrayitem_gc_i(arraydescr, array, index)) def do_setarrayitem_gc(cpu, _, arraybox, indexbox, itembox, arraydescr): array = arraybox.getref_base() @@ -153,6 +153,12 @@ def do_new_with_vtable(cpu, _, clsbox): return BoxPtr(exec_new_with_vtable(cpu, clsbox)) +def do_arraycopy(cpu, _, calldescr, funcbox, x1box, x2box, + x3box, x4box, x5box, arraydescr): + cpu.bh_call_v(funcbox.getint(), calldescr, + [x3box.getint(), x4box.getint(), x5box.getint()], + [x1box.getref_base(), x2box.getref_base()], None) + def do_int_add_ovf(cpu, metainterp, box1, box2): a = box1.getint() b = box2.getint() @@ -275,6 +281,8 @@ for argtype in func.argtypes: if argtype not in ('i', 'r', 'f', 'd', 'cpu'): return None + if list(func.argtypes).count('d') > 1: + return None if func.resulttype not in ('i', 'r', 'f', None): return None argtypes = unrolling_iterable(func.argtypes) @@ -319,7 +327,8 @@ # constant-folded away. Only works if opnum and num_args are # constants, of course. func = EXECUTE_BY_NUM_ARGS[num_args][opnum] - assert func is not None, "EXECUTE_BY_NUM_ARGS[%s][%s]" % (num_args, opnum) + assert func is not None, "EXECUTE_BY_NUM_ARGS[%s][%s]" % ( + num_args, resoperation.opname[opnum]) return func get_execute_function._annspecialcase_ = 'specialize:memo' Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Sat May 22 12:29:39 2010 @@ -394,34 +394,103 @@ def opimpl_new_array(self, itemsizedescr, countbox): return self.execute_with_descr(rop.NEW_ARRAY, itemsizedescr, countbox) - @FixME #arguments("box", "descr", "box") - def opimpl_getarrayitem_gc(self, arraybox, arraydesc, indexbox): - self.execute_with_descr(rop.GETARRAYITEM_GC, arraydesc, arraybox, indexbox) - - @FixME #arguments("box", "descr", "box") - def opimpl_getarrayitem_gc_pure(self, arraybox, arraydesc, indexbox): - self.execute_with_descr(rop.GETARRAYITEM_GC_PURE, arraydesc, arraybox, indexbox) - - @arguments("descr", "box", "box", "box") - def _opimpl_setarrayitem_gc(self, arraydesc, arraybox, indexbox, itembox): - self.execute_with_descr(rop.SETARRAYITEM_GC, arraydesc, arraybox, + @arguments("box", "descr", "box") + def _opimpl_getarrayitem_gc_any(self, arraybox, arraydescr, indexbox): + return self.execute_with_descr(rop.GETARRAYITEM_GC, + arraydescr, arraybox, indexbox) + + opimpl_getarrayitem_gc_i = _opimpl_getarrayitem_gc_any + opimpl_getarrayitem_gc_r = _opimpl_getarrayitem_gc_any + opimpl_getarrayitem_gc_f = _opimpl_getarrayitem_gc_any + + @arguments("box", "descr", "box") + def _opimpl_getarrayitem_gc_pure_any(self, arraybox, arraydescr, indexbox): + return self.execute_with_descr(rop.GETARRAYITEM_GC_PURE, + arraydescr, arraybox, indexbox) + + opimpl_getarrayitem_gc_pure_i = _opimpl_getarrayitem_gc_pure_any + opimpl_getarrayitem_gc_pure_r = _opimpl_getarrayitem_gc_pure_any + opimpl_getarrayitem_gc_pure_f = _opimpl_getarrayitem_gc_pure_any + + @arguments("box", "descr", "box", "box") + def _opimpl_setarrayitem_gc_any(self, arraybox, arraydescr, + indexbox, itembox): + self.execute_with_descr(rop.SETARRAYITEM_GC, arraydescr, arraybox, indexbox, itembox) - opimpl_setarrayitem_gc_i = _opimpl_setarrayitem_gc - opimpl_setarrayitem_gc_r = _opimpl_setarrayitem_gc - opimpl_setarrayitem_gc_f = _opimpl_setarrayitem_gc - - @FixME #arguments("box", "descr") - def opimpl_arraylen_gc(self, arraybox, arraydesc): - self.execute_with_descr(rop.ARRAYLEN_GC, arraydesc, arraybox) + opimpl_setarrayitem_gc_i = _opimpl_setarrayitem_gc_any + opimpl_setarrayitem_gc_r = _opimpl_setarrayitem_gc_any + opimpl_setarrayitem_gc_f = _opimpl_setarrayitem_gc_any - @FixME #arguments("descr", "box", "box", "box", "box", "box", "box", "descr") + @arguments("box", "descr") + def opimpl_arraylen_gc(self, arraybox, arraydescr): + return self.execute_with_descr(rop.ARRAYLEN_GC, arraydescr, arraybox) + + @arguments("descr", "box", "box", "box", "box", "box", "box", "descr") def opimpl_arraycopy(self, calldescr, fnptr, sourcebox, destbox, - source_startbox, dest_startbox, lengthbox, arraydescr): + source_startbox, dest_startbox, lengthbox, + arraydescr): self.execute_with_descr(rop.ARRAYCOPY, arraydescr, calldescr, fnptr, sourcebox, destbox, source_startbox, dest_startbox, lengthbox) + @arguments("orgpc", "box", "descr", "box") + def opimpl_check_neg_index(self, orgpc, arraybox, arraydescr, indexbox): + negbox = self.metainterp.execute_and_record( + rop.INT_LT, None, indexbox, ConstInt(0)) + negbox = self.implement_guard_value(orgpc, negbox) + if negbox.getint(): + # the index is < 0; add the array length to it + lenbox = self.metainterp.execute_and_record( + rop.ARRAYLEN_GC, arraydescr, arraybox) + indexbox = self.metainterp.execute_and_record( + rop.INT_ADD, None, indexbox, lenbox) + return indexbox + + @FixME #arguments("descr", "descr", "descr", "descr", "box") + def opimpl_newlist(self, structdescr, lengthdescr, itemsdescr, arraydescr, + sizebox): + sbox = self.metainterp.execute_and_record(rop.NEW, structdescr) + self.metainterp.execute_and_record(rop.SETFIELD_GC, lengthdescr, + sbox, sizebox) + abox = self.metainterp.execute_and_record(rop.NEW_ARRAY, arraydescr, + sizebox) + self.metainterp.execute_and_record(rop.SETFIELD_GC, itemsdescr, + sbox, abox) + self.make_result_box(sbox) + + @FixME #arguments("box", "descr", "descr", "box") + def opimpl_getlistitem_gc(self, listbox, itemsdescr, arraydescr, indexbox): + arraybox = self.metainterp.execute_and_record(rop.GETFIELD_GC, + itemsdescr, listbox) + self.execute_with_descr(rop.GETARRAYITEM_GC, arraydescr, arraybox, indexbox) + + @arguments("box", "descr", "descr", "box", "box") + def _opimpl_setlistitem_gc_any(self, listbox, itemsdescr, arraydescr, + indexbox, valuebox): + arraybox = self.metainterp.execute_and_record(rop.GETFIELD_GC, + itemsdescr, listbox) + self.execute_with_descr(rop.SETARRAYITEM_GC, arraydescr, arraybox, + indexbox, valuebox) + + opimpl_setlistitem_gc_i = _opimpl_setlistitem_gc_any + opimpl_setlistitem_gc_r = _opimpl_setlistitem_gc_any + opimpl_setlistitem_gc_f = _opimpl_setlistitem_gc_any + + @arguments("orgpc", "box", "descr", "box") + def opimpl_check_resizable_neg_index(self, orgpc, listbox, lengthdescr, + indexbox): + negbox = self.metainterp.execute_and_record( + rop.INT_LT, None, indexbox, history.CONST_FALSE) + negbox = self.implement_guard_value(orgpc, negbox) + if negbox.getint(): + # the index is < 0; add the array length to it + lenbox = self.metainterp.execute_and_record( + rop.GETFIELD_GC, lengthdescr, listbox) + indexbox = self.metainterp.execute_and_record( + rop.INT_ADD, None, indexbox, lenbox) + return indexbox + @arguments("box", "descr") def _opimpl_getfield_gc_any(self, box, fielddescr): return self.execute_with_descr(rop.GETFIELD_GC, fielddescr, box) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py Sat May 22 12:29:39 2010 @@ -208,14 +208,14 @@ 'NEW_WITH_VTABLE/1', 'NEW_ARRAY/1d', 'FORCE_TOKEN/0', - 'VIRTUAL_REF/2', + 'VIRTUAL_REF/2', # removed before it's passed to the backend '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations ----- 'SETARRAYITEM_GC/3d', 'SETARRAYITEM_RAW/3d',#only added by backend.llsupport.gc.rewrite_assembler 'SETFIELD_GC/2d', 'SETFIELD_RAW/2d', - 'ARRAYCOPY/7d', + 'ARRAYCOPY/7d', # removed before it's passed to the backend 'NEWSTR/1', 'STRSETITEM/3', 'UNICODESETITEM/3', @@ -223,7 +223,7 @@ #'RUNTIMENEW/1', # ootype operation 'COND_CALL_GC_WB', # [objptr, newvalue] (for the write barrier) 'DEBUG_MERGE_POINT/1', # debugging only - 'VIRTUAL_REF_FINISH/2', + 'VIRTUAL_REF_FINISH/2', # removed before it's passed to the backend '_CANRAISE_FIRST', # ----- start of can_raise operations ----- 'CALL', Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py Sat May 22 12:29:39 2010 @@ -472,8 +472,9 @@ AbstractVirtualStructInfo.__init__(self, fielddescrs) self.typedescr = typedescr - def allocate(self, metainterp): - return metainterp.execute_and_record(rop.NEW, self.typedescr) + @specialize.argtype(1) + def allocate(self, decoder): + return decoder.allocate_struct(self.typedescr) def debug_prints(self): debug_print("\tvstructinfo", self.typedescr.repr_rpython()) @@ -484,17 +485,16 @@ self.arraydescr = arraydescr #self.fieldnums = ... - def allocate(self, metainterp): + @specialize.argtype(1) + def allocate(self, decoder): length = len(self.fieldnums) - return metainterp.execute_and_record(rop.NEW_ARRAY, - self.arraydescr, - ConstInt(length)) + return decoder.allocate_array(self.arraydescr, length) @specialize.argtype(1) def setfields(self, decoder, array): arraydescr = self.arraydescr for i in range(len(self.fieldnums)): - decoder.setarrayitem(descr, array, i, self.fieldnums[i]) + decoder.setarrayitem(arraydescr, array, i, self.fieldnums[i]) def debug_prints(self): debug_print("\tvarrayinfo", self.arraydescr) @@ -614,6 +614,10 @@ def allocate_with_vtable(self, known_class): xxx + def allocate_struct(self, typedescr): + xxx + def allocate_array(self, arraydescr, length): + xxx def setfield(self, descr, struct, fieldnum): xxx def setarrayitem(self, arraydescr, array, index, fieldnum): @@ -718,6 +722,12 @@ from pypy.jit.metainterp.executor import exec_new_with_vtable return exec_new_with_vtable(self.cpu, known_class) + def allocate_struct(self, typedescr): + return self.cpu.bh_new(typedescr) + + def allocate_array(self, arraydescr, length): + return self.cpu.bh_new_array(arraydescr, length) + def setfield(self, descr, struct, fieldnum): if descr.is_pointer_field(): newvalue = self.decode_ref(fieldnum) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/simple_optimize.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/simple_optimize.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/simple_optimize.py Sat May 22 12:29:39 2010 @@ -10,12 +10,16 @@ def transform(op): from pypy.jit.metainterp.history import AbstractDescr # change ARRAYCOPY to call, so we don't have to pass around - # unnecessary information to the backend + # unnecessary information to the backend. Do the same with VIRTUAL_REF_*. if op.opnum == rop.ARRAYCOPY: descr = op.args[0] assert isinstance(descr, AbstractDescr) - return ResOperation(rop.CALL, op.args[1:], op.result, descr=descr) - return op + op = ResOperation(rop.CALL, op.args[1:], op.result, descr=descr) + elif op.opnum == rop.VIRTUAL_REF: + op = ResOperation(rop.SAME_AS, [op.args[0]], op.result) + elif op.opnum == rop.VIRTUAL_REF_FINISH: + return [] + return [op] def optimize_loop(metainterp_sd, old_loops, loop): if old_loops: @@ -35,7 +39,7 @@ modifier = resume.ResumeDataVirtualAdder(descr, memo) newboxes = modifier.finish(EMPTY_VALUES) descr.store_final_boxes(op, newboxes) - newoperations.append(transform(op)) + newoperations.extend(transform(op)) loop.operations = newoperations return None Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/specnode.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/specnode.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/specnode.py Sat May 22 12:29:39 2010 @@ -57,7 +57,8 @@ from pypy.jit.metainterp import executor, history, resoperation for ofs, subspecnode in self.fields: assert isinstance(ofs, history.AbstractDescr) - fieldbox = executor.execute(cpu, resoperation.rop.GETFIELD_GC, + fieldbox = executor.execute(cpu, None, + resoperation.rop.GETFIELD_GC, ofs, valuebox) subspecnode.extract_runtime_data(cpu, fieldbox, resultlist) @@ -95,7 +96,8 @@ def extract_runtime_data(self, cpu, valuebox, resultlist): from pypy.jit.metainterp import executor, history, resoperation for i in range(len(self.items)): - itembox = executor.execute(cpu, resoperation.rop.GETARRAYITEM_GC, + itembox = executor.execute(cpu, None, + resoperation.rop.GETARRAYITEM_GC, self.arraydescr, valuebox, history.ConstInt(i)) subspecnode = self.items[i] Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_list.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_list.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_list.py Sat May 22 12:29:39 2010 @@ -1,7 +1,5 @@ import py from pypy.rlib.jit import JitDriver -from pypy.jit.metainterp.policy import StopAtXPolicy -from pypy.rpython.ootypesystem import ootype from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin From arigo at codespeak.net Sat May 22 12:45:29 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 May 2010 12:45:29 +0200 (CEST) Subject: [pypy-svn] r74671 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100522104529.8726C36C538@codespeak.net> Author: arigo Date: Sat May 22 12:45:28 2010 New Revision: 74671 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Log: Complete and test array operations in the codewriter. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Sat May 22 12:45:28 2010 @@ -386,25 +386,35 @@ return SpaceOperation('new_array', [arraydescr, op.args[2]], op.result) + def rewrite_op_getarrayitem(self, op): + ARRAY = op.args[0].concretetype.TO + assert ARRAY._gckind == 'gc' + if self._array_of_voids(ARRAY): + return [] + arraydescr = self.cpu.arraydescrof(ARRAY) + kind = getkind(op.result.concretetype) + return SpaceOperation('getarrayitem_gc_%s' % kind[0], + [op.args[0], arraydescr, op.args[1]], + op.result) + def rewrite_op_setarrayitem(self, op): ARRAY = op.args[0].concretetype.TO assert ARRAY._gckind == 'gc' if self._array_of_voids(ARRAY): - return -## if op.args[0] in self.vable_array_vars: # for virtualizables -## (v_base, arrayindex) = self.vable_array_vars[op.args[0]] -## self.emit('setarrayitem_vable', -## self.var_position(v_base), -## arrayindex, -## self.var_position(op.args[1]), -## self.var_position(op.args[2])) -## return + return [] arraydescr = self.cpu.arraydescrof(ARRAY) kind = getkind(op.args[2].concretetype) return SpaceOperation('setarrayitem_gc_%s' % kind[0], [op.args[0], arraydescr, op.args[1], op.args[2]], None) + def rewrite_op_getarraysize(self, op): + ARRAY = op.args[0].concretetype.TO + assert ARRAY._gckind == 'gc' + arraydescr = self.cpu.arraydescrof(ARRAY) + return SpaceOperation('arraylen_gc', [op.args[0], arraydescr], + op.result) + def _array_of_voids(self, ARRAY): #if isinstance(ARRAY, ootype.Array): # return ARRAY.ITEM == ootype.Void @@ -835,7 +845,7 @@ itemsdescr, structdescr): v_index, extraop = self._prepare_list_getset(op, lengthdescr, args, 'check_resizable_neg_index') - kind = getkind(op.args[2].concretetype)[0] + kind = getkind(args[2].concretetype)[0] op = SpaceOperation('setlistitem_gc_%s' % kind, [args[0], itemsdescr, arraydescr, v_index, args[2]], None) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Sat May 22 12:45:28 2010 @@ -50,6 +50,8 @@ return FakeDescr() def sizeof(self, STRUCT): return FakeDescr() + def arraydescrof(self, ARRAY): + return FakeDescr() class FakeCallControl: _descr_cannot_raise = FakeDescr() @@ -605,3 +607,46 @@ residual_call_r_r $<* fn jit_force_virtual>, , R[%r1] -> %r2 ref_return %r2 """, transform=True, cc=FakeCallControlWithVRefInfo()) + + def test_array_operations(self): + A = lltype.GcArray(lltype.Signed) + def f(): + array = lltype.malloc(A, 5) + array[2] = 5 + return array[2] + len(array) + self.encoding_test(f, [], """ + new_array , $5 -> %r0 + setarrayitem_gc_i %r0, , $2, $5 + getarrayitem_gc_i %r0, , $2 -> %i0 + arraylen_gc %r0, -> %i1 + int_add %i0, %i1 -> %i2 + int_return %i2 + """, transform=True) + + def test_void_array_operations(self): + A = lltype.GcArray(lltype.Void) + def f(): + array = lltype.malloc(A, 5) + array[2] = None + x = array[2] + return len(array) + self.encoding_test(f, [], """ + new_array , $5 -> %r0 + arraylen_gc %r0, -> %i0 + int_return %i0 + """, transform=True) + + def test_string_operations(self): + from pypy.rpython.lltypesystem import rstr + def f(n): + s = lltype.malloc(rstr.STR, 2) + s.chars[1] = chr(n) + return ord(s.chars[1]) + len(s.chars) + self.encoding_test(f, [512], """ + newstr $2 -> %r0 + strsetitem %r0, $1, %i0 + strgetitem %r0, $1 -> %i1 + strlen %r0 -> %i2 + int_add %i1, %i2 -> %i3 + int_return %i3 + """, transform=True) From arigo at codespeak.net Sat May 22 13:01:03 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 May 2010 13:01:03 +0200 (CEST) Subject: [pypy-svn] r74672 - in pypy/branch/blackhole-improvement/pypy/jit/metainterp: . test Message-ID: <20100522110103.9AC9836C538@codespeak.net> Author: arigo Date: Sat May 22 13:01:01 2010 New Revision: 74672 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_slist.py Log: Make test_slist pass. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Sat May 22 13:01:01 2010 @@ -783,33 +783,52 @@ # ---------- # list operations - @arguments("r", "d", "i", returns="i") - def bhimpl_check_neg_index(array, arraydescr, index): - xxx - - @arguments("r", "d", "i", returns="i") - def bhimpl_check_resizable_neg_index(list, lengthdescr, index): - xxx - - @arguments() - def bhimpl_getlistitem_gc_i(yyy): - xxx - @arguments() - def bhimpl_getlistitem_gc_r(yyy): - xxx - @arguments() - def bhimpl_getlistitem_gc_f(yyy): - xxx + @arguments("cpu", "r", "d", "i", returns="i") + def bhimpl_check_neg_index(cpu, array, arraydescr, index): + if index < 0: + index += cpu.bh_arraylen_gc(arraydescr, array) + return index - @arguments() - def bhimpl_setlistitem_gc_i(yyy): - xxx - @arguments() - def bhimpl_setlistitem_gc_r(yyy): - xxx - @arguments() - def bhimpl_setlistitem_gc_f(yyy): - xxx + @arguments("cpu", "r", "d", "i", returns="i") + def bhimpl_check_resizable_neg_index(cpu, lst, lengthdescr, index): + if index < 0: + index += cpu.bh_getfield_gc_i(lst, lengthdescr) + return index + + @arguments("cpu", "d", "d", "d", "d", "i", returns="r") + def bhimpl_newlist(cpu, structdescr, lengthdescr, itemsdescr, + arraydescr, length): + result = cpu.bh_new(structdescr) + cpu.bh_setfield_gc_i(result, lengthdescr, length) + items = cpu.bh_new_array(arraydescr, length) + cpu.bh_setfield_gc_r(result, itemsdescr, items) + return result + + @arguments("cpu", "r", "d", "d", "i", returns="i") + def bhimpl_getlistitem_gc_i(cpu, lst, itemsdescr, arraydescr, index): + items = cpu.bh_getfield_gc_r(lst, itemsdescr) + return cpu.bh_getarrayitem_gc_i(arraydescr, items, index) + @arguments("cpu", "r", "d", "d", "i", returns="r") + def bhimpl_getlistitem_gc_r(cpu, lst, itemsdescr, arraydescr, index): + items = cpu.bh_getfield_gc_r(lst, itemsdescr) + return cpu.bh_getarrayitem_gc_r(arraydescr, items, index) + @arguments("cpu", "r", "d", "d", "i", returns="f") + def bhimpl_getlistitem_gc_f(cpu, lst, itemsdescr, arraydescr, index): + items = cpu.bh_getfield_gc_r(lst, itemsdescr) + return cpu.bh_getarrayitem_gc_f(arraydescr, items, index) + + @arguments("cpu", "r", "d", "d", "i", "i") + def bhimpl_setlistitem_gc_i(cpu, lst, itemsdescr, arraydescr, index, nval): + items = cpu.bh_getfield_gc_r(lst, itemsdescr) + cpu.bh_setarrayitem_gc_i(arraydescr, items, index, nval) + @arguments("cpu", "r", "d", "d", "i", "r") + def bhimpl_setlistitem_gc_r(cpu, lst, itemsdescr, arraydescr, index, nval): + items = cpu.bh_getfield_gc_r(lst, itemsdescr) + cpu.bh_setarrayitem_gc_r(arraydescr, items, index, nval) + @arguments("cpu", "r", "d", "d", "i", "f") + def bhimpl_setlistitem_gc_f(cpu, lst, itemsdescr, arraydescr, index, nval): + items = cpu.bh_getfield_gc_r(lst, itemsdescr) + cpu.bh_setarrayitem_gc_f(arraydescr, items, index, nval) # ---------- # the following operations are directly implemented by the backend Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Sat May 22 13:01:01 2010 @@ -437,7 +437,7 @@ @arguments("orgpc", "box", "descr", "box") def opimpl_check_neg_index(self, orgpc, arraybox, arraydescr, indexbox): negbox = self.metainterp.execute_and_record( - rop.INT_LT, None, indexbox, ConstInt(0)) + rop.INT_LT, None, indexbox, history.CONST_FALSE) negbox = self.implement_guard_value(orgpc, negbox) if negbox.getint(): # the index is < 0; add the array length to it @@ -447,7 +447,7 @@ rop.INT_ADD, None, indexbox, lenbox) return indexbox - @FixME #arguments("descr", "descr", "descr", "descr", "box") + @arguments("descr", "descr", "descr", "descr", "box") def opimpl_newlist(self, structdescr, lengthdescr, itemsdescr, arraydescr, sizebox): sbox = self.metainterp.execute_and_record(rop.NEW, structdescr) @@ -457,13 +457,19 @@ sizebox) self.metainterp.execute_and_record(rop.SETFIELD_GC, itemsdescr, sbox, abox) - self.make_result_box(sbox) + return sbox - @FixME #arguments("box", "descr", "descr", "box") - def opimpl_getlistitem_gc(self, listbox, itemsdescr, arraydescr, indexbox): + @arguments("box", "descr", "descr", "box") + def _opimpl_getlistitem_gc_any(self, listbox, itemsdescr, arraydescr, + indexbox): arraybox = self.metainterp.execute_and_record(rop.GETFIELD_GC, itemsdescr, listbox) - self.execute_with_descr(rop.GETARRAYITEM_GC, arraydescr, arraybox, indexbox) + return self.execute_with_descr(rop.GETARRAYITEM_GC, + arraydescr, arraybox, indexbox) + + opimpl_getlistitem_gc_i = _opimpl_getlistitem_gc_any + opimpl_getlistitem_gc_r = _opimpl_getlistitem_gc_any + opimpl_getlistitem_gc_f = _opimpl_getlistitem_gc_any @arguments("box", "descr", "descr", "box", "box") def _opimpl_setlistitem_gc_any(self, listbox, itemsdescr, arraydescr, Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_slist.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_slist.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_slist.py Sat May 22 13:01:01 2010 @@ -1,5 +1,4 @@ import py -from pypy.jit.metainterp.policy import StopAtXPolicy from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin from pypy.rlib.jit import JitDriver, OPTIMIZER_SIMPLE From arigo at codespeak.net Sat May 22 13:05:10 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 May 2010 13:05:10 +0200 (CEST) Subject: [pypy-svn] r74673 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100522110510.228BF36C538@codespeak.net> Author: arigo Date: Sat May 22 13:05:08 2010 New Revision: 74673 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py Log: Missing -live- before 'check_neg_index'. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Sat May 22 13:05:08 2010 @@ -766,9 +766,10 @@ else: v_posindex = Variable('posindex') v_posindex.concretetype = lltype.Signed - op = SpaceOperation(checkname, [args[0], - descr, args[1]], v_posindex) - return v_posindex, [op] + op0 = SpaceOperation('-live-', [], None) + op1 = SpaceOperation(checkname, [args[0], + descr, args[1]], v_posindex) + return v_posindex, [op0, op1] def _get_initial_newlist_length(self, op, args): # normalize number of arguments to the 'newlist' function Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py Sat May 22 13:05:08 2010 @@ -109,6 +109,7 @@ builtin_test('list.getitem/NEG', [varoftype(FIXEDLIST), varoftype(lltype.Signed)], lltype.Signed, """ + -live- check_neg_index %r0, , %i0 -> %i1 getarrayitem_gc_i %r0, , %i1 -> %i2 """) @@ -125,6 +126,7 @@ builtin_test('list.getitem_foldable/NEG', [varoftype(FIXEDLIST), varoftype(lltype.Signed)], lltype.Signed, """ + -live- check_neg_index %r0, , %i0 -> %i1 getarrayitem_gc_pure_i %r0, , %i1 -> %i2 """) @@ -143,6 +145,7 @@ varoftype(lltype.Signed), varoftype(lltype.Signed)], lltype.Void, """ + -live- check_neg_index %r0, , %i0 -> %i1 setarrayitem_gc_i %r0, , %i1, %i2 """) @@ -190,6 +193,7 @@ builtin_test('list.getitem/NEG', [varoftype(VARLIST), varoftype(lltype.Signed)], lltype.Signed, """ + -live- check_resizable_neg_index %r0, , %i0 -> %i1 getlistitem_gc_i %r0, , , %i1 -> %i2 """) @@ -208,6 +212,7 @@ varoftype(lltype.Signed), varoftype(lltype.Signed)], lltype.Void, """ + -live- check_resizable_neg_index %r0, , %i0 -> %i1 setlistitem_gc_i %r0, , , %i1, %i2 """) From arigo at codespeak.net Sat May 22 13:11:18 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 May 2010 13:11:18 +0200 (CEST) Subject: [pypy-svn] r74674 - pypy/branch/blackhole-improvement/pypy/jit/metainterp/test Message-ID: <20100522111118.F3AE036C538@codespeak.net> Author: arigo Date: Sat May 22 13:11:16 2010 New Revision: 74674 Added: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_immutable.py - copied, changed from r74615, pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_codewriter.py Removed: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_codewriter.py Log: Kill most of test_codewriter, now in the codewriter/ directory. Remaining test is about handling _immutable_fields_. Copied: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_immutable.py (from r74615, pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_codewriter.py) ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_immutable.py Sat May 22 13:11:16 2010 @@ -1,440 +1,6 @@ -import py -from pypy.rlib import jit -from pypy.jit.codewriter import support -from pypy.jit.metainterp import typesystem -from pypy.jit.metainterp.policy import JitPolicy -from pypy.jit.metainterp.history import ConstInt -from pypy.jit.metainterp.codewriter import CodeWriter, BytecodeMaker from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin -from pypy.translator.translator import graphof -from pypy.rpython.lltypesystem.rbuiltin import ll_instantiate -class SomeLabel(object): - def __eq__(self, other): - return repr(other).startswith('label') # :-/ - - -class TestCodeWriter: - - def make_graphs(self, func, values, type_system='lltype'): - class FakeMetaInterpSd: - virtualizable_info = None - class options: - listops = True - def find_opcode(self, name): - default = len(self.opname_to_index) - return self.opname_to_index.setdefault(name, default) - def _register_indirect_call_target(self, fnaddress, jitcode): - self.indirectcalls.append((fnaddress, jitcode)) - - class FakeMethDescr: - def __init__(self1, CLASS, methname): - self.methdescrs.append(self1) - self1.CLASS = CLASS - self1.methname = methname - self1.jitcodes = None - def setup(self1, jitcodes): - self1.jitcodes = jitcodes - self.methdescrs = [] - - class FakeCPU: - supports_floats = False - def fielddescrof(self, STRUCT, fieldname): - return ('fielddescr', STRUCT, fieldname) - def calldescrof(self, FUNC, NON_VOID_ARGS, RESULT, effectinfo=None): - return ('calldescr', FUNC, NON_VOID_ARGS, RESULT, effectinfo) - def typedescrof(self, CLASS): - return ('typedescr', CLASS) - def methdescrof(self, CLASS, methname): - return FakeMethDescr(CLASS, methname) - def arraydescrof(self, ARRAY): - return ('arraydescr', ARRAY) - def sizeof(self, STRUCT): - return ('sizeof', STRUCT) - - if type_system == 'lltype': - FakeCPU.ts = typesystem.llhelper - else: - FakeCPU.ts = typesystem.oohelper - - self.metainterp_sd = FakeMetaInterpSd() - self.metainterp_sd.opcode_implementations = None - self.metainterp_sd.opname_to_index = {} - self.metainterp_sd.indirectcalls = [] - self.metainterp_sd.cpu = FakeCPU() - - self.rtyper = support.annotate(func, values, type_system=type_system) - self.metainterp_sd.cpu.rtyper = self.rtyper - return self.rtyper.annotator.translator.graphs - - def graphof(self, func): - rtyper = self.metainterp_sd.cpu.rtyper - return graphof(rtyper.annotator.translator, func) - - def test_basic(self): - def f(n): - return n + 10 - graphs = self.make_graphs(f, [5]) - cw = CodeWriter(self.rtyper) - cw.candidate_graphs = graphs - cw._start(self.metainterp_sd, None) - jitcode = cw.make_one_bytecode((graphs[0], None), False) - assert jitcode._source == [ - SomeLabel(), - 'int_add', 0, 1, '# => r1', - 'make_new_vars_1', 2, - 'return'] - - def test_guess_call_kind_and_calls_from_graphs(self): - from pypy.objspace.flow.model import SpaceOperation, Constant, Variable - - portal_runner_ptr = object() - g = object() - g1 = object() - cw = CodeWriter(None) - cw.candidate_graphs = [g, g1] - cw.portal_runner_ptr = portal_runner_ptr - - op = SpaceOperation('direct_call', [Constant(portal_runner_ptr)], - Variable()) - assert cw.guess_call_kind(op) == 'recursive' - - op = SpaceOperation('direct_call', [Constant(object())], - Variable()) - assert cw.guess_call_kind(op) == 'residual' - - class funcptr: - class graph: - class func: - oopspec = "spec" - op = SpaceOperation('direct_call', [Constant(funcptr)], - Variable()) - assert cw.guess_call_kind(op) == 'builtin' - - class funcptr: - graph = g - op = SpaceOperation('direct_call', [Constant(funcptr)], - Variable()) - res = cw.graphs_from(op) - assert res == [g] - assert cw.guess_call_kind(op) == 'regular' - - class funcptr: - graph = object() - op = SpaceOperation('direct_call', [Constant(funcptr)], - Variable()) - res = cw.graphs_from(op) - assert res is None - assert cw.guess_call_kind(op) == 'residual' - - h = object() - op = SpaceOperation('indirect_call', [Variable(), - Constant([g, g1, h])], - Variable()) - res = cw.graphs_from(op) - assert res == [g, g1] - assert cw.guess_call_kind(op) == 'regular' - - op = SpaceOperation('indirect_call', [Variable(), - Constant([h])], - Variable()) - res = cw.graphs_from(op) - assert res is None - assert cw.guess_call_kind(op) == 'residual' - - def test_direct_call(self): - def g(m): - return 123 - def f(n): - return g(n+1) - graphs = self.make_graphs(f, [5]) - cw = CodeWriter(self.rtyper) - cw.candidate_graphs = graphs - cw._start(self.metainterp_sd, None) - jitcode = cw.make_one_bytecode((graphs[0], None), False) - assert len(cw.all_graphs) == 2 - - def test_indirect_call_target(self): - def g(m): - return 123 - def h(m): - return 456 - def f(n): - if n > 3: - call = g - else: - call = h - return call(n+1) + call(n+2) - graphs = self.make_graphs(f, [5]) - cw = CodeWriter(self.rtyper) - cw.candidate_graphs = graphs - cw._start(self.metainterp_sd, None) - jitcode = cw.make_one_bytecode((graphs[0], None), False) - assert len(self.metainterp_sd.indirectcalls) == 2 - names = [jitcode.name for (fnaddress, jitcode) - in self.metainterp_sd.indirectcalls] - assert dict.fromkeys(names) == {'g': None, 'h': None} - - def test_indirect_look_inside_only_one(self): - def g(m): - return 123 - @jit.dont_look_inside - def h(m): - return 456 - def f(n): - if n > 3: - call = g - else: - call = h - return call(n+1) + call(n+2) - graphs = self.make_graphs(f, [5]) - graphs = [g for g in graphs if getattr(g.func, '_jit_look_inside_', - True)] - cw = CodeWriter(self.rtyper) - cw.candidate_graphs = graphs - cw._start(self.metainterp_sd, None) - jitcode = cw.make_one_bytecode((graphs[0], None), False) - assert len(self.metainterp_sd.indirectcalls) == 1 - names = [jitcode1.name for (fnaddress, jitcode1) - in self.metainterp_sd.indirectcalls] - assert dict.fromkeys(names) == {'g': None} - calldescrs = [calldescr for calldescr in jitcode.constants - if isinstance(calldescr, tuple) and - calldescr[0] == 'calldescr'] - assert len(calldescrs) == 1 - assert calldescrs[0][4] is not None - assert not calldescrs[0][4].write_descrs_fields - assert not calldescrs[0][4].write_descrs_arrays - assert not calldescrs[0][4].forces_virtual_or_virtualizable - - def test_oosend_look_inside_only_one(self): - class A: - pass - class B(A): - def g(self): - return 123 - class C(A): - @jit.dont_look_inside - def g(self): - return 456 - def f(n): - if n > 3: - x = B() - else: - x = C() - return x.g() + x.g() - graphs = self.make_graphs(f, [5], type_system='ootype') - graphs = [g for g in graphs if getattr(g.func, '_jit_look_inside_', - True)] - cw = CodeWriter(self.rtyper) - cw.candidate_graphs = graphs - cw._start(self.metainterp_sd, None) - jitcode = cw.make_one_bytecode((graphs[0], None), False) - assert len(self.methdescrs) == 1 - assert self.methdescrs[0].CLASS._name.endswith('.A') - assert self.methdescrs[0].methname == 'og' - assert len(self.methdescrs[0].jitcodes.keys()) == 2 - values = self.methdescrs[0].jitcodes.values() - values.sort() - assert values[0] is None - assert values[1].name == 'B.g' - for graph, _ in cw.all_graphs.keys(): - assert graph.name in ['f', 'B.g'] - - def test_instantiate(self): - class A1: id = 651 - class A2(A1): id = 652 - class B1: id = 661 - class B2(B1): id = 662 - def f(n): - if n > 5: - x, y = A1, B1 - else: - x, y = A2, B2 - n += 1 - return x().id + y().id + n - graphs = self.make_graphs(f, [5]) - cw = CodeWriter(self.rtyper) - cw.candidate_graphs = graphs - cw._start(self.metainterp_sd, None) - cw.make_one_bytecode((graphs[0], None), False) - graph2 = self.graphof(ll_instantiate) - jitcode = cw.make_one_bytecode((graph2, None), False) - assert 'residual_call' not in jitcode._source - names = [jitcode.name for (fnaddress, jitcode) - in self.metainterp_sd.indirectcalls] - names = dict.fromkeys(names) - assert len(names) >= 4 - for expected in ['A1', 'A2', 'B1', 'B2']: - for name in names: - if name.startswith('instantiate_') and name.endswith(expected): - break - else: - assert 0, "missing instantiate_*_%s in:\n%r" % (expected, - names) - - def test_oois_constant_null(self): - from pypy.rpython.lltypesystem import lltype - - S = lltype.GcStruct('S') - s = lltype.malloc(S) - NULL = lltype.nullptr(S) - def f(p, i): - if i % 2: - return p == NULL - elif i % 3: - return NULL == p - elif i % 4: - return p != NULL - else: - return NULL != p - graphs = self.make_graphs(f, [s, 5]) - cw = CodeWriter(self.rtyper) - cw.candidate_graphs = graphs - cw._start(self.metainterp_sd, None) - jitcode = cw.make_one_bytecode((graphs[0], None), False) - assert 'ptr_eq' not in jitcode._source - assert 'ptr_ne' not in jitcode._source - assert jitcode._source.count('oononnull') == 2 - assert jitcode._source.count('ooisnull') == 2 - - def test_list_of_addr2name(self): - class A1: - def g(self): - self.x = 123 - return 5 - def f(): - a = A1() - a.y = a.g() - return a - graphs = self.make_graphs(f, []) - cw = CodeWriter(self.rtyper) - cw.candidate_graphs = [graphs[0]] - cw._start(self.metainterp_sd, None) - jitcode = cw.make_one_bytecode((graphs[0], None), False) - assert len(cw.list_of_addr2name) == 2 - assert cw.list_of_addr2name[0][1].endswith('.A1') - assert cw.list_of_addr2name[1][1] == 'A1.g' - - def test_jit_force_virtualizable_effectinfo(self): - class Frame(object): - _virtualizable2_ = ['x'] - - def __init__(self, x, y): - self.x = x - self.y = y - - def g1(f): - f.x += 1 - - def g2(f): - return f.x - - def h(f): - f.y -= 1 - - def f(n): - f_inst = Frame(n+1, n+2) - g1(f_inst) - r = g2(f_inst) - h(f_inst) - return r - - graphs = self.make_graphs(f, [5]) - cw = CodeWriter(self.rtyper) - cw.candidate_graphs = [graphs[0]] - cw._start(self.metainterp_sd, None) - jitcode = cw.make_one_bytecode((graphs[0], None), False) - calldescrs = [calldescr for calldescr in jitcode.constants - if isinstance(calldescr, tuple) and - calldescr[0] == 'calldescr'] - assert len(calldescrs) == 4 # for __init__, g1, g2, h. - effectinfo_g1 = calldescrs[1][4] - effectinfo_g2 = calldescrs[2][4] - effectinfo_h = calldescrs[3][4] - assert effectinfo_g1.forces_virtual_or_virtualizable - assert effectinfo_g2.forces_virtual_or_virtualizable - assert not effectinfo_h.forces_virtual_or_virtualizable - - def make_vrefinfo(self): - from pypy.jit.metainterp.virtualref import VirtualRefInfo - class FakeWarmRunnerDesc: - cpu = self.metainterp_sd.cpu - self.metainterp_sd.virtualref_info = VirtualRefInfo(FakeWarmRunnerDesc) - - def test_vref_simple(self): - class X: - pass - def f(): - return jit.virtual_ref(X()) - graphs = self.make_graphs(f, []) - assert graphs[0].func is f - assert graphs[1].func is jit.virtual_ref - self.make_vrefinfo() - cw = CodeWriter(self.rtyper) - cw.candidate_graphs = [graphs[0]] - cw._start(self.metainterp_sd, None) - jitcode = cw.make_one_bytecode((graphs[0], None), False) - assert 'virtual_ref' in jitcode._source - - def test_vref_forced(self): - class X: - pass - def f(): - vref = jit.virtual_ref(X()) - return vref() - graphs = self.make_graphs(f, []) - assert graphs[0].func is f - assert graphs[1].func is jit.virtual_ref - self.make_vrefinfo() - cw = CodeWriter(self.rtyper) - cw.candidate_graphs = [graphs[0]] - cw._start(self.metainterp_sd, None) - jitcode = cw.make_one_bytecode((graphs[0], None), False) - assert 'virtual_ref' in jitcode._source - # the call vref() becomes a residual call to a helper that contains - # itself a copy of the call. - assert 'residual_call' in jitcode._source - - def test_we_are_jitted(self): - def f(): - if jit.we_are_jitted(): - return 55 - else: - return 66 - graphs = self.make_graphs(f, []) - cw = CodeWriter(self.rtyper) - cw.candidate_graphs = [graphs[0]] - cw._start(self.metainterp_sd, None) - jitcode = cw.make_one_bytecode((graphs[0], None), False) - assert 'goto_if_not' not in jitcode._source - assert ConstInt(55) in jitcode.constants - assert ConstInt(66) not in jitcode.constants - - def test_ll_arraycopy(self): - from pypy.rlib import rgc - from pypy.rpython.lltypesystem import lltype - - A = lltype.GcArray(lltype.Signed) - - def f(): - l = lltype.malloc(A, 3) - l[0] = 1 - l[1] = 2 - l[3] = 3 - l2 = lltype.malloc(A, 3) - rgc.ll_arraycopy(l, l2, 0, 0, 3) - return l2[0] + l2[1] + l2[3] - - graphs = self.make_graphs(f, []) - cw = CodeWriter(self.rtyper) - cw.candidate_graphs = [graphs[0]] - cw._start(self.metainterp_sd, None) - jitcode = cw.make_one_bytecode((graphs[0], None), False) - assert 'residual_call' not in jitcode._source - assert 'arraycopy' in jitcode._source - class ImmutableFieldsTests: def test_fields(self): From arigo at codespeak.net Sat May 22 13:27:33 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 May 2010 13:27:33 +0200 (CEST) Subject: [pypy-svn] r74675 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter metainterp metainterp/test Message-ID: <20100522112733.0957436C53A@codespeak.net> Author: arigo Date: Sat May 22 13:27:31 2010 New Revision: 74675 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/effectinfo.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_compile.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_optimizefindnode.py Log: Tweaks and fixes. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py Sat May 22 13:27:31 2010 @@ -7,7 +7,7 @@ from pypy.jit.codewriter.jitcode import JitCode from pypy.jit.codewriter.effectinfo import VirtualizableAnalyzer from pypy.jit.codewriter.effectinfo import effectinfo_from_writeanalyze -from pypy.jit.codewriter import effectinfo as ef +from pypy.jit.codewriter.effectinfo import EffectInfo from pypy.translator.simplify import get_funcobj, get_functype from pypy.rpython.lltypesystem import lltype, llmemory from pypy.translator.backendopt.canraise import RaiseAnalyzer @@ -199,23 +199,23 @@ "loop-invariant function!") # build the extraeffect if self.virtualizable_analyzer.analyze(op): - extraeffect = ef.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE + extraeffect = EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE elif loopinvariant: - extraeffect = ef.EF_LOOPINVARIANT + extraeffect = EffectInfo.EF_LOOPINVARIANT elif pure: # XXX check what to do about exceptions (also MemoryError?) - extraeffect = ef.EF_PURE + extraeffect = EffectInfo.EF_PURE elif self._canraise(op): - extraeffect = ef.EF_CAN_RAISE + extraeffect = EffectInfo.EF_CAN_RAISE else: - extraeffect = ef.EF_CANNOT_RAISE + extraeffect = EffectInfo.EF_CANNOT_RAISE # effectinfo = effectinfo_from_writeanalyze( self.readwrite_analyzer.analyze(op), self.cpu, extraeffect) # if pure or loopinvariant: assert effectinfo is not None - assert extraeffect != ef.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE + assert extraeffect != EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE # return self.cpu.calldescrof(FUNC, tuple(NON_VOID_ARGS), RESULT, effectinfo) @@ -229,7 +229,8 @@ def calldescr_canraise(self, calldescr): effectinfo = calldescr.get_extra_info() - return effectinfo is None or effectinfo.extraeffect >= ef.EF_CAN_RAISE + return (effectinfo is None or + effectinfo.extraeffect >= EffectInfo.EF_CAN_RAISE) def found_jitdriver(self, jitdriver): if self.jitdriver is None: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/effectinfo.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/effectinfo.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/effectinfo.py Sat May 22 13:27:31 2010 @@ -4,17 +4,17 @@ from pypy.rpython.ootypesystem import ootype from pypy.translator.backendopt.graphanalyze import BoolGraphAnalyzer -# the 'extraeffect' field is one of the following values: -EF_PURE = 0 # pure function (and cannot raise) -EF_CANNOT_RAISE = 1 # a function which cannot raise -EF_CAN_RAISE = 2 # normal function (can raise) -EF_LOOPINVARIANT = 3 # special: call it only once per loop -EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE = 4 # can raise and force virtualizables - class EffectInfo(object): _cache = {} + # the 'extraeffect' field is one of the following values: + EF_PURE = 0 #pure function (and cannot raise) + EF_CANNOT_RAISE = 1 #a function which cannot raise + EF_CAN_RAISE = 2 #normal function (can raise) + EF_LOOPINVARIANT = 3 #special: call it only once per loop + EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE = 4 #can raise and force virtualizables + def __new__(cls, readonly_descrs_fields, write_descrs_fields, write_descrs_arrays, extraeffect=EF_CAN_RAISE): @@ -33,9 +33,10 @@ return result def check_forces_virtual_or_virtualizable(self): - return self.extraeffect >= EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE + return self.extraeffect >= self.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE -def effectinfo_from_writeanalyze(effects, cpu, extraeffect=EF_CAN_RAISE): +def effectinfo_from_writeanalyze(effects, cpu, + extraeffect=EffectInfo.EF_CAN_RAISE): from pypy.translator.backendopt.writeanalyze import top_set if effects is top_set: return None Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Sat May 22 13:27:31 2010 @@ -42,8 +42,11 @@ # def do_rename(var, var_or_const): renamings[var] = var_or_const - if isinstance(var_or_const, Constant): - renamings_constants[var] = var_or_const + if (isinstance(var_or_const, Constant) + and var.concretetype != lltype.Void): + value = var_or_const.value + value = lltype._cast_whatever(var.concretetype, value) + renamings_constants[var] = Constant(value, var.concretetype) # for op in block.operations: if renamings_constants: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py Sat May 22 13:27:31 2010 @@ -241,7 +241,7 @@ def build_ll_0_alloc_with_del(RESULT, vtable): def _ll_0_alloc_with_del(): - p = lltype.malloc(RESULT) + p = lltype.malloc(RESULT.TO) lltype.cast_pointer(rclass.OBJECTPTR, p).typeptr = vtable return p return _ll_0_alloc_with_del Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Sat May 22 13:27:31 2010 @@ -22,7 +22,6 @@ from pypy.rlib.jit import DEBUG_OFF, DEBUG_PROFILE, DEBUG_STEPS, DEBUG_DETAILED from pypy.jit.metainterp.compile import GiveUp from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr -from pypy.jit.codewriter import effectinfo as ef # ____________________________________________________________ @@ -1037,8 +1036,8 @@ def do_residual_call(self, funcbox, descr, argboxes): allboxes = [funcbox] + argboxes effectinfo = descr.get_extra_info() - if (effectinfo is None or - effectinfo.extraeffect == ef.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE): + if (effectinfo is None or effectinfo.extraeffect == + effectinfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE): # residual calls require attention to keep virtualizables in-sync self.metainterp.vable_and_vrefs_before_residual_call() # xxx do something about code duplication @@ -1054,11 +1053,11 @@ return self.metainterp.assert_no_exception() else: effect = effectinfo.extraeffect - if effect == ef.EF_CANNOT_RAISE: + if effect == effectinfo.EF_CANNOT_RAISE: opnum, exc = rop.CALL, False - elif effect == ef.EF_PURE: + elif effect == effectinfo.EF_PURE: opnum, exc = rop.CALL_PURE, False - elif effect == ef.EF_LOOPINVARIANT: + elif effect == effectinfo.EF_LOOPINVARIANT: opnum, exc = rop.CALL_LOOPINVARIANT, True else: opnum, exc = rop.CALL, True Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_compile.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_compile.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_compile.py Sat May 22 13:27:31 2010 @@ -2,7 +2,8 @@ from pypy.jit.metainterp.history import BoxInt from pypy.jit.metainterp.specnode import NotSpecNode, ConstantSpecNode from pypy.jit.metainterp.compile import insert_loop_token, compile_new_loop -from pypy.jit.metainterp.compile import ResumeGuardDescr, ResumeGuardCounters +from pypy.jit.metainterp.compile import ResumeGuardDescr +from pypy.jit.metainterp.compile import ResumeGuardCountersInt from pypy.jit.metainterp import optimize, jitprof, typesystem from pypy.jit.metainterp.test.oparser import parse from pypy.jit.metainterp.test.test_optimizefindnode import LLtypeMixin @@ -107,49 +108,49 @@ def test_resume_guard_counters(): - rgc = ResumeGuardCounters() + rgc = ResumeGuardCountersInt() # fill in the table for i in range(5): - count = rgc.see(BoxInt(100+i)) + count = rgc.see_int(100+i) assert count == 1 - count = rgc.see(BoxInt(100+i)) + count = rgc.see_int(100+i) assert count == 2 assert rgc.counters == [0] * (4-i) + [2] * (1+i) for i in range(5): - count = rgc.see(BoxInt(100+i)) + count = rgc.see_int(100+i) assert count == 3 # make a distribution: [5, 4, 7, 6, 3] assert rgc.counters == [3, 3, 3, 3, 3] - count = rgc.see(BoxInt(101)) + count = rgc.see_int(101) assert count == 4 - count = rgc.see(BoxInt(101)) + count = rgc.see_int(101) assert count == 5 - count = rgc.see(BoxInt(101)) + count = rgc.see_int(101) assert count == 6 - count = rgc.see(BoxInt(102)) + count = rgc.see_int(102) assert count == 4 - count = rgc.see(BoxInt(102)) + count = rgc.see_int(102) assert count == 5 - count = rgc.see(BoxInt(102)) + count = rgc.see_int(102) assert count == 6 - count = rgc.see(BoxInt(102)) + count = rgc.see_int(102) assert count == 7 - count = rgc.see(BoxInt(103)) + count = rgc.see_int(103) assert count == 4 - count = rgc.see(BoxInt(104)) + count = rgc.see_int(104) assert count == 4 - count = rgc.see(BoxInt(104)) + count = rgc.see_int(104) assert count == 5 assert rgc.counters == [5, 4, 7, 6, 3] # the next new item should throw away 104, as 5 is the middle counter - count = rgc.see(BoxInt(190)) + count = rgc.see_int(190) assert count == 1 assert rgc.counters == [1, 4, 7, 6, 3] # the next new item should throw away 103, as 4 is the middle counter - count = rgc.see(BoxInt(191)) + count = rgc.see_int(191) assert count == 1 assert rgc.counters == [1, 1, 7, 6, 3] # the next new item should throw away 100, as 3 is the middle counter - count = rgc.see(BoxInt(192)) + count = rgc.see_int(192) assert count == 1 assert rgc.counters == [1, 1, 7, 6, 1] Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_optimizefindnode.py Sat May 22 13:27:31 2010 @@ -6,7 +6,7 @@ from pypy.jit.backend.llgraph import runner from pypy.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstPtr, - Const, ConstAddr, TreeLoop, BoxObj, + Const, TreeLoop, BoxObj, ConstObj, AbstractDescr) from pypy.jit.metainterp.optimizefindnode import PerfectSpecializationFinder from pypy.jit.metainterp.optimizefindnode import BridgeSpecializationFinder @@ -16,7 +16,7 @@ from pypy.jit.metainterp.specnode import VirtualArraySpecNode from pypy.jit.metainterp.specnode import VirtualStructSpecNode from pypy.jit.metainterp.specnode import ConstantSpecNode -from pypy.jit.metainterp.effectinfo import EffectInfo +from pypy.jit.codewriter.effectinfo import EffectInfo from pypy.jit.metainterp.test.oparser import parse def test_sort_descrs(): @@ -113,7 +113,7 @@ EffectInfo([adescr], [], [])) mayforcevirtdescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([nextdescr], [], [], - forces_virtual_or_virtualizable=True)) + EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE)) class LoopToken(AbstractDescr): pass asmdescr = LoopToken() # it can be whatever, it's not a descr though @@ -129,83 +129,83 @@ jit_virtual_ref_vtable = vrefinfo.jit_virtual_ref_vtable jvr_vtable_adr = llmemory.cast_ptr_to_adr(jit_virtual_ref_vtable) - cpu.class_sizes = { - cpu.cast_adr_to_int(node_vtable_adr): cpu.sizeof(NODE), - cpu.cast_adr_to_int(node_vtable_adr2): cpu.sizeof(NODE2), - cpu.cast_adr_to_int(u_vtable_adr): cpu.sizeof(U), - cpu.cast_adr_to_int(jvr_vtable_adr): cpu.sizeof( - vrefinfo.JIT_VIRTUAL_REF), - } +## cpu.class_sizes = { +## llmemory.cast_adr_to_int(node_vtable_adr): cpu.sizeof(NODE), +## llmemory.cast_adr_to_int(node_vtable_adr2): cpu.sizeof(NODE2), +## llmemory.cast_adr_to_int(u_vtable_adr): cpu.sizeof(U), +## llmemory.cast_adr_to_int(jvr_vtable_adr): cpu.sizeof( +## vrefinfo.JIT_VIRTUAL_REF), +## } namespace = locals() -class OOtypeMixin(object): +class OOtypeMixin_xxx_disabled(object): type_system = 'ootype' - def get_class_of_box(self, box): - root = box.getref(ootype.ROOT) - return ootype.classof(root) +## def get_class_of_box(self, box): +## root = box.getref(ootype.ROOT) +## return ootype.classof(root) - cpu = runner.OOtypeCPU(None) - NODE = ootype.Instance('NODE', ootype.ROOT, {}) - NODE._add_fields({'value': ootype.Signed, - 'floatval' : ootype.Float, - 'next': NODE}) - NODE2 = ootype.Instance('NODE2', NODE, {'other': NODE}) - - node_vtable = ootype.runtimeClass(NODE) - node_vtable_adr = ootype.cast_to_object(node_vtable) - node_vtable2 = ootype.runtimeClass(NODE2) - node_vtable_adr2 = ootype.cast_to_object(node_vtable2) - - node = ootype.new(NODE) - nodebox = BoxObj(ootype.cast_to_object(node)) - myptr = nodebox.value - myptr2 = ootype.cast_to_object(ootype.new(NODE)) - nodebox2 = BoxObj(ootype.cast_to_object(node)) - valuedescr = cpu.fielddescrof(NODE, 'value') - floatdescr = cpu.fielddescrof(NODE, 'floatval') - nextdescr = cpu.fielddescrof(NODE, 'next') - otherdescr = cpu.fielddescrof(NODE2, 'other') - nodesize = cpu.typedescrof(NODE) - nodesize2 = cpu.typedescrof(NODE2) - - arraydescr = cpu.arraydescrof(ootype.Array(ootype.Signed)) - floatarraydescr = cpu.arraydescrof(ootype.Array(ootype.Float)) - - # a plain Record - S = ootype.Record({'a': ootype.Signed, 'b': NODE}) - ssize = cpu.typedescrof(S) - adescr = cpu.fielddescrof(S, 'a') - bdescr = cpu.fielddescrof(S, 'b') - sbox = BoxObj(ootype.cast_to_object(ootype.new(S))) - arraydescr2 = cpu.arraydescrof(ootype.Array(S)) - - T = ootype.Record({'c': ootype.Signed, - 'd': ootype.Array(NODE)}) - tsize = cpu.typedescrof(T) - cdescr = cpu.fielddescrof(T, 'c') - ddescr = cpu.fielddescrof(T, 'd') - arraydescr3 = cpu.arraydescrof(ootype.Array(NODE)) - - U = ootype.Instance('U', ootype.ROOT, {'one': ootype.Array(NODE)}) - usize = cpu.typedescrof(U) - onedescr = cpu.fielddescrof(U, 'one') - u_vtable = ootype.runtimeClass(U) - u_vtable_adr = ootype.cast_to_object(u_vtable) - - # force a consistent order - valuedescr.sort_key() - nextdescr.sort_key() - adescr.sort_key() - bdescr.sort_key() - - FUNC = lltype.FuncType([lltype.Signed], lltype.Signed) - nonwritedescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) # XXX fix ootype - - cpu.class_sizes = {node_vtable_adr: cpu.typedescrof(NODE), - node_vtable_adr2: cpu.typedescrof(NODE2), - u_vtable_adr: cpu.typedescrof(U)} - namespace = locals() +## cpu = runner.OOtypeCPU(None) +## NODE = ootype.Instance('NODE', ootype.ROOT, {}) +## NODE._add_fields({'value': ootype.Signed, +## 'floatval' : ootype.Float, +## 'next': NODE}) +## NODE2 = ootype.Instance('NODE2', NODE, {'other': NODE}) + +## node_vtable = ootype.runtimeClass(NODE) +## node_vtable_adr = ootype.cast_to_object(node_vtable) +## node_vtable2 = ootype.runtimeClass(NODE2) +## node_vtable_adr2 = ootype.cast_to_object(node_vtable2) + +## node = ootype.new(NODE) +## nodebox = BoxObj(ootype.cast_to_object(node)) +## myptr = nodebox.value +## myptr2 = ootype.cast_to_object(ootype.new(NODE)) +## nodebox2 = BoxObj(ootype.cast_to_object(node)) +## valuedescr = cpu.fielddescrof(NODE, 'value') +## floatdescr = cpu.fielddescrof(NODE, 'floatval') +## nextdescr = cpu.fielddescrof(NODE, 'next') +## otherdescr = cpu.fielddescrof(NODE2, 'other') +## nodesize = cpu.typedescrof(NODE) +## nodesize2 = cpu.typedescrof(NODE2) + +## arraydescr = cpu.arraydescrof(ootype.Array(ootype.Signed)) +## floatarraydescr = cpu.arraydescrof(ootype.Array(ootype.Float)) + +## # a plain Record +## S = ootype.Record({'a': ootype.Signed, 'b': NODE}) +## ssize = cpu.typedescrof(S) +## adescr = cpu.fielddescrof(S, 'a') +## bdescr = cpu.fielddescrof(S, 'b') +## sbox = BoxObj(ootype.cast_to_object(ootype.new(S))) +## arraydescr2 = cpu.arraydescrof(ootype.Array(S)) + +## T = ootype.Record({'c': ootype.Signed, +## 'd': ootype.Array(NODE)}) +## tsize = cpu.typedescrof(T) +## cdescr = cpu.fielddescrof(T, 'c') +## ddescr = cpu.fielddescrof(T, 'd') +## arraydescr3 = cpu.arraydescrof(ootype.Array(NODE)) + +## U = ootype.Instance('U', ootype.ROOT, {'one': ootype.Array(NODE)}) +## usize = cpu.typedescrof(U) +## onedescr = cpu.fielddescrof(U, 'one') +## u_vtable = ootype.runtimeClass(U) +## u_vtable_adr = ootype.cast_to_object(u_vtable) + +## # force a consistent order +## valuedescr.sort_key() +## nextdescr.sort_key() +## adescr.sort_key() +## bdescr.sort_key() + +## FUNC = lltype.FuncType([lltype.Signed], lltype.Signed) +## nonwritedescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) # XXX fix ootype + +## cpu.class_sizes = {node_vtable_adr: cpu.typedescrof(NODE), +## node_vtable_adr2: cpu.typedescrof(NODE2), +## u_vtable_adr: cpu.typedescrof(U)} +## namespace = locals() class BaseTest(object): invent_fail_descr = None @@ -220,6 +220,7 @@ # def constclass(cls_vtable): if self.type_system == 'lltype': + XXX return ConstAddr(llmemory.cast_ptr_to_adr(cls_vtable), self.cpu) else: @@ -1160,14 +1161,13 @@ class TestLLtype(BaseTestOptimizeFindNode, LLtypeMixin): pass -class TestOOtype(BaseTestOptimizeFindNode, OOtypeMixin): - - def test_find_nodes_instanceof(self): - ops = """ - [i0] - p0 = new_with_vtable(ConstClass(node_vtable)) - i1 = instanceof(p0, descr=nodesize) - jump(i1) - """ - boxes, getnode = self.find_nodes(ops, 'Not') - assert not getnode(boxes.p0).escaped +##class TestOOtype(BaseTestOptimizeFindNode, OOtypeMixin): +## def test_find_nodes_instanceof(self): +## ops = """ +## [i0] +## p0 = new_with_vtable(ConstClass(node_vtable)) +## i1 = instanceof(p0, descr=nodesize) +## jump(i1) +## """ +## boxes, getnode = self.find_nodes(ops, 'Not') +## assert not getnode(boxes.p0).escaped From hpk at codespeak.net Sat May 22 13:29:15 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sat, 22 May 2010 13:29:15 +0200 (CEST) Subject: [pypy-svn] r74676 - in pypy/trunk: . lib-python py py/_cmdline py/_code py/_io py/_plugin py/_test pypy pypy/interpreter pypy/interpreter/test pypy/jit/metainterp/test pypy/tool/pytest pypy/tool/pytest/test Message-ID: <20100522112915.68850282BAD@codespeak.net> Author: hpk Date: Sat May 22 13:29:12 2010 New Revision: 74676 Removed: pypy/trunk/py/__init__.py.orig pypy/trunk/py/_plugin/pytest_pytester.py.orig pypy/trunk/py/_plugin/pytest_terminal.py.orig Modified: pypy/trunk/ (props changed) pypy/trunk/lib-python/conftest.py pypy/trunk/py/__init__.py pypy/trunk/py/_builtin.py pypy/trunk/py/_cmdline/pytest.py pypy/trunk/py/_code/code.py pypy/trunk/py/_code/source.py pypy/trunk/py/_io/capture.py pypy/trunk/py/_plugin/pytest__pytest.py pypy/trunk/py/_plugin/pytest_capture.py pypy/trunk/py/_plugin/pytest_genscript.py pypy/trunk/py/_plugin/pytest_junitxml.py pypy/trunk/py/_plugin/pytest_mark.py pypy/trunk/py/_plugin/pytest_pytester.py pypy/trunk/py/_plugin/pytest_resultlog.py pypy/trunk/py/_plugin/pytest_runner.py pypy/trunk/py/_plugin/pytest_skipping.py pypy/trunk/py/_plugin/pytest_terminal.py pypy/trunk/py/_test/cmdline.py pypy/trunk/py/_test/pycollect.py pypy/trunk/pypy/conftest.py pypy/trunk/pypy/interpreter/gateway.py pypy/trunk/pypy/interpreter/test/test_code.py pypy/trunk/pypy/jit/metainterp/test/test_warmspot.py pypy/trunk/pypy/tool/pytest/appsupport.py pypy/trunk/pypy/tool/pytest/test/test_appsupport.py Log: merge py131 branch (which merges py1.3.1a1 - release pending) relevant news: * fixes keyboardinterrupt issues - hopefully strange internal TBs when doing ctrl-c are a thing of the past now * be more robust against bogus source code and fix/improve reporting of failing applevel tests * fix some xfail/skipping issues and introduce a new imperative way to xfail a test by py.test.xfail() # or py.test.xfail("reasonstring") I strongly recommend to use this instead of py.test.skip("xxx") because it more clearly communicates that not just some platform/dependency problem but an implementation issue is present. Note that it is also possible to do: @py.test.mark.xfail def test_function() and with Python2.6 it can also be applied to a test class like this: @py.test.mark.xfail(reason="xyz needs improvement/refactoring") class TestClass: def test_... And btw, you can issue "py.test --runxfail" to still run those tests and see tracebacks. And finally, @py.test.mark.xfail(run=False) def test_function() will not try to run the function at all. Modified: pypy/trunk/lib-python/conftest.py ============================================================================== --- pypy/trunk/lib-python/conftest.py (original) +++ pypy/trunk/lib-python/conftest.py Sat May 22 13:29:12 2010 @@ -14,7 +14,6 @@ # the following adds command line options as a side effect! from pypy.conftest import gettestobjspace, option as pypy_option -from test import pystone from pypy.tool.pytest import appsupport from pypy.tool.pytest.confpath import pypydir, libpythondir, \ @@ -40,6 +39,7 @@ option = py.test.config.option def gettimeout(): + from test import pystone timeout = option.timeout.lower() if timeout.endswith('mp'): megapystone = float(timeout[:-2]) Modified: pypy/trunk/py/__init__.py ============================================================================== --- pypy/trunk/py/__init__.py (original) +++ pypy/trunk/py/__init__.py Sat May 22 13:29:12 2010 @@ -8,7 +8,7 @@ (c) Holger Krekel and others, 2004-2010 """ -__version__ = version = "1.3.0" +__version__ = version = "1.3.1a1" import py.apipkg @@ -25,7 +25,6 @@ 'pytest': '._cmdline.pytest:main', 'pylookup': '._cmdline.pylookup:main', 'pycountloc': '._cmdline.pycountlog:main', - 'pytest': '._test.cmdline:main', 'pylookup': '._cmdline.pylookup:main', 'pycountloc': '._cmdline.pycountloc:main', 'pycleanup': '._cmdline.pycleanup:main', Modified: pypy/trunk/py/_builtin.py ============================================================================== --- pypy/trunk/py/_builtin.py (original) +++ pypy/trunk/py/_builtin.py Sat May 22 13:29:12 2010 @@ -151,7 +151,10 @@ return getattr(function, "__dict__", None) def _getcode(function): - return getattr(function, "func_code", None) + try: + return getattr(function, "__code__") + except AttributeError: + return getattr(function, "func_code", None) def print_(*args, **kwargs): """ minimal backport of py3k print statement. """ @@ -175,6 +178,7 @@ def exec_(obj, globals=None, locals=None): """ minimal backport of py3k exec statement. """ + __tracebackhide__ = True if globals is None: frame = sys._getframe(1) globals = frame.f_globals @@ -187,14 +191,17 @@ if sys.version_info >= (3,0): exec (""" def _reraise(cls, val, tb): + __tracebackhide__ = True assert hasattr(val, '__traceback__') raise val """) else: exec (""" def _reraise(cls, val, tb): + __tracebackhide__ = True raise cls, val, tb def exec2(obj, globals, locals): + __tracebackhide__ = True exec obj in globals, locals """) Modified: pypy/trunk/py/_cmdline/pytest.py ============================================================================== --- pypy/trunk/py/_cmdline/pytest.py (original) +++ pypy/trunk/py/_cmdline/pytest.py Sat May 22 13:29:12 2010 @@ -1,5 +1,5 @@ #!/usr/bin/env python import py -def main(args): - py.test.cmdline.main(args) +def main(args=None): + raise SystemExit(py.test.cmdline.main(args)) Modified: pypy/trunk/py/_code/code.py ============================================================================== --- pypy/trunk/py/_code/code.py (original) +++ pypy/trunk/py/_code/code.py Sat May 22 13:29:12 2010 @@ -23,64 +23,14 @@ def __ne__(self, other): return not self == other - def new(self, rec=False, **kwargs): - """ return new code object with modified attributes. - if rec-cursive is true then dive into code - objects contained in co_consts. - """ - if sys.platform.startswith("java"): - # XXX jython does not support the below co_filename hack - return self.raw - names = [x for x in dir(self.raw) if x[:3] == 'co_'] - for name in kwargs: - if name not in names: - raise TypeError("unknown code attribute: %r" %(name, )) - if rec and hasattr(self.raw, 'co_consts'): # jython - newconstlist = [] - co = self.raw - cotype = type(co) - for c in co.co_consts: - if isinstance(c, cotype): - c = self.__class__(c).new(rec=True, **kwargs) - newconstlist.append(c) - return self.new(rec=False, co_consts=tuple(newconstlist), **kwargs) - for name in names: - if name not in kwargs: - kwargs[name] = getattr(self.raw, name) - arglist = [ - kwargs['co_argcount'], - kwargs['co_nlocals'], - kwargs.get('co_stacksize', 0), # jython - kwargs.get('co_flags', 0), # jython - kwargs.get('co_code', ''), # jython - kwargs.get('co_consts', ()), # jython - kwargs.get('co_names', []), # - kwargs['co_varnames'], - kwargs['co_filename'], - kwargs['co_name'], - kwargs['co_firstlineno'], - kwargs.get('co_lnotab', ''), #jython - kwargs.get('co_freevars', None), #jython - kwargs.get('co_cellvars', None), # jython - ] - if sys.version_info >= (3,0): - arglist.insert(1, kwargs['co_kwonlyargcount']) - return self.raw.__class__(*arglist) - else: - return py.std.new.code(*arglist) - def path(self): """ return a path object pointing to source code""" - fn = self.raw.co_filename - try: - return fn.__path__ - except AttributeError: - p = py.path.local(self.raw.co_filename) - if not p.check(): - # XXX maybe try harder like the weird logic - # in the standard lib [linecache.updatecache] does? - p = self.raw.co_filename - return p + p = py.path.local(self.raw.co_filename) + if not p.check(): + # XXX maybe try harder like the weird logic + # in the standard lib [linecache.updatecache] does? + p = self.raw.co_filename + return p path = property(path, None, None, "path of this code object") Modified: pypy/trunk/py/_code/source.py ============================================================================== --- pypy/trunk/py/_code/source.py (original) +++ pypy/trunk/py/_code/source.py Sat May 22 13:29:12 2010 @@ -2,6 +2,7 @@ import sys import inspect, tokenize import py +from types import ModuleType cpy_compile = compile try: @@ -25,6 +26,8 @@ partlines = [] if isinstance(part, Source): partlines = part.lines + elif isinstance(part, (tuple, list)): + partlines = [x.rstrip("\n") for x in part] elif isinstance(part, py.builtin._basestring): partlines = part.split('\n') if rstrip: @@ -171,7 +174,9 @@ try: #compile(source+'\n', "x", "exec") syntax_checker(source+'\n') - except SyntaxError: + except KeyboardInterrupt: + raise + except Exception: return False else: return True @@ -212,10 +217,17 @@ else: if flag & _AST_FLAG: return co - co_filename = MyStr(filename) - co_filename.__source__ = self - return py.code.Code(co).new(rec=1, co_filename=co_filename) - #return newcode_withfilename(co, co_filename) + lines = [(x + "\n") for x in self.lines] + if sys.version_info[0] >= 3: + # XXX py3's inspect.getsourcefile() checks for a module + # and a pep302 __loader__ ... we don't have a module + # at code compile-time so we need to fake it here + m = ModuleType("_pycodecompile_pseudo_module") + py.std.inspect.modulesbyfile[filename] = None + py.std.sys.modules[None] = m + m.__loader__ = 1 + py.std.linecache.cache[filename] = (1, None, lines, filename) + return co # # public API shortcut functions @@ -224,11 +236,9 @@ def compile_(source, filename=None, mode='exec', flags= generators.compiler_flag, dont_inherit=0): """ compile the given source to a raw code object, - which points back to the source code through - "co_filename.__source__". All code objects - contained in the code object will recursively - also have this special subclass-of-string - filename. + and maintain an internal cache which allows later + retrieval of the source code for the code object + and any recursively created code objects. """ if _ast is not None and isinstance(source, _ast.AST): # XXX should Source support having AST? @@ -262,44 +272,26 @@ # # helper functions # -class MyStr(str): - """ custom string which allows to add attributes. """ def findsource(obj): - obj = py.code.getrawcode(obj) try: - fullsource = obj.co_filename.__source__ - except AttributeError: - try: - sourcelines, lineno = py.std.inspect.findsource(obj) - except (KeyboardInterrupt, SystemExit): - raise - except: - return None, None - source = Source() - source.lines = [line.rstrip() for line in sourcelines] - return source, lineno - else: - lineno = obj.co_firstlineno - 1 - return fullsource, lineno - + sourcelines, lineno = py.std.inspect.findsource(obj) + except (KeyboardInterrupt, SystemExit): + raise + except: + return None, None + source = Source() + source.lines = [line.rstrip() for line in sourcelines] + return source, lineno def getsource(obj, **kwargs): obj = py.code.getrawcode(obj) try: - fullsource = obj.co_filename.__source__ - except AttributeError: - try: - strsrc = inspect.getsource(obj) - except IndentationError: - strsrc = "\"Buggy python version consider upgrading, cannot get source\"" - assert isinstance(strsrc, str) - return Source(strsrc, **kwargs) - else: - lineno = obj.co_firstlineno - 1 - end = fullsource.getblockend(lineno) - return Source(fullsource[lineno:end+1], deident=True) - + strsrc = inspect.getsource(obj) + except IndentationError: + strsrc = "\"Buggy python version consider upgrading, cannot get source\"" + assert isinstance(strsrc, str) + return Source(strsrc, **kwargs) def deindent(lines, offset=None): if offset is None: Modified: pypy/trunk/py/_io/capture.py ============================================================================== --- pypy/trunk/py/_io/capture.py (original) +++ pypy/trunk/py/_io/capture.py Sat May 22 13:29:12 2010 @@ -26,46 +26,56 @@ raise TypeError("not a byte value: %r" %(data,)) StringIO.write(self, data) +patchsysdict = {0: 'stdin', 1: 'stdout', 2: 'stderr'} + class FDCapture: """ Capture IO to/from a given os-level filedescriptor. """ - def __init__(self, targetfd, tmpfile=None): + def __init__(self, targetfd, tmpfile=None, now=True, patchsys=False): """ save targetfd descriptor, and open a new temporary file there. If no tmpfile is specified a tempfile.Tempfile() will be opened in text mode. """ self.targetfd = targetfd - if tmpfile is None: + if tmpfile is None and targetfd != 0: f = tempfile.TemporaryFile('wb+') tmpfile = dupfile(f, encoding="UTF-8") f.close() self.tmpfile = tmpfile - self._savefd = os.dup(targetfd) - os.dup2(self.tmpfile.fileno(), targetfd) - self._patched = [] - - def setasfile(self, name, module=sys): - """ patch . to self.tmpfile - """ - key = (module, name) - self._patched.append((key, getattr(module, name))) - setattr(module, name, self.tmpfile) + self._savefd = os.dup(self.targetfd) + if patchsys: + self._oldsys = getattr(sys, patchsysdict[targetfd]) + if now: + self.start() - def unsetfiles(self): - """ unpatch all patched items - """ - while self._patched: - (module, name), value = self._patched.pop() - setattr(module, name, value) + def start(self): + try: + os.fstat(self._savefd) + except OSError: + raise ValueError("saved filedescriptor not valid, " + "did you call start() twice?") + if self.targetfd == 0 and not self.tmpfile: + fd = os.open(devnullpath, os.O_RDONLY) + os.dup2(fd, 0) + os.close(fd) + if hasattr(self, '_oldsys'): + setattr(sys, patchsysdict[self.targetfd], DontReadFromInput()) + else: + fd = self.tmpfile.fileno() + os.dup2(self.tmpfile.fileno(), self.targetfd) + if hasattr(self, '_oldsys'): + setattr(sys, patchsysdict[self.targetfd], self.tmpfile) def done(self): """ unpatch and clean up, returns the self.tmpfile (file object) """ os.dup2(self._savefd, self.targetfd) - self.unsetfiles() os.close(self._savefd) - self.tmpfile.seek(0) + if self.targetfd != 0: + self.tmpfile.seek(0) + if hasattr(self, '_oldsys'): + setattr(sys, patchsysdict[self.targetfd], self._oldsys) return self.tmpfile def writeorg(self, data): @@ -100,7 +110,7 @@ if encoding is not None: mode = mode.replace("b", "") buffering = True - return os.fdopen(newfd, mode, buffering, encoding, closefd=False) + return os.fdopen(newfd, mode, buffering, encoding, closefd=True) else: f = os.fdopen(newfd, mode, buffering) if encoding is not None: @@ -146,90 +156,94 @@ def reset(self): """ reset sys.stdout/stderr and return captured output as strings. """ - if hasattr(self, '_suspended'): - outfile = self._kwargs['out'] - errfile = self._kwargs['err'] - del self._kwargs - else: - outfile, errfile = self.done() + outfile, errfile = self.done() out, err = "", "" - if outfile: + if outfile and not outfile.closed: out = outfile.read() outfile.close() - if errfile and errfile != outfile: + if errfile and errfile != outfile and not errfile.closed: err = errfile.read() errfile.close() return out, err def suspend(self): """ return current snapshot captures, memorize tempfiles. """ - assert not hasattr(self, '_suspended') - self._suspended = True outerr = self.readouterr() outfile, errfile = self.done() - self._kwargs['out'] = outfile - self._kwargs['err'] = errfile return outerr - def resume(self): - """ resume capturing with original temp files. """ - assert self._suspended - self._initialize(**self._kwargs) - del self._suspended - class StdCaptureFD(Capture): """ This class allows to capture writes to FD1 and FD2 and may connect a NULL file to FD0 (and prevent - reads from sys.stdin) + reads from sys.stdin). If any of the 0,1,2 file descriptors + is invalid it will not be captured. """ - def __init__(self, out=True, err=True, - mixed=False, in_=True, patchsys=True): - self._kwargs = locals().copy() - del self._kwargs['self'] - self._initialize(**self._kwargs) - - def _initialize(self, out=True, err=True, - mixed=False, in_=True, patchsys=True): + def __init__(self, out=True, err=True, mixed=False, + in_=True, patchsys=True, now=True): + self._options = locals() + self._save() + if now: + self.startall() + + def _save(self): + in_ = self._options['in_'] + out = self._options['out'] + err = self._options['err'] + mixed = self._options['mixed'] + patchsys = self._options['patchsys'] if in_: - self._oldin = (sys.stdin, os.dup(0)) - sys.stdin = DontReadFromInput() - fd = os.open(devnullpath, os.O_RDONLY) - os.dup2(fd, 0) - os.close(fd) - if out: + try: + self.in_ = FDCapture(0, tmpfile=None, now=False, + patchsys=patchsys) + except OSError: + pass + if out: tmpfile = None if hasattr(out, 'write'): tmpfile = out - self.out = py.io.FDCapture(1, tmpfile=tmpfile) - if patchsys: - self.out.setasfile('stdout') - if err: - if mixed and out: + try: + self.out = FDCapture(1, tmpfile=tmpfile, + now=False, patchsys=patchsys) + self._options['out'] = self.out.tmpfile + except OSError: + pass + if err: + if out and mixed: tmpfile = self.out.tmpfile elif hasattr(err, 'write'): tmpfile = err else: tmpfile = None - self.err = py.io.FDCapture(2, tmpfile=tmpfile) - if patchsys: - self.err.setasfile('stderr') + try: + self.err = FDCapture(2, tmpfile=tmpfile, + now=False, patchsys=patchsys) + self._options['err'] = self.err.tmpfile + except OSError: + pass + + def startall(self): + if hasattr(self, 'in_'): + self.in_.start() + if hasattr(self, 'out'): + self.out.start() + if hasattr(self, 'err'): + self.err.start() + + def resume(self): + """ resume capturing with original temp files. """ + self.startall() def done(self): """ return (outfile, errfile) and stop capturing. """ - if hasattr(self, 'out'): + outfile = errfile = None + if hasattr(self, 'out') and not self.out.tmpfile.closed: outfile = self.out.done() - else: - outfile = None - if hasattr(self, 'err'): + if hasattr(self, 'err') and not self.err.tmpfile.closed: errfile = self.err.done() - else: - errfile = None - if hasattr(self, '_oldin'): - oldsys, oldfd = self._oldin - os.dup2(oldfd, 0) - os.close(oldfd) - sys.stdin = oldsys + if hasattr(self, 'in_'): + tmpfile = self.in_.done() + self._save() return outfile, errfile def readouterr(self): @@ -252,69 +266,61 @@ modifies sys.stdout|stderr|stdin attributes and does not touch underlying File Descriptors (use StdCaptureFD for that). """ - def __init__(self, out=True, err=True, in_=True, mixed=False): - self._kwargs = locals().copy() - del self._kwargs['self'] - self._initialize(**self._kwargs) - - def _initialize(self, out, err, in_, mixed): - self._out = out - self._err = err - self._in = in_ - if out: - self._oldout = sys.stdout - if not hasattr(out, 'write'): - out = TextIO() - sys.stdout = self.out = out - if err: - self._olderr = sys.stderr - if out and mixed: - err = self.out + def __init__(self, out=True, err=True, in_=True, mixed=False, now=True): + self._oldout = sys.stdout + self._olderr = sys.stderr + self._oldin = sys.stdin + if out and not hasattr(out, 'file'): + out = TextIO() + self.out = out + if err: + if mixed: + err = out elif not hasattr(err, 'write'): err = TextIO() - sys.stderr = self.err = err - if in_: - self._oldin = sys.stdin - sys.stdin = self.newin = DontReadFromInput() + self.err = err + self.in_ = in_ + if now: + self.startall() + + def startall(self): + if self.out: + sys.stdout = self.out + if self.err: + sys.stderr = self.err + if self.in_: + sys.stdin = self.in_ = DontReadFromInput() def done(self): """ return (outfile, errfile) and stop capturing. """ - o,e = sys.stdout, sys.stderr - if self._out: - try: - sys.stdout = self._oldout - except AttributeError: - raise IOError("stdout capturing already reset") - del self._oldout + outfile = errfile = None + if self.out and not self.out.closed: + sys.stdout = self._oldout outfile = self.out outfile.seek(0) - else: - outfile = None - if self._err: - try: - sys.stderr = self._olderr - except AttributeError: - raise IOError("stderr capturing already reset") - del self._olderr + if self.err and not self.err.closed: + sys.stderr = self._olderr errfile = self.err errfile.seek(0) - else: - errfile = None - if self._in: + if self.in_: sys.stdin = self._oldin return outfile, errfile + def resume(self): + """ resume capturing with original temp files. """ + self.startall() + def readouterr(self): """ return snapshot value of stdout/stderr capturings. """ out = err = "" - if self._out: - out = sys.stdout.getvalue() - sys.stdout.truncate(0) - sys.stdout.seek(0) - if self._err: - err = sys.stderr.getvalue() - sys.stderr.truncate(0) - sys.stderr.seek(0) + if self.out: + out = self.out.getvalue() + self.out.truncate(0) + self.out.seek(0) + if self.err: + err = self.err.getvalue() + self.err.truncate(0) + self.err.seek(0) return out, err class DontReadFromInput: @@ -344,5 +350,3 @@ devnullpath = 'NUL' else: devnullpath = '/dev/null' - - Modified: pypy/trunk/py/_plugin/pytest__pytest.py ============================================================================== --- pypy/trunk/py/_plugin/pytest__pytest.py (original) +++ pypy/trunk/py/_plugin/pytest__pytest.py Sat May 22 13:29:12 2010 @@ -46,7 +46,8 @@ recorder = RecordCalls() self._recorders[hookspec] = recorder self._registry.register(recorder) - self.hook = HookRelay(hookspecs, registry=self._registry) + self.hook = HookRelay(hookspecs, registry=self._registry, + prefix="pytest_") def finish_recording(self): for recorder in self._recorders.values(): Modified: pypy/trunk/py/_plugin/pytest_capture.py ============================================================================== --- pypy/trunk/py/_plugin/pytest_capture.py (original) +++ pypy/trunk/py/_plugin/pytest_capture.py Sat May 22 13:29:12 2010 @@ -106,6 +106,14 @@ def pytest_configure(config): config.pluginmanager.register(CaptureManager(), 'capturemanager') +class NoCapture: + def startall(self): + pass + def resume(self): + pass + def suspend(self): + return "", "" + class CaptureManager: def __init__(self): self._method2capture = {} @@ -118,15 +126,17 @@ def _makestringio(self): return py.io.TextIO() - def _startcapture(self, method): + def _getcapture(self, method): if method == "fd": - return py.io.StdCaptureFD( + return py.io.StdCaptureFD(now=False, out=self._maketempfile(), err=self._maketempfile() ) elif method == "sys": - return py.io.StdCapture( + return py.io.StdCapture(now=False, out=self._makestringio(), err=self._makestringio() ) + elif method == "no": + return NoCapture() else: raise ValueError("unknown capturing method: %r" % method) @@ -152,27 +162,25 @@ if hasattr(self, '_capturing'): raise ValueError("cannot resume, already capturing with %r" % (self._capturing,)) - if method != "no": - cap = self._method2capture.get(method) - if cap is None: - cap = self._startcapture(method) - self._method2capture[method] = cap - else: - cap.resume() + cap = self._method2capture.get(method) self._capturing = method + if cap is None: + self._method2capture[method] = cap = self._getcapture(method) + cap.startall() + else: + cap.resume() def suspendcapture(self, item=None): self.deactivate_funcargs() if hasattr(self, '_capturing'): method = self._capturing - if method != "no": - cap = self._method2capture[method] + cap = self._method2capture.get(method) + if cap is not None: outerr = cap.suspend() - else: - outerr = "", "" del self._capturing if item: - outerr = (item.outerr[0] + outerr[0], item.outerr[1] + outerr[1]) + outerr = (item.outerr[0] + outerr[0], + item.outerr[1] + outerr[1]) return outerr return "", "" @@ -180,19 +188,17 @@ if not hasattr(pyfuncitem, 'funcargs'): return assert not hasattr(self, '_capturing_funcargs') - l = [] - for name, obj in pyfuncitem.funcargs.items(): - if name == 'capfd' and not hasattr(os, 'dup'): - py.test.skip("capfd funcarg needs os.dup") + self._capturing_funcargs = capturing_funcargs = [] + for name, capfuncarg in pyfuncitem.funcargs.items(): if name in ('capsys', 'capfd'): - obj._start() - l.append(obj) - if l: - self._capturing_funcargs = l + capturing_funcargs.append(capfuncarg) + capfuncarg._start() def deactivate_funcargs(self): - if hasattr(self, '_capturing_funcargs'): - for capfuncarg in self._capturing_funcargs: + capturing_funcargs = getattr(self, '_capturing_funcargs', None) + if capturing_funcargs is not None: + while capturing_funcargs: + capfuncarg = capturing_funcargs.pop() capfuncarg._finalize() del self._capturing_funcargs @@ -256,16 +262,19 @@ platform does not have ``os.dup`` (e.g. Jython) tests using this funcarg will automatically skip. """ + if not hasattr(os, 'dup'): + py.test.skip("capfd funcarg needs os.dup") return CaptureFuncarg(request, py.io.StdCaptureFD) class CaptureFuncarg: def __init__(self, request, captureclass): self._cclass = captureclass + self.capture = self._cclass(now=False) #request.addfinalizer(self._finalize) def _start(self): - self.capture = self._cclass() + self.capture.startall() def _finalize(self): if hasattr(self, 'capture'): @@ -276,6 +285,4 @@ return self.capture.readouterr() def close(self): - self.capture.reset() - del self.capture - + self._finalize() Modified: pypy/trunk/py/_plugin/pytest_genscript.py ============================================================================== --- pypy/trunk/py/_plugin/pytest_genscript.py (original) +++ pypy/trunk/py/_plugin/pytest_genscript.py Sat May 22 13:29:12 2010 @@ -4,14 +4,7 @@ """ import os -import zlib -import base64 import sys -try: - import pickle -except Importerror: - import cPickle as pickle - def pytest_addoption(parser): group = parser.getgroup("debugconfig") group.addoption("--genscript", action="store", default=None, @@ -30,6 +23,13 @@ raise SystemExit(0) def main(pybasedir, outfile, infile): + import base64 + import zlib + try: + import pickle + except Importerror: + import cPickle as pickle + outfile = str(outfile) infile = str(infile) assert os.path.isabs(outfile) Modified: pypy/trunk/py/_plugin/pytest_junitxml.py ============================================================================== --- pypy/trunk/py/_plugin/pytest_junitxml.py (original) +++ pypy/trunk/py/_plugin/pytest_junitxml.py Sat May 22 13:29:12 2010 @@ -56,10 +56,15 @@ def append_failure(self, report): self._opentestcase(report) #msg = str(report.longrepr.reprtraceback.extraline) - self.appendlog('%s', - report.longrepr) + if "xfail" in report.keywords: + self.appendlog( + '') + self.skipped += 1 + else: + self.appendlog('%s', + report.longrepr) + self.failed += 1 self._closetestcase() - self.failed += 1 def _opentestcase_collectfailure(self, report): node = report.collector @@ -95,7 +100,12 @@ def append_skipped(self, report): self._opentestcase(report) - self.appendlog("") + if "xfail" in report.keywords: + self.appendlog( + '%s', + report.keywords['xfail']) + else: + self.appendlog("") self._closetestcase() self.skipped += 1 Modified: pypy/trunk/py/_plugin/pytest_mark.py ============================================================================== --- pypy/trunk/py/_plugin/pytest_mark.py (original) +++ pypy/trunk/py/_plugin/pytest_mark.py Sat May 22 13:29:12 2010 @@ -34,38 +34,43 @@ .. _`scoped-marking`: -Marking classes or modules +Marking whole classes or modules ---------------------------------------------------- -To mark all methods of a class set a ``pytestmark`` attribute like this:: +If you are programming with Python2.6 you may use ``py.test.mark`` decorators +with classes to apply markers to all its test methods:: + + @py.test.mark.webtest + class TestClass: + def test_startup(self): + ... + +This is equivalent to directly applying the decorator to the +``test_startup`` function. + +To remain compatible with Python2.5 you can instead set a +``pytestmark`` attribute on a TestClass like this:: import py class TestClass: pytestmark = py.test.mark.webtest -You can re-use the same markers that you would use for decorating -a function - in fact this marker decorator will be applied -to all test methods of the class. +or if you need to use multiple markers:: + + import py + + class TestClass: + pytestmark = [py.test.mark.webtest, pytest.mark.slowtest] You can also set a module level marker:: import py pytestmark = py.test.mark.webtest -in which case then the marker decorator will be applied to all functions and +in which case then it will be applied to all functions and methods defined in the module. -The order in which marker functions are called is this:: - - per-function (upon import of module already) - per-class - per-module - -Later called markers may overwrite previous key-value settings. -Positional arguments are all appended to the same 'args' list -of the Marker object. - Using "-k MARKNAME" to select tests ---------------------------------------------------- @@ -105,15 +110,23 @@ """ if passed a single callable argument: decorate it with mark info. otherwise add *args/**kwargs in-place to mark information. """ if args: - if len(args) == 1 and hasattr(args[0], '__call__'): - func = args[0] - holder = getattr(func, self.markname, None) - if holder is None: - holder = MarkInfo(self.markname, self.args, self.kwargs) - setattr(func, self.markname, holder) + func = args[0] + if len(args) == 1 and hasattr(func, '__call__') or \ + hasattr(func, '__bases__'): + if hasattr(func, '__bases__'): + l = func.__dict__.setdefault("pytestmark", []) + if not isinstance(l, list): + func.pytestmark = [l, self] + else: + l.append(self) else: - holder.kwargs.update(self.kwargs) - holder.args.extend(self.args) + holder = getattr(func, self.markname, None) + if holder is None: + holder = MarkInfo(self.markname, self.args, self.kwargs) + setattr(func, self.markname, holder) + else: + holder.kwargs.update(self.kwargs) + holder.args.extend(self.args) return func else: self.args.extend(args) @@ -147,6 +160,10 @@ func = getattr(func, 'im_func', func) # py2 for parent in [x for x in (mod, cls) if x]: marker = getattr(parent.obj, 'pytestmark', None) - if isinstance(marker, MarkDecorator): - marker(func) + if marker is not None: + if not isinstance(marker, list): + marker = [marker] + for mark in marker: + if isinstance(mark, MarkDecorator): + mark(func) return item Modified: pypy/trunk/py/_plugin/pytest_pytester.py ============================================================================== --- pypy/trunk/py/_plugin/pytest_pytester.py (original) +++ pypy/trunk/py/_plugin/pytest_pytester.py Sat May 22 13:29:12 2010 @@ -110,7 +110,7 @@ def _makefile(self, ext, args, kwargs): items = list(kwargs.items()) if args: - source = "\n".join(map(str, args)) + source = "\n".join(map(str, args)) + "\n" basename = self.request.function.__name__ items.insert(0, (basename, source)) ret = None @@ -294,8 +294,10 @@ ret = popen.wait() f1.close() f2.close() - out = p1.read("rb").decode("utf-8").splitlines() - err = p2.read("rb").decode("utf-8").splitlines() + out = p1.read("rb") + out = getdecoded(out).splitlines() + err = p2.read("rb") + err = getdecoded(err).splitlines() def dump_lines(lines, fp): try: for line in lines: @@ -360,6 +362,13 @@ child.timeout = expect_timeout return child +def getdecoded(out): + try: + return out.decode("utf-8") + except UnicodeDecodeError: + return "INTERNAL not-utf8-decodeable, truncated string:\n%s" % ( + py.io.saferepr(out),) + class PseudoPlugin: def __init__(self, vars): self.__dict__.update(vars) Modified: pypy/trunk/py/_plugin/pytest_resultlog.py ============================================================================== --- pypy/trunk/py/_plugin/pytest_resultlog.py (original) +++ pypy/trunk/py/_plugin/pytest_resultlog.py Sat May 22 13:29:12 2010 @@ -73,7 +73,7 @@ code = report.shortrepr if code == 'x': longrepr = str(report.longrepr) - elif code == 'P': + elif code == 'X': longrepr = '' elif report.passed: longrepr = "" Modified: pypy/trunk/py/_plugin/pytest_runner.py ============================================================================== --- pypy/trunk/py/_plugin/pytest_runner.py (original) +++ pypy/trunk/py/_plugin/pytest_runner.py Sat May 22 13:29:12 2010 @@ -10,6 +10,7 @@ 'skip' : skip, 'importorskip' : importorskip, 'fail' : fail, + 'xfail' : xfail, 'exit' : exit, } @@ -295,6 +296,10 @@ """ raised from an explicit call to py.test.fail() """ __module__ = 'builtins' +class XFailed(OutcomeException): + """ raised from an explicit call to py.test.xfail() """ + __module__ = 'builtins' + class ExceptionFailure(Failed): """ raised by py.test.raises on an exception-assertion mismatch. """ def __init__(self, expr, expected, msg=None, excinfo=None): @@ -335,6 +340,14 @@ fail.Exception = Failed +def xfail(reason=""): + """ xfail an executing test or setup functions, taking an optional + reason string. + """ + __tracebackhide__ = True + raise XFailed(reason) +xfail.Exception = XFailed + def raises(ExpectedException, *args, **kwargs): """ if args[0] is callable: raise AssertionError if calling it with the remaining arguments does not raise the expected exception. Modified: pypy/trunk/py/_plugin/pytest_skipping.py ============================================================================== --- pypy/trunk/py/_plugin/pytest_skipping.py (original) +++ pypy/trunk/py/_plugin/pytest_skipping.py Sat May 22 13:29:12 2010 @@ -60,6 +60,19 @@ # The ``pytestmark`` decorator will be applied to each test function. +If your code targets python2.6 or above you can equivalently use +the skipif decorator on classes:: + + @py.test.mark.skipif("sys.platform == 'win32'") + class TestPosixCalls: + + def test_function(self): + # will not be setup or run under 'win32' platform + # + +It is fine in general to apply multiple "skipif" decorators +on a single function - this means that if any of the conditions +apply the function will be skipped. .. _`whole class- or module level`: mark.html#scoped-marking @@ -82,10 +95,17 @@ depending on platform:: @py.test.mark.xfail("sys.version_info >= (3,0)") - def test_function(): ... +To not run a test and still regard it as "xfailed":: + + @py.test.mark.xfail(..., run=False) + +To specify an explicit reason to be shown with xfailure detail:: + + @py.test.mark.xfail(..., reason="my reason") + skipping on a missing import dependency -------------------------------------------------- @@ -115,11 +135,14 @@ py.test.skip("unsuppored configuration") """ -# XXX py.test.skip, .importorskip and the Skipped class -# should also be defined in this plugin, requires thought/changes import py +def pytest_addoption(parser): + group = parser.getgroup("general") + group.addoption('--runxfail', + action="store_true", dest="runxfail", default=False, + help="run tests even if they are marked xfail") class MarkEvaluator: def __init__(self, item, name): @@ -134,17 +157,20 @@ def istrue(self): if self.holder: d = {'os': py.std.os, 'sys': py.std.sys, 'config': self.item.config} - self.result = True - for expr in self.holder.args: - self.expr = expr - if isinstance(expr, str): - result = cached_eval(self.item.config, expr, d) - else: - result = expr - if not result: - self.result = False + if self.holder.args: + self.result = False + for expr in self.holder.args: self.expr = expr - break + if isinstance(expr, str): + result = cached_eval(self.item.config, expr, d) + else: + result = expr + if result: + self.result = True + self.expr = expr + break + else: + self.result = True return getattr(self, 'result', False) def get(self, attr, default=None): @@ -161,27 +187,53 @@ def pytest_runtest_setup(item): - expr, result = evalexpression(item, 'skipif') - if result: - py.test.skip(expr) + if not isinstance(item, py.test.collect.Function): + return + evalskip = MarkEvaluator(item, 'skipif') + if evalskip.istrue(): + py.test.skip(evalskip.getexplanation()) + item._evalxfail = MarkEvaluator(item, 'xfail') + if not item.config.getvalue("runxfail"): + if item._evalxfail.istrue(): + if not item._evalxfail.get('run', True): + py.test.skip("xfail") def pytest_runtest_makereport(__multicall__, item, call): - if call.when != "call": + if not isinstance(item, py.test.collect.Function): return - expr, result = evalexpression(item, 'xfail') - rep = __multicall__.execute() - if result: - if call.excinfo: - rep.skipped = True - rep.failed = rep.passed = False + if not (call.excinfo and + call.excinfo.errisinstance(py.test.xfail.Exception)): + evalxfail = getattr(item, '_evalxfail', None) + if not evalxfail: + return + if call.excinfo and call.excinfo.errisinstance(py.test.xfail.Exception): + rep = __multicall__.execute() + rep.keywords['xfail'] = "reason: " + call.excinfo.value.msg + rep.skipped = True + rep.failed = False + return rep + if call.when == "setup": + rep = __multicall__.execute() + if rep.skipped and evalxfail.istrue(): + expl = evalxfail.getexplanation() + if not evalxfail.get("run", True): + expl = "[NOTRUN] " + expl + rep.keywords['xfail'] = expl + return rep + elif call.when == "call": + rep = __multicall__.execute() + if not item.config.getvalue("runxfail") and evalxfail.istrue(): + if call.excinfo: + rep.skipped = True + rep.failed = rep.passed = False + else: + rep.skipped = rep.passed = False + rep.failed = True + rep.keywords['xfail'] = evalxfail.getexplanation() else: - rep.skipped = rep.passed = False - rep.failed = True - rep.keywords['xfail'] = expr - else: - if 'xfail' in rep.keywords: - del rep.keywords['xfail'] - return rep + if 'xfail' in rep.keywords: + del rep.keywords['xfail'] + return rep # called by terminalreporter progress reporting def pytest_report_teststatus(report): @@ -189,7 +241,7 @@ if report.skipped: return "xfailed", "x", "xfail" elif report.failed: - return "xpassed", "P", "xpass" + return "xpassed", "X", "XPASS" # called by the terminalreporter instance/plugin def pytest_terminal_summary(terminalreporter): @@ -229,13 +281,8 @@ xfailed = terminalreporter.stats.get("xfailed") if xfailed: for rep in xfailed: - entry = rep.longrepr.reprcrash - modpath = rep.item.getmodpath(includemodule=True) - pos = "%s %s:%d: " %(modpath, entry.path, entry.lineno) - reason = rep.longrepr.reprcrash.message - i = reason.find("\n") - if i != -1: - reason = reason[:i] + pos = terminalreporter.gettestid(rep.item) + reason = rep.keywords['xfail'] lines.append("XFAIL %s %s" %(pos, reason)) def show_xpassed(terminalreporter, lines): @@ -246,24 +293,6 @@ reason = rep.keywords['xfail'] lines.append("XPASS %s %s" %(pos, reason)) - -def evalexpression(item, keyword): - if isinstance(item, py.test.collect.Function): - markholder = getattr(item.obj, keyword, None) - result = False - if markholder: - d = {'os': py.std.os, 'sys': py.std.sys, 'config': item.config} - expr, result = None, True - for expr in markholder.args: - if isinstance(expr, str): - result = cached_eval(item.config, expr, d) - else: - result = expr - if not result: - break - return expr, result - return None, False - def cached_eval(config, expr, d): if not hasattr(config, '_evalcache'): config._evalcache = {} Modified: pypy/trunk/py/_plugin/pytest_terminal.py ============================================================================== --- pypy/trunk/py/_plugin/pytest_terminal.py (original) +++ pypy/trunk/py/_plugin/pytest_terminal.py Sat May 22 13:29:12 2010 @@ -141,6 +141,20 @@ else: return "???", dict(red=True) + def gettestid(self, item, relative=True): + fspath = item.fspath + chain = [x for x in item.listchain() if x.fspath == fspath] + chain = chain[1:] + names = [x.name for x in chain if x.name != "()"] + path = item.fspath + if relative: + relpath = path.relto(self.curdir) + if relpath: + path = relpath + names.insert(0, str(path)) + return "::".join(names) + + def pytest_internalerror(self, excrepr): for line in str(excrepr).split("\n"): self.write_line("INTERNALERROR> " + line) Modified: pypy/trunk/py/_test/cmdline.py ============================================================================== --- pypy/trunk/py/_test/cmdline.py (original) +++ pypy/trunk/py/_test/cmdline.py Sat May 22 13:29:12 2010 @@ -16,8 +16,9 @@ colitems = config.getinitialnodes() exitstatus = session.main(colitems) config.pluginmanager.do_unconfigure(config) - raise SystemExit(exitstatus) except config.Error: e = sys.exc_info()[1] sys.stderr.write("ERROR: %s\n" %(e.args[0],)) - raise SystemExit(3) + exitstatus = 3 + py.test.config = py.test.config.__class__() + return exitstatus Modified: pypy/trunk/py/_test/pycollect.py ============================================================================== --- pypy/trunk/py/_test/pycollect.py (original) +++ pypy/trunk/py/_test/pycollect.py Sat May 22 13:29:12 2010 @@ -393,5 +393,5 @@ def hasinit(obj): init = getattr(obj, '__init__', None) if init: - if not isinstance(init, type(object.__init__)): + if init != object.__init__: return True Modified: pypy/trunk/pypy/conftest.py ============================================================================== --- pypy/trunk/pypy/conftest.py (original) +++ pypy/trunk/pypy/conftest.py Sat May 22 13:29:12 2010 @@ -383,10 +383,15 @@ if option.runappdirect: return target() space = gettestobjspace() - func = app2interp_temp(target) + filename = self._getdynfilename(target) + func = app2interp_temp(target, filename=filename) print "executing", func self.execute_appex(space, func, space) + def _getdynfilename(self, func): + code = getattr(func, 'im_func', func).func_code + return "[%s:%s]" % (code.co_filename, code.co_firstlineno) + class AppTestMethod(AppTestFunction): def setup(self): @@ -410,7 +415,8 @@ if option.runappdirect: return target() space = target.im_self.space - func = app2interp_temp(target.im_func) + filename = self._getdynfilename(target) + func = app2interp_temp(target.im_func, filename=filename) w_instance = self.parent.w_instance self.execute_appex(space, func, space, w_instance) Modified: pypy/trunk/pypy/interpreter/gateway.py ============================================================================== --- pypy/trunk/pypy/interpreter/gateway.py (original) +++ pypy/trunk/pypy/interpreter/gateway.py Sat May 22 13:29:12 2010 @@ -821,9 +821,6 @@ # and now for something completely different ... # -class MyStr(str): - pass - class ApplevelClass: """NOT_RPYTHON A container for app-level source code that should be executed @@ -837,8 +834,7 @@ # HAAACK (but a good one) if filename is None: f = sys._getframe(1) - filename = MyStr('<%s:%d>' % (f.f_code.co_filename, f.f_lineno)) - filename.__source__ = py.code.Source(source) + filename = '<%s:%d>' % (f.f_code.co_filename, f.f_lineno) self.filename = filename self.source = str(py.code.Source(source).deindent()) self.modname = modname @@ -848,6 +844,9 @@ self.can_use_geninterp = False else: self.can_use_geninterp = True + # make source code available for tracebacks + lines = [x + "\n" for x in source.split("\n")] + py.std.linecache.cache[filename] = (1, None, lines, filename) def __repr__(self): return "" % (self.filename, self.can_use_geninterp) @@ -1081,14 +1080,14 @@ # ____________________________________________________________ -def appdef(source, applevel=ApplevelClass): +def appdef(source, applevel=ApplevelClass, filename=None): """ NOT_RPYTHON: build an app-level helper function, like for example: myfunc = appdef('''myfunc(x, y): return x+y ''') """ if not isinstance(source, str): - source = str(py.code.Source(source).strip()) + source = py.std.inspect.getsource(source).lstrip() while source.startswith('@py.test.mark.'): # these decorators are known to return the same function # object, we may ignore them @@ -1100,7 +1099,11 @@ assert p >= 0 funcname = source[:p].strip() source = source[p:] - return applevel("def %s%s\n" % (funcname, source)).interphook(funcname) + assert source.strip() + funcsource = "def %s%s\n" % (funcname, source) + #for debugging of wrong source code: py.std.parser.suite(funcsource) + a = applevel(funcsource, filename=filename) + return a.interphook(funcname) applevel = ApplevelClass # backward compatibility app2interp = appdef # backward compatibility @@ -1118,6 +1121,6 @@ return PyPyCacheDir.build_applevelinterp_dict(self, space) # app2interp_temp is used for testing mainly -def app2interp_temp(func, applevel_temp=applevel_temp): +def app2interp_temp(func, applevel_temp=applevel_temp, filename=None): """ NOT_RPYTHON """ - return appdef(func, applevel_temp) + return appdef(func, applevel_temp, filename=filename) Modified: pypy/trunk/pypy/interpreter/test/test_code.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_code.py (original) +++ pypy/trunk/pypy/interpreter/test/test_code.py Sat May 22 13:29:12 2010 @@ -6,11 +6,7 @@ def setup_class(cls): space = gettestobjspace() cls.space = space - if py.test.config.option.runappdirect: - filename = __file__ - else: - filename = gateway.__file__ - + filename = __file__ if filename[-3:] != '.py': filename = filename[:-1] Modified: pypy/trunk/pypy/jit/metainterp/test/test_warmspot.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_warmspot.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_warmspot.py Sat May 22 13:29:12 2010 @@ -131,7 +131,7 @@ assert warmrunnerdescr.state.optimize_loop is optimize.optimize_loop assert warmrunnerdescr.state.optimize_bridge is optimize.optimize_bridge - def test_static_debug_level(self): + def test_static_debug_level(self, capfd): py.test.skip("debug_level is being deprecated") from pypy.rlib.jit import DEBUG_PROFILE, DEBUG_OFF, DEBUG_STEPS from pypy.jit.metainterp.jitprof import EmptyProfiler, Profiler @@ -144,36 +144,31 @@ n -= 1 return n - outerr = py.io.StdCaptureFD() + capfd.readouterr() self.meta_interp(f, [10], debug_level=DEBUG_OFF, ProfilerClass=Profiler) - out, errf = outerr.done() - err = errf.read() + out, err = capfd.readouterr() assert not 'ENTER' in err assert not 'LEAVE' in err assert not "Running asm" in err - outerr = py.io.StdCaptureFD() self.meta_interp(f, [10], debug_level=DEBUG_PROFILE, ProfilerClass=Profiler) - out, errf = outerr.done() - err = errf.read() + out, err = capfd.readouterr() assert not 'ENTER' in err assert not 'LEAVE' in err assert not 'compiled new' in err assert "Running asm" in err - outerr = py.io.StdCaptureFD() + self.meta_interp(f, [10], debug_level=DEBUG_STEPS, ProfilerClass=Profiler) - out, errf = outerr.done() - err = errf.read() + out, err = capfd.readouterr() assert 'ENTER' in err assert 'LEAVE' in err assert "Running asm" in err - outerr = py.io.StdCaptureFD() + self.meta_interp(f, [10], debug_level=DEBUG_STEPS, ProfilerClass=EmptyProfiler) - out, errf = outerr.done() - err = errf.read() + out, err = capfd.readouterr() assert 'ENTER' in err assert 'LEAVE' in err assert not "Running asm" in err Modified: pypy/trunk/pypy/tool/pytest/appsupport.py ============================================================================== --- pypy/trunk/pypy/tool/pytest/appsupport.py (original) +++ pypy/trunk/pypy/tool/pytest/appsupport.py Sat May 22 13:29:12 2010 @@ -11,7 +11,8 @@ self.raw = pycode self.w_file = space.getattr(pycode, space.wrap('co_filename')) self.name = space.getattr(pycode, space.wrap('co_name')) - self.firstlineno = space.unwrap(space.getattr(pycode, space.wrap('co_firstlineno'))) + self.firstlineno = space.unwrap( + space.getattr(pycode, space.wrap('co_firstlineno'))) - 1 #try: # self.path = space.unwrap(space.getattr(self.w_file, space.wrap('__path__'))) #except OperationError: @@ -20,13 +21,14 @@ self.space = space def fullsource(self): + filename = self.space.str_w(self.w_file) + source = py.code.Source(py.std.linecache.getlines(filename)) + if source.lines: + return source try: - return self.space.str_w(self.w_file).__source__ - except AttributeError: - try: - return py.code.Source(self.path.read(mode="rU")) - except py.error.Error: - return None + return py.code.Source(self.path.read(mode="rU")) + except py.error.Error: + return None fullsource = property(fullsource, None, None, "Full source of AppCode") def getargs(self): @@ -211,12 +213,19 @@ source = py.code.Source(expr) frame = space.getexecutioncontext().gettopframe() w_locals = frame.getdictscope() + pycode = frame.pycode + filename = "<%s:%s>" %(pycode.co_filename, frame.f_lineno) + lines = [x + "\n" for x in expr.split("\n")] + py.std.linecache.cache[filename] = (1, None, lines, filename) w_locals = space.call_method(w_locals, 'copy') for key, w_value in kwds_w.items(): space.setitem(w_locals, space.wrap(key), w_value) + #filename = __file__ + #if filename.endswith("pyc"): + # filename = filename[:-1] try: space.exec_(str(source), frame.w_globals, w_locals, - filename=__file__) + filename=filename) except OperationError, e: if e.match(space, w_ExpectedException): return _exc_info(space, e) Modified: pypy/trunk/pypy/tool/pytest/test/test_appsupport.py ============================================================================== --- pypy/trunk/pypy/tool/pytest/test/test_appsupport.py (original) +++ pypy/trunk/pypy/tool/pytest/test/test_appsupport.py Sat May 22 13:29:12 2010 @@ -70,6 +70,49 @@ assert result.ret == 0 result.stdout.fnmatch_lines(["*2 passed*"]) +def test_applevel_raises_simple_display(testdir): + setpypyconftest(testdir) + p = testdir.makepyfile(""" + def app_test_raises(): + raises(ValueError, x) + class AppTestRaises: + def test_func(self): + raises (ValueError, x) + # + """) + result = testdir.runpytest(p, "-s") + assert result.ret == 1 + result.stdout.fnmatch_lines([ + "*E*application-level*NameError*x*not defined", + "*test_func(self)*", + ">*raises*ValueError*", + "*E*application-level*NameError*x*not defined", + "*test_applevel_raises_simple_display*", + ]) + result = testdir.runpytest(p) # this time we may run the pyc file + assert result.ret == 1 + result.stdout.fnmatch_lines([ + "*E*application-level*NameError*x*not defined", + ]) + +def test_applevel_raises_display(testdir): + setpypyconftest(testdir) + p = testdir.makepyfile(""" + def app_test_raises(): + raises(ValueError, "x") + pass + """) + result = testdir.runpytest(p, "-s") + assert result.ret == 1 + result.stdout.fnmatch_lines([ + "*E*application-level*NameError*x*not defined", + ]) + result = testdir.runpytest(p) # this time we may run the pyc file + assert result.ret == 1 + result.stdout.fnmatch_lines([ + "*E*application-level*NameError*x*not defined", + ]) + def app_test_raises(): info = raises(TypeError, id) assert info.type is TypeError From hpk at codespeak.net Sat May 22 13:41:45 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sat, 22 May 2010 13:41:45 +0200 (CEST) Subject: [pypy-svn] r74677 - pypy/branch/py131 Message-ID: <20100522114145.AA07036C53B@codespeak.net> Author: hpk Date: Sat May 22 13:41:44 2010 New Revision: 74677 Removed: pypy/branch/py131/ Log: remove merged branch From afa at codespeak.net Sat May 22 15:24:10 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 22 May 2010 15:24:10 +0200 (CEST) Subject: [pypy-svn] r74678 - in pypy/trunk/pypy/module/cpyext: . include test Message-ID: <20100522132410.1237A36C227@codespeak.net> Author: afa Date: Sat May 22 15:24:08 2010 New Revision: 74678 Modified: pypy/trunk/pypy/module/cpyext/include/methodobject.h pypy/trunk/pypy/module/cpyext/methodobject.py pypy/trunk/pypy/module/cpyext/modsupport.py pypy/trunk/pypy/module/cpyext/test/test_cpyext.py pypy/trunk/pypy/module/cpyext/typeobject.py Log: Fix the signature of PyCFunction_NewEx, and expose it. Modified: pypy/trunk/pypy/module/cpyext/include/methodobject.h ============================================================================== --- pypy/trunk/pypy/module/cpyext/include/methodobject.h (original) +++ pypy/trunk/pypy/module/cpyext/include/methodobject.h Sat May 22 15:24:08 2010 @@ -49,6 +49,8 @@ #define METH_COEXIST 0x0040 +#define PyCFunction_New(ml, self) PyCFunction_NewEx((ml), (self), NULL) + /* Macros for direct access to these values. Type checks are *not* done, so use with care. */ #define PyCFunction_GET_FUNCTION(func) \ Modified: pypy/trunk/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/methodobject.py (original) +++ pypy/trunk/pypy/module/cpyext/methodobject.py Sat May 22 15:24:08 2010 @@ -1,5 +1,5 @@ from pypy.interpreter.baseobjspace import Wrappable, W_Root -from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.gateway import ObjSpace, W_Root from pypy.interpreter.argument import Arguments from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w @@ -57,10 +57,10 @@ PyObject_dealloc(space, py_obj) class W_PyCFunctionObject(Wrappable): - def __init__(self, space, ml, w_self, doc=None): + def __init__(self, space, ml, w_self, w_module=None): self.ml = ml self.w_self = w_self - self.doc = doc + self.w_module = w_module def call(self, space, w_self, w_args, w_kw): # Call the C function @@ -100,6 +100,9 @@ w_arg = w_args return generic_cpy_call(space, self.ml.c_ml_meth, w_self, w_arg) + def get_doc(space, self): + return space.wrap(rffi.charp2str(self.ml.c_ml_doc)) + class W_PyCMethodObject(W_PyCFunctionObject): w_self = None @@ -192,7 +195,8 @@ W_PyCFunctionObject.typedef = TypeDef( 'builtin_function_or_method', __call__ = interp2app(cfunction_descr_call), - __doc__ = interp_attrproperty('doc', cls=W_PyCFunctionObject), + __doc__ = GetSetProperty(W_PyCFunctionObject.get_doc), + __module__ = interp_attrproperty_w('w_module', cls=W_PyCFunctionObject), ) W_PyCFunctionObject.typedef.acceptable_as_base_class = False @@ -220,9 +224,9 @@ W_PyCWrapperObject.typedef.acceptable_as_base_class = False -def PyCFunction_NewEx(space, ml, w_self): # not exactly the API sig - return space.wrap(W_PyCFunctionObject(space, ml, w_self)) - + at cpython_api([lltype.Ptr(PyMethodDef), PyObject, PyObject], PyObject) +def PyCFunction_NewEx(space, ml, w_self, w_name): + return space.wrap(W_PyCFunctionObject(space, ml, w_self, w_name)) def PyDescr_NewMethod(space, w_type, method): return space.wrap(W_PyCMethodObject(space, method, w_type)) @@ -233,7 +237,7 @@ wrapper_func, doc, flags, func)) @cpython_api([lltype.Ptr(PyMethodDef), PyObject, CONST_STRING], PyObject) -def Py_FindMethod(space, table, w_ob, name_ptr): +def Py_FindMethod(space, table, w_obj, name_ptr): """Return a bound method object for an extension type implemented in C. This can be useful in the implementation of a tp_getattro or tp_getattr handler that does not use the @@ -253,7 +257,7 @@ if name == "__methods__": method_list_w.append(space.wrap(rffi.charp2str(method.c_ml_name))) elif rffi.charp2str(method.c_ml_name) == name: # XXX expensive copying - return PyCFunction_NewEx(space, method, w_ob) + return space.wrap(W_PyCFunctionObject(space, method, w_obj)) if name == "__methods__": return space.newlist(method_list_w) raise OperationError(space.w_AttributeError, space.wrap(name)) Modified: pypy/trunk/pypy/module/cpyext/modsupport.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/modsupport.py (original) +++ pypy/trunk/pypy/module/cpyext/modsupport.py Sat May 22 15:24:08 2010 @@ -52,7 +52,7 @@ w_mod = PyImport_AddModule(space, state.package_context) dict_w = {} - convert_method_defs(space, dict_w, methods, None, w_self) + convert_method_defs(space, dict_w, methods, None, w_self, modname) for key, w_value in dict_w.items(): space.setattr(w_mod, space.wrap(key), w_value) if doc: @@ -61,7 +61,8 @@ return borrow_from(None, w_mod) -def convert_method_defs(space, dict_w, methods, w_type, w_self=None): +def convert_method_defs(space, dict_w, methods, w_type, w_self=None, name=None): + w_name = space.wrap(name) methods = rffi.cast(rffi.CArrayPtr(PyMethodDef), methods) if methods: i = -1 @@ -72,16 +73,12 @@ methodname = rffi.charp2str(method.c_ml_name) flags = rffi.cast(lltype.Signed, method.c_ml_flags) - if method.c_ml_doc: - doc = rffi.charp2str(method.c_ml_doc) - else: - doc = None if w_type is None: if flags & METH_CLASS or flags & METH_STATIC: raise OperationError(space.w_ValueError, space.wrap("module functions cannot set METH_CLASS or METH_STATIC")) - w_obj = space.wrap(W_PyCFunctionObject(space, method, w_self, doc)) + w_obj = space.wrap(W_PyCFunctionObject(space, method, w_self, w_name)) else: if methodname in dict_w and not (flags & METH_COEXIST): continue @@ -92,7 +89,7 @@ #w_obj = PyDescr_NewClassMethod(space, w_type, method) w_obj = space.w_Ellipsis # XXX elif flags & METH_STATIC: - w_func = PyCFunction_NewEx(space, method, None) + w_func = PyCFunction_NewEx(space, method, None, None) w_obj = space.w_Ellipsis # XXX #w_obj = PyStaticMethod_New(space, w_func) else: Modified: pypy/trunk/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Sat May 22 15:24:08 2010 @@ -298,6 +298,7 @@ assert 'return_pi' in dir(module) assert module.return_pi is not None assert module.return_pi() == 3.14 + assert module.return_pi.__module__ == 'foo' def test_export_docstring(self): Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Sat May 22 15:24:08 2010 @@ -23,8 +23,8 @@ from pypy.module.cpyext import structmemberdefs from pypy.module.cpyext.modsupport import convert_method_defs, PyCFunction from pypy.module.cpyext.state import State -from pypy.module.cpyext.methodobject import PyDescr_NewWrapper, \ - PyCFunction_NewEx +from pypy.module.cpyext.methodobject import ( + PyDescr_NewWrapper, PyCFunction_NewEx) from pypy.module.cpyext.pyobject import Py_IncRef, Py_DecRef, _Py_Dealloc from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr, PyTypeObject, \ @@ -199,7 +199,7 @@ return pyo = rffi.cast(PyObject, pto) dict_w["__new__"] = PyCFunction_NewEx(space, get_new_method_def(space), - from_ref(space, pyo)) + from_ref(space, pyo), None) def inherit_special(space, pto, base_pto): # XXX missing: copy basicsize and flags in a magical way From hpk at codespeak.net Sat May 22 17:20:40 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sat, 22 May 2010 17:20:40 +0200 (CEST) Subject: [pypy-svn] r74680 - in pypy/trunk/py: . _code _plugin _test Message-ID: <20100522152040.B2204282BAD@codespeak.net> Author: hpk Date: Sat May 22 17:20:39 2010 New Revision: 74680 Modified: pypy/trunk/py/__init__.py pypy/trunk/py/_code/code.py pypy/trunk/py/_plugin/pytest_mark.py pypy/trunk/py/_plugin/pytest_runner.py pypy/trunk/py/_plugin/pytest_skipping.py pypy/trunk/py/_plugin/pytest_terminal.py pypy/trunk/py/_test/collect.py pypy/trunk/py/_test/pycollect.py Log: update to py-1.3.1 release candidate Modified: pypy/trunk/py/__init__.py ============================================================================== --- pypy/trunk/py/__init__.py (original) +++ pypy/trunk/py/__init__.py Sat May 22 17:20:39 2010 @@ -8,7 +8,7 @@ (c) Holger Krekel and others, 2004-2010 """ -__version__ = version = "1.3.1a1" +__version__ = version = "1.3.1" import py.apipkg Modified: pypy/trunk/py/_code/code.py ============================================================================== --- pypy/trunk/py/_code/code.py (original) +++ pypy/trunk/py/_code/code.py Sat May 22 17:20:39 2010 @@ -416,7 +416,7 @@ args.append((argname, self._saferepr(argvalue))) return ReprFuncArgs(args) - def get_source(self, source, line_index=-1, excinfo=None): + def get_source(self, source, line_index=-1, excinfo=None, short=False): """ return formatted and marked up source lines. """ lines = [] if source is None: @@ -428,6 +428,8 @@ if i == line_index: prefix = self.flow_marker + " " else: + if short: + continue prefix = " " line = prefix + source[i] lines.append(line) @@ -482,24 +484,26 @@ line_index = entry.lineno - max(entry.getfirstlinesource(), 0) lines = [] - if self.style == "long": - reprargs = self.repr_args(entry) - lines.extend(self.get_source(source, line_index, excinfo)) - message = excinfo and excinfo.typename or "" + if self.style in ("short", "long"): + short = self.style == "short" + reprargs = None + if not short: + reprargs = self.repr_args(entry) + s = self.get_source(source, line_index, excinfo, short=short) + lines.extend(s) + if short: + message = "in %s" %(entry.name) + else: + message = excinfo and excinfo.typename or "" path = self._makepath(entry.path) filelocrepr = ReprFileLocation(path, entry.lineno+1, message) - localsrepr = self.repr_locals(entry.locals) - return ReprEntry(lines, reprargs, localsrepr, filelocrepr) - else: - if self.style == "short": - line = source[line_index].lstrip() - basename = os.path.basename(entry.frame.code.filename) - lines.append(' File "%s", line %d, in %s' % ( - basename, entry.lineno+1, entry.name)) - lines.append(" " + line) - if excinfo: - lines.extend(self.get_exconly(excinfo, indent=4)) - return ReprEntry(lines, None, None, None) + localsrepr = None + if not short: + localsrepr = self.repr_locals(entry.locals) + return ReprEntry(lines, reprargs, localsrepr, filelocrepr, short) + if excinfo: + lines.extend(self.get_exconly(excinfo, indent=4)) + return ReprEntry(lines, None, None, None, False) def _makepath(self, path): if not self.abspath: @@ -595,13 +599,21 @@ class ReprEntry(TerminalRepr): localssep = "_ " - def __init__(self, lines, reprfuncargs, reprlocals, filelocrepr): + def __init__(self, lines, reprfuncargs, reprlocals, filelocrepr, short): self.lines = lines self.reprfuncargs = reprfuncargs self.reprlocals = reprlocals self.reprfileloc = filelocrepr + self.short = short def toterminal(self, tw): + if self.short: + self.reprfileloc.toterminal(tw) + for line in self.lines: + red = line.startswith("E ") + tw.line(line, bold=True, red=red) + #tw.line("") + return if self.reprfuncargs: self.reprfuncargs.toterminal(tw) for line in self.lines: Modified: pypy/trunk/py/_plugin/pytest_mark.py ============================================================================== --- pypy/trunk/py/_plugin/pytest_mark.py (original) +++ pypy/trunk/py/_plugin/pytest_mark.py Sat May 22 17:20:39 2010 @@ -44,11 +44,13 @@ class TestClass: def test_startup(self): ... + def test_startup_and_more(self): + ... This is equivalent to directly applying the decorator to the -``test_startup`` function. +two test functions. -To remain compatible with Python2.5 you can instead set a +To remain compatible with Python2.5 you can also set a ``pytestmark`` attribute on a TestClass like this:: import py @@ -56,7 +58,7 @@ class TestClass: pytestmark = py.test.mark.webtest -or if you need to use multiple markers:: +or if you need to use multiple markers you can use a list:: import py @@ -68,7 +70,7 @@ import py pytestmark = py.test.mark.webtest -in which case then it will be applied to all functions and +in which case it will be applied to all functions and methods defined in the module. Using "-k MARKNAME" to select tests @@ -114,11 +116,14 @@ if len(args) == 1 and hasattr(func, '__call__') or \ hasattr(func, '__bases__'): if hasattr(func, '__bases__'): - l = func.__dict__.setdefault("pytestmark", []) - if not isinstance(l, list): - func.pytestmark = [l, self] - else: - l.append(self) + if hasattr(func, 'pytestmark'): + l = func.pytestmark + if not isinstance(l, list): + func.pytestmark = [l, self] + else: + l.append(self) + else: + func.pytestmark = [self] else: holder = getattr(func, self.markname, None) if holder is None: Modified: pypy/trunk/py/_plugin/pytest_runner.py ============================================================================== --- pypy/trunk/py/_plugin/pytest_runner.py (original) +++ pypy/trunk/py/_plugin/pytest_runner.py Sat May 22 17:20:39 2010 @@ -188,7 +188,11 @@ self.passed = True self.result = result else: - self.longrepr = self.collector._repr_failure_py(excinfo) + style = "short" + if collector.config.getvalue("fulltrace"): + style = "long" + self.longrepr = self.collector._repr_failure_py(excinfo, + style=style) if excinfo.errisinstance(py.test.skip.Exception): self.skipped = True self.reason = str(excinfo.value) Modified: pypy/trunk/py/_plugin/pytest_skipping.py ============================================================================== --- pypy/trunk/py/_plugin/pytest_skipping.py (original) +++ pypy/trunk/py/_plugin/pytest_skipping.py Sat May 22 17:20:39 2010 @@ -106,6 +106,17 @@ @py.test.mark.xfail(..., reason="my reason") +imperative xfail from within a test or setup function +------------------------------------------------------ + +If you cannot declare xfail-conditions at import time +you can also imperatively produce an XFail-outcome from +within test or setup code. Example:: + + def test_function(): + if not valid_config(): + py.test.xfail("unsuppored configuration") + skipping on a missing import dependency -------------------------------------------------- @@ -207,11 +218,12 @@ if not evalxfail: return if call.excinfo and call.excinfo.errisinstance(py.test.xfail.Exception): - rep = __multicall__.execute() - rep.keywords['xfail'] = "reason: " + call.excinfo.value.msg - rep.skipped = True - rep.failed = False - return rep + if not item.config.getvalue("runxfail"): + rep = __multicall__.execute() + rep.keywords['xfail'] = "reason: " + call.excinfo.value.msg + rep.skipped = True + rep.failed = False + return rep if call.when == "setup": rep = __multicall__.execute() if rep.skipped and evalxfail.istrue(): Modified: pypy/trunk/py/_plugin/pytest_terminal.py ============================================================================== --- pypy/trunk/py/_plugin/pytest_terminal.py (original) +++ pypy/trunk/py/_plugin/pytest_terminal.py Sat May 22 17:20:39 2010 @@ -284,7 +284,7 @@ self._sessionstarttime = py.std.time.time() verinfo = ".".join(map(str, sys.version_info[:3])) - msg = "python: platform %s -- Python %s" % (sys.platform, verinfo) + msg = "platform %s -- Python %s" % (sys.platform, verinfo) msg += " -- pytest-%s" % (py.__version__) if self.config.option.verbose or self.config.option.debug or getattr(self.config.option, 'pastebin', None): msg += " -- " + str(sys.executable) Modified: pypy/trunk/py/_test/collect.py ============================================================================== --- pypy/trunk/py/_test/collect.py (original) +++ pypy/trunk/py/_test/collect.py Sat May 22 17:20:39 2010 @@ -172,14 +172,15 @@ def _prunetraceback(self, traceback): return traceback - def _repr_failure_py(self, excinfo): + def _repr_failure_py(self, excinfo, style=None): excinfo.traceback = self._prunetraceback(excinfo.traceback) # XXX should excinfo.getrepr record all data and toterminal() # process it? - if self.config.option.tbstyle == "short": - style = "short" - else: - style = "long" + if style is None: + if self.config.option.tbstyle == "short": + style = "short" + else: + style = "long" return excinfo.getrepr(funcargs=True, showlocals=self.config.option.showlocals, style=style) Modified: pypy/trunk/py/_test/pycollect.py ============================================================================== --- pypy/trunk/py/_test/pycollect.py (original) +++ pypy/trunk/py/_test/pycollect.py Sat May 22 17:20:39 2010 @@ -253,7 +253,7 @@ traceback = ntraceback.filter() return traceback - def _repr_failure_py(self, excinfo): + def _repr_failure_py(self, excinfo, style="long"): if excinfo.errisinstance(funcargs.FuncargRequest.LookupError): fspath, lineno, msg = self.reportinfo() lines, _ = inspect.getsourcelines(self.obj) @@ -261,11 +261,13 @@ if line.strip().startswith('def'): return FuncargLookupErrorRepr(fspath, lineno, lines[:i+1], str(excinfo.value)) - return super(FunctionMixin, self)._repr_failure_py(excinfo) + return super(FunctionMixin, self)._repr_failure_py(excinfo, + style=style) def repr_failure(self, excinfo, outerr=None): assert outerr is None, "XXX outerr usage is deprecated" - return self._repr_failure_py(excinfo) + return self._repr_failure_py(excinfo, + style=self.config.getvalue("tbstyle")) shortfailurerepr = "F" From hpk at codespeak.net Sat May 22 18:13:38 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sat, 22 May 2010 18:13:38 +0200 (CEST) Subject: [pypy-svn] r74681 - pypy/trunk/lib-python Message-ID: <20100522161338.7C2B2282BAD@codespeak.net> Author: hpk Date: Sat May 22 18:13:36 2010 New Revision: 74681 Modified: pypy/trunk/lib-python/conftest.py Log: import test.pystone only when needed as otherwise globally importable 'test' can get in the way in some environments. Modified: pypy/trunk/lib-python/conftest.py ============================================================================== --- pypy/trunk/lib-python/conftest.py (original) +++ pypy/trunk/lib-python/conftest.py Sat May 22 18:13:36 2010 @@ -42,6 +42,7 @@ from test import pystone timeout = option.timeout.lower() if timeout.endswith('mp'): + from test import pystone megapystone = float(timeout[:-2]) t, stone = pystone.Proc0(10000) pystonetime = t/stone From hpk at codespeak.net Sat May 22 18:20:40 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sat, 22 May 2010 18:20:40 +0200 (CEST) Subject: [pypy-svn] r74682 - pypy/trunk/pypy/module/cpyext/test Message-ID: <20100522162040.7C86D282BAD@codespeak.net> Author: hpk Date: Sat May 22 18:20:39 2010 New Revision: 74682 Modified: pypy/trunk/pypy/module/cpyext/test/conftest.py Log: don't use deprecated "class Directory" Modified: pypy/trunk/pypy/module/cpyext/test/conftest.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/conftest.py (original) +++ pypy/trunk/pypy/module/cpyext/test/conftest.py Sat May 22 18:20:39 2010 @@ -1,15 +1,12 @@ import py from pypy.conftest import option, gettestobjspace -class Directory(py.test.collect.Directory): - def collect(self): - if option.runappdirect: - py.test.skip("cannot be run by py.test -A") +def pytest_collect_directory(parent): + if parent.config.option.runappdirect: + py.test.skip("cannot be run by py.test -A") - # ensure additional functions are registered - import pypy.module.cpyext.test.test_cpyext - - return super(Directory, self).collect() + # ensure additional functions are registered + import pypy.module.cpyext.test.test_cpyext def pytest_funcarg__space(request): return gettestobjspace(usemodules=['cpyext', 'thread']) From hpk at codespeak.net Sat May 22 22:30:43 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sat, 22 May 2010 22:30:43 +0200 (CEST) Subject: [pypy-svn] r74684 - pypy/trunk/lib-python Message-ID: <20100522203043.387E6282BAD@codespeak.net> Author: hpk Date: Sat May 22 22:30:41 2010 New Revision: 74684 Modified: pypy/trunk/lib-python/conftest.py Log: forgot to svn-up, this line is redundant Modified: pypy/trunk/lib-python/conftest.py ============================================================================== --- pypy/trunk/lib-python/conftest.py (original) +++ pypy/trunk/lib-python/conftest.py Sat May 22 22:30:41 2010 @@ -42,7 +42,6 @@ from test import pystone timeout = option.timeout.lower() if timeout.endswith('mp'): - from test import pystone megapystone = float(timeout[:-2]) t, stone = pystone.Proc0(10000) pystonetime = t/stone From benjamin at codespeak.net Mon May 24 00:55:52 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 24 May 2010 00:55:52 +0200 (CEST) Subject: [pypy-svn] r74702 - pypy/trunk/pypy/rlib Message-ID: <20100523225552.9BA51282BD8@codespeak.net> Author: benjamin Date: Mon May 24 00:55:51 2010 New Revision: 74702 Modified: pypy/trunk/pypy/rlib/rweakref.py Log: ref isn't used Modified: pypy/trunk/pypy/rlib/rweakref.py ============================================================================== --- pypy/trunk/pypy/rlib/rweakref.py (original) +++ pypy/trunk/pypy/rlib/rweakref.py Mon May 24 00:55:51 2010 @@ -5,7 +5,6 @@ """ import weakref -from weakref import ref class RWeakValueDictionary(object): From arigo at codespeak.net Mon May 24 14:03:24 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 24 May 2010 14:03:24 +0200 (CEST) Subject: [pypy-svn] r74704 - pypy/extradoc/planning Message-ID: <20100524120324.497F2282BDB@codespeak.net> Author: arigo Date: Mon May 24 14:03:19 2010 New Revision: 74704 Modified: pypy/extradoc/planning/jit.txt Log: Add a task. Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Mon May 24 14:03:19 2010 @@ -24,6 +24,10 @@ that are never changed, but read from (an into which is a loop constant will stay as a getfield). +- kill GUARD_(NO)_EXCEPTION; replace that by LAST_EXC_VALUE to load the + current exception from the struct in memory, followed by a regular + GUARD_CLASS. + TASKS ----- From arigo at codespeak.net Mon May 24 14:23:50 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 24 May 2010 14:23:50 +0200 (CEST) Subject: [pypy-svn] r74705 - pypy/branch/blackhole-improvement/pypy/rpython Message-ID: <20100524122350.C287E282BDB@codespeak.net> Author: arigo Date: Mon May 24 14:23:49 2010 New Revision: 74705 Modified: pypy/branch/blackhole-improvement/pypy/rpython/llinterp.py Log: Fix printing the traceback from the llinterp. Modified: pypy/branch/blackhole-improvement/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/llinterp.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/llinterp.py Mon May 24 14:23:49 2010 @@ -110,16 +110,14 @@ self.traceback_frames = [] lines = [] for frame in frames: - logline = frame.graph.name + logline = frame.graph.name + "()" if frame.curr_block is None: logline += " " lines.append(logline) continue try: - logline += " " + self.typer.annotator.annotated[frame.curr_block].__module__ + logline += " " + self.typer.annotator.annotated[frame.curr_block].func.__module__ except (KeyError, AttributeError): - # if the graph is from the GC it was not produced by the same - # translator :-( logline += " " lines.append(logline) for i, operation in enumerate(frame.curr_block.operations): From arigo at codespeak.net Mon May 24 15:37:30 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 24 May 2010 15:37:30 +0200 (CEST) Subject: [pypy-svn] r74706 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter codewriter/test metainterp metainterp/test Message-ID: <20100524133730.67F65282BDB@codespeak.net> Author: arigo Date: Mon May 24 15:37:28 2010 New Revision: 74706 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_exception.py Log: First set of fixes to exception handling. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Mon May 24 15:37:28 2010 @@ -128,6 +128,8 @@ self.emitline("%s_return" % kind, self.getcolor(args[0])) elif len(args) == 2: # exception block, raising an exception from a function + if isinstance(args[1], Variable): + self.emitline("-live-") # xxx hack self.emitline("raise", self.getcolor(args[1])) else: raise Exception("?") Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Mon May 24 15:37:28 2010 @@ -115,6 +115,7 @@ def encoding_test(self, func, args, expected, transform=False, liveness=False, cc=None): graphs = self.make_graphs(func, args) + #graphs[0].show() if transform: from pypy.jit.codewriter.jtransform import transform_graph cc = cc or FakeCallControl() @@ -382,6 +383,37 @@ raise $<* struct object> """) + def test_exc_finally(self): + def get_exception(n): + if n > 5: + raise ValueError + class Foo: + pass + Foo.__module__ = "test" + foo = Foo() + def f(i): + try: + get_exception(i) + finally: + foo.sideeffect = 5 + + self.encoding_test(f, [65], """ + residual_call_ir_v $<* fn get_exception>, , I[%i0], R[] + -live- + catch_exception L1 + setfield_gc_i $<* struct test.Foo>, , $5 + void_return + --- + L1: + last_exception -> %i1 + last_exc_value -> %r0 + int_copy %i1 -> %i2 + ref_copy %r0 -> %r1 + setfield_gc_i $<* struct test.Foo>, , $5 + -live- + raise %r1 + """, transform=True) + def test_goto_if_not_int_is_true(self): def f(i): return not i Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Mon May 24 15:37:28 2010 @@ -305,8 +305,8 @@ except LeaveFrame: break except Exception, e: - e = get_llexception(self.cpu, e) - self.handle_exception_in_frame(e) + lle = get_llexception(self.cpu, e) + self.handle_exception_in_frame(lle) def get_tmpreg_i(self): return self.tmpreg_i @@ -358,19 +358,20 @@ # the exception is handled in the frame itself. code = self.jitcode.code position = self.position - opcode = ord(code[position]) - if opcode != self.op_catch_exception: - # no 'catch_exception' insn follows: just reraise - if we_are_translated(): - raise Exception, e - else: - etype = rclass.ll_type(e) - raise LLException(etype, e) - else: - # else store the exception on 'self', and jump to the handler - self.exception_last_value = e - target = ord(code[position+1]) | (ord(code[position+2])<<8) - self.position = target + if position < len(code): + opcode = ord(code[position]) + if opcode == self.op_catch_exception: + # store the exception on 'self', and jump to the handler + self.exception_last_value = e + target = ord(code[position+1]) | (ord(code[position+2])<<8) + self.position = target + return + # no 'catch_exception' insn follows: just reraise + if we_are_translated(): + raise Exception, e + else: + etype = rclass.ll_type(e) + raise LLException(etype, e) # XXX must be specialized def copy_constants(self, registers, constants): @@ -761,15 +762,23 @@ @arguments("self", "r") def bhimpl_raise(self, excvalue): - import pdb; pdb.set_trace() - XXX - raise real_instance + e = lltype.cast_opaque_ptr(rclass.OBJECTPTR, excvalue) + assert e + if we_are_translated(): + raise Exception, e + else: + etype = rclass.ll_type(e) + raise LLException(etype, e) @arguments("self") def bhimpl_reraise(self): - real_instance = self.exception_last_value - assert real_instance - raise real_instance + e = self.exception_last_value + assert e + if we_are_translated(): + raise Exception, e + else: + etype = rclass.ll_type(e) + raise LLException(etype, e) @arguments() def bhimpl_can_enter_jit(): @@ -1094,18 +1103,29 @@ opnum == rop.GUARD_ISNULL or opnum == rop.GUARD_NONNULL_CLASS): # Produced by goto_if_not_ptr_{non,is}zero(). The pc is at the - # start of the opcode (so it will be redone). This is needed + # start of the opcode (so it will be redone); this is needed # because of GUARD_NONNULL_CLASS. pass + # elif (opnum == rop.GUARD_NO_EXCEPTION or opnum == rop.GUARD_EXCEPTION or - opnum == rop.GUARD_NOT_FORCED or - opnum == rop.GUARD_NO_OVERFLOW or - opnum == rop.GUARD_OVERFLOW): + opnum == rop.GUARD_NOT_FORCED): + return self.cpu.grab_exc_value() + # + elif opnum == rop.GUARD_NO_OVERFLOW: + # Produced by int_xxx_ovf(). The pc is just after the opcode. + # We get here because it did not used to overflow, but now it does. + return get_llexception(self.cpu, OverflowError()) + # + elif opnum == rop.GUARD_OVERFLOW: + # Produced by int_xxx_ovf(). The pc is just after the opcode. + # We get here because it used to overflow, but now it no longer + # does. pass else: from pypy.jit.metainterp.resoperation import opname raise NotImplementedError(opname[opnum]) + return NULL # connect the return of values from the called frame to the # 'xxx_call_yyy' instructions from the caller frame @@ -1116,9 +1136,6 @@ def _setup_return_value_f(self, result): self.registers_f[ord(self.jitcode.code[self.position-1])] = result - def _exit_frame_with_exception(self, e): - xxx - def _done_with_this_frame(self): # rare case: we only get there if the blackhole interps all returned # normally (in general we get a ContinueRunningNormally exception). @@ -1135,6 +1152,12 @@ else: assert False + def _exit_frame_with_exception(self, e): + # rare case + sd = self.builder.metainterp_sd + e = lltype.cast_opaque_ptr(llmemory.GCREF, e) + raise sd.ExitFrameWithExceptionRef(self.cpu, e) + # ____________________________________________________________ def resume_in_blackhole(metainterp_sd, resumedescr): @@ -1147,9 +1170,9 @@ False) # XXX # XXX virtualrefs # XXX virtualizable - blackholeinterp._prepare_resume_from_failure(resumedescr.guard_opnum) + current_exc = blackholeinterp._prepare_resume_from_failure( + resumedescr.guard_opnum) try: - current_exc = blackholeinterp.cpu.grab_exc_value() current_exc = lltype.cast_opaque_ptr(rclass.OBJECTPTR, current_exc) while True: current_exc = blackholeinterp._resume_mainloop(current_exc) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Mon May 24 15:37:28 2010 @@ -159,6 +159,8 @@ assert oldbox not in registers[count:] def make_result_of_lastop(self, resultbox): + if resultbox is None: + return target_index = ord(self.bytecode[self.pc-1]) if resultbox.type == history.INT: self.registers_i[target_index] = resultbox @@ -194,6 +196,7 @@ @arguments("box", "box") def opimpl_%s(self, b1, b2): resbox = self.execute(rop.%s, b1, b2) + self.make_result_of_lastop(resbox) # same as execute_varargs() self.metainterp.handle_possible_overflow_error() return resbox ''' % (_opimpl, _opimpl.upper())).compile() @@ -680,7 +683,8 @@ @FixME #arguments("descr", "varargs") def opimpl_residual_call_loopinvariant(self, calldescr, varargs): - return self.execute_varargs(rop.CALL_LOOPINVARIANT, varargs, calldescr, exc=True) + return self.execute_varargs(rop.CALL_LOOPINVARIANT, varargs, calldescr, + exc=True) @FixME #arguments("orgpc", "descr", "varargs") def opimpl_recursive_call(self, pc, calldescr, varargs): @@ -767,15 +771,18 @@ @FixME #arguments("descr", "varargs") def opimpl_residual_oosend_canraise(self, methdescr, varargs): - return self.execute_varargs(rop.OOSEND, varargs, descr=methdescr, exc=True) + return self.execute_varargs(rop.OOSEND, varargs, descr=methdescr, + exc=True) @FixME #arguments("descr", "varargs") def opimpl_residual_oosend_noraise(self, methdescr, varargs): - self.execute_varargs(rop.OOSEND, varargs, descr=methdescr, exc=False) + return self.execute_varargs(rop.OOSEND, varargs, descr=methdescr, + exc=False) @FixME #arguments("descr", "varargs") def opimpl_residual_oosend_pure(self, methdescr, boxes): - self.execute_varargs(rop.OOSEND_PURE, boxes, descr=methdescr, exc=False) + return self.execute_varargs(rop.OOSEND_PURE, boxes, descr=methdescr, + exc=False) @arguments("orgpc", "box",) def _opimpl_guard_value(self, orgpc, box): @@ -791,16 +798,6 @@ self.generate_guard(rop.GUARD_CLASS, box, [clsbox], resumepc=orgpc) return clsbox -## @FixME #arguments("orgpc", "box", "builtin") -## def opimpl_guard_builtin(self, pc, box, builtin): -## self.generate_guard(pc, "guard_builtin", box, [builtin]) - -## @FixME #arguments("orgpc", "box", "builtin") -## def opimpl_guard_len(self, pc, box, builtin): -## intbox = self.metainterp.cpu.execute_operation( -## 'len', [builtin.len_func, box], 'int') -## self.generate_guard(pc, "guard_len", box, [intbox]) - def verify_green_args(self, varargs): num_green_args = self.metainterp.staticdata.num_green_args for i in range(num_green_args): @@ -861,11 +858,17 @@ metainterp = self.metainterp last_exc_value_box = metainterp.last_exc_value_box assert last_exc_value_box is not None + assert metainterp.class_of_last_exc_is_const if not metainterp.cpu.ts.instanceOf(last_exc_value_box, vtablebox): self.pc = next_exc_target - @arguments("box") - def opimpl_raise(self, exc_value_box): + @arguments("orgpc", "box") + def opimpl_raise(self, orgpc, exc_value_box): + # xxx hack + clsbox = self.cls_of_box(exc_value_box) + self.generate_guard(rop.GUARD_CLASS, exc_value_box, [clsbox], + resumepc=orgpc) + self.metainterp.class_of_last_exc_is_const = True self.metainterp.last_exc_value_box = exc_value_box self.metainterp.popframe() self.metainterp.finishframe_exception() @@ -876,6 +879,20 @@ self.metainterp.popframe() self.metainterp.finishframe_exception() + @arguments() + def opimpl_last_exception(self): + # Same comment as in opimpl_goto_if_exception_mismatch(). + exc_value_box = self.metainterp.last_exc_value_box + assert exc_value_box is not None + assert self.metainterp.class_of_last_exc_is_const + return self.metainterp.cpu.ts.cls_of_box(exc_value_box) + + @arguments() + def opimpl_last_exc_value(self): + exc_value_box = self.metainterp.last_exc_value_box + assert exc_value_box is not None + return exc_value_box + @FixME #arguments("box") def opimpl_virtual_ref(self, box): # Details on the content of metainterp.virtualref_boxes: @@ -902,10 +919,9 @@ # Note: we allocate a JIT_VIRTUAL_REF here # (in virtual_ref_during_tracing()), in order to detect when # the virtual escapes during tracing already. We record it as a - # VIRTUAL_REF operation, although the backend sees this operation - # as a no-op. The point is that the backend should not really see - # it in practice, as optimizeopt.py should either kill it or - # replace it with a NEW_WITH_VTABLE followed by SETFIELD_GCs. + # VIRTUAL_REF operation. Later, optimizeopt.py should either kill + # that operation or replace it with a NEW_WITH_VTABLE followed by + # SETFIELD_GCs. metainterp.virtualref_boxes.append(box) metainterp.virtualref_boxes.append(resbox) self.make_result_box(resbox) @@ -1027,6 +1043,9 @@ def execute_varargs(self, opnum, argboxes, descr, exc): resbox = self.metainterp.execute_and_record_varargs(opnum, argboxes, descr=descr) + self.make_result_of_lastop(resbox) + # ^^^ this is done before handle_possible_exception() because we need + # the box to show up in get_list_of_active_boxes() if exc: self.metainterp.handle_possible_exception() else: @@ -1259,6 +1278,7 @@ self.portal_trace_positions = [] self.greenkey_of_huge_function = None self.free_frames_list = [] + self.last_exc_value_box = None def is_blackholing(self): return False # XXX get rid of this method @@ -1299,8 +1319,7 @@ def finishframe(self, resultbox): self.popframe() if self.framestack: - if resultbox is not None: - self.framestack[-1].make_result_of_lastop(resultbox) + self.framestack[-1].make_result_of_lastop(resultbox) raise ChangeFrame else: if not self.is_blackholing(): @@ -1335,12 +1354,14 @@ frame = self.framestack[-1] code = frame.bytecode position = frame.pc # <-- just after the insn that raised - opcode = ord(code[position]) - if opcode == self.staticdata.op_catch_exception: - # found a 'catch_exception' instruction; jump to the handler - target = ord(code[position+1]) | (ord(code[position+2])<<8) - frame.pc = target - raise ChangeFrame + if position < len(code): + opcode = ord(code[position]) + if opcode == self.staticdata.op_catch_exception: + # found a 'catch_exception' instruction; + # jump to the handler + target = ord(code[position+1]) | (ord(code[position+2])<<8) + frame.pc = target + raise ChangeFrame self.popframe() if not self.is_blackholing(): try: @@ -1461,6 +1482,10 @@ if constant: exc_value_box = exc_value_box.constbox() self.last_exc_value_box = exc_value_box + self.class_of_last_exc_is_const = constant + # 'class_of_last_exc_is_const' means that the class of the value + # stored in the exc_value Box can be assumed to be a Const. This + # is only True after a GUARD_EXCEPTION or GUARD_CLASS. def execute_did_not_raise(self): self.last_exc_value_box = None @@ -1647,8 +1672,7 @@ opnum == rop.GUARD_ISNULL or opnum == rop.GUARD_NONNULL_CLASS): pass # the pc is already set to the *start* of the opcode - elif (opnum == rop.GUARD_NO_EXCEPTION or opnum == rop.GUARD_EXCEPTION - or opnum == rop.GUARD_NOT_FORCED): + elif opnum == rop.GUARD_NO_EXCEPTION or opnum == rop.GUARD_EXCEPTION: exception = self.cpu.grab_exc_value() if exception: self.execute_raised(exception) @@ -1657,9 +1681,12 @@ self.handle_possible_exception() elif opnum == rop.GUARD_NO_OVERFLOW: # an overflow now detected self.execute_raised(OverflowError(), constant=True) - self.finishframe_exception() + try: + self.finishframe_exception() + except ChangeFrame: + pass elif opnum == rop.GUARD_OVERFLOW: # no longer overflowing - pass + self.execute_did_not_raise() else: from pypy.jit.metainterp.resoperation import opname raise NotImplementedError(opname[opnum]) @@ -1855,27 +1882,29 @@ def handle_possible_exception(self): frame = self.framestack[-1] - if self.last_exc_value_box: + if self.last_exc_value_box is not None: exception_box = self.cpu.ts.cls_of_box(self.last_exc_value_box) op = frame.generate_guard(rop.GUARD_EXCEPTION, None, [exception_box]) - if op: - op.result = self.last_exc_value_box + assert op is not None + op.result = self.last_exc_value_box + self.class_of_last_exc_is_const = True self.finishframe_exception() else: frame.generate_guard(rop.GUARD_NO_EXCEPTION, None, []) def handle_possible_overflow_error(self): frame = self.framestack[-1] - if self.last_exc_value_box: + if self.last_exc_value_box is not None: frame.generate_guard(rop.GUARD_OVERFLOW, None) assert isinstance(self.last_exc_value_box, Const) + assert self.class_of_last_exc_is_const self.finishframe_exception() else: frame.generate_guard(rop.GUARD_NO_OVERFLOW, None) def assert_no_exception(self): - assert not self.last_exc_value_box + assert self.last_exc_value_box is None def rebuild_state_after_failure(self, resumedescr): vinfo = self.staticdata.virtualizable_info @@ -2130,10 +2159,8 @@ num_return_args = len(argcodes) - next_argcode assert num_return_args == 0 or num_return_args == 2 if num_return_args: - # Save the type of the resulting box. This is needed if the - # operation is an inlined call and we need to get the list of - # all alive boxes in it, to know that the result box was not - # written yet. + # Save the type of the resulting box. This is needed if there is + # a get_list_of_active_boxes(). See comments there. self._result_argcode = argcodes[next_argcode + 1] position += 1 else: @@ -2160,8 +2187,7 @@ else: resultbox = unboundmethod(self, *args) # - if resultbox is not None: - self.make_result_of_lastop(resultbox) + self.make_result_of_lastop(resultbox) # unboundmethod = getattr(MIFrame, 'opimpl_' + name).im_func argtypes = unrolling_iterable(unboundmethod.argtypes) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py Mon May 24 15:37:28 2010 @@ -32,10 +32,13 @@ def _ensure_parent_resumedata(framestack, n): target = framestack[n] - if n == 0 or target.parent_resumedata_frame_info_list is not None: + if n == 0: return - _ensure_parent_resumedata(framestack, n-1) back = framestack[n-1] + if target.parent_resumedata_frame_info_list is not None: + assert target.parent_resumedata_frame_info_list.pc == back.pc + return + _ensure_parent_resumedata(framestack, n-1) target.parent_resumedata_frame_info_list = FrameInfo( back.parent_resumedata_frame_info_list, back.jitcode, Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_exception.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_exception.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_exception.py Mon May 24 15:37:28 2010 @@ -2,7 +2,7 @@ from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin from pypy.rlib.jit import JitDriver, OPTIMIZER_SIMPLE from pypy.rlib.rarithmetic import ovfcheck, LONG_BIT, intmask -from pypy.jit.metainterp.policy import StopAtXPolicy +from pypy.jit.codewriter.policy import StopAtXPolicy class ExceptionTests: From arigo at codespeak.net Mon May 24 15:55:38 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 24 May 2010 15:55:38 +0200 (CEST) Subject: [pypy-svn] r74707 - in pypy/branch/blackhole-improvement/pypy/jit/metainterp: . test Message-ID: <20100524135538.2DC87282BDB@codespeak.net> Author: arigo Date: Mon May 24 15:55:36 2010 New Revision: 74707 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_exception.py Log: Fixes. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Mon May 24 15:55:36 2010 @@ -1344,6 +1344,7 @@ # detect and propagate some exceptions early: # - AssertionError # - all subclasses of JitException + excvaluebox = self.last_exc_value_box if we_are_translated(): from pypy.jit.metainterp.warmspot import JitException e = self.cpu.ts.get_exception_obj(excvaluebox) @@ -1675,10 +1676,17 @@ elif opnum == rop.GUARD_NO_EXCEPTION or opnum == rop.GUARD_EXCEPTION: exception = self.cpu.grab_exc_value() if exception: + if not we_are_translated(): + from pypy.rpython.lltypesystem import rclass + etype = rclass.ll_type(exception) + exception = LLException(etype, exception) self.execute_raised(exception) else: self.execute_did_not_raise() - self.handle_possible_exception() + try: + self.handle_possible_exception() + except ChangeFrame: + pass elif opnum == rop.GUARD_NO_OVERFLOW: # an overflow now detected self.execute_raised(OverflowError(), constant=True) try: Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_exception.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_exception.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_exception.py Mon May 24 15:55:36 2010 @@ -516,11 +516,10 @@ assert res == -1 self.check_tree_loop_count(2) # the loop and the entry path # we get: - # ENTER - compile the new loop - # ENTER (BlackHole) - leave - # ENTER - compile the entry bridge - # ENTER - compile the leaving path (raising MyError) - self.check_enter_count(4) + # ENTER - compile the new loop + # ENTER - compile the entry bridge + # ENTER - compile the leaving path (raising MyError) + self.check_enter_count(3) def test_bridge_from_interpreter_exc_2(self): From arigo at codespeak.net Mon May 24 16:16:49 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 24 May 2010 16:16:49 +0200 (CEST) Subject: [pypy-svn] r74708 - in pypy/branch/blackhole-improvement/pypy/jit: backend backend/llgraph metainterp/test Message-ID: <20100524141649.0A238282BDB@codespeak.net> Author: arigo Date: Mon May 24 16:16:48 2010 New Revision: 74708 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py pypy/branch/blackhole-improvement/pypy/jit/backend/model.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_executor.py Log: Pass test_executor. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py Mon May 24 16:16:48 2010 @@ -1091,9 +1091,9 @@ if _last_exception is not None: result = _last_exception.args[1] _last_exception = None - return lltype.cast_opaque_ptr(llmemory.GCREF, result) + return result else: - return lltype.nullptr(llmemory.GCREF.TO) + return lltype.nullptr(rclass.OBJECTPTR.TO) ##_pseudo_exceptions = {} Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/model.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/model.py Mon May 24 16:16:48 2010 @@ -101,7 +101,9 @@ raise NotImplementedError def grab_exc_value(self): - """Return and clear the exception set by the latest execute_token().""" + """Return and clear the exception set by the latest execute_token(), + when it exits due to a failure of a GUARD_EXCEPTION or + GUARD_NO_EXCEPTION.""" # XXX remove me raise NotImplementedError @staticmethod @@ -212,5 +214,18 @@ def bh_cast_ptr_to_int(self, ptr): raise NotImplementedError + def bh_strlen(self, string): + raise NotImplementedError + def bh_strgetitem(self, string, index): + raise NotImplementedError + def bh_unicodelen(self, string): + raise NotImplementedError + def bh_unicodegetitem(self, string, index): + raise NotImplementedError + def bh_strsetitem(self, string, index, newvalue): + raise NotImplementedError + def bh_unicodesetitem(self, string, index, newvalue): + raise NotImplementedError + def force(self, force_token): raise NotImplementedError Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_executor.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_executor.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_executor.py Mon May 24 16:16:48 2010 @@ -1,60 +1,90 @@ import py import sys, random from pypy.rlib.rarithmetic import r_uint, intmask -from pypy.jit.metainterp.executor import make_execute_list, execute +from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.jit.metainterp.executor import execute from pypy.jit.metainterp.executor import execute_varargs, execute_nonspec from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.history import BoxInt, ConstInt +from pypy.jit.metainterp.history import BoxPtr, ConstPtr from pypy.jit.metainterp.history import BoxFloat, ConstFloat from pypy.jit.metainterp.history import AbstractDescr, Box +from pypy.jit.metainterp import history from pypy.jit.backend.model import AbstractCPU class FakeDescr(AbstractDescr): pass -class FakeBox(Box): +class FakeCallDescr(FakeDescr): + def get_return_type(self): + return history.FLOAT + +class FakeFieldDescr(FakeDescr): + def is_pointer_field(self): + return False + def is_float_field(self): + return True + +class FakeArrayDescr(FakeDescr): + def is_array_of_pointers(self): + return False + def is_array_of_floats(self): + return True + +class FakeResultR: + _TYPE = llmemory.GCREF def __init__(self, *args): - self.args = args + self.fakeargs = args + +class FakeMetaInterp: + def execute_did_not_raise(self): + self.didraise = False class FakeCPU(AbstractCPU): supports_floats = True - def do_new(self, descr): - return FakeBox('new', descr) - - def do_arraylen_gc(self, box1, descr): - return FakeBox('arraylen_gc', box1, descr) - - def do_setfield_gc(self, box1, box2, descr): - return FakeBox('setfield_gc', box1, box2, descr) + def bh_new(self, descr): + return FakeResultR('new', descr) - def do_setarrayitem_gc(self, box1, box2, box3, descr): - return FakeBox('setarrayitem_gc', box1, box2, box3, descr) + def bh_arraylen_gc(self, descr, array): + assert not array + assert isinstance(descr, FakeDescr) + return 55 + + def bh_setfield_gc_f(self, struct, fielddescr, newvalue): + self.fakesetfield = (struct, newvalue, fielddescr) + + def bh_setarrayitem_gc_f(self, arraydescr, array, index, newvalue): + self.fakesetarrayitem = (array, index, newvalue, arraydescr) + + def bh_call_f(self, func, calldescr, args_i, args_r, args_f): + self.fakecalled = (func, calldescr, args_i, args_r, args_f) + return 42.5 - def do_call(self, args, descr): - return FakeBox('call', args, descr) - - def do_strsetitem(self, box1, box2, box3): - return FakeBox('strsetitem', box1, box2, box3) - -make_execute_list(FakeCPU) + def bh_strsetitem(self, string, index, newvalue): + self.fakestrsetitem = (string, index, newvalue) def test_execute(): cpu = FakeCPU() descr = FakeDescr() - box = execute(cpu, rop.INT_ADD, None, BoxInt(40), ConstInt(2)) + box = execute(cpu, None, rop.INT_ADD, None, BoxInt(40), ConstInt(2)) assert box.value == 42 - box = execute(cpu, rop.NEW, descr) - assert box.args == ('new', descr) + box = execute(cpu, None, rop.NEW, descr) + assert box.value.fakeargs == ('new', descr) def test_execute_varargs(): cpu = FakeCPU() - descr = FakeDescr() - argboxes = [BoxInt(321), ConstInt(123)] - box = execute_varargs(cpu, rop.CALL, argboxes, descr) - assert box.args == ('call', argboxes, descr) + metainterp = FakeMetaInterp() + descr = FakeCallDescr() + argboxes = [BoxInt(99999), BoxInt(321), ConstFloat(2.25), ConstInt(123), + BoxPtr(), BoxFloat(5.5)] + box = execute_varargs(cpu, metainterp, rop.CALL, argboxes, descr) + assert box.value == 42.5 + assert cpu.fakecalled == (99999, descr, [321, 123], + [ConstPtr.value], [2.25, 5.5]) + assert not metainterp.didraise def test_execute_nonspec(): cpu = FakeCPU() @@ -62,33 +92,38 @@ # cases with a descr # arity == -1 argboxes = [BoxInt(321), ConstInt(123)] - box = execute_nonspec(cpu, rop.CALL, argboxes, descr) - assert box.args == ('call', argboxes, descr) + box = execute_nonspec(cpu, FakeMetaInterp(), rop.CALL, + argboxes, FakeCallDescr()) + assert box.value == 42.5 # arity == 0 - box = execute_nonspec(cpu, rop.NEW, [], descr) - assert box.args == ('new', descr) + box = execute_nonspec(cpu, None, rop.NEW, [], descr) + assert box.value.fakeargs == ('new', descr) # arity == 1 - box1 = BoxInt(515) - box = execute_nonspec(cpu, rop.ARRAYLEN_GC, [box1], descr) - assert box.args == ('arraylen_gc', box1, descr) + box1 = BoxPtr() + box = execute_nonspec(cpu, None, rop.ARRAYLEN_GC, [box1], descr) + assert box.value == 55 # arity == 2 - box2 = BoxInt(222) - box = execute_nonspec(cpu, rop.SETFIELD_GC, [box1, box2], descr) - assert box.args == ('setfield_gc', box1, box2, descr) + box2 = BoxFloat(222.2) + fielddescr = FakeFieldDescr() + execute_nonspec(cpu, None, rop.SETFIELD_GC, [box1, box2], fielddescr) + assert cpu.fakesetfield == (box1.value, box2.value, fielddescr) # arity == 3 - box3 = BoxInt(-33) - box = execute_nonspec(cpu, rop.SETARRAYITEM_GC, [box1, box2, box3], descr) - assert box.args == ('setarrayitem_gc', box1, box2, box3, descr) + box3 = BoxInt(33) + arraydescr = FakeArrayDescr() + execute_nonspec(cpu, None, rop.SETARRAYITEM_GC, [box1, box3, box2], + arraydescr) + assert cpu.fakesetarrayitem == (box1.value, box3.value, box2.value, + arraydescr) # cases without descr # arity == 1 - box = execute_nonspec(cpu, rop.INT_INVERT, [box1]) - assert box.value == ~515 + box = execute_nonspec(cpu, None, rop.INT_INVERT, [box3]) + assert box.value == ~33 # arity == 2 - box = execute_nonspec(cpu, rop.INT_LSHIFT, [box1, BoxInt(3)]) - assert box.value == 515 << 3 + box = execute_nonspec(cpu, None, rop.INT_LSHIFT, [box3, BoxInt(3)]) + assert box.value == 33 << 3 # arity == 3 - box = execute_nonspec(cpu, rop.STRSETITEM, [box1, box2, box3]) - assert box.args == ('strsetitem', box1, box2, box3) + execute_nonspec(cpu, None, rop.STRSETITEM, [box1, BoxInt(3), box3]) + assert cpu.fakestrsetitem == (box1.value, 3, box3.value) # ints @@ -181,7 +216,7 @@ (rop.INT_IS_TRUE, [(0, 0), (1, 1), (2, 1), (-1, 1), (minint, 1)]), (rop.INT_NEG, [(0, 0), (123, -123), (-23127, 23127)]), (rop.INT_INVERT, [(0, ~0), (-1, ~(-1)), (123, ~123)]), - (rop.BOOL_NOT, [(0, 1), (1, 0)]), + (rop.INT_IS_ZERO, [(0, 1), (1, 0), (2, 0), (-1, 0), (minint, 0)]), ]: for x, y in testcases: yield opnum, [x], y @@ -204,7 +239,7 @@ def test_int_ops(): cpu = FakeCPU() for opnum, boxargs, retvalue in get_int_tests(): - box = execute_nonspec(cpu, opnum, boxargs) + box = execute_nonspec(cpu, None, opnum, boxargs) assert box.getint() == retvalue # floats @@ -242,9 +277,6 @@ yield (rop.FLOAT_NEG, [15.9], 'float', -15.9) yield (rop.FLOAT_ABS, [-5.9], 'float', 5.9) yield (rop.FLOAT_ABS, [15.9], 'float', 15.9) - yield (rop.FLOAT_IS_TRUE, [-5.9], 'int', 1) - yield (rop.FLOAT_IS_TRUE, [0.0], 'int', 0) - yield (rop.FLOAT_IS_TRUE, [-0.0], 'int', 0) yield (rop.CAST_FLOAT_TO_INT, [-5.9], 'int', -5) yield (rop.CAST_FLOAT_TO_INT, [5.9], 'int', 5) yield (rop.CAST_INT_TO_FLOAT, [123], 'float', 123.0) @@ -276,7 +308,7 @@ def test_float_ops(): cpu = FakeCPU() for opnum, boxargs, rettype, retvalue in get_float_tests(cpu): - box = execute_nonspec(cpu, opnum, boxargs) + box = execute_nonspec(cpu, None, opnum, boxargs) if rettype == 'float': assert box.getfloat() == retvalue elif rettype == 'int': From arigo at codespeak.net Mon May 24 16:21:06 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 24 May 2010 16:21:06 +0200 (CEST) Subject: [pypy-svn] r74709 - pypy/branch/blackhole-improvement/pypy/jit/codewriter Message-ID: <20100524142106.F04FA282BDB@codespeak.net> Author: arigo Date: Mon May 24 16:21:05 2010 New Revision: 74709 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Log: Fix (see metainterp.test.test_immutable). Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Mon May 24 16:21:05 2010 @@ -36,6 +36,7 @@ def optimize_block(self, block): if block.operations == (): return + self.immutable_arrays = {} renamings = {} renamings_constants = {} # subset of 'renamings', {Var:Const} only newoperations = [] @@ -802,7 +803,7 @@ v_index, extraop = self._prepare_list_getset(op, arraydescr, args, 'check_neg_index') extra = getkind(op.result.concretetype)[0] - if pure: + if pure or args[0] in self.immutable_arrays: extra = 'pure_' + extra op = SpaceOperation('getarrayitem_gc_%s' % extra, [args[0], arraydescr, v_index], op.result) From arigo at codespeak.net Mon May 24 16:31:38 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 24 May 2010 16:31:38 +0200 (CEST) Subject: [pypy-svn] r74710 - in pypy/branch/blackhole-improvement/pypy/jit/metainterp: . test Message-ID: <20100524143138.854A5282BDB@codespeak.net> Author: arigo Date: Mon May 24 16:31:36 2010 New Revision: 74710 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/jitprof.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_jitprof.py Log: Fix jitprof. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/jitprof.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/jitprof.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/jitprof.py Mon May 24 16:31:36 2010 @@ -96,7 +96,7 @@ self.t1 = self.starttime self.times = [0, 0] self.counters = [0] * ncounters - self.calls = [[0, 0], [0, 0], [0, 0]] + self.calls = [[0, 0], [0, 0]] self.current = [] def finish(self): @@ -149,10 +149,10 @@ def count_ops(self, opnum, kind=OPS): from pypy.jit.metainterp.resoperation import rop self.counters[kind] += 1 - if opnum == rop.CALL or opnum == rop.OOSEND: + if opnum == rop.CALL: # or opnum == rop.OOSEND: self.calls[kind-OPS][0] += 1 - elif opnum == rop.CALL_PURE or opnum == rop.OOSEND_PURE: - self.calls[kind-OPS][1] += 1 + elif opnum == rop.CALL_PURE: # or opnum == rop.OOSEND_PURE: + self.calls[kind-OPS][1] += 1 def print_stats(self): cnt = self.counters @@ -171,8 +171,6 @@ self._print_intline(" calls", calls[1][0]) self._print_intline(" pure calls", calls[1][1]) self._print_intline("guards", cnt[GUARDS]) - self._print_intline("blackholed ops", calls[2][0]) - self._print_intline(" pure calls", calls[2][1]) self._print_intline("opt ops", cnt[OPT_OPS]) self._print_intline("opt guards", cnt[OPT_GUARDS]) self._print_intline("forcings", cnt[OPT_FORCINGS]) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_jitprof.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_jitprof.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_jitprof.py Mon May 24 16:31:36 2010 @@ -63,7 +63,7 @@ ] assert profiler.events == expected assert profiler.times == [2, 1, 1, 1] - assert profiler.counters == [1, 1, 1, 1, 4, 3, 1, 1, 7, 1, 0, 0, 0, + assert profiler.counters == [1, 1, 1, 1, 3, 3, 1, 7, 1, 0, 0, 0, 0, 0, 0, 0] def test_simple_loop_with_call(self): @@ -84,8 +84,8 @@ res = self.meta_interp(f, [6, 7]) assert res == 84 profiler = pyjitpl._warmrunnerdesc.metainterp_sd.profiler - # calls = (executed, recorded, blackholed) x (inpure, pure) - assert profiler.calls == [[1, 0], [1, 0], [0, 0]] + # calls = (executed, recorded) x (inpure, pure) + assert profiler.calls == [[1, 0], [1, 0]] def test_blackhole_pure(self): @purefunction @@ -99,12 +99,11 @@ myjitdriver.can_enter_jit(x=x, y=y, res=res, z=z) myjitdriver.jit_merge_point(x=x, y=y, res=res, z=z) res += x - if y == 1: - res += g(z) + res += g(z) y -= 1 return res * 2 res = self.meta_interp(f, [6, 7, 2]) - assert res == 90 + assert res == f(6, 7, 2) profiler = pyjitpl._warmrunnerdesc.metainterp_sd.profiler - # calls = (executed, recorded, blackholed) x (inpure, pure) - assert profiler.calls == [[0, 1], [0, 0], [0, 1]] + # calls = (executed, recorded) x (inpure, pure) + assert profiler.calls == [[0, 1], [0, 0]] From arigo at codespeak.net Mon May 24 16:53:35 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 24 May 2010 16:53:35 +0200 (CEST) Subject: [pypy-svn] r74711 - pypy/branch/blackhole-improvement/pypy/jit/metainterp/test Message-ID: <20100524145335.3C26A282BDB@codespeak.net> Author: arigo Date: Mon May 24 16:53:33 2010 New Revision: 74711 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_resume.py Log: Bah. It's going to be a pain to fix these tests. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_resume.py Mon May 24 16:53:33 2010 @@ -3,7 +3,7 @@ from pypy.jit.metainterp.optimizeopt import VirtualValue, OptValue, VArrayValue from pypy.jit.metainterp.optimizeopt import VStructValue from pypy.jit.metainterp.resume import * -from pypy.jit.metainterp.history import BoxInt, BoxPtr, ConstInt, ConstAddr +from pypy.jit.metainterp.history import BoxInt, BoxPtr, ConstInt from pypy.jit.metainterp.history import ConstPtr, ConstFloat from pypy.jit.metainterp.test.test_optimizefindnode import LLtypeMixin from pypy.jit.metainterp import executor @@ -74,30 +74,102 @@ self.resboxes.append(resbox) return resbox +S = lltype.GcStruct('S') +gcref1 = lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S)) +gcref2 = lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S)) +gcrefnull = lltype.nullptr(llmemory.GCREF.TO) + +class MyCPU: + class ts: + NULLREF = gcrefnull + CONST_NULL = ConstPtr(gcrefnull) + def __init__(self, values): + self.values = values + def get_latest_value_count(self): + return len(self.values) + def get_latest_value_int(self, index): + return self.values[index] + def get_latest_value_ref(self, index): + return self.values[index] + def get_latest_value_float(self, index): + return self.values[index] + +class MyBlackholeInterp: + def __init__(self): + self.written_i = {} + self.written_r = {} + self.written_f = {} + + def setarg_i(self, index, value): + assert index not in self.written_i + self.written_i[index] = value + + def setarg_r(self, index, value): + assert index not in self.written_r + self.written_r[index] = value + + def setarg_f(self, index, value): + assert index not in self.written_f + self.written_f[index] = value + def test_simple_read(): - b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()] - c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)] + #b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()] + c1, c2, c3 = [ConstInt(111), ConstInt(222), ConstInt(333)] storage = Storage() storage.rd_consts = [c1, c2, c3] - numb = Numbering(None, [tag(0, TAGBOX), tag(1, TAGBOX), tag(2, TAGBOX)]) - numb = Numbering(numb, [tag(1, TAGCONST), tag(2, TAGCONST)]) + numb = Numbering(None, [tag(0, TAGBOX), tag(2, TAGBOX), TYPEBARRIER, + tag(1, TAGBOX)]) + numb = Numbering(numb, [tag(1, TAGCONST), tag(2, TAGCONST), TYPEBARRIER]) numb = Numbering(numb, [tag(0, TAGBOX), tag(0, TAGCONST), - NULLREF, tag(0, TAGBOX), + TYPEBARRIER, + NULLREF, tag(1, TAGBOX)]) storage.rd_numb = numb - - b1s, b2s, b3s = [BoxInt(), BoxPtr(), BoxInt()] - assert b1s != b3s - reader = ResumeDataReader(storage, [b1s, b2s, b3s], MyMetaInterp()) - lst = reader.consume_boxes() - assert lst == [b1s, ConstInt(1), LLtypeMixin.cpu.ts.CONST_NULL, b1s, b2s] - lst = reader.consume_boxes() - assert lst == [ConstInt(2), ConstInt(3)] - lst = reader.consume_boxes() - assert lst == [b1s, b2s, b3s] + # + cpu = MyCPU([42, gcref1, -66]) + reader = ResumeDataDirectReader(cpu, storage) + bh = MyBlackholeInterp() + reader.consume_one_section(bh) + assert bh.written_i == {0: 42, + 1: 111, + 2: 42, + } + assert bh.written_r == {0: gcrefnull, + 1: gcref1, + } + bh = MyBlackholeInterp() + reader.consume_one_section(bh) + assert bh.written_i == {0: 222, + 1: 333, + } + assert bh.written_r == {} + bh = MyBlackholeInterp() + reader.consume_one_section(bh) + assert bh.written_i == {0: 42, + 1: -66, + } + assert bh.written_r == {0: gcref1} + # + metainterp = MyMetaInterp(cpu) + reader = ResumeDataBoxReader(storage, metainterp) + bi, br, bf = [None]*3, [None]*2, [None]*0 + reader.consume_boxes(bi, br, bf) + b1s = reader.liveboxes[0] + b2s = reader.liveboxes[1] + assert bi == [b1s, ConstInt(111), b1s] + assert br == [ConstPtr(gcrefnull), b2s] + bi, br, bf = [None]*2, [None]*0, [None]*0 + reader.consume_boxes(bi, br, bf) + assert bi == [ConstInt(222), ConstInt(333)] + bi, br, bf = [None]*2, [None]*1, [None]*0 + reader.consume_boxes(bi, br, bf) + b3s = reader.liveboxes[2] + assert bi == [b1s, b3s] + assert br == [b2s] + # def test_simple_read_tagged_ints(): b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()] @@ -379,6 +451,9 @@ pass fakeoptimizer = FakeOptimizer_VirtualValue() +def ConstAddr(addr, cpu=None): # compatibility + return ConstInt(llmemory.cast_adr_to_int(addr)) + def virtual_value(keybox, value, next): vv = VirtualValue(fakeoptimizer, ConstAddr(LLtypeMixin.node_vtable_adr, LLtypeMixin.cpu), keybox) From arigo at codespeak.net Mon May 24 17:22:26 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 24 May 2010 17:22:26 +0200 (CEST) Subject: [pypy-svn] r74712 - pypy/branch/blackhole-improvement/pypy/jit/metainterp/test Message-ID: <20100524152226.F2226282BDB@codespeak.net> Author: arigo Date: Mon May 24 17:22:25 2010 New Revision: 74712 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_resume.py Log: Skip some tests, fix others. XXX rewrite... Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_resume.py Mon May 24 17:22:25 2010 @@ -51,16 +51,14 @@ class MyMetaInterp: _already_allocated_resume_virtuals = None - def __init__(self, cpu=None): - if cpu is None: - cpu = LLtypeMixin.cpu + def __init__(self, cpu): self.cpu = cpu self.trace = [] self.framestack = [] self.resboxes = [] def newframe(self, jitcode): - frame = FakeFrame(jitcode, -1, -1) + frame = FakeFrame(jitcode, -1) self.framestack.append(frame) return frame @@ -96,21 +94,29 @@ class MyBlackholeInterp: def __init__(self): - self.written_i = {} - self.written_r = {} - self.written_f = {} + self.written_i = [] + self.written_r = [] + self.written_f = [] def setarg_i(self, index, value): - assert index not in self.written_i - self.written_i[index] = value + assert index == len(self.written_i) + self.written_i.append(value) def setarg_r(self, index, value): - assert index not in self.written_r - self.written_r[index] = value + assert index == len(self.written_r) + self.written_r.append(value) def setarg_f(self, index, value): - assert index not in self.written_f - self.written_f[index] = value + assert index == len(self.written_f) + self.written_f.append(value) + +def _next_section(reader, *expected): + bh = MyBlackholeInterp() + reader.consume_one_section(bh) + lst_i = bh.written_i + lst_r = bh.written_r + lst_f = bh.written_f + assert lst_i + lst_r + lst_f == list(expected) def test_simple_read(): @@ -131,27 +137,9 @@ # cpu = MyCPU([42, gcref1, -66]) reader = ResumeDataDirectReader(cpu, storage) - bh = MyBlackholeInterp() - reader.consume_one_section(bh) - assert bh.written_i == {0: 42, - 1: 111, - 2: 42, - } - assert bh.written_r == {0: gcrefnull, - 1: gcref1, - } - bh = MyBlackholeInterp() - reader.consume_one_section(bh) - assert bh.written_i == {0: 222, - 1: 333, - } - assert bh.written_r == {} - bh = MyBlackholeInterp() - reader.consume_one_section(bh) - assert bh.written_i == {0: 42, - 1: -66, - } - assert bh.written_r == {0: gcref1} + _next_section(reader, 42, 111, 42, gcrefnull, gcref1) + _next_section(reader, 222, 333) + _next_section(reader, 42, -66, gcref1) # metainterp = MyMetaInterp(cpu) reader = ResumeDataBoxReader(storage, metainterp) @@ -172,32 +160,21 @@ # def test_simple_read_tagged_ints(): - b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()] storage = Storage() storage.rd_consts = [] - numb = Numbering(None, [tag(0, TAGBOX), tag(1, TAGBOX), tag(2, TAGBOX)]) - numb = Numbering(numb, [tag(2, TAGINT), tag(3, TAGINT)]) - numb = Numbering(numb, [tag(0, TAGBOX), - tag(1, TAGINT), - tag(0, TAGBOX), - tag(1, TAGBOX)]) + numb = Numbering(None, [tag(100, TAGINT), TYPEBARRIER]) storage.rd_numb = numb - b1s, b2s, b3s = [BoxInt(), BoxPtr(), BoxInt()] - assert b1s != b3s - reader = ResumeDataReader(storage, [b1s, b2s, b3s], MyMetaInterp()) - lst = reader.consume_boxes() - assert lst == [b1s, ConstInt(1), b1s, b2s] - lst = reader.consume_boxes() - assert lst == [ConstInt(2), ConstInt(3)] - lst = reader.consume_boxes() - assert lst == [b1s, b2s, b3s] + # + cpu = MyCPU([]) + reader = ResumeDataDirectReader(cpu, storage) + _next_section(reader, 100) def test_prepare_virtuals(): class FakeVinfo(object): - def allocate(self, metainterp): + def allocate(self, decoder): return "allocated" - def setfields(self, metainterp, virtual, func): + def setfields(self, decoder, virtual): assert virtual == "allocated" class FakeStorage(object): rd_virtuals = [FakeVinfo(), None] @@ -207,7 +184,7 @@ class FakeMetainterp(object): _already_allocated_resume_virtuals = None cpu = None - reader = ResumeDataReader(FakeStorage(), [], FakeMetainterp()) + reader = ResumeDataDirectReader(None, FakeStorage()) assert reader.virtuals == ["allocated", None] # ____________________________________________________________ @@ -218,11 +195,13 @@ parent_resumedata_snapshot = None parent_resumedata_frame_info_list = None - def __init__(self, code, pc, exc_target, *boxes): + def __init__(self, code, pc, *boxes): self.jitcode = code self.pc = pc - self.exception_target = exc_target - self.env = list(boxes) + self._env = list(boxes) + + def get_list_of_active_boxes(self, flag): + return self._env def setup_resume_at_op(self, pc, exception_target, env): self.__init__(self.jitcode, pc, exception_target, *env) @@ -232,8 +211,7 @@ def __ne__(self, other): return self.__dict__ != other.__dict__ def __repr__(self): - return "" % (self.jitcode, self.pc, - self.exception_target, self.env) + return "" % (self.jitcode, self.pc, self._env) def test_Snapshot_create(): l = ['b0', 'b1'] @@ -248,20 +226,16 @@ def test_FrameInfo_create(): jitcode = "JITCODE" - frame = FakeFrame(jitcode, 1, 2) - fi = FrameInfo(None, frame) + fi = FrameInfo(None, jitcode, 1) assert fi.prev is None assert fi.jitcode is jitcode assert fi.pc == 1 - assert fi.exception_target == 2 jitcode1 = "JITCODE1" - frame1 = FakeFrame(jitcode, 3, 4) - fi1 = FrameInfo(fi, frame1) + fi1 = FrameInfo(fi, jitcode1, 3) assert fi1.prev is fi - assert fi1.jitcode is jitcode + assert fi1.jitcode is jitcode1 assert fi1.pc == 3 - assert fi1.exception_target == 4 def test_Numbering_create(): l = [1, 2] @@ -277,7 +251,7 @@ def test_capture_resumedata(): b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()] c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)] - fs = [FakeFrame("code0", 0, -1, b1, c1, b2)] + fs = [FakeFrame("code0", 0, b1, c1, b2)] storage = Storage() capture_resumedata(fs, None, [], storage) @@ -290,13 +264,12 @@ assert storage.rd_snapshot.boxes == [] # for virtualrefs snapshot = storage.rd_snapshot.prev assert snapshot.prev is None - assert snapshot.boxes == fs[0].env - assert snapshot.boxes is not fs[0].env + assert snapshot.boxes == fs[0]._env storage = Storage() - fs = [FakeFrame("code0", 0, -1, b1, c1, b2), - FakeFrame("code1", 3, 7, b3, c2, b1), - FakeFrame("code2", 9, -1, c3, b2)] + fs = [FakeFrame("code0", 0, b1, c1, b2), + FakeFrame("code1", 3, b3, c2, b1), + FakeFrame("code2", 9, c3, b2)] capture_resumedata(fs, None, [], storage) frame_info_list = storage.rd_frame_info_list @@ -307,26 +280,23 @@ assert storage.rd_snapshot.boxes == [] # for virtualrefs snapshot = storage.rd_snapshot.prev assert snapshot.prev is fs[2].parent_resumedata_snapshot - assert snapshot.boxes == fs[2].env - assert snapshot.boxes is not fs[2].env + assert snapshot.boxes == fs[2]._env frame_info_list = frame_info_list.prev assert frame_info_list.prev is fs[1].parent_resumedata_frame_info_list assert frame_info_list.jitcode == 'code1' snapshot = snapshot.prev assert snapshot.prev is fs[1].parent_resumedata_snapshot - assert snapshot.boxes == fs[1].env - assert snapshot.boxes is not fs[1].env + assert snapshot.boxes == fs[1]._env frame_info_list = frame_info_list.prev assert frame_info_list.prev is None assert frame_info_list.jitcode == 'code0' snapshot = snapshot.prev assert snapshot.prev is None - assert snapshot.boxes == fs[0].env - assert snapshot.boxes is not fs[0].env + assert snapshot.boxes == fs[0]._env - fs[2].env = [b2, b3] + fs[2]._env = [b2, b3] fs[2].pc = 15 vbs = [b1, b2] vrs = [b3] @@ -347,7 +317,7 @@ snapshot = snapshot.prev assert snapshot.prev is fs[2].parent_resumedata_snapshot - assert snapshot.boxes == fs[2].env + assert snapshot.boxes == fs[2]._env class FakeMetaInterpStaticData: @@ -357,12 +327,13 @@ failargs_limit = 100 def test_rebuild_from_resumedata(): + py.test.skip("XXX rewrite") b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()] c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)] storage = Storage() - fs = [FakeFrame("code0", 0, -1, b1, c1, b2), - FakeFrame("code1", 3, 7, b3, c2, b1), - FakeFrame("code2", 9, -1, c3, b2)] + fs = [FakeFrame("code0", 0, b1, c1, b2), + FakeFrame("code1", 3, b3, c2, b1), + FakeFrame("code2", 9, c3, b2)] capture_resumedata(fs, None, [], storage) memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) @@ -372,21 +343,21 @@ b1t, b2t, b3t = [BoxInt(), BoxPtr(), BoxInt()] newboxes = _resume_remap(liveboxes, [b1, b2, b3], b1t, b2t, b3t) - result = rebuild_from_resumedata(metainterp, newboxes, storage, - False) + result = rebuild_from_resumedata(metainterp, storage, False) assert result == (None, []) - fs2 = [FakeFrame("code0", 0, -1, b1t, c1, b2t), - FakeFrame("code1", 3, 7, b3t, c2, b1t), - FakeFrame("code2", 9, -1, c3, b2t)] + fs2 = [FakeFrame("code0", 0, b1t, c1, b2t), + FakeFrame("code1", 3, b3t, c2, b1t), + FakeFrame("code2", 9, c3, b2t)] assert metainterp.framestack == fs2 def test_rebuild_from_resumedata_with_virtualizable(): + py.test.skip("XXX rewrite") b1, b2, b3, b4 = [BoxInt(), BoxPtr(), BoxInt(), BoxPtr()] c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)] storage = Storage() - fs = [FakeFrame("code0", 0, -1, b1, c1, b2), - FakeFrame("code1", 3, 7, b3, c2, b1), - FakeFrame("code2", 9, -1, c3, b2)] + fs = [FakeFrame("code0", 0, b1, c1, b2), + FakeFrame("code1", 3, b3, c2, b1), + FakeFrame("code2", 9, c3, b2)] capture_resumedata(fs, [b4], [], storage) memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) @@ -399,21 +370,22 @@ result = rebuild_from_resumedata(metainterp, newboxes, storage, True) assert result == ([b4t], []) - fs2 = [FakeFrame("code0", 0, -1, b1t, c1, b2t), - FakeFrame("code1", 3, 7, b3t, c2, b1t), - FakeFrame("code2", 9, -1, c3, b2t)] + fs2 = [FakeFrame("code0", 0, b1t, c1, b2t), + FakeFrame("code1", 3, b3t, c2, b1t), + FakeFrame("code2", 9, c3, b2t)] assert metainterp.framestack == fs2 def test_rebuild_from_resumedata_two_guards(): + py.test.skip("XXX rewrite") b1, b2, b3, b4 = [BoxInt(), BoxPtr(), BoxInt(), BoxInt()] c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)] storage = Storage() - fs = [FakeFrame("code0", 0, -1, b1, c1, b2), - FakeFrame("code1", 3, 7, b3, c2, b1), - FakeFrame("code2", 9, -1, c3, b2)] + fs = [FakeFrame("code0", 0, b1, c1, b2), + FakeFrame("code1", 3, b3, c2, b1), + FakeFrame("code2", 9, c3, b2)] capture_resumedata(fs, None, [], storage) storage2 = Storage() - fs = fs[:-1] + [FakeFrame("code2", 10, -1, c3, b2, b4)] + fs = fs[:-1] + [FakeFrame("code2", 10, c3, b2, b4)] capture_resumedata(fs, None, [], storage2) memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) @@ -431,9 +403,9 @@ result = rebuild_from_resumedata(metainterp, newboxes, storage, False) assert result == (None, []) - fs2 = [FakeFrame("code0", 0, -1, b1t, c1, b2t), - FakeFrame("code1", 3, 7, b3t, c2, b1t), - FakeFrame("code2", 9, -1, c3, b2t)] + fs2 = [FakeFrame("code0", 0, b1t, c1, b2t), + FakeFrame("code1", 3, b3t, c2, b1t), + FakeFrame("code2", 9, c3, b2t)] assert metainterp.framestack == fs2 newboxes = _resume_remap(liveboxes2, [b1, b2, b3, b4], b1t, b2t, b3t, b4t) @@ -442,7 +414,7 @@ result = rebuild_from_resumedata(metainterp, newboxes, storage2, False) assert result == (None, []) - fs2 = fs2[:-1] + [FakeFrame("code2", 10, -1, c3, b2t, b4t)] + fs2 = fs2[:-1] + [FakeFrame("code2", 10, c3, b2t, b4t)] assert metainterp.framestack == fs2 @@ -464,17 +436,18 @@ return vv def test_rebuild_from_resumedata_two_guards_w_virtuals(): + py.test.skip("XXX rewrite") b1, b2, b3, b4, b5 = [BoxInt(), BoxPtr(), BoxInt(), BoxInt(), BoxInt()] c1, c2, c3, c4 = [ConstInt(1), ConstInt(2), ConstInt(3), LLtypeMixin.nodebox.constbox()] storage = Storage() - fs = [FakeFrame("code0", 0, -1, b1, c1, b2), - FakeFrame("code1", 3, 7, b3, c2, b1), - FakeFrame("code2", 9, -1, c3, b2)] + fs = [FakeFrame("code0", 0, b1, c1, b2), + FakeFrame("code1", 3, b3, c2, b1), + FakeFrame("code2", 9, c3, b2)] capture_resumedata(fs, None, [], storage) storage2 = Storage() - fs = fs[:-1] + [FakeFrame("code2", 10, -1, c3, b2, b4)] + fs = fs[:-1] + [FakeFrame("code2", 10, c3, b2, b4)] capture_resumedata(fs, None, [], storage2) memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) @@ -508,9 +481,9 @@ False) b2t = metainterp.resboxes[0] - fs2 = [FakeFrame("code0", 0, -1, b1t, c1, b2t), - FakeFrame("code1", 3, 7, b3t, c2, b1t), - FakeFrame("code2", 9, -1, c3, b2t)] + fs2 = [FakeFrame("code0", 0, b1t, c1, b2t), + FakeFrame("code1", 3, b3t, c2, b1t), + FakeFrame("code2", 9, c3, b2t)] assert metainterp.framestack == fs2 newboxes = _resume_remap(liveboxes2, [b1, b3, b4], b1t, b3t, b4t) @@ -520,17 +493,18 @@ False) b2t = metainterp.resboxes[0] assert len(metainterp.resboxes) == 2 - fs2 = [FakeFrame("code0", 0, -1, b1t, c1, b2t), - FakeFrame("code1", 3, 7, b3t, c2, b1t), - FakeFrame("code2", 10, -1, c3, b2t, b4t)] + fs2 = [FakeFrame("code0", 0, b1t, c1, b2t), + FakeFrame("code1", 3, b3t, c2, b1t), + FakeFrame("code2", 10, c3, b2t, b4t)] assert metainterp.framestack == fs2 def test_rebuild_from_resumedata_two_guards_w_shared_virtuals(): + py.test.skip("XXX rewrite") b1, b2, b3, b4, b5, b6 = [BoxPtr(), BoxPtr(), BoxInt(), BoxPtr(), BoxInt(), BoxInt()] c1, c2, c3, c4 = [ConstInt(1), ConstInt(2), ConstInt(3), LLtypeMixin.nodebox.constbox()] storage = Storage() - fs = [FakeFrame("code0", 0, -1, c1, b2, b3)] + fs = [FakeFrame("code0", 0, c1, b2, b3)] capture_resumedata(fs, None, [], storage) memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) @@ -542,7 +516,7 @@ tag(0, TAGCONST)] storage2 = Storage() - fs = [FakeFrame("code0", 0, -1, b1, b4, b2)] + fs = [FakeFrame("code0", 0, b1, b4, b2)] capture_resumedata(fs, None, [], storage2) values[b4] = virtual_value(b4, b6, c4) modifier = ResumeDataVirtualAdder(storage2, memo) @@ -553,9 +527,10 @@ def test_resumedata_top_recursive_virtuals(): + py.test.skip("XXX rewrite") b1, b2, b3 = [BoxPtr(), BoxPtr(), BoxInt()] storage = Storage() - fs = [FakeFrame("code0", 0, -1, b1, b2)] + fs = [FakeFrame("code0", 0, b1, b2)] capture_resumedata(fs, None, [], storage) memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) @@ -643,9 +618,9 @@ assert liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), b3: tag(2, TAGBOX)} assert numb.nums == [tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX), - tag(1, TAGINT)] + tag(1, TAGINT), TYPEBARRIER] assert numb.prev.nums == [tag(0, TAGBOX), tag(1, TAGINT), tag(1, TAGBOX), - tag(0, TAGBOX), tag(2, TAGINT)] + tag(0, TAGBOX), tag(2, TAGINT), TYPEBARRIER] assert numb.prev.prev is None numb2, liveboxes2, v = memo.number({}, snap2) @@ -655,7 +630,7 @@ b3: tag(2, TAGBOX)} assert liveboxes2 is not liveboxes assert numb2.nums == [tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX), - tag(3, TAGINT)] + tag(3, TAGINT), TYPEBARRIER] assert numb2.prev is numb.prev env3 = [c3, b3, b1, c3] @@ -678,11 +653,11 @@ assert liveboxes3 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX)} assert numb3.nums == [tag(3, TAGINT), tag(4, TAGINT), tag(0, TAGBOX), - tag(3, TAGINT)] + tag(3, TAGINT), TYPEBARRIER] assert numb3.prev is numb.prev # virtual - env4 = [c3, b4, b1, c3] + env4 = [c3, b1, c3, b4] snap4 = Snapshot(snap, env4) numb4, liveboxes4, v = memo.number({b4: FakeValue(True, b4)}, snap4) @@ -690,8 +665,8 @@ assert liveboxes4 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), b4: tag(0, TAGVIRTUAL)} - assert numb4.nums == [tag(3, TAGINT), tag(0, TAGVIRTUAL), tag(0, TAGBOX), - tag(3, TAGINT)] + assert numb4.nums == [tag(3, TAGINT), tag(0, TAGBOX), + tag(3, TAGINT), TYPEBARRIER, tag(0, TAGVIRTUAL)] assert numb4.prev is numb.prev env5 = [b1, b4, b5] @@ -703,8 +678,8 @@ assert liveboxes5 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), b4: tag(0, TAGVIRTUAL), b5: tag(1, TAGVIRTUAL)} - assert numb5.nums == [tag(0, TAGBOX), tag(0, TAGVIRTUAL), - tag(1, TAGVIRTUAL)] + assert numb5.nums == [tag(0, TAGBOX), TYPEBARRIER, + tag(0, TAGVIRTUAL), tag(1, TAGVIRTUAL)] assert numb5.prev is numb4 def test_ResumeDataLoopMemo_number_boxes(): @@ -802,16 +777,11 @@ modifier = ResumeDataVirtualAdder(storage, memo) liveboxes = modifier.finish({}) assert storage.rd_snapshot is None - metainterp = MyMetaInterp() - reader = ResumeDataReader(storage, [], metainterp) - lst = reader.consume_boxes() - assert lst == [b1s, b2s, b3s] - lst = reader.consume_boxes() - assert lst == [ConstInt(2), ConstInt(3)] - lst = reader.consume_boxes() - assert lst == [ConstInt(sys.maxint), ConstInt(1), ConstInt(sys.maxint), - ConstInt(2**16)] - assert metainterp.trace == [] + cpu = MyCPU([]) + reader = ResumeDataDirectReader(cpu, storage) + _next_section(reader, sys.maxint, 2**16, -65) + _next_section(reader, 2, 3) + _next_section(reader, sys.maxint, 1, sys.maxint, 2**16) def test_virtual_adder_memo_const_sharing(): @@ -832,6 +802,7 @@ def test_virtual_adder_no_op_renaming(): + py.test.skip("XXX rewrite") b1s, b2s, b3s = [BoxInt(1), BoxInt(2), BoxInt(3)] storage = make_storage(b1s, b2s, b3s) memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) @@ -863,6 +834,7 @@ def test_virtual_adder_make_constant(): + py.test.skip("XXX rewrite") b1s, b2s, b3s = [BoxInt(1), BoxPtr(), BoxInt(3)] b1s = ConstInt(111) storage = make_storage(b1s, b2s, b3s) @@ -884,6 +856,7 @@ def test_virtual_adder_make_virtual(): + py.test.skip("XXX rewrite") b2s, b3s, b4s, b5s = [BoxPtr(), BoxInt(3), BoxPtr(), BoxPtr()] c1s = ConstInt(111) storage = Storage() @@ -957,6 +930,7 @@ assert ptr2.parent.next == ptr def test_virtual_adder_make_varray(): + py.test.skip("XXX rewrite") b2s, b4s = [BoxPtr(), BoxInt(4)] c1s = ConstInt(111) storage = Storage() @@ -1009,6 +983,7 @@ def test_virtual_adder_make_vstruct(): + py.test.skip("XXX rewrite") b2s, b4s = [BoxPtr(), BoxPtr()] c1s = ConstInt(111) storage = Storage() @@ -1052,6 +1027,7 @@ def test_virtual_adder_pending_fields(): + py.test.skip("XXX rewrite") b2s, b4s = [BoxPtr(), BoxPtr()] storage = Storage() memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) From arigo at codespeak.net Mon May 24 17:45:54 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 24 May 2010 17:45:54 +0200 (CEST) Subject: [pypy-svn] r74713 - in pypy/branch/blackhole-improvement/pypy/jit/metainterp: . test Message-ID: <20100524154554.62244282BDB@codespeak.net> Author: arigo Date: Mon May 24 17:45:52 2010 New Revision: 74713 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_send.py Log: Complete resume.py. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py Mon May 24 17:45:52 2010 @@ -446,9 +446,6 @@ for i in range(len(self.fielddescrs)): descr = self.fielddescrs[i] decoder.setfield(descr, struct, self.fieldnums[i]) -## fieldbox = fn_decode_box(self.fieldnums[i], kind) -## metainterp.execute_and_record(rop.SETFIELD_GC, -## descr, box, fieldbox) def debug_prints(self): assert len(self.fielddescrs) == len(self.fieldnums) @@ -496,8 +493,20 @@ @specialize.argtype(1) def setfields(self, decoder, array): arraydescr = self.arraydescr - for i in range(len(self.fieldnums)): - decoder.setarrayitem(arraydescr, array, i, self.fieldnums[i]) + length = len(self.fieldnums) + # NB. the check for the kind of array elements is moved out of the loop + if arraydescr.is_array_of_pointers(): + for i in range(length): + decoder.setarrayitem_ptr(arraydescr, array, i, + self.fieldnums[i]) + elif arraydescr.is_array_of_floats(): + for i in range(length): + decoder.setarrayitem_float(arraydescr, array, i, + self.fieldnums[i]) + else: + for i in range(length): + decoder.setarrayitem_int(arraydescr, array, i, + self.fieldnums[i]) def debug_prints(self): debug_print("\tvarrayinfo", self.arraydescr) @@ -616,15 +625,40 @@ self._prepare_next_section() def allocate_with_vtable(self, known_class): - xxx + return self.metainterp.execute_and_record(rop.NEW_WITH_VTABLE, + None, known_class) + def allocate_struct(self, typedescr): - xxx + return self.metainterp.execute_and_record(rop.NEW, typedescr) + def allocate_array(self, arraydescr, length): - xxx - def setfield(self, descr, struct, fieldnum): - xxx - def setarrayitem(self, arraydescr, array, index, fieldnum): - xxx + return metainterp.execute_and_record(rop.NEW_ARRAY, arraydescr, length) + + def setfield(self, descr, structbox, fieldnum): + if descr.is_pointer_field(): + kind = REF + elif descr.is_float_field(): + kind = FLOAT + else: + kind = INT + fieldbox = self.decode_box(fieldnum, kind) + self.metainterp.execute_and_record(rop.SETFIELD_GC, descr, + structbox, fieldbox) + + def setarrayitem_int(self, arraydescr, arraybox, index, fieldnum): + self.setarrayitem(arraydescr, arraybox, index, fieldnum, INT) + + def setarrayitem_ref(self, arraydescr, arraybox, index, fieldnum): + self.setarrayitem(arraydescr, arraybox, index, fieldnum, REF) + + def setarrayitem_float(self, arraydescr, arraybox, index, fieldnum): + self.setarrayitem(arraydescr, arraybox, index, fieldnum, FLOAT) + + def setarrayitem(self, arraydescr, arraybox, index, fieldnum, kind): + itembox = self.decode_box(fieldnum, kind) + metainterp.execute_and_record(rop.SETARRAYITEM_GC, + arraydescr, arraybox, + ConstInt(index), itembox) def decode_int(self, tagged): return self.decode_box(tagged, INT) @@ -637,20 +671,22 @@ num, tag = untag(tagged) if tag == TAGCONST: if tagged_eq(tagged, NULLREF): - return self.cpu.ts.CONST_NULL - return self.consts[num] + box = self.cpu.ts.CONST_NULL + else: + box = self.consts[num] elif tag == TAGVIRTUAL: virtuals = self.virtuals assert virtuals is not None - return virtuals[num] + box = virtuals[num] elif tag == TAGINT: - return ConstInt(num) + box = ConstInt(num) else: assert tag == TAGBOX box = self.liveboxes[num] if box is None: box = self.load_box_from_cpu(num, kind) - return box + assert box.type == kind + return box def load_box_from_cpu(self, num, kind): if num < 0: @@ -742,16 +778,17 @@ newvalue = self.decode_int(fieldnum) self.cpu.bh_setfield_gc_i(struct, descr, newvalue) - def setarrayitem(self, arraydescr, array, index, fieldnum): - if arraydescr.is_array_of_pointers(): - newvalue = self.decode_ref(fieldnum) - self.cpu.bh_setarrayitem_gc_r(arraydescr, array, index, newvalue) - elif arraydescr.is_array_of_floats(): - newvalue = self.decode_float(fieldnum) - self.cpu.bh_setarrayitem_gc_f(arraydescr, array, index, newvalue) - else: - newvalue = self.decode_int(fieldnum) - self.cpu.bh_setarrayitem_gc_i(arraydescr, array, index, newvalue) + def setarrayitem_int(self, arraydescr, array, index, fieldnum): + newvalue = self.decode_int(fieldnum) + self.cpu.bh_setarrayitem_gc_i(arraydescr, array, index, newvalue) + + def setarrayitem_ref(self, arraydescr, array, index, fieldnum): + newvalue = self.decode_ref(fieldnum) + self.cpu.bh_setarrayitem_gc_r(arraydescr, array, index, newvalue) + + def setarrayitem_float(self, arraydescr, array, index, fieldnum): + newvalue = self.decode_float(fieldnum) + self.cpu.bh_setarrayitem_gc_f(arraydescr, array, index, newvalue) def decode_int(self, tagged): num, tag = untag(tagged) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_send.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_send.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_send.py Mon May 24 17:45:52 2010 @@ -1,7 +1,6 @@ import py from pypy.rlib.jit import JitDriver, hint, purefunction -from pypy.jit.metainterp.policy import StopAtXPolicy -from pypy.rpython.ootypesystem import ootype +from pypy.jit.codewriter.policy import StopAtXPolicy from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin From arigo at codespeak.net Mon May 24 18:08:48 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 24 May 2010 18:08:48 +0200 (CEST) Subject: [pypy-svn] r74714 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter metainterp metainterp/test Message-ID: <20100524160848.1A53F282BDB@codespeak.net> Author: arigo Date: Mon May 24 18:08:46 2010 New Revision: 74714 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_recursive.py Log: Automatically sorting the lists given to the JitDriver gives troubles later when we need to reorder them back to match the one expected by the callbacks. Instead, just give the lists in the right order and that's it. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Mon May 24 18:08:46 2010 @@ -295,7 +295,8 @@ def insert_renamings(self, link): renamings = {} lst = [(self.getcolor(v), self.getcolor(link.target.inputargs[i])) - for i, v in enumerate(link.args)] + for i, v in enumerate(link.args) + if v.concretetype is not lltype.Void] lst.sort(key=lambda(v, w): w.index) for v, w in lst: if v == w: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py Mon May 24 18:08:46 2010 @@ -78,7 +78,7 @@ def decode_hp_hint_args(op): # Returns (list-of-green-vars, list-of-red-vars) without Voids. - # Both lists are sorted: first INT, then REF, then FLOAT. + # Both lists must be sorted: first INT, then REF, then FLOAT. assert op.opname == 'jit_marker' jitdriver = op.args[1].value numgreens = len(jitdriver.greens) @@ -91,7 +91,11 @@ from pypy.jit.metainterp.history import getkind lst = [v for v in args_v if v.concretetype is not lltype.Void] _kind2count = {'int': 1, 'ref': 2, 'float': 3} - lst.sort(key=lambda v: _kind2count[getkind(v.concretetype)]) + lst2 = sorted(lst, key=lambda v: _kind2count[getkind(v.concretetype)]) + # a crash here means that you have to reorder the variable named in + # the JitDriver. Indeed, greens and reds must both be sorted: first + # all INTs, followed by all REFs, followed by all FLOATs. + assert lst == lst2 return lst # return (_sort(greens_v), _sort(reds_v)) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Mon May 24 18:08:46 2010 @@ -1146,14 +1146,13 @@ elif sd.result_type == 'int': raise sd.DoneWithThisFrameInt(self.final_result_i()) elif sd.result_type == 'ref': - raise sd.DoneWithThisFrameRef(self.final_result_r()) + raise sd.DoneWithThisFrameRef(self.cpu, self.final_result_r()) elif sd.result_type == 'float': raise sd.DoneWithThisFrameFloat(self.final_result_f()) else: assert False def _exit_frame_with_exception(self, e): - # rare case sd = self.builder.metainterp_sd e = lltype.cast_opaque_ptr(llmemory.GCREF, e) raise sd.ExitFrameWithExceptionRef(self.cpu, e) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Mon May 24 18:08:46 2010 @@ -854,7 +854,7 @@ assert not res def test_isinstance_2(self): - driver = JitDriver(greens = [], reds = ['x', 'n', 'sum']) + driver = JitDriver(greens = [], reds = ['n', 'sum', 'x']) class A: pass class B(A): @@ -1078,10 +1078,10 @@ def test_residual_external_call(self): import math - myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'res']) + myjitdriver = JitDriver(greens = [], reds = ['y', 'x', 'res']) def f(x, y): x = float(x) - res = 0 + res = 0.0 while y > 0: myjitdriver.can_enter_jit(x=x, y=y, res=res) myjitdriver.jit_merge_point(x=x, y=y, res=res) @@ -1320,7 +1320,7 @@ self.check_loops(call=1) def test_bug_optimizeopt_mutates_ops(self): - myjitdriver = JitDriver(greens = [], reds = ['x', 'res', 'a', 'const']) + myjitdriver = JitDriver(greens = [], reds = ['x', 'res', 'const', 'a']) class A(object): pass class B(A): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_recursive.py Mon May 24 18:08:46 2010 @@ -1,7 +1,7 @@ import py from pypy.rlib.jit import JitDriver, we_are_jitted, OPTIMIZER_SIMPLE, hint from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin -from pypy.jit.metainterp.policy import StopAtXPolicy +from pypy.jit.codewriter.policy import StopAtXPolicy from pypy.rpython.annlowlevel import hlstr from pypy.jit.metainterp.warmspot import CannotInlineCanEnterJit, get_stats @@ -105,11 +105,11 @@ def can_inline(*args): return True else: - def can_inline(code, i): + def can_inline(i, code): code = hlstr(code) return not JUMP_BACK in code - jitdriver = JitDriver(greens = ['code', 'i'], reds = ['n'], + jitdriver = JitDriver(greens = ['i', 'code'], reds = ['n'], can_inline = can_inline) def interpret(codenum, n, i): From electronicru at codespeak.net Mon May 24 18:26:22 2010 From: electronicru at codespeak.net (electronicru at codespeak.net) Date: Mon, 24 May 2010 18:26:22 +0200 (CEST) Subject: [pypy-svn] r74715 - pypy/branch/port-to-2.6 Message-ID: <20100524162622.9D37A282BDB@codespeak.net> Author: electronicru Date: Mon May 24 18:26:20 2010 New Revision: 74715 Removed: pypy/branch/port-to-2.6/ Log: Removing duplicate branch. From arigo at codespeak.net Mon May 24 19:09:57 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 24 May 2010 19:09:57 +0200 (CEST) Subject: [pypy-svn] r74716 - in pypy/branch/blackhole-improvement/pypy/jit: backend/llgraph metainterp metainterp/test Message-ID: <20100524170957.36FA1282BDB@codespeak.net> Author: arigo Date: Mon May 24 19:09:55 2010 New Revision: 74716 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizefindnode.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizeopt.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_resume.py Log: General progress. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py Mon May 24 19:09:55 2010 @@ -74,12 +74,12 @@ 'int_eq' : (('int', 'int'), 'bool'), 'int_ne' : (('int', 'int'), 'bool'), 'int_is_true' : (('int',), 'bool'), + 'int_is_zero' : (('int',), 'bool'), 'int_neg' : (('int',), 'int'), 'int_invert' : (('int',), 'int'), 'int_add_ovf' : (('int', 'int'), 'int'), 'int_sub_ovf' : (('int', 'int'), 'int'), 'int_mul_ovf' : (('int', 'int'), 'int'), - 'bool_not' : (('bool',), 'bool'), 'uint_add' : (('int', 'int'), 'int'), 'uint_sub' : (('int', 'int'), 'int'), 'uint_mul' : (('int', 'int'), 'int'), Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py Mon May 24 19:09:55 2010 @@ -217,15 +217,13 @@ # to get the other behavior (i.e. using this __eq__). if self.__class__ is not other.__class__: return False - if isinstance(self.value, Symbolic): - v1 = "symbolic", id(self.value) - else: - v1 = self.value - if isinstance(other.value, Symbolic): - v2 = "symbolic", id(other.value) - else: - v2 = other.value - return v1 == v2 + try: + return self.value == other.value + except TypeError: + if (isinstance(self.value, Symbolic) and + isinstance(other.value, Symbolic)): + return self.value is other.value + raise def __ne__(self, other): return not (self == other) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizefindnode.py Mon May 24 19:09:55 2010 @@ -169,8 +169,8 @@ def find_nodes_no_escape(self, op): pass # for operations that don't escape their arguments - find_nodes_OOIS = find_nodes_no_escape - find_nodes_OOISNOT = find_nodes_no_escape + find_nodes_PTR_EQ = find_nodes_no_escape + find_nodes_PTR_NE = find_nodes_no_escape find_nodes_INSTANCEOF = find_nodes_no_escape find_nodes_GUARD_NONNULL = find_nodes_no_escape find_nodes_GUARD_ISNULL = find_nodes_no_escape Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizeopt.py Mon May 24 19:09:55 2010 @@ -89,7 +89,7 @@ if level == LEVEL_KNOWNCLASS: return self.known_class elif level == LEVEL_CONSTANT: - return cpu.ts.cls_of_box(cpu, self.box) + return cpu.ts.cls_of_box(self.box) else: return None @@ -785,10 +785,10 @@ return self.optimize_default(op) - def optimize_OOISNOT(self, op): + def optimize_PTR_NE(self, op): self._optimize_oois_ooisnot(op, True) - def optimize_OOIS(self, op): + def optimize_PTR_EQ(self, op): self._optimize_oois_ooisnot(op, False) def optimize_VIRTUAL_REF(self, op): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Mon May 24 19:09:55 2010 @@ -547,7 +547,7 @@ standard_box = self.metainterp.virtualizable_boxes[-1] if standard_box is box: return False - eqbox = self.metainterp.execute_and_record(rop.OOIS, None, + eqbox = self.metainterp.execute_and_record(rop.PTR_EQ, None, box, standard_box) eqbox = self.implement_guard_value(pc, eqbox) isstandard = eqbox.getint() Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py Mon May 24 19:09:55 2010 @@ -19,6 +19,7 @@ __slots__ = ('prev', 'boxes') def __init__(self, prev, boxes): + _assert_order(boxes) self.prev = prev self.boxes = boxes @@ -170,14 +171,7 @@ numb1, liveboxes, v = self.number(values, snapshot.prev) n = len(liveboxes)-v boxes = snapshot.boxes - # - if not we_are_translated(): - # verifies that 'boxes' are in order: all INTs, then all REFs, - # and finally all FLOATs. - _kind2count = {INT: 1, REF: 2, FLOAT: 3} - kinds = [_kind2count[box.type] for box in boxes] - assert kinds == sorted(kinds) - # + _assert_order(boxes) length = len(boxes) numslength = length + 1 + (length > 0 and boxes[-1].type == FLOAT) nums = [UNASSIGNED] * numslength @@ -422,6 +416,14 @@ return self.liveboxes_from_env[box] return self.liveboxes[box] +def _assert_order(boxes): + if not we_are_translated(): + # verifies that 'boxes' are in order: all INTs, then all REFs, + # and finally all FLOATs. + _kind2count = {INT: 1, REF: 2, FLOAT: 3} + kinds = [_kind2count[box.type] for box in boxes] + assert kinds == sorted(kinds) + class AbstractVirtualInfo(object): #def allocate(self, metainterp): @@ -497,7 +499,7 @@ # NB. the check for the kind of array elements is moved out of the loop if arraydescr.is_array_of_pointers(): for i in range(length): - decoder.setarrayitem_ptr(arraydescr, array, i, + decoder.setarrayitem_ref(arraydescr, array, i, self.fieldnums[i]) elif arraydescr.is_array_of_floats(): for i in range(length): @@ -632,7 +634,8 @@ return self.metainterp.execute_and_record(rop.NEW, typedescr) def allocate_array(self, arraydescr, length): - return metainterp.execute_and_record(rop.NEW_ARRAY, arraydescr, length) + return self.metainterp.execute_and_record(rop.NEW_ARRAY, + arraydescr, ConstInt(length)) def setfield(self, descr, structbox, fieldnum): if descr.is_pointer_field(): @@ -656,9 +659,9 @@ def setarrayitem(self, arraydescr, arraybox, index, fieldnum, kind): itembox = self.decode_box(fieldnum, kind) - metainterp.execute_and_record(rop.SETARRAYITEM_GC, - arraydescr, arraybox, - ConstInt(index), itembox) + self.metainterp.execute_and_record(rop.SETARRAYITEM_GC, + arraydescr, arraybox, + ConstInt(index), itembox) def decode_int(self, tagged): return self.decode_box(tagged, INT) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_optimizefindnode.py Mon May 24 19:09:55 2010 @@ -17,6 +17,7 @@ from pypy.jit.metainterp.specnode import VirtualStructSpecNode from pypy.jit.metainterp.specnode import ConstantSpecNode from pypy.jit.codewriter.effectinfo import EffectInfo +from pypy.jit.codewriter.heaptracker import register_known_gctype from pypy.jit.metainterp.test.oparser import parse def test_sort_descrs(): @@ -129,13 +130,11 @@ jit_virtual_ref_vtable = vrefinfo.jit_virtual_ref_vtable jvr_vtable_adr = llmemory.cast_ptr_to_adr(jit_virtual_ref_vtable) -## cpu.class_sizes = { -## llmemory.cast_adr_to_int(node_vtable_adr): cpu.sizeof(NODE), -## llmemory.cast_adr_to_int(node_vtable_adr2): cpu.sizeof(NODE2), -## llmemory.cast_adr_to_int(u_vtable_adr): cpu.sizeof(U), -## llmemory.cast_adr_to_int(jvr_vtable_adr): cpu.sizeof( -## vrefinfo.JIT_VIRTUAL_REF), -## } + register_known_gctype(cpu, node_vtable, NODE) + register_known_gctype(cpu, node_vtable2, NODE2) + register_known_gctype(cpu, u_vtable, U) + register_known_gctype(cpu, jit_virtual_ref_vtable,vrefinfo.JIT_VIRTUAL_REF) + namespace = locals() class OOtypeMixin_xxx_disabled(object): @@ -220,9 +219,9 @@ # def constclass(cls_vtable): if self.type_system == 'lltype': - XXX - return ConstAddr(llmemory.cast_ptr_to_adr(cls_vtable), - self.cpu) + return ConstInt( + llmemory.cast_adr_to_int( + llmemory.cast_ptr_to_adr(cls_vtable))) else: return ConstObj(ootype.cast_to_object(cls_vtable)) def constant(value): @@ -358,7 +357,7 @@ assert not boxp2.fromstart assert boxp1.knownclsbox is None - assert boxp2.knownclsbox.value == self.node_vtable_adr + assert boxp2.knownclsbox.getaddr() == self.node_vtable_adr def test_find_nodes_new_2(self): ops = """ @@ -414,8 +413,8 @@ assert not boxp2.fromstart assert not boxp3.fromstart - assert boxp2.knownclsbox.value == self.node_vtable_adr - assert boxp3.knownclsbox.value == self.node_vtable_adr2 + assert boxp2.knownclsbox.getaddr() == self.node_vtable_adr + assert boxp3.knownclsbox.getaddr() == self.node_vtable_adr2 def test_find_nodes_new_aliasing_0(self): ops = """ @@ -504,31 +503,31 @@ nextdescr=Virtual(node_vtable, nextdescr=Virtual(node_vtable)))''') - def test_find_nodes_oois(self): + def test_find_nodes_ptr_eq(self): ops = """ [p3, p4, p2] p0 = new_with_vtable(ConstClass(node_vtable)) p1 = new_with_vtable(ConstClass(node_vtable)) guard_nonnull(p0) [] - i3 = ooisnot(p0, NULL) + i3 = ptr_ne(p0, NULL) guard_true(i3) [] - i4 = oois(p0, NULL) + i4 = ptr_eq(p0, NULL) guard_false(i4) [] - i5 = ooisnot(NULL, p0) + i5 = ptr_ne(NULL, p0) guard_true(i5) [] - i6 = oois(NULL, p0) + i6 = ptr_eq(NULL, p0) guard_false(i6) [] - i7 = ooisnot(p0, p1) + i7 = ptr_ne(p0, p1) guard_true(i7) [] - i8 = oois(p0, p1) + i8 = ptr_eq(p0, p1) guard_false(i8) [] - i9 = ooisnot(p0, p2) + i9 = ptr_ne(p0, p2) guard_true(i9) [] - i10 = oois(p0, p2) + i10 = ptr_eq(p0, p2) guard_false(i10) [] - i11 = ooisnot(p2, p1) + i11 = ptr_ne(p2, p1) guard_true(i11) [] - i12 = oois(p2, p1) + i12 = ptr_eq(p2, p1) guard_false(i12) [] jump(p0, p1, p2) """ Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_optimizeopt.py Mon May 24 19:09:55 2010 @@ -1,8 +1,7 @@ import py from pypy.rlib.objectmodel import instantiate -from pypy.jit.metainterp.test.test_resume import MyMetaInterp from pypy.jit.metainterp.test.test_optimizefindnode import (LLtypeMixin, - OOtypeMixin, + #OOtypeMixin, BaseTest) from pypy.jit.metainterp.optimizefindnode import PerfectSpecializationFinder from pypy.jit.metainterp import optimizeopt @@ -10,18 +9,17 @@ from pypy.jit.metainterp.optimizeutil import InvalidLoop from pypy.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt from pypy.jit.metainterp.jitprof import EmptyProfiler -from pypy.jit.metainterp import executor, compile, resume +from pypy.jit.metainterp import executor, compile, resume, history from pypy.jit.metainterp.resoperation import rop, opname, ResOperation from pypy.jit.metainterp.test.oparser import pure_parse -class FakeFrame(object): - parent_resumedata_snapshot = None - parent_resumedata_frame_info_list = None - - def __init__(self, code="", pc=0, exc_target=-1): - self.jitcode = code - self.pc = pc - self.exception_target = exc_target +##class FakeFrame(object): +## parent_resumedata_snapshot = None +## parent_resumedata_frame_info_list = None + +## def __init__(self, code="", pc=0): +## self.jitcode = code +## self.pc = pc class Fake(object): failargs_limit = 1000 @@ -37,7 +35,7 @@ def test_store_final_boxes_in_guard(): from pypy.jit.metainterp.compile import ResumeGuardDescr - from pypy.jit.metainterp.resume import tag, TAGBOX + from pypy.jit.metainterp.resume import tag, TAGBOX, TYPEBARRIER b0 = BoxInt() b1 = BoxInt() opt = optimizeopt.Optimizer(FakeMetaInterpStaticData(LLtypeMixin.cpu), @@ -45,20 +43,19 @@ fdescr = ResumeGuardDescr(None, None) op = ResOperation(rop.GUARD_TRUE, [], None, descr=fdescr) # setup rd data - fi0 = resume.FrameInfo(None, FakeFrame("code0", 1, 2)) - fdescr.rd_frame_info_list = resume.FrameInfo(fi0, - FakeFrame("code1", 3, -1)) + fi0 = resume.FrameInfo(None, "code0", 11) + fdescr.rd_frame_info_list = resume.FrameInfo(fi0, "code1", 33) snapshot0 = resume.Snapshot(None, [b0]) fdescr.rd_snapshot = resume.Snapshot(snapshot0, [b1]) # opt.store_final_boxes_in_guard(op) if op.fail_args == [b0, b1]: - assert fdescr.rd_numb.nums == [tag(1, TAGBOX)] - assert fdescr.rd_numb.prev.nums == [tag(0, TAGBOX)] + assert fdescr.rd_numb.nums == [tag(1, TAGBOX), TYPEBARRIER] + assert fdescr.rd_numb.prev.nums == [tag(0, TAGBOX), TYPEBARRIER] else: assert op.fail_args == [b1, b0] - assert fdescr.rd_numb.nums == [tag(0, TAGBOX)] - assert fdescr.rd_numb.prev.nums == [tag(1, TAGBOX)] + assert fdescr.rd_numb.nums == [tag(0, TAGBOX), TYPEBARRIER] + assert fdescr.rd_numb.prev.nums == [tag(1, TAGBOX), TYPEBARRIER] assert fdescr.rd_virtuals is None assert fdescr.rd_consts == [] @@ -215,14 +212,18 @@ def __eq__(self, other): return type(self) is type(other) # xxx obscure +def _sortboxes(boxes): + _kind2count = {history.INT: 1, history.REF: 2, history.FLOAT: 3} + return sorted(boxes, key=lambda box: _kind2count[box.type]) + class BaseTestOptimizeOpt(BaseTest): def invent_fail_descr(self, fail_args): if fail_args is None: return None descr = Storage() - descr.rd_frame_info_list = resume.FrameInfo(None, FakeFrame()) - descr.rd_snapshot = resume.Snapshot(None, fail_args) + descr.rd_frame_info_list = resume.FrameInfo(None, "code", 11) + descr.rd_snapshot = resume.Snapshot(None, _sortboxes(fail_args)) return descr def assert_equal(self, optimized, expected): @@ -273,7 +274,7 @@ i0 = int_add(2, 3) i1 = int_is_true(i0) guard_true(i1) [] - i2 = bool_not(i1) + i2 = int_is_zero(i1) guard_false(i2) [] guard_value(i0, 5) [] jump() @@ -289,7 +290,7 @@ from pypy.jit.metainterp.executor import execute_nonspec from pypy.jit.metainterp.history import BoxInt import random - for opnum in range(rop.INT_ADD, rop.BOOL_NOT+1): + for opnum in range(rop.INT_ADD, rop.SAME_AS+1): try: op = opname[opnum] except KeyError: @@ -309,7 +310,7 @@ jump() """ % (op.lower(), ', '.join(map(str, args))) argboxes = [BoxInt(a) for a in args] - expected_value = execute_nonspec(self.cpu, opnum, + expected_value = execute_nonspec(self.cpu, None, opnum, argboxes).getint() expected = """ [] @@ -478,13 +479,13 @@ ops = """ [p0] guard_class(p0, ConstClass(node_vtable)) [] - i0 = ooisnot(p0, NULL) + i0 = ptr_ne(p0, NULL) guard_true(i0) [] - i1 = oois(p0, NULL) + i1 = ptr_eq(p0, NULL) guard_false(i1) [] - i2 = ooisnot(NULL, p0) + i2 = ptr_ne(NULL, p0) guard_true(i0) [] - i3 = oois(NULL, p0) + i3 = ptr_eq(NULL, p0) guard_false(i1) [] jump(p0) """ @@ -499,13 +500,13 @@ ops = """ [p0] setfield_gc(p0, 5, descr=valuedescr) # forces p0 != NULL - i0 = ooisnot(p0, NULL) + i0 = ptr_ne(p0, NULL) guard_true(i0) [] - i1 = oois(p0, NULL) + i1 = ptr_eq(p0, NULL) guard_false(i1) [] - i2 = ooisnot(NULL, p0) + i2 = ptr_ne(NULL, p0) guard_true(i0) [] - i3 = oois(NULL, p0) + i3 = ptr_eq(NULL, p0) guard_false(i1) [] guard_nonnull(p0) [] jump(p0) @@ -728,25 +729,25 @@ ops = """ [p0, p1, p2] guard_nonnull(p0) [] - i3 = ooisnot(p0, NULL) + i3 = ptr_ne(p0, NULL) guard_true(i3) [] - i4 = oois(p0, NULL) + i4 = ptr_eq(p0, NULL) guard_false(i4) [] - i5 = ooisnot(NULL, p0) + i5 = ptr_ne(NULL, p0) guard_true(i5) [] - i6 = oois(NULL, p0) + i6 = ptr_eq(NULL, p0) guard_false(i6) [] - i7 = ooisnot(p0, p1) + i7 = ptr_ne(p0, p1) guard_true(i7) [] - i8 = oois(p0, p1) + i8 = ptr_eq(p0, p1) guard_false(i8) [] - i9 = ooisnot(p0, p2) + i9 = ptr_ne(p0, p2) guard_true(i9) [] - i10 = oois(p0, p2) + i10 = ptr_eq(p0, p2) guard_false(i10) [] - i11 = ooisnot(p2, p1) + i11 = ptr_ne(p2, p1) guard_true(i11) [] - i12 = oois(p2, p1) + i12 = ptr_eq(p2, p1) guard_false(i12) [] jump(p0, p1, p2) """ @@ -766,17 +767,17 @@ expected2 = """ [p0, p1, p2] guard_nonnull(p0) [] - i7 = ooisnot(p0, p1) + i7 = ptr_ne(p0, p1) guard_true(i7) [] - i8 = oois(p0, p1) + i8 = ptr_eq(p0, p1) guard_false(i8) [] - i9 = ooisnot(p0, p2) + i9 = ptr_ne(p0, p2) guard_true(i9) [] - i10 = oois(p0, p2) + i10 = ptr_eq(p0, p2) guard_false(i10) [] - i11 = ooisnot(p2, p1) + i11 = ptr_ne(p2, p1) guard_true(i11) [] - i12 = oois(p2, p1) + i12 = ptr_eq(p2, p1) guard_false(i12) [] jump(p0, p1, p2) """ @@ -870,7 +871,7 @@ p0 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p0, NULL, descr=nextdescr) p2 = getfield_gc(p0, descr=nextdescr) - i1 = oois(p2, NULL) + i1 = ptr_eq(p2, NULL) jump(i1) """ expected = """ @@ -886,7 +887,7 @@ p0 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p0, ConstPtr(myptr), descr=nextdescr) p2 = getfield_gc(p0, descr=nextdescr) - i1 = oois(p2, NULL) + i1 = ptr_eq(p2, NULL) jump(i1) """ expected = """ @@ -1514,7 +1515,7 @@ """ expected = """ [p1, i2, i3] - guard_true(i3) [p1, i2] + guard_true(i3) [i2, p1] i4 = int_neg(i2) setfield_gc(p1, NULL, descr=nextdescr) jump(p1, i2, i4) @@ -1879,7 +1880,7 @@ ops = """ [p1] guard_class(p1, ConstClass(node_vtable2)) [] - i = ooisnot(ConstPtr(myptr), p1) + i = ptr_ne(ConstPtr(myptr), p1) guard_true(i) [] jump(p1) """ @@ -1895,9 +1896,9 @@ [p0] p1 = getfield_gc(p0, descr=nextdescr) p2 = getfield_gc(p0, descr=nextdescr) - i1 = oois(p1, p2) + i1 = ptr_eq(p1, p2) guard_true(i1) [] - i2 = ooisnot(p1, p2) + i2 = ptr_ne(p1, p2) guard_false(i2) [] jump(p0) """ @@ -1911,8 +1912,8 @@ def test_remove_duplicate_pure_op(self): ops = """ [p1, p2] - i1 = oois(p1, p2) - i2 = oois(p1, p2) + i1 = ptr_eq(p1, p2) + i2 = ptr_eq(p1, p2) i3 = int_add(i1, 1) i3b = int_is_true(i3) guard_true(i3b) [] @@ -1927,7 +1928,7 @@ """ expected = """ [p1, p2] - i1 = oois(p1, p2) + i1 = ptr_eq(p1, p2) i3 = int_add(i1, 1) i3b = int_is_true(i3) guard_true(i3b) [] @@ -1967,7 +1968,7 @@ # typically called twice, before and after optimization if self.oparse is None: fdescr.rd_frame_info_list = resume.FrameInfo(None, - FakeFrame()) + "code", 11) fdescr.rd_snapshot = resume.Snapshot(None, fail_args) self.oparse = oparse # @@ -2039,7 +2040,7 @@ for pvar, pfieldname, pfieldvar in pendingfields: box = oparse.getvar(pvar) fielddescr = self.namespace[pfieldname.strip()] - fieldbox = executor.execute(self.cpu, + fieldbox = executor.execute(self.cpu, None, rop.GETFIELD_GC, fielddescr, box) @@ -2057,13 +2058,13 @@ if tag[0] in ('virtual', 'vstruct'): fieldname, fieldvalue = fieldtext.split('=') fielddescr = self.namespace[fieldname.strip()] - fieldbox = executor.execute(self.cpu, + fieldbox = executor.execute(self.cpu, None, rop.GETFIELD_GC, fielddescr, resolved) elif tag[0] == 'varray': fieldvalue = fieldtext - fieldbox = executor.execute(self.cpu, + fieldbox = executor.execute(self.cpu, None, rop.GETARRAYITEM_GC, tag[1], resolved, ConstInt(index)) @@ -2073,12 +2074,14 @@ index += 1 def check_expanded_fail_descr(self, expectedtext, guard_opnum): + from pypy.jit.metainterp.test.test_resume import ResumeDataFakeReader + from pypy.jit.metainterp.test.test_resume import MyMetaInterp guard_op, = [op for op in self.loop.operations if op.is_guard()] fail_args = guard_op.fail_args fdescr = guard_op.descr assert fdescr.guard_opnum == guard_opnum - reader = resume.ResumeDataReader(fdescr, fail_args, - MyMetaInterp(self.cpu)) + reader = ResumeDataFakeReader(fdescr, fail_args, + MyMetaInterp(self.cpu)) boxes = reader.consume_boxes() self._verify_fail_args(boxes, fdescr.oparse, expectedtext) @@ -2133,7 +2136,7 @@ setfield_gc(p1, p2, descr=nextdescr) setfield_gc(p2, i2, descr=valuedescr) setfield_gc(p2, p3, descr=nextdescr) - guard_true(i1, descr=fdescr) [p1, i3] + guard_true(i1, descr=fdescr) [i3, p1] jump(i2, i1, i3, p3) """ expected = """ @@ -2142,15 +2145,14 @@ jump(i2, 1, i3, p3) """ self.optimize_loop(ops, 'Not, Not, Not, Not', expected) - self.check_expanded_fail_descr('''p1, i3 + self.check_expanded_fail_descr('''i3, p1 where p1 is a node_vtable, valuedescr=1, nextdescr=p2 where p2 is a node_vtable, valuedescr=i2, nextdescr=p3 ''', rop.GUARD_TRUE) def test_expand_fail_4(self): - for arg in ['p1', 'p1,i2', 'i2,p1', 'p1,p2', 'p2,p1', - 'p1,p2,i2', 'p1,i2,p2', 'p2,p1,i2', - 'p2,i2,p1', 'i2,p1,p2', 'i2,p2,p1']: + for arg in ['p1', 'i2,p1', 'p1,p2', 'p2,p1', + 'i2,p1,p2', 'i2,p2,p1']: self.make_fail_descr() ops = """ [i1, i2, i3] @@ -2161,7 +2163,7 @@ setfield_gc(p1, i2, descr=valuedescr) setfield_gc(p1, p2, descr=nextdescr) setfield_gc(p2, i2, descr=valuedescr) - guard_true(i1, descr=fdescr) [i4, %s, i3] + guard_true(i1, descr=fdescr) [i4, i3, %s] jump(i1, i2, i3) """ expected = """ @@ -2170,7 +2172,7 @@ jump(1, i2, i3) """ self.optimize_loop(ops % arg, 'Not, Not, Not', expected) - self.check_expanded_fail_descr('''i3, %s, i3 + self.check_expanded_fail_descr('''i3, i3, %s where p1 is a node_vtable, valuedescr=i2, nextdescr=p2 where p2 is a node_vtable, valuedescr=i2''' % arg, rop.GUARD_TRUE) @@ -2185,7 +2187,7 @@ setfield_gc(p1, p2, descr=nextdescr) setfield_gc(p2, i2, descr=valuedescr) setfield_gc(p2, p1, descr=nextdescr) # a cycle - guard_true(i1, descr=fdescr) [p1, i3, p2, i4] + guard_true(i1, descr=fdescr) [i3, i4, p1, p2] jump(i2, i1, i3, i4) """ expected = """ @@ -2194,7 +2196,7 @@ jump(i2, 1, i3, i4) """ self.optimize_loop(ops, 'Not, Not, Not, Not', expected) - self.check_expanded_fail_descr('''p1, i3, p2, i4 + self.check_expanded_fail_descr('''i3, i4, p1, p2 where p1 is a node_vtable, valuedescr=i4, nextdescr=p2 where p2 is a node_vtable, valuedescr=i2, nextdescr=p1 ''', rop.GUARD_TRUE) @@ -2813,31 +2815,31 @@ ''' self.optimize_loop(ops, 'Not', expected) -class TestOOtype(BaseTestOptimizeOpt, OOtypeMixin): +##class TestOOtype(BaseTestOptimizeOpt, OOtypeMixin): - def test_instanceof(self): - ops = """ - [i0] - p0 = new_with_vtable(ConstClass(node_vtable)) - i1 = instanceof(p0, descr=nodesize) - jump(i1) - """ - expected = """ - [i0] - jump(1) - """ - self.optimize_loop(ops, 'Not', expected) +## def test_instanceof(self): +## ops = """ +## [i0] +## p0 = new_with_vtable(ConstClass(node_vtable)) +## i1 = instanceof(p0, descr=nodesize) +## jump(i1) +## """ +## expected = """ +## [i0] +## jump(1) +## """ +## self.optimize_loop(ops, 'Not', expected) - def test_instanceof_guard_class(self): - ops = """ - [i0, p0] - guard_class(p0, ConstClass(node_vtable)) [] - i1 = instanceof(p0, descr=nodesize) - jump(i1, p0) - """ - expected = """ - [i0, p0] - guard_class(p0, ConstClass(node_vtable)) [] - jump(1, p0) - """ - self.optimize_loop(ops, 'Not, Not', expected) +## def test_instanceof_guard_class(self): +## ops = """ +## [i0, p0] +## guard_class(p0, ConstClass(node_vtable)) [] +## i1 = instanceof(p0, descr=nodesize) +## jump(i1, p0) +## """ +## expected = """ +## [i0, p0] +## guard_class(p0, ConstClass(node_vtable)) [] +## jump(1, p0) +## """ +## self.optimize_loop(ops, 'Not, Not', expected) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_resume.py Mon May 24 19:09:55 2010 @@ -51,7 +51,9 @@ class MyMetaInterp: _already_allocated_resume_virtuals = None - def __init__(self, cpu): + def __init__(self, cpu=None): + if cpu is None: + cpu = LLtypeMixin.cpu self.cpu = cpu self.trace = [] self.framestack = [] @@ -63,7 +65,7 @@ return frame def execute_and_record(self, opnum, descr, *argboxes): - resbox = executor.execute(self.cpu, opnum, descr, *argboxes) + resbox = executor.execute(self.cpu, None, opnum, descr, *argboxes) self.trace.append((opnum, list(argboxes), resbox, @@ -214,12 +216,12 @@ return "" % (self.jitcode, self.pc, self._env) def test_Snapshot_create(): - l = ['b0', 'b1'] + l = [BoxInt(), BoxPtr()] snap = Snapshot(None, l) assert snap.prev is None assert snap.boxes is l - l1 = ['b3'] + l1 = [BoxInt()] snap1 = Snapshot(snap, l1) assert snap1.prev is snap assert snap1.boxes is l1 @@ -296,7 +298,7 @@ assert snapshot.prev is None assert snapshot.boxes == fs[0]._env - fs[2]._env = [b2, b3] + fs[2]._env = [b3, b2] fs[2].pc = 15 vbs = [b1, b2] vrs = [b3] @@ -765,7 +767,7 @@ storage = Storage() snapshot = Snapshot(None, [b1, ConstInt(1), b1, b2]) snapshot = Snapshot(snapshot, [ConstInt(2), ConstInt(3)]) - snapshot = Snapshot(snapshot, [b1, b2, b3]) + snapshot = Snapshot(snapshot, [b1, b3, b2]) storage.rd_snapshot = snapshot storage.rd_frame_info_list = None return storage @@ -779,7 +781,7 @@ assert storage.rd_snapshot is None cpu = MyCPU([]) reader = ResumeDataDirectReader(cpu, storage) - _next_section(reader, sys.maxint, 2**16, -65) + _next_section(reader, sys.maxint, -65, 2**16) _next_section(reader, 2, 3) _next_section(reader, sys.maxint, 1, sys.maxint, 2**16) @@ -801,8 +803,30 @@ assert storage2.rd_consts is memo.consts +class ResumeDataFakeReader(ResumeDataBoxReader): + """Another subclass of AbstractResumeDataReader meant for tests.""" + def __init__(self, storage, newboxes, metainterp): + self.liveboxes = newboxes + self.metainterp = metainterp + self._prepare(metainterp.cpu, storage) + + def consume_boxes(self): + self.lst = [] + self._prepare_next_section() + return self.lst + + def write_an_int(self, count_i, box): + assert box.type == INT + self.lst.append(box) + def write_a_ref(self, count_r, box): + assert box.type == REF + self.lst.append(box) + def write_a_float(self, count_f, box): + assert box.type == FLOAT + self.lst.append(box) + + def test_virtual_adder_no_op_renaming(): - py.test.skip("XXX rewrite") b1s, b2s, b3s = [BoxInt(1), BoxInt(2), BoxInt(3)] storage = make_storage(b1s, b2s, b3s) memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) @@ -823,9 +847,9 @@ b1t, b3t = [BoxInt(11), BoxInt(33)] newboxes = _resume_remap(liveboxes, [b1_2, b3s], b1t, b3t) metainterp = MyMetaInterp() - reader = ResumeDataReader(storage, newboxes, metainterp) + reader = ResumeDataFakeReader(storage, newboxes, metainterp) lst = reader.consume_boxes() - assert lst == [b1t, b1t, b3t] + assert lst == [b1t, b3t, b1t] lst = reader.consume_boxes() assert lst == [ConstInt(2), ConstInt(3)] lst = reader.consume_boxes() @@ -834,7 +858,6 @@ def test_virtual_adder_make_constant(): - py.test.skip("XXX rewrite") b1s, b2s, b3s = [BoxInt(1), BoxPtr(), BoxInt(3)] b1s = ConstInt(111) storage = make_storage(b1s, b2s, b3s) @@ -844,10 +867,10 @@ b2t, b3t = [BoxPtr(demo55o), BoxInt(33)] newboxes = _resume_remap(liveboxes, [b2s, b3s], b2t, b3t) metainterp = MyMetaInterp() - reader = ResumeDataReader(storage, newboxes, metainterp) + reader = ResumeDataFakeReader(storage, newboxes, metainterp) lst = reader.consume_boxes() c1t = ConstInt(111) - assert lst == [c1t, b2t, b3t] + assert lst == [c1t, b3t, b2t] lst = reader.consume_boxes() assert lst == [ConstInt(2), ConstInt(3)] lst = reader.consume_boxes() @@ -856,7 +879,6 @@ def test_virtual_adder_make_virtual(): - py.test.skip("XXX rewrite") b2s, b3s, b4s, b5s = [BoxPtr(), BoxInt(3), BoxPtr(), BoxPtr()] c1s = ConstInt(111) storage = Storage() @@ -897,10 +919,10 @@ b5s], b3t, b5t) metainterp = MyMetaInterp() - reader = ResumeDataReader(storage, newboxes, metainterp) + reader = ResumeDataFakeReader(storage, newboxes, metainterp) assert len(reader.virtuals) == 2 - b2t = reader._decode_box(modifier._gettagged(b2s)) - b4t = reader._decode_box(modifier._gettagged(b4s)) + b2t = reader.decode_ref(modifier._gettagged(b2s)) + b4t = reader.decode_ref(modifier._gettagged(b4s)) trace = metainterp.trace b2new = (rop.NEW_WITH_VTABLE, [ConstAddr(LLtypeMixin.node_vtable_adr, LLtypeMixin.cpu)], @@ -930,7 +952,6 @@ assert ptr2.parent.next == ptr def test_virtual_adder_make_varray(): - py.test.skip("XXX rewrite") b2s, b4s = [BoxPtr(), BoxInt(4)] c1s = ConstInt(111) storage = Storage() @@ -961,9 +982,9 @@ b4t) # resume metainterp = MyMetaInterp() - reader = ResumeDataReader(storage, newboxes, metainterp) + reader = ResumeDataFakeReader(storage, newboxes, metainterp) assert len(reader.virtuals) == 1 - b2t = reader._decode_box(tag(0, TAGVIRTUAL)) + b2t = reader.decode_ref(tag(0, TAGVIRTUAL)) trace = metainterp.trace expected = [ (rop.NEW_ARRAY, [ConstInt(2)], b2t, LLtypeMixin.arraydescr), @@ -983,7 +1004,6 @@ def test_virtual_adder_make_vstruct(): - py.test.skip("XXX rewrite") b2s, b4s = [BoxPtr(), BoxPtr()] c1s = ConstInt(111) storage = Storage() @@ -1007,9 +1027,9 @@ # NULL = ConstPtr.value metainterp = MyMetaInterp() - reader = ResumeDataReader(storage, newboxes, metainterp) + reader = ResumeDataFakeReader(storage, newboxes, metainterp) assert len(reader.virtuals) == 1 - b2t = reader._decode_box(tag(0, TAGVIRTUAL)) + b2t = reader.decode_ref(tag(0, TAGVIRTUAL)) trace = metainterp.trace expected = [ @@ -1027,7 +1047,6 @@ def test_virtual_adder_pending_fields(): - py.test.skip("XXX rewrite") b2s, b4s = [BoxPtr(), BoxPtr()] storage = Storage() memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) @@ -1055,7 +1074,7 @@ newboxes = _resume_remap(liveboxes, [b2s, b4s], b2t, b4t) metainterp = MyMetaInterp() - reader = ResumeDataReader(storage, newboxes, metainterp) + reader = ResumeDataFakeReader(storage, newboxes, metainterp) assert reader.virtuals is None trace = metainterp.trace b2set = (rop.SETFIELD_GC, [b2t, b4t], None, LLtypeMixin.nextdescr) From arigo at codespeak.net Mon May 24 19:14:31 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 24 May 2010 19:14:31 +0200 (CEST) Subject: [pypy-svn] r74717 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100524171431.0A060282BDB@codespeak.net> Author: arigo Date: Mon May 24 19:14:30 2010 New Revision: 74717 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/logger.py Log: Revert this change. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/logger.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/logger.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/logger.py Mon May 24 19:14:30 2010 @@ -52,7 +52,7 @@ addr = arg.getaddr() name = self.metainterp_sd.get_name_from_address(addr) if name: - return 'ConstAddr(' + name + ')' + return 'ConstClass(' + name + ')' return str(arg.value) elif isinstance(arg, BoxInt): return 'i' + str(mv) From wildchild at codespeak.net Mon May 24 19:31:14 2010 From: wildchild at codespeak.net (wildchild at codespeak.net) Date: Mon, 24 May 2010 19:31:14 +0200 (CEST) Subject: [pypy-svn] r74718 - pypy/trunk/pypy/lib Message-ID: <20100524173114.9731E282BDB@codespeak.net> Author: wildchild Date: Mon May 24 19:31:13 2010 New Revision: 74718 Modified: pypy/trunk/pypy/lib/stackless.py Log: Pickling a tasklet halted with stackless.schedule() shouldn't save the state passed the call to schedule(). Too many things (read other tasklets) are then pickled when they shouldn't. Modified: pypy/trunk/pypy/lib/stackless.py ============================================================================== --- pypy/trunk/pypy/lib/stackless.py (original) +++ pypy/trunk/pypy/lib/stackless.py Mon May 24 19:31:13 2010 @@ -488,6 +488,19 @@ # we want to get rid of the parent thing. # for now, we just drop it a, b, c, d = coro_state + + # Removing all frames related to stackless.py. + # They point to stuff we don't want to be pickled. + frame_list = list(b) + new_frame_list = [] + for frame in frame_list: + if frame.f_code == schedule.func_code: + # Removing everything including and after the + # call to stackless.schedule() + break + new_frame_list.append(frame) + b = tuple(new_frame_list) + if d: assert isinstance(d, coroutine) coro_state = a, b, c, None From wildchild at codespeak.net Mon May 24 19:52:25 2010 From: wildchild at codespeak.net (wildchild at codespeak.net) Date: Mon, 24 May 2010 19:52:25 +0200 (CEST) Subject: [pypy-svn] r74719 - pypy/trunk/pypy/lib/test2 Message-ID: <20100524175225.29E56282BDB@codespeak.net> Author: wildchild Date: Mon May 24 19:52:23 2010 New Revision: 74719 Modified: pypy/trunk/pypy/lib/test2/test_stackless.py Log: Test to show the problem resolved in r74718 about pickling tasklets halted on stackless.schedule(). Modified: pypy/trunk/pypy/lib/test2/test_stackless.py ============================================================================== --- pypy/trunk/pypy/lib/test2/test_stackless.py (original) +++ pypy/trunk/pypy/lib/test2/test_stackless.py Mon May 24 19:52:23 2010 @@ -54,3 +54,36 @@ ''' in mod.__dict__ finally: del sys.modules['mod'] + + def test_pickle2(self): + # To test a bug where too much stuff gets pickled when + # a tasklet halted on stackless.schedule() is pickled. + import new, sys + + mod = new.module('mod') + sys.modules['mod'] = mod + mod.can_unpickle = self.can_unpickle + mod.skip = skip + try: + exec ''' +import pickle, sys +import stackless +import socket + +def task_should_be_picklable(): + stackless.schedule() + +def task_socket(): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + stackless.schedule() + +def task_pickle(ref_task): + p = pickle.dumps(ref_task) + +ref_task = stackless.tasklet(task_should_be_picklable)() +stackless.tasklet(task_socket)() +stackless.tasklet(task_pickle)(ref_task) +stackless.run() +''' in mod.__dict__ + finally: + del sys.modules['mod'] From cfbolz at codespeak.net Mon May 24 19:55:40 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 24 May 2010 19:55:40 +0200 (CEST) Subject: [pypy-svn] r74720 - pypy/extradoc/talk/s3-2010 Message-ID: <20100524175540.16F59282BDB@codespeak.net> Author: cfbolz Date: Mon May 24 19:55:38 2010 New Revision: 74720 Added: pypy/extradoc/talk/s3-2010/ pypy/extradoc/talk/s3-2010/virtuals.txt - copied unchanged from r74719, user/fijal/txt/virtuals.txt Log: a directory for the s3 blog posts and later paper. From jcreigh at codespeak.net Mon May 24 22:24:35 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Mon, 24 May 2010 22:24:35 +0200 (CEST) Subject: [pypy-svn] r74721 - in pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86: . tool Message-ID: <20100524202435.01A9F282BDB@codespeak.net> Author: jcreigh Date: Mon May 24 22:24:33 2010 New Revision: 74721 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regalloc.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/tool/instruction_encoding.sh Log: a little progress on using rx86 from assembler.py Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Mon May 24 22:24:33 2010 @@ -16,7 +16,7 @@ xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, RegLoc, StackLoc, - ImmedLoc, AddressLoc, imm) + ImmedLoc, AddressLoc, imm, rel32) from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.jit.backend.x86 import rx86, regloc, codebuf @@ -527,14 +527,15 @@ def _cmpop(cond, rev_cond): def genop_cmp(self, op, arglocs, result_loc): + # Clear high bits + self.mc.MOV_ri(result_loc.value, 0) rl = result_loc.lowest8bits() if isinstance(op.args[0], Const): self.mc.CMP(arglocs[1], arglocs[0]) - getattr(self.mc, 'SET' + rev_cond)(rl) + self.mc.SET_ir(rx86.Conditions[rev_cond], rl.value) else: self.mc.CMP(arglocs[0], arglocs[1]) - getattr(self.mc, 'SET' + cond)(rl) - self.mc.MOVZX(result_loc, rl) + self.mc.SET_ir(rx86.Conditions[cond], rl.value) return genop_cmp def _cmpop_float(cond, is_ne=False): @@ -617,11 +618,12 @@ mc.MOV_sr(p, tmp.value) p += round_up_to_4(loc.width) self._regalloc.reserve_param(p//WORD) - mc.CALL_l(x) + # x is a location + mc.CALL(x) self.mark_gc_roots() def call(self, addr, args, res): - self._emit_call(addr, args) + self._emit_call(rel32(addr), args) assert res is eax genop_int_neg = _unaryop("NEG") @@ -940,16 +942,16 @@ basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) assert itemsize == 1 - self.mc.MOVZX(resloc, addr8_add(base_loc, ofs_loc, basesize)) + self.mc.MOVZX8(resloc, AddressLoc(base_loc, ofs_loc, 0, basesize)) def genop_unicodegetitem(self, op, arglocs, resloc): base_loc, ofs_loc = arglocs basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) if itemsize == 4: - self.mc.MOV(resloc, addr_add(base_loc, ofs_loc, basesize, 2)) + self.mc.MOV(resloc, AddressLoc(base_loc, ofs_loc, 2, basesize)) elif itemsize == 2: - self.mc.MOVZX(resloc, addr_add(base_loc, ofs_loc, basesize, 1)) + self.mc.MOVZX16(resloc, AddressLoc(base_loc, ofs_loc, 1, basesize)) else: assert 0, itemsize @@ -961,7 +963,7 @@ def genop_guard_guard_no_exception(self, ign_1, guard_op, addr, locs, ign_2): - self.mc.CMP(heap(self.cpu.pos_exception()), imm(0)) + self.mc.CMP_ji(self.cpu.pos_exception(), 0) return self.implement_guard(addr, 'NZ') def genop_guard_guard_exception(self, ign_1, guard_op, addr, @@ -1421,7 +1423,7 @@ self._emit_call(x, arglocs, 2, tmp=tmp) - if isinstance(resloc, MODRM64): + if isinstance(resloc, StackLoc) and resloc.width == 8: self.mc.FSTP(resloc) elif size == 1: self.mc.AND(eax, imm(0xff)) @@ -1590,6 +1592,14 @@ num = getattr(rop, opname.upper()) genop_list[num] = value +def addr_add_const(reg_or_imm1, offset): + # XXX: ri386 migration shim + return AddressLoc(reg_or_imm1, ImmedLoc(0), 0, offset) + +def mem(loc, offset): + # XXX: ri386 migration shim + return AddressLoc(loc, ImmedLoc(0), (0), offset) + def round_up_to_4(size): if size < 4: return 4 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regalloc.py Mon May 24 22:24:33 2010 @@ -95,7 +95,7 @@ def convert_to_imm(self, c): adr = self.float_constants.record_float(c.getfloat()) - return heap64(adr) + return AddressLoc(ImmedLoc(adr), ImmedLoc(0), 0, 0) def after_call(self, v): # the result is stored in st0, but we don't have this around, Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Mon May 24 22:24:33 2010 @@ -124,6 +124,11 @@ getattr(self, name + '_' + code1 + code2)(loc1.value, loc2.value) return INSN + def _unaryop(name): + def INSN(self, loc): + getattr(self, name + '_' + loc.location_code())(loc.value) + return INSN + ADD = _binaryop('ADD') OR = _binaryop('OR') XOR = _binaryop('XOR') @@ -131,10 +136,29 @@ AND = _binaryop('AND') SUB = _binaryop('SUB') IMUL = _binaryop('IMUL') + NEG = _unaryop('NEG') + CMP = _binaryop('CMP') MOV = _binaryop('MOV') MOV8 = _binaryop('MOV8') + MOVZX8 = _binaryop("MOVZX8") + MOVZX16 = _binaryop("MOVZX16") + MOVSD = _binaryop('MOVSD') + ADDSD = _binaryop('ADDSD') + SUBSD = _binaryop('SUBSD') + MULSD = _binaryop('MULSD') + DIVSD = _binaryop('DIVSD') + UCOMISD = _binaryop('UCOMISD') + + + def CALL(self, loc): + # FIXME: Kludge that works in 32-bit because the "relative" CALL is + # actually absolute on i386 + if loc.location_code() == 'j': + self.CALL_l(loc.value) + else: + getattr(self, 'CALL_' + loc.location_code())(loc.value) def MOV16(self, dest_loc, src_loc): # Select 16-bit operand mode @@ -149,23 +173,6 @@ assert isinstance(loc, RegLoc) self.POP_r(loc.value) - def CMP(self, loc0, loc1): - if isinstance(loc0, RegLoc): - val0 = loc0.value - if isinstance(loc1, RegLoc): - self.CMP_rr(val0, loc1.value) - elif isinstance(loc1, StackLoc): - self.CMP_rb(val0, loc1.value) - else: - self.CMP_ri(val0, loc1.getint()) - else: - assert isinstance(loc0, StackLoc) - val0 = loc0.value - if isinstance(loc1, RegLoc): - self.CMP_br(val0, loc1.value) - else: - self.CMP_bi(val0, loc1.getint()) - def CMPi(self, loc0, loc1): # like CMP, but optimized for the case of loc1 being a Const assert isinstance(loc1, Const) @@ -182,6 +189,10 @@ else: return ImmedLoc(x) +def rel32(x): + # XXX: ri386 migration shim + return AddressLoc(ImmedLoc(x), ImmedLoc(0)) + all_extra_instructions = [name for name in LocationCodeBuilder.__dict__ if name[0].isupper()] all_extra_instructions.sort() Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py Mon May 24 22:24:33 2010 @@ -310,6 +310,17 @@ return INSN_ri, INSN_rr, INSN_rb, INSN_bi, INSN_br +def select_8_or_32_bit_immed(insn_8, insn_32): + def INSN(*args): + immed = args[-1] + if single_byte(immed): + insn_8(*args) + else: + assert fits_in_32bits(immed) + insn_32(*args) + + return INSN + # ____________________________________________________________ @@ -357,11 +368,15 @@ # "MOV reg1, [immediate2]" and the opposite direction MOV_rj = insn(rex_w, '\x8B', register(1,8), '\x05', immediate(2)) MOV_jr = insn(rex_w, '\x89', register(2,8), '\x05', immediate(1)) + MOV_ji = insn(rex_w, '\xC7', '\x05', immediate(1), immediate(2)) MOV8_mr = insn(rex_w, '\x88', register(2, 8), mem_reg_plus_const(1)) MOVZX8_rm = insn(rex_w, '\x0F\xB6', register(1,8), mem_reg_plus_const(2)) + MOVZX8_ra = insn(rex_w, '\x0F\xB6', register(1,8), mem_reg_plus_scaled_reg_plus_const(2)) + MOVZX16_rm = insn(rex_w, '\x0F\xB7', register(1,8), mem_reg_plus_const(2)) + MOVZX16_ra = insn(rex_w, '\x0F\xB7', register(1,8), mem_reg_plus_scaled_reg_plus_const(2)) # ------------------------------ Arithmetic ------------------------------ @@ -372,25 +387,28 @@ XOR_ri, XOR_rr, XOR_rb, _, _ = common_modes(6) CMP_ri, CMP_rr, CMP_rb, CMP_bi, CMP_br = common_modes(7) + _CMP_mi8 = insn(rex_w, '\x83', orbyte(7<<3), mem_reg_plus_const(1), immediate(2, 'b')) + _CMP_mi32 = insn(rex_w, '\x81', orbyte(7<<3), mem_reg_plus_const(1), immediate(2)) + CMP_mi = select_8_or_32_bit_immed(_CMP_mi8, _CMP_mi32) + + _CMP_ji8 = insn(rex_w, '\x83', '\x3D', immediate(1), immediate(2, 'b')) + _CMP_ji32 = insn(rex_w, '\x81', '\x3D', immediate(1), immediate(2)) + CMP_ji = select_8_or_32_bit_immed(_CMP_ji8, _CMP_ji32) + + NEG_r = insn(rex_w, '\xF7', register(1), '\xD8') + DIV_r = insn(rex_w, '\xF7', register(1), '\xF0') IDIV_r = insn(rex_w, '\xF7', register(1), '\xF8') IMUL_rr = insn(rex_w, '\x0F\xAF', register(1, 8), register(2), '\xC0') IMUL_rb = insn(rex_w, '\x0F\xAF', register(1, 8), stack_bp(2)) - # 8-bit immediate + _IMUL_rri8 = insn(rex_w, '\x6B', register(1, 8), register(2), '\xC0', immediate(3, 'b')) - # 32-bit immediate _IMUL_rri32 = insn(rex_w, '\x69', register(1, 8), register(2), '\xC0', immediate(3)) - - def IMUL_rri(self, reg1, reg2, immed): - if single_byte(immed): - self._IMUL_rri8(reg1, reg2, immed) - else: - assert fits_in_32bits(immed) - self._IMUL_rri32(reg1, reg2, immed) + IMUL_rri = select_8_or_32_bit_immed(_IMUL_rri8, _IMUL_rri32) def IMUL_ri(self, reg, immed): - return self.IMUL_rri(reg, reg, immed) + self.IMUL_rri(reg, reg, immed) # ------------------------------ Misc stuff ------------------------------ @@ -433,6 +451,28 @@ MOVSD_rj = xmminsn('\xF2', rex_nw, '\x0F\x10', register(1, 8), '\x05', immediate(2)) MOVSD_jr = xmminsn('\xF2', rex_nw, '\x0F\x11', register(2, 8), '\x05', immediate(1)) + # Arithmetic + ADDSD_rr = xmminsn('\xF2', rex_nw, '\x0F\x58', register(1, 8), register(2), '\xC0') + ADDSD_rb = xmminsn('\xF2', rex_nw, '\x0F\x58', register(1, 8), stack_bp(2)) + ADDSD_rj = xmminsn('\xF2', rex_nw, '\x0F\x58', register(1, 8), '\x05', immediate(2)) + + SUBSD_rr = xmminsn('\xF2', rex_nw, '\x0F\x5C', register(1, 8), register(2), '\xC0') + SUBSD_rb = xmminsn('\xF2', rex_nw, '\x0F\x5C', register(1, 8), stack_bp(2)) + SUBSD_rj = xmminsn('\xF2', rex_nw, '\x0F\x5C', register(1, 8), '\x05', immediate(2)) + + MULSD_rr = xmminsn('\xF2', rex_nw, '\x0F\x59', register(1, 8), register(2), '\xC0') + MULSD_rb = xmminsn('\xF2', rex_nw, '\x0F\x59', register(1, 8), stack_bp(2)) + MULSD_rj = xmminsn('\xF2', rex_nw, '\x0F\x59', register(1, 8), '\x05', immediate(2)) + + DIVSD_rr = xmminsn('\xF2', rex_nw, '\x0F\x5E', register(1, 8), register(2), '\xC0') + DIVSD_rb = xmminsn('\xF2', rex_nw, '\x0F\x5E', register(1, 8), stack_bp(2)) + DIVSD_rj = xmminsn('\xF2', rex_nw, '\x0F\x5E', register(1, 8), '\x05', immediate(2)) + + # Comparision + UCOMISD_rr = xmminsn('\x66', rex_nw, '\x0F\x2E', register(1, 8), register(2), '\xC0') + UCOMISD_rb = xmminsn('\x66', rex_nw, '\x0F\x2E', register(1, 8), stack_bp(2)) + UCOMISD_rj = xmminsn('\x66', rex_nw, '\x0F\x2E', register(1, 8), '\x05', immediate(2)) + # ------------------------------------------------------------ Conditions = { @@ -499,12 +539,27 @@ py.test.skip("MOV_rj unsupported") def MOV_jr(self, mem_immed, reg): py.test.skip("MOV_jr unsupported") + def MOV_ji(self, mem_immed, immed): + py.test.skip("MOV_ji unsupported") def XCHG_rj(self, reg, mem_immed): py.test.skip("XCGH_rj unsupported") + def CMP_ji(self, addr, immed): + py.test.skip("CMP_ji unsupported") def MOVSD_rj(self, xmm_reg, mem_immed): py.test.skip("MOVSD_rj unsupported") def MOVSD_jr(self, xmm_reg, mem_immed): py.test.skip("MOVSD_jr unsupported") + def ADDSD_rj(self, xxm_reg, mem_immed): + py.test.skip("ADDSD_rj unsupported") + def SUBSD_rj(self, xxm_reg, mem_immed): + py.test.skip("SUBSD_rj unsupported") + def MULSD_rj(self, xxm_reg, mem_immed): + py.test.skip("MULSD_rj unsupported") + def DIVSD_rj(self, xxm_reg, mem_immed): + py.test.skip("DIVSD_rj unsupported") + def UCOMISD_rj(self, xxm_reg, mem_immed): + py.test.skip("UCOMISD_rj unsupported") + # ____________________________________________________________ Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/tool/instruction_encoding.sh ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/tool/instruction_encoding.sh (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/tool/instruction_encoding.sh Mon May 24 22:24:33 2010 @@ -3,16 +3,18 @@ # Tool to quickly see how the GNU assembler encodes an instruction # (AT&T syntax only for now) +# Command line options are passed on to "as" + # Provide readline if available if which rlwrap > /dev/null && [ "$INSIDE_RLWRAP" = "" ]; then export INSIDE_RLWRAP=1 - exec rlwrap "$0" + exec rlwrap "$0" "$@" fi while :; do echo -n '? ' read instruction - echo "$instruction" | as + echo "$instruction" | as "$@" objdump --disassemble ./a.out | grep '^ *[0-9a-f]\+:' rm -f ./a.out done From afa at codespeak.net Mon May 24 23:09:51 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 24 May 2010 23:09:51 +0200 (CEST) Subject: [pypy-svn] r74722 - pypy/trunk/pypy/module/cpyext Message-ID: <20100524210951.971E8282BDB@codespeak.net> Author: afa Date: Mon May 24 23:09:50 2010 New Revision: 74722 Modified: pypy/trunk/pypy/module/cpyext/api.py Log: Fix a random failure in tests: on unix, Py_VaBuildValue() clashes with the one defined by CPython... Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Mon May 24 23:09:50 2010 @@ -281,7 +281,9 @@ '_PyArg_NoKeywords', 'PyString_FromFormat', 'PyString_FromFormatV', 'PyModule_AddObject', 'PyModule_AddIntConstant', 'PyModule_AddStringConstant', - 'Py_BuildValue', 'PyTuple_Pack', 'PyErr_Format', 'PyErr_NewException', + 'Py_BuildValue', 'Py_VaBuildValue', 'PyTuple_Pack', + + 'PyErr_Format', 'PyErr_NewException', 'PyEval_CallFunction', 'PyEval_CallMethod', 'PyObject_CallFunction', 'PyObject_CallMethod', 'PyObject_CallFunctionObjArgs', 'PyObject_CallMethodObjArgs', From jcreigh at codespeak.net Tue May 25 00:50:50 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Tue, 25 May 2010 00:50:50 +0200 (CEST) Subject: [pypy-svn] r74723 - in pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86: . test Message-ID: <20100524225050.1F6E0282BDB@codespeak.net> Author: jcreigh Date: Tue May 25 00:50:48 2010 New Revision: 74723 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py Log: more progress. some tests actually pass Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Tue May 25 00:50:48 2010 @@ -1048,7 +1048,8 @@ addr, locs, ign_2): mc = self._start_block() mc.CMP(locs[0], imm8(1)) - mc.JB(rel8_patched_later) + # Patched below + mc.J_il8(rx86.Conditions['B'], 0) jb_location = mc.get_relative_pos() self._cmp_guard_class(mc, locs) # patch the JB above @@ -1592,15 +1593,23 @@ num = getattr(rop, opname.upper()) genop_list[num] = value +def round_up_to_4(size): + if size < 4: + return 4 + return size + +# XXX: ri386 migration shims: + +def addr_add(reg_or_imm1, reg_or_imm2, offset=0, scale=0): + return AddressLoc(reg_or_imm1, reg_or_imm2, scale, offset) + def addr_add_const(reg_or_imm1, offset): - # XXX: ri386 migration shim return AddressLoc(reg_or_imm1, ImmedLoc(0), 0, offset) def mem(loc, offset): - # XXX: ri386 migration shim - return AddressLoc(loc, ImmedLoc(0), (0), offset) + return AddressLoc(loc, ImmedLoc(0), 0, offset) -def round_up_to_4(size): - if size < 4: - return 4 - return size +def heap(addr): + return AddressLoc(ImmedLoc(addr), ImmedLoc(0), 0, 0) + +imm8 = imm Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Tue May 25 00:50:48 2010 @@ -76,6 +76,9 @@ def getint(self): return self.value + def __repr__(self): + return "ImmedLoc(%d)" % (self.value) + class AddressLoc(AssemblerLocation): _immutable_ = True @@ -90,8 +93,8 @@ self._location_code = 'j' self.value = base_loc.value + (scaled_loc.value << scale) + static_offset else: - # FIXME - raise AssertionError("Don't know how to handle this case yet") + self._location_code = 'a' + self.value = (None, scaled_loc.value, scale, static_offset) else: if isinstance(scaled_loc, ImmedLoc): # FIXME: What if base_loc is ebp or esp? @@ -132,6 +135,7 @@ ADD = _binaryop('ADD') OR = _binaryop('OR') XOR = _binaryop('XOR') + TEST = _binaryop('TEST') AND = _binaryop('AND') SUB = _binaryop('SUB') @@ -141,8 +145,10 @@ CMP = _binaryop('CMP') MOV = _binaryop('MOV') MOV8 = _binaryop('MOV8') - MOVZX8 = _binaryop("MOVZX8") - MOVZX16 = _binaryop("MOVZX16") + MOVZX8 = _binaryop('MOVZX8') + MOVZX16 = _binaryop('MOVZX16') + + LEA = _binaryop('LEA') MOVSD = _binaryop('MOVSD') ADDSD = _binaryop('ADDSD') Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py Tue May 25 00:50:48 2010 @@ -189,8 +189,20 @@ # emit "reg1 + (reg2 << scaleshift) + offset" assert reg1 != R.ebp and reg2 != R.esp assert 0 <= scaleshift < 4 - reg1 = reg_number_3bits(mc, reg1) reg2 = reg_number_3bits(mc, reg2) + + # Special case for no base register + if reg1 == None: + # modrm + mc.writechar(chr(0x04 | orbyte)) + # SIB + mc.writechar(chr((scaleshift<<6) | (reg2<<3) | 5)) + # We're forced to output a disp32, even if offset == 0 + mc.writeimm32(offset) + return 0 + + reg1 = reg_number_3bits(mc, reg1) + SIB = chr((scaleshift<<6) | (reg2<<3) | reg1) # no_offset = offset == 0 @@ -348,6 +360,7 @@ MOV_ri = insn(rex_w, register(1), '\xB8', immediate(2, 'q')) MOV_rr = insn(rex_w, '\x89', register(2,8), register(1), '\xC0') + MOV_bi = insn(rex_w, '\xC7', stack_bp(1), immediate(2)) MOV_br = insn(rex_w, '\x89', register(2,8), stack_bp(1)) MOV_rb = insn(rex_w, '\x8B', register(1,8), stack_bp(2)) MOV_sr = insn(rex_w, '\x89', register(2,8), stack_sp(1)) @@ -387,13 +400,13 @@ XOR_ri, XOR_rr, XOR_rb, _, _ = common_modes(6) CMP_ri, CMP_rr, CMP_rb, CMP_bi, CMP_br = common_modes(7) - _CMP_mi8 = insn(rex_w, '\x83', orbyte(7<<3), mem_reg_plus_const(1), immediate(2, 'b')) - _CMP_mi32 = insn(rex_w, '\x81', orbyte(7<<3), mem_reg_plus_const(1), immediate(2)) - CMP_mi = select_8_or_32_bit_immed(_CMP_mi8, _CMP_mi32) - - _CMP_ji8 = insn(rex_w, '\x83', '\x3D', immediate(1), immediate(2, 'b')) - _CMP_ji32 = insn(rex_w, '\x81', '\x3D', immediate(1), immediate(2)) - CMP_ji = select_8_or_32_bit_immed(_CMP_ji8, _CMP_ji32) + CMP_mi8 = insn(rex_w, '\x83', orbyte(7<<3), mem_reg_plus_const(1), immediate(2, 'b')) + CMP_mi32 = insn(rex_w, '\x81', orbyte(7<<3), mem_reg_plus_const(1), immediate(2)) + CMP_mi = select_8_or_32_bit_immed(CMP_mi8, CMP_mi32) + + CMP_ji8 = insn(rex_w, '\x83', '\x3D', immediate(1), immediate(2, 'b')) + CMP_ji32 = insn(rex_w, '\x81', '\x3D', immediate(1), immediate(2)) + CMP_ji = select_8_or_32_bit_immed(CMP_ji8, CMP_ji32) NEG_r = insn(rex_w, '\xF7', register(1), '\xD8') @@ -403,9 +416,9 @@ IMUL_rr = insn(rex_w, '\x0F\xAF', register(1, 8), register(2), '\xC0') IMUL_rb = insn(rex_w, '\x0F\xAF', register(1, 8), stack_bp(2)) - _IMUL_rri8 = insn(rex_w, '\x6B', register(1, 8), register(2), '\xC0', immediate(3, 'b')) - _IMUL_rri32 = insn(rex_w, '\x69', register(1, 8), register(2), '\xC0', immediate(3)) - IMUL_rri = select_8_or_32_bit_immed(_IMUL_rri8, _IMUL_rri32) + IMUL_rri8 = insn(rex_w, '\x6B', register(1, 8), register(2), '\xC0', immediate(3, 'b')) + IMUL_rri32 = insn(rex_w, '\x69', register(1, 8), register(2), '\xC0', immediate(3)) + IMUL_rri = select_8_or_32_bit_immed(IMUL_rri8, IMUL_rri32) def IMUL_ri(self, reg, immed): self.IMUL_rri(reg, reg, immed) @@ -420,6 +433,7 @@ LEA_rb = insn(rex_w, '\x8D', register(1,8), stack_bp(2)) LEA32_rb = insn(rex_w, '\x8D', register(1,8),stack_bp(2,force_32bits=True)) + LEA_ra = insn(rex_w, '\x8D', register(1, 8), mem_reg_plus_scaled_reg_plus_const(2)) CALL_l = insn('\xE8', relative(1)) CALL_r = insn(rex_nw, '\xFF', register(1), chr(0xC0 | (2<<3))) @@ -429,12 +443,18 @@ XCHG_rj = insn(rex_w, '\x87', register(1,8), '\x05', immediate(2)) JMP_l = insn('\xE9', relative(1)) + # FIXME: J_il8 assume the caller will do the appropriate calculation + # to find the displacement, but J_il does it for the caller. + # We need to be consistent. + J_il8 = insn(immediate(1, 'o'), '\x70', immediate(2, 'b')) J_il = insn('\x0F', immediate(1,'o'), '\x80', relative(2)) SET_ir = insn('\x0F', immediate(1,'o'),'\x90', register(2), '\xC0') # The 64-bit version of this, CQO, is defined in X86_64_CodeBuilder CDQ = insn(rex_nw, '\x99') + TEST_rr = insn(rex_w, '\x85', register(2,8), register(1), '\xC0') + # ------------------------------ SSE2 ------------------------------ MOVSD_rr = xmminsn('\xF2', rex_nw, '\x0F\x10', register(1,8), register(2), Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py Tue May 25 00:50:48 2010 @@ -63,6 +63,15 @@ '\x8B\x54\xBE\x80' + '\x8B\x94\xBE\x80\x00\x00\x00') +def test_mov_ra_no_base(): + s = CodeBuilder32() + s.MOV_ra(edx, (None, edi, 2, 0)) + assert s.getvalue() == '\x8B\x14\xBD\x00\x00\x00\x00' + + s = CodeBuilder32() + s.MOV_ra(edx, (None, edi, 2, 0xCD)) + assert s.getvalue() == '\x8B\x14\xBD\xCD\x00\x00\x00' + def test_mov_ar(): s = CodeBuilder32() s.MOV_ar((esi, edi, 2, 0), edx) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py Tue May 25 00:50:48 2010 @@ -230,13 +230,14 @@ return X86_CodeBuilder def complete_test(self, methname): - if methname.split('_')[0][-1].isdigit(): - print "artificial instruction: %r" % (methname,) - return if '_' in methname: instrname, argmodes = methname.split('_') else: instrname, argmodes = methname, '' + if instrname[-1].isdigit() or (argmodes != '' and argmodes[-1].isdigit()): + print "artificial instruction: %r" % (methname,) + return + print "Testing %s with argmodes=%r" % (instrname, argmodes) self.methname = methname self.is_xmm_insn = getattr(getattr(rx86.AbstractX86CodeBuilder, From getxsick at codespeak.net Tue May 25 02:11:18 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Tue, 25 May 2010 02:11:18 +0200 (CEST) Subject: [pypy-svn] r74724 - pypy/trunk/dotviewer Message-ID: <20100525001118.08ABA282BDB@codespeak.net> Author: getxsick Date: Tue May 25 02:11:17 2010 New Revision: 74724 Modified: pypy/trunk/dotviewer/graphparse.py Log: replace deprecated popen2() Modified: pypy/trunk/dotviewer/graphparse.py ============================================================================== --- pypy/trunk/dotviewer/graphparse.py (original) +++ pypy/trunk/dotviewer/graphparse.py Tue May 25 02:11:17 2010 @@ -2,7 +2,9 @@ Graph file parsing. """ -import os, sys, re +import sys, re +import subprocess + import msgstruct re_nonword = re.compile(r'([^0-9a-zA-Z_.]+)') @@ -45,7 +47,9 @@ else: cmdline = 'neato -Tplain' #print >> sys.stderr, '* running:', cmdline - child_in, child_out = os.popen2(cmdline, 'b') + p = subprocess.Popen(cmdline, shell=True, close_fds=True, + stdin=subprocess.PIPE, stdout=subprocess.PIPE) + (child_in, child_out) = (p.stdin, p.stdout) try: import thread except ImportError: From afa at codespeak.net Tue May 25 09:10:08 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 25 May 2010 09:10:08 +0200 (CEST) Subject: [pypy-svn] r74725 - pypy/trunk/pypy/lib Message-ID: <20100525071008.F1749282B9D@codespeak.net> Author: afa Date: Tue May 25 09:10:06 2010 New Revision: 74725 Modified: pypy/trunk/pypy/lib/_locale.py Log: Apply last patch of issue361. This is not very important, now that the module has been rewritten in module/_locale, (the code there already use size_t for strxfrm) Modified: pypy/trunk/pypy/lib/_locale.py ============================================================================== --- pypy/trunk/pypy/lib/_locale.py (original) +++ pypy/trunk/pypy/lib/_locale.py Tue May 25 09:10:06 2010 @@ -4,7 +4,7 @@ """ from ctypes import (Structure, POINTER, create_string_buffer, - c_ubyte, c_int, c_char_p, c_wchar_p) + c_ubyte, c_int, c_char_p, c_wchar_p, c_size_t) from ctypes_support import standard_c_lib as libc from ctypes_support import get_errno @@ -12,9 +12,6 @@ from ctypes_config_cache._locale_cache import * -size_t = c_int - - # Ubuntu Gusty i386 structure class lconv(Structure): _fields_ = ( @@ -94,8 +91,8 @@ _wcscoll.restype = c_int _strxfrm = libc.strxfrm -_strxfrm.argtypes = (c_char_p, c_char_p, size_t) -_strxfrm.restype = size_t +_strxfrm.argtypes = (c_char_p, c_char_p, c_size_t) +_strxfrm.restype = c_size_t HAS_LIBINTL = hasattr(libc, 'gettext') if HAS_LIBINTL: From arigo at codespeak.net Tue May 25 12:18:32 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 25 May 2010 12:18:32 +0200 (CEST) Subject: [pypy-svn] r74726 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100525101832.4021C282B9D@codespeak.net> Author: arigo Date: Tue May 25 12:18:30 2010 New Revision: 74726 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitcode.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py Log: * more debugging support in dump() * replace enumerate_live_vars() with get_live_vars_info() Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Tue May 25 12:18:30 2010 @@ -21,7 +21,9 @@ it inside the 'jitcode'. If jitcode is None, make a new one. """ self.setup() + ssarepr._insns_pos = [] for insn in ssarepr.insns: + ssarepr._insns_pos.append(len(self.code)) self.write_insn(insn) self.fix_labels() self.check_result() Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Tue May 25 12:18:30 2010 @@ -7,6 +7,7 @@ def __init__(self, name): self.name = name self.insns = [] + self._insns_pos = None # after being assembled class Label(object): def __init__(self, name): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py Tue May 25 12:18:30 2010 @@ -52,7 +52,9 @@ insns = ssarepr.insns if insns and insns[-1] == ('---',): insns = insns[:-1] - for asm in insns: + for i, asm in enumerate(insns): + if ssarepr._insns_pos: + print >> output, '%4d ' % ssarepr._insns_pos[i], if isinstance(asm[0], Label): if asm[0].name in seenlabels: print >> output, '%s:' % getlabelname(asm[0]) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py Tue May 25 12:18:30 2010 @@ -47,39 +47,25 @@ def has_liveness_info(self, pc): return pc in self.liveness - def enumerate_live_vars(self, pc, callback, arg, - registers_i, registers_r, registers_f): - # 'pc' gives a position in this bytecode. This invokes - # 'callback' for each variable that is live across the - # instruction boundary at 'pc'. More precisely, - # this invokes 'callback(arg, box)' where 'box' comes from one - # of the three lists of registers. + def get_live_vars_info(self, pc): + # 'pc' gives a position in this bytecode. This returns an object + # that describes all variables that are live across the instruction + # boundary at 'pc'. To decode the object, use the global functions + # 'get_register_{count,index}_{i,r,f}()'. if not we_are_translated() and pc not in self.liveness: self._missing_liveness(pc) - live_i, live_r, live_f = self.liveness[pc] # XXX compactify!! - for c in live_i: - x = callback(arg, registers_i[ord(c)]) - assert x is None - for c in live_r: - x = callback(arg, registers_r[ord(c)]) - assert x is None - for c in live_f: - x = callback(arg, registers_f[ord(c)]) - assert x is None - enumerate_live_vars._annspecialcase_ = 'specialize:arg(2)' + return self.liveness[pc] # XXX compactify!! def _live_vars(self, pc): # for testing only - class Names: - def __init__(self, kind): - self.kind = kind - def __getitem__(self, index): - return '%%%s%d' % (self.kind, index) - lst = [] - self.enumerate_live_vars(pc, list.append, lst, - Names('i'), Names('r'), Names('f')) - lst.sort() - return ' '.join(lst) + info = self.get_live_vars_info(pc) + lst_i = ['%%i%d' % get_register_index_i(info, index) + for index in range(get_register_count_i(info))] + lst_r = ['%%r%d' % get_register_index_r(info, index) + for index in range(get_register_count_r(info))] + lst_f = ['%%f%d' % get_register_index_f(info, index) + for index in range(get_register_count_f(info))] + return ' '.join(lst_i + lst_r + lst_f) def _missing_liveness(self, pc): raise MissingLiveness("missing liveness[%d]\n%s" % (pc, self.dump())) @@ -116,3 +102,18 @@ def __repr__(self): dict = getattr(self, 'dict', '?') return '' % (dict,) + + +def get_register_count_i((live_i, live_r, live_f)): + return len(live_i) +def get_register_count_r((live_i, live_r, live_f)): + return len(live_r) +def get_register_count_f((live_i, live_r, live_f)): + return len(live_f) + +def get_register_index_i((live_i, live_r, live_f), index): + return ord(live_i[index]) +def get_register_index_r((live_i, live_r, live_f), index): + return ord(live_r[index]) +def get_register_index_f((live_i, live_r, live_f), index): + return ord(live_f[index]) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitcode.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitcode.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitcode.py Tue May 25 12:18:30 2010 @@ -21,14 +21,4 @@ j.setup(liveness={5: (" A", "b", "CD")}) assert j.has_liveness_info(5) assert not j.has_liveness_info(4) - # - seen = [] - def callback(arg, value): - assert arg == "foo" - seen.append(value) - # - j.enumerate_live_vars(5, callback, "foo", - {ord(" "): "i10", ord("A"): "i20"}, - {ord("b"): "r30"}, - {ord("C"): "f40", ord("D"): "f50"}) - assert seen == ["i10", "i20", "r30", "f40", "f50"] + assert j._live_vars(5) == "%i32 %i65 %r98 %f67 %f68" Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py Tue May 25 12:18:30 2010 @@ -44,6 +44,7 @@ def builtin_test(oopspec_name, args, RESTYPE, expected): v_result = varoftype(RESTYPE) tr = Transformer(FakeCPU(), FakeCallControl()) + tr.immutable_arrays = {} if '/' in oopspec_name: oopspec_name, property = oopspec_name.split('/') def force_flags(op): From arigo at codespeak.net Tue May 25 12:27:50 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 25 May 2010 12:27:50 +0200 (CEST) Subject: [pypy-svn] r74727 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100525102750.97153282B9D@codespeak.net> Author: arigo Date: Tue May 25 12:27:49 2010 New Revision: 74727 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: Fix pyjitpl for r74726. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Tue May 25 12:27:49 2010 @@ -114,32 +114,47 @@ elif argcode == 'f': self.registers_f[index] = None self._result_argcode = 'v' # done # - self._tmp_count = 0 - count = self.jitcode.enumerate_live_vars( - self.pc, MIFrame._count_boxes, self, - self.registers_i, self.registers_r, self.registers_f) - # - self._tmp_env = [None] * self._tmp_count - self._tmp_count = 0 - self.jitcode.enumerate_live_vars( - self.pc, MIFrame._store_in_env, self, - self.registers_i, self.registers_r, self.registers_f) - # - env = self._tmp_env - self._tmp_env = None + from pypy.jit.codewriter.jitcode import get_register_count_i + from pypy.jit.codewriter.jitcode import get_register_count_r + from pypy.jit.codewriter.jitcode import get_register_count_f + from pypy.jit.codewriter.jitcode import get_register_index_i + from pypy.jit.codewriter.jitcode import get_register_index_r + from pypy.jit.codewriter.jitcode import get_register_index_f + info = self.jitcode.get_live_vars_info(self.pc) + # first count how many boxes there are, skipping None + total = 0 + for index in range(get_register_count_i(info)): + if self.registers_i[get_register_index_i(info, index)] is not None: + total += 1 + for index in range(get_register_count_r(info)): + if self.registers_r[get_register_index_r(info, index)] is not None: + total += 1 + for index in range(get_register_count_f(info)): + if self.registers_f[get_register_index_f(info, index)] is not None: + total += 1 + # allocate a list of the correct size + env = [None] * total make_sure_not_resized(env) + # fill it now + count = 0 + for index in range(get_register_count_i(info)): + box = self.registers_i[get_register_index_i(info, index)] + if box is not None: + env[count] = box + count += 1 + for index in range(get_register_count_r(info)): + box = self.registers_r[get_register_index_r(info, index)] + if box is not None: + env[count] = box + count += 1 + for index in range(get_register_count_f(info)): + box = self.registers_f[get_register_index_f(info, index)] + if box is not None: + env[count] = box + count += 1 + assert count == total return env - def _count_boxes(self, box): - if box is not None: # just used to count how many boxes there are - self._tmp_count += 1 - - def _store_in_env(self, box): - if box is not None: - index = self._tmp_count - self._tmp_env[index] = box - self._tmp_count = index + 1 - def replace_active_box_in_frame(self, oldbox, newbox): if isinstance(oldbox, history.BoxInt): count = self.jitcode.num_regs_i() From arigo at codespeak.net Tue May 25 12:44:42 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 25 May 2010 12:44:42 +0200 (CEST) Subject: [pypy-svn] r74728 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100525104442.EB8BF282B9D@codespeak.net> Author: arigo Date: Tue May 25 12:44:41 2010 New Revision: 74728 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: Simplify the encoding. That seems to be necessary to fix the next issue. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py Tue May 25 12:44:41 2010 @@ -310,6 +310,8 @@ def repr_rpython(self): return repr_rpython(self, 'cf') +CONST_FZERO = ConstFloat(0.0) + class ConstPtr(Const): type = REF value = lltype.nullptr(llmemory.GCREF.TO) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Tue May 25 12:44:41 2010 @@ -109,10 +109,10 @@ # is not defined yet. argcode = self._result_argcode index = ord(self.bytecode[self.pc - 1]) - if argcode == 'i': self.registers_i[index] = None - elif argcode == 'r': self.registers_r[index] = None - elif argcode == 'f': self.registers_f[index] = None - self._result_argcode = 'v' # done + if argcode == 'i': self.registers_i[index] = history.CONST_FALSE + elif argcode == 'r': self.registers_r[index] = history.CONST_NULL + elif argcode == 'f': self.registers_f[index] = history.CONST_FZERO + self._result_argcode = '?' # done # from pypy.jit.codewriter.jitcode import get_register_count_i from pypy.jit.codewriter.jitcode import get_register_count_r @@ -121,17 +121,9 @@ from pypy.jit.codewriter.jitcode import get_register_index_r from pypy.jit.codewriter.jitcode import get_register_index_f info = self.jitcode.get_live_vars_info(self.pc) - # first count how many boxes there are, skipping None - total = 0 - for index in range(get_register_count_i(info)): - if self.registers_i[get_register_index_i(info, index)] is not None: - total += 1 - for index in range(get_register_count_r(info)): - if self.registers_r[get_register_index_r(info, index)] is not None: - total += 1 - for index in range(get_register_count_f(info)): - if self.registers_f[get_register_index_f(info, index)] is not None: - total += 1 + total = (get_register_count_i(info) + + get_register_count_r(info) + + get_register_count_f(info)) # allocate a list of the correct size env = [None] * total make_sure_not_resized(env) @@ -139,19 +131,19 @@ count = 0 for index in range(get_register_count_i(info)): box = self.registers_i[get_register_index_i(info, index)] - if box is not None: - env[count] = box - count += 1 + assert box is not None + env[count] = box + count += 1 for index in range(get_register_count_r(info)): box = self.registers_r[get_register_index_r(info, index)] - if box is not None: - env[count] = box - count += 1 + assert box is not None + env[count] = box + count += 1 for index in range(get_register_count_f(info)): box = self.registers_f[get_register_index_f(info, index)] - if box is not None: - env[count] = box - count += 1 + assert box is not None + env[count] = box + count += 1 assert count == total return env From arigo at codespeak.net Tue May 25 16:00:23 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 25 May 2010 16:00:23 +0200 (CEST) Subject: [pypy-svn] r74734 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter codewriter/test metainterp metainterp/test Message-ID: <20100525140023.32331282B9D@codespeak.net> Author: arigo Date: Tue May 25 16:00:22 2010 New Revision: 74734 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitcode.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_loop.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_resume.py Log: Fix a subtle bug, involving a mismatch between reality and expectation in the order of boxes. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Tue May 25 16:00:22 2010 @@ -165,16 +165,16 @@ self.code[startposition] = chr(num) self.startpoints.add(startposition) - def get_liveness_info(self, prevstring, args, kind): + def get_liveness_info(self, prevlives, args, kind): """Return a string whose characters are register numbers. We sort the numbers, too, to increase the chances of duplicate strings (which are collapsed into a single string during translation). """ - lives = set(prevstring) # set of characters + lives = set(prevlives) # set of characters for reg in args: if isinstance(reg, Register) and reg.kind == kind: lives.add(chr(reg.index)) - return ''.join(sorted(lives)) + return lives def fix_labels(self): for name, pos in self.tlabel_positions: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py Tue May 25 16:00:22 2010 @@ -28,7 +28,7 @@ self.num_regs_encoded = ((num_regs_i << 16) | (num_regs_f << 8) | (num_regs_r << 0)) - self.liveness = liveness + self.liveness = make_liveness_cache(liveness) self._startpoints = startpoints # debugging self._alllabels = alllabels # debugging @@ -49,9 +49,8 @@ def get_live_vars_info(self, pc): # 'pc' gives a position in this bytecode. This returns an object - # that describes all variables that are live across the instruction - # boundary at 'pc'. To decode the object, use the global functions - # 'get_register_{count,index}_{i,r,f}()'. + # of class LiveVarsInfo that describes all variables that are live + # across the instruction boundary at 'pc'. if not we_are_translated() and pc not in self.liveness: self._missing_liveness(pc) return self.liveness[pc] # XXX compactify!! @@ -59,12 +58,12 @@ def _live_vars(self, pc): # for testing only info = self.get_live_vars_info(pc) - lst_i = ['%%i%d' % get_register_index_i(info, index) - for index in range(get_register_count_i(info))] - lst_r = ['%%r%d' % get_register_index_r(info, index) - for index in range(get_register_count_r(info))] - lst_f = ['%%f%d' % get_register_index_f(info, index) - for index in range(get_register_count_f(info))] + lst_i = ['%%i%d' % info.get_register_index_i(index) + for index in range(info.get_register_count_i())] + lst_r = ['%%r%d' % info.get_register_index_r(index) + for index in range(info.get_register_count_r())] + lst_f = ['%%f%d' % info.get_register_index_f(index) + for index in range(info.get_register_count_f())] return ' '.join(lst_i + lst_r + lst_f) def _missing_liveness(self, pc): @@ -104,16 +103,54 @@ return '' % (dict,) -def get_register_count_i((live_i, live_r, live_f)): - return len(live_i) -def get_register_count_r((live_i, live_r, live_f)): - return len(live_r) -def get_register_count_f((live_i, live_r, live_f)): - return len(live_f) - -def get_register_index_i((live_i, live_r, live_f), index): - return ord(live_i[index]) -def get_register_index_r((live_i, live_r, live_f), index): - return ord(live_r[index]) -def get_register_index_f((live_i, live_r, live_f), index): - return ord(live_f[index]) +class LiveVarsInfo(object): + def __init__(self, live_i, live_r, live_f): + self.live_i = live_i + self.live_r = live_r + self.live_f = live_f + + def get_register_count_i(self): + return len(self.live_i) + def get_register_count_r(self): + return len(self.live_r) + def get_register_count_f(self): + return len(self.live_f) + + def get_register_index_i(self, index): + return ord(self.live_i[index]) + def get_register_index_r(self, index): + return ord(self.live_r[index]) + def get_register_index_f(self, index): + return ord(self.live_f[index]) + + def enumerate_vars(self, callback_i, callback_r, callback_f): + index = 0 + for i in range(self.get_register_count_i()): + callback_i(index, self.get_register_index_i(i)) + index += 1 + for i in range(self.get_register_count_r()): + callback_r(index, self.get_register_index_r(i)) + index += 1 + for i in range(self.get_register_count_f()): + callback_f(index, self.get_register_index_f(i)) + index += 1 + +_liveness_cache = {} + +def make_liveness_cache(liveness): + if liveness is None: + return None + result = {} + for key, (value_i, value_r, value_f) in liveness.items(): + # Sort the lists to increase the chances of sharing between unrelated + # strings that happen to contain the same characters. We sort in the + # reversed order just to reduce the risks of tests passing by chance. + value = (''.join(sorted(value_i, reverse=True)), + ''.join(sorted(value_r, reverse=True)), + ''.join(sorted(value_f, reverse=True))) + try: + info = _liveness_cache[value] + except KeyError: + info = _liveness_cache[value] = LiveVarsInfo(*value) + result[key] = info + return result Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitcode.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitcode.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitcode.py Tue May 25 16:00:22 2010 @@ -21,4 +21,4 @@ j.setup(liveness={5: (" A", "b", "CD")}) assert j.has_liveness_info(5) assert not j.has_liveness_info(4) - assert j._live_vars(5) == "%i32 %i65 %r98 %f67 %f68" + assert j._live_vars(5) == "%i65 %i32 %r98 %f68 %f67" Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Tue May 25 16:00:22 2010 @@ -353,6 +353,9 @@ self.registers_r[i] = NULL self.exception_last_value = None + def get_current_position_info(self): + return self.jitcode.get_live_vars_info(self.position) + def handle_exception_in_frame(self, e): # This frame raises an exception. First try to see if # the exception is handled in the frame itself. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Tue May 25 16:00:22 2010 @@ -101,6 +101,9 @@ outvalue[startindex+i] = reg prepare_list_of_boxes._annspecialcase_ = 'specialize:arg(4)' + def get_current_position_info(self): + return self.jitcode.get_live_vars_info(self.pc) + def get_list_of_active_boxes(self, in_a_call): if in_a_call: # If we are not the topmost frame, self._result_argcode contains @@ -114,37 +117,24 @@ elif argcode == 'f': self.registers_f[index] = history.CONST_FZERO self._result_argcode = '?' # done # - from pypy.jit.codewriter.jitcode import get_register_count_i - from pypy.jit.codewriter.jitcode import get_register_count_r - from pypy.jit.codewriter.jitcode import get_register_count_f - from pypy.jit.codewriter.jitcode import get_register_index_i - from pypy.jit.codewriter.jitcode import get_register_index_r - from pypy.jit.codewriter.jitcode import get_register_index_f - info = self.jitcode.get_live_vars_info(self.pc) - total = (get_register_count_i(info) + - get_register_count_r(info) + - get_register_count_f(info)) + info = self.get_current_position_info() + start_i = 0 + start_r = start_i + info.get_register_count_i() + start_f = start_r + info.get_register_count_r() + total = start_f + info.get_register_count_f() # allocate a list of the correct size env = [None] * total make_sure_not_resized(env) # fill it now - count = 0 - for index in range(get_register_count_i(info)): - box = self.registers_i[get_register_index_i(info, index)] - assert box is not None - env[count] = box - count += 1 - for index in range(get_register_count_r(info)): - box = self.registers_r[get_register_index_r(info, index)] - assert box is not None - env[count] = box - count += 1 - for index in range(get_register_count_f(info)): - box = self.registers_f[get_register_index_f(info, index)] - assert box is not None - env[count] = box - count += 1 - assert count == total + for i in range(info.get_register_count_i()): + index = info.get_register_index_i(i) + env[start_i + i] = self.registers_i[index] + for i in range(info.get_register_count_r()): + index = info.get_register_index_r(i) + env[start_r + i] = self.registers_r[index] + for i in range(info.get_register_count_f()): + index = info.get_register_index_f(i) + env[start_f + i] = self.registers_f[index] return env def replace_active_box_in_frame(self, oldbox, newbox): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py Tue May 25 16:00:22 2010 @@ -19,7 +19,6 @@ __slots__ = ('prev', 'boxes') def __init__(self, prev, boxes): - _assert_order(boxes) self.prev = prev self.boxes = boxes @@ -58,7 +57,7 @@ storage.rd_frame_info_list = frame_info_list snapshot = Snapshot(top.parent_resumedata_snapshot, top.get_list_of_active_boxes(False)) - snapshot = Snapshot(snapshot, virtualref_boxes[:]) # xxx for now + #snapshot = Snapshot(snapshot, virtualref_boxes[:]) # xxx for now if virtualizable_boxes is not None: snapshot = Snapshot(snapshot, virtualizable_boxes[:]) # xxx for now storage.rd_snapshot = snapshot @@ -105,9 +104,6 @@ UNASSIGNED = tag(-1<<13, TAGBOX) UNASSIGNEDVIRTUAL = tag(-1<<13, TAGVIRTUAL) NULLREF = tag(-1, TAGCONST) -TYPEBARRIER = tag(-2, TAGCONST) - # nums = [..BoxInts.., TYPEBARRIER, ..BoxPtrs..] - # and optionally an extra [TYPEBARRIER, ..BoxFloats..] class ResumeDataLoopMemo(object): @@ -171,12 +167,8 @@ numb1, liveboxes, v = self.number(values, snapshot.prev) n = len(liveboxes)-v boxes = snapshot.boxes - _assert_order(boxes) length = len(boxes) - numslength = length + 1 + (length > 0 and boxes[-1].type == FLOAT) - nums = [UNASSIGNED] * numslength - prevkind = INT - j = 0 + nums = [UNASSIGNED] * length for i in range(length): box = boxes[i] value = values.get(box, None) @@ -195,22 +187,7 @@ tagged = tag(n, TAGBOX) n += 1 liveboxes[box] = tagged - # - if box.type != prevkind: - nums[j] = TYPEBARRIER - j += 1 - if prevkind == INT and box.type == FLOAT: - nums[j] = TYPEBARRIER - j += 1 - prevkind = box.type - # - nums[j] = tagged - j += 1 - # - if j == length: - nums[j] = TYPEBARRIER - j += 1 - assert j == numslength + nums[i] = tagged # numb = Numbering(numb1, nums) self.numberings[snapshot] = numb, liveboxes, v @@ -416,14 +393,6 @@ return self.liveboxes_from_env[box] return self.liveboxes[box] -def _assert_order(boxes): - if not we_are_translated(): - # verifies that 'boxes' are in order: all INTs, then all REFs, - # and finally all FLOATs. - _kind2count = {INT: 1, REF: 2, FLOAT: 3} - kinds = [_kind2count[box.type] for box in boxes] - assert kinds == sorted(kinds) - class AbstractVirtualInfo(object): #def allocate(self, metainterp): @@ -552,33 +521,25 @@ struct = self.decode_ref(num) self.setfield(descr, struct, fieldnum) - def _prepare_next_section(self): - numb = self.cur_numb - self.cur_numb = numb.prev - nums = numb.nums - length = len(nums) - count_i = count_r = count_f = 0 - i = 0 - while True: - num = nums[i] - i += 1 - if tagged_eq(num, TYPEBARRIER): - break - self.write_an_int(count_i, self.decode_int(num)) - count_i += 1 - while i < length: - num = nums[i] - i += 1 - if tagged_eq(num, TYPEBARRIER): - break - self.write_a_ref(count_r, self.decode_ref(num)) - count_r += 1 - while i < length: - num = nums[i] - i += 1 - assert not tagged_eq(num, TYPEBARRIER) - self.write_a_float(count_f, self.decode_float(num)) - count_f += 1 + def _prepare_next_section(self, info): + # Use info.enumerate_vars(), normally dispatching to + # pypy.jit.codewriter.jitcode. Some tests give a different 'info'. + info.enumerate_vars(self._callback_i, + self._callback_r, + self._callback_f) + self.cur_numb = self.cur_numb.prev + + def _callback_i(self, index, register_index): + value = self.decode_int(self.cur_numb.nums[index]) + self.write_an_int(register_index, value) + + def _callback_r(self, index, register_index): + value = self.decode_ref(self.cur_numb.nums[index]) + self.write_a_ref(register_index, value) + + def _callback_f(self, index, register_index): + value = self.decode_float(self.cur_numb.nums[index]) + self.write_a_float(register_index, value) def done(self): self.cpu.clear_latest_values() @@ -591,12 +552,13 @@ virtualizable_boxes = None if expects_virtualizables: XXX # virtualizable_boxes = resumereader.consume_boxes() - resumereader.consume_boxes([], [], []) # XXX virtualref + #resumereader.consume_boxes([], [], []) # XXX virtualref frameinfo = storage.rd_frame_info_list while True: f = metainterp.newframe(frameinfo.jitcode) f.setup_resume_at_op(frameinfo.pc) - resumereader.consume_boxes(f.registers_i, f.registers_r, f.registers_f) + resumereader.consume_boxes(f.get_current_position_info(), + f.registers_i, f.registers_r, f.registers_f) frameinfo = frameinfo.prev if frameinfo is None: break @@ -610,7 +572,8 @@ virtualizable_boxes = None if expects_virtualizables: virtualizable_boxes = resumereader.consume_boxes() - virtualref_boxes = resumereader.consume_boxes() + #virtualref_boxes = resumereader.consume_boxes() + virtualref_boxes = [] # XXX virtualrefs return virtualizable_boxes, virtualref_boxes, resumereader.virtuals class ResumeDataBoxReader(AbstractResumeDataReader): @@ -620,11 +583,11 @@ self.liveboxes = [None] * metainterp.cpu.get_latest_value_count() self._prepare(metainterp.cpu, storage) - def consume_boxes(self, boxes_i, boxes_r, boxes_f): + def consume_boxes(self, info, boxes_i, boxes_r, boxes_f): self.boxes_i = boxes_i self.boxes_r = boxes_r self.boxes_f = boxes_f - self._prepare_next_section() + self._prepare_next_section(info) def allocate_with_vtable(self, known_class): return self.metainterp.execute_and_record(rop.NEW_WITH_VTABLE, @@ -720,8 +683,7 @@ resumereader = ResumeDataDirectReader(blackholeinterpbuilder.cpu, storage) if expects_virtualizables: XXX - #virtualref_boxes = resumereader.consume_boxes() - resumereader.consume_one_section(None) # virtualref XXX + #virtualref_boxes = resumereader.consume_boxes() # virtualref XXX # # First get a chain of blackhole interpreters whose length is given # by the depth of rd_frame_info_list. The first one we get must be @@ -758,7 +720,8 @@ def consume_one_section(self, blackholeinterp): self.blackholeinterp = blackholeinterp - self._prepare_next_section() + info = blackholeinterp.get_current_position_info() + self._prepare_next_section(info) def allocate_with_vtable(self, known_class): from pypy.jit.metainterp.executor import exec_new_with_vtable Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_loop.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_loop.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_loop.py Tue May 25 16:00:22 2010 @@ -4,7 +4,7 @@ from pypy.jit.metainterp.warmspot import ll_meta_interp, get_stats from pypy.rpython.lltypesystem import lltype from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin -from pypy.jit.metainterp.policy import StopAtXPolicy +from pypy.jit.codewriter.policy import StopAtXPolicy from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp import history @@ -342,7 +342,7 @@ assert res == expected def test_loop_unicode(self): - myjitdriver = JitDriver(greens = [], reds = ['x', 'n']) + myjitdriver = JitDriver(greens = [], reds = ['n', 'x']) def f(n): x = u'' while n > 13: @@ -356,7 +356,7 @@ assert res == expected def test_loop_string(self): - myjitdriver = JitDriver(greens = [], reds = ['x', 'n']) + myjitdriver = JitDriver(greens = [], reds = ['n', 'x']) def f(n): x = '' while n > 13: Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_resume.py Tue May 25 16:00:22 2010 @@ -95,10 +95,26 @@ return self.values[index] class MyBlackholeInterp: - def __init__(self): + def __init__(self, ARGS): self.written_i = [] self.written_r = [] self.written_f = [] + self.ARGS = ARGS + + def get_current_position_info(self): + class MyInfo: + def enumerate_vars(_, callback_i, callback_r, callback_f): + count_i = count_r = count_f = 0 + for index, ARG in enumerate(self.ARGS): + if ARG == lltype.Signed: + callback_i(index, count_i); count_i += 1 + elif ARG == llmemory.GCREF: + callback_r(index, count_r); count_r += 1 + elif ARG == lltype.Float: + callback_f(index, count_f); count_f += 1 + else: + assert 0 + return MyInfo() def setarg_i(self, index, value): assert index == len(self.written_i) @@ -113,12 +129,14 @@ self.written_f.append(value) def _next_section(reader, *expected): - bh = MyBlackholeInterp() + bh = MyBlackholeInterp(map(lltype.typeOf, expected)) reader.consume_one_section(bh) - lst_i = bh.written_i - lst_r = bh.written_r - lst_f = bh.written_f - assert lst_i + lst_r + lst_f == list(expected) + expected_i = [x for x in expected if lltype.typeOf(x) == lltype.Signed] + expected_r = [x for x in expected if lltype.typeOf(x) == llmemory.GCREF] + expected_f = [x for x in expected if lltype.typeOf(x) == lltype.Float] + assert bh.written_i == expected_i + assert bh.written_r == expected_r + assert bh.written_f == expected_f def test_simple_read(): @@ -126,36 +144,41 @@ c1, c2, c3 = [ConstInt(111), ConstInt(222), ConstInt(333)] storage = Storage() storage.rd_consts = [c1, c2, c3] - numb = Numbering(None, [tag(0, TAGBOX), tag(2, TAGBOX), TYPEBARRIER, - tag(1, TAGBOX)]) - numb = Numbering(numb, [tag(1, TAGCONST), tag(2, TAGCONST), TYPEBARRIER]) + numb = Numbering(None, [tag(0, TAGBOX), tag(1, TAGBOX), tag(2, TAGBOX)]) + numb = Numbering(numb, [tag(1, TAGCONST), tag(2, TAGCONST)]) numb = Numbering(numb, [tag(0, TAGBOX), tag(0, TAGCONST), - tag(0, TAGBOX), - TYPEBARRIER, NULLREF, + tag(0, TAGBOX), tag(1, TAGBOX)]) storage.rd_numb = numb # cpu = MyCPU([42, gcref1, -66]) reader = ResumeDataDirectReader(cpu, storage) - _next_section(reader, 42, 111, 42, gcrefnull, gcref1) + _next_section(reader, 42, 111, gcrefnull, 42, gcref1) _next_section(reader, 222, 333) - _next_section(reader, 42, -66, gcref1) + _next_section(reader, 42, gcref1, -66) # metainterp = MyMetaInterp(cpu) reader = ResumeDataBoxReader(storage, metainterp) bi, br, bf = [None]*3, [None]*2, [None]*0 - reader.consume_boxes(bi, br, bf) + info = MyBlackholeInterp([lltype.Signed, lltype.Signed, + llmemory.GCREF, lltype.Signed, + llmemory.GCREF]).get_current_position_info() + reader.consume_boxes(info, bi, br, bf) b1s = reader.liveboxes[0] b2s = reader.liveboxes[1] assert bi == [b1s, ConstInt(111), b1s] assert br == [ConstPtr(gcrefnull), b2s] bi, br, bf = [None]*2, [None]*0, [None]*0 - reader.consume_boxes(bi, br, bf) + info = MyBlackholeInterp([lltype.Signed, + lltype.Signed]).get_current_position_info() + reader.consume_boxes(info, bi, br, bf) assert bi == [ConstInt(222), ConstInt(333)] bi, br, bf = [None]*2, [None]*1, [None]*0 - reader.consume_boxes(bi, br, bf) + info = MyBlackholeInterp([lltype.Signed, llmemory.GCREF, + lltype.Signed]).get_current_position_info() + reader.consume_boxes(info, bi, br, bf) b3s = reader.liveboxes[2] assert bi == [b1s, b3s] assert br == [b2s] @@ -164,7 +187,7 @@ def test_simple_read_tagged_ints(): storage = Storage() storage.rd_consts = [] - numb = Numbering(None, [tag(100, TAGINT), TYPEBARRIER]) + numb = Numbering(None, [tag(100, TAGINT)]) storage.rd_numb = numb # cpu = MyCPU([]) @@ -216,12 +239,12 @@ return "" % (self.jitcode, self.pc, self._env) def test_Snapshot_create(): - l = [BoxInt(), BoxPtr()] + l = ['b0', 'b1'] snap = Snapshot(None, l) assert snap.prev is None assert snap.boxes is l - l1 = [BoxInt()] + l1 = ['b3'] snap1 = Snapshot(snap, l1) assert snap1.prev is snap assert snap1.boxes is l1 @@ -620,9 +643,9 @@ assert liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), b3: tag(2, TAGBOX)} assert numb.nums == [tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX), - tag(1, TAGINT), TYPEBARRIER] + tag(1, TAGINT)] assert numb.prev.nums == [tag(0, TAGBOX), tag(1, TAGINT), tag(1, TAGBOX), - tag(0, TAGBOX), tag(2, TAGINT), TYPEBARRIER] + tag(0, TAGBOX), tag(2, TAGINT)] assert numb.prev.prev is None numb2, liveboxes2, v = memo.number({}, snap2) @@ -632,7 +655,7 @@ b3: tag(2, TAGBOX)} assert liveboxes2 is not liveboxes assert numb2.nums == [tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX), - tag(3, TAGINT), TYPEBARRIER] + tag(3, TAGINT)] assert numb2.prev is numb.prev env3 = [c3, b3, b1, c3] @@ -655,11 +678,11 @@ assert liveboxes3 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX)} assert numb3.nums == [tag(3, TAGINT), tag(4, TAGINT), tag(0, TAGBOX), - tag(3, TAGINT), TYPEBARRIER] + tag(3, TAGINT)] assert numb3.prev is numb.prev # virtual - env4 = [c3, b1, c3, b4] + env4 = [c3, b4, b1, c3] snap4 = Snapshot(snap, env4) numb4, liveboxes4, v = memo.number({b4: FakeValue(True, b4)}, snap4) @@ -667,8 +690,8 @@ assert liveboxes4 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), b4: tag(0, TAGVIRTUAL)} - assert numb4.nums == [tag(3, TAGINT), tag(0, TAGBOX), - tag(3, TAGINT), TYPEBARRIER, tag(0, TAGVIRTUAL)] + assert numb4.nums == [tag(3, TAGINT), tag(0, TAGVIRTUAL), tag(0, TAGBOX), + tag(3, TAGINT)] assert numb4.prev is numb.prev env5 = [b1, b4, b5] @@ -680,8 +703,8 @@ assert liveboxes5 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), b4: tag(0, TAGVIRTUAL), b5: tag(1, TAGVIRTUAL)} - assert numb5.nums == [tag(0, TAGBOX), TYPEBARRIER, - tag(0, TAGVIRTUAL), tag(1, TAGVIRTUAL)] + assert numb5.nums == [tag(0, TAGBOX), tag(0, TAGVIRTUAL), + tag(1, TAGVIRTUAL)] assert numb5.prev is numb4 def test_ResumeDataLoopMemo_number_boxes(): @@ -767,7 +790,7 @@ storage = Storage() snapshot = Snapshot(None, [b1, ConstInt(1), b1, b2]) snapshot = Snapshot(snapshot, [ConstInt(2), ConstInt(3)]) - snapshot = Snapshot(snapshot, [b1, b3, b2]) + snapshot = Snapshot(snapshot, [b1, b2, b3]) storage.rd_snapshot = snapshot storage.rd_frame_info_list = None return storage @@ -781,7 +804,7 @@ assert storage.rd_snapshot is None cpu = MyCPU([]) reader = ResumeDataDirectReader(cpu, storage) - _next_section(reader, sys.maxint, -65, 2**16) + _next_section(reader, sys.maxint, 2**16, -65) _next_section(reader, 2, 3) _next_section(reader, sys.maxint, 1, sys.maxint, 2**16) @@ -812,7 +835,22 @@ def consume_boxes(self): self.lst = [] - self._prepare_next_section() + class Whatever: + def __eq__(self, other): + return True + class MyInfo: + def enumerate_vars(_, callback_i, callback_r, callback_f): + for index, tagged in enumerate(self.cur_numb.nums): + box = self.decode_box(tagged, Whatever()) + if box.type == INT: + callback_i(index, index) + elif box.type == REF: + callback_r(index, index) + elif box.type == FLOAT: + callback_f(index, index) + else: + assert 0 + self._prepare_next_section(MyInfo()) return self.lst def write_an_int(self, count_i, box): @@ -849,7 +887,7 @@ metainterp = MyMetaInterp() reader = ResumeDataFakeReader(storage, newboxes, metainterp) lst = reader.consume_boxes() - assert lst == [b1t, b3t, b1t] + assert lst == [b1t, b1t, b3t] lst = reader.consume_boxes() assert lst == [ConstInt(2), ConstInt(3)] lst = reader.consume_boxes() @@ -870,7 +908,7 @@ reader = ResumeDataFakeReader(storage, newboxes, metainterp) lst = reader.consume_boxes() c1t = ConstInt(111) - assert lst == [c1t, b3t, b2t] + assert lst == [c1t, b2t, b3t] lst = reader.consume_boxes() assert lst == [ConstInt(2), ConstInt(3)] lst = reader.consume_boxes() From hpk at codespeak.net Tue May 25 16:58:27 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 25 May 2010 16:58:27 +0200 (CEST) Subject: [pypy-svn] r74735 - in pypy/trunk/py: _plugin _test Message-ID: <20100525145827.140F9282B9D@codespeak.net> Author: hpk Date: Tue May 25 16:58:25 2010 New Revision: 74735 Modified: pypy/trunk/py/_plugin/pytest_default.py pypy/trunk/py/_plugin/pytest_helpconfig.py pypy/trunk/py/_plugin/pytest_pytester.py pypy/trunk/py/_plugin/pytest_terminal.py pypy/trunk/py/_test/session.py Log: introducing --maxfail=NUM option , hopefully the last 1.3.1 candidate Modified: pypy/trunk/py/_plugin/pytest_default.py ============================================================================== --- pypy/trunk/py/_plugin/pytest_default.py (original) +++ pypy/trunk/py/_plugin/pytest_default.py Tue May 25 16:58:25 2010 @@ -62,9 +62,12 @@ def pytest_addoption(parser): group = parser.getgroup("general", "running and selection options") - group._addoption('-x', '--exitfirst', - action="store_true", dest="exitfirst", default=False, + group._addoption('-x', '--exitfirst', action="store_true", default=False, + dest="exitfirst", help="exit instantly on first error or failed test."), + group._addoption('--maxfail', metavar="num", + action="store", type="int", dest="maxfail", default=0, + help="exit after first num failures or errors.") group._addoption('-k', action="store", dest="keyword", default='', help="only run test items matching the given " @@ -89,6 +92,9 @@ def pytest_configure(config): setsession(config) + # compat + if config.getvalue("exitfirst"): + config.option.maxfail = 1 def setsession(config): val = config.getvalue Modified: pypy/trunk/py/_plugin/pytest_helpconfig.py ============================================================================== --- pypy/trunk/py/_plugin/pytest_helpconfig.py (original) +++ pypy/trunk/py/_plugin/pytest_helpconfig.py Tue May 25 16:58:25 2010 @@ -45,7 +45,7 @@ options = [opt for opt in options if opt._long_opts] options.sort(key=lambda x: x._long_opts) for opt in options: - if not opt._long_opts: + if not opt._long_opts or not opt.dest: continue optstrings = list(opt._long_opts) # + list(opt._short_opts) optstrings = filter(None, optstrings) Modified: pypy/trunk/py/_plugin/pytest_pytester.py ============================================================================== --- pypy/trunk/py/_plugin/pytest_pytester.py (original) +++ pypy/trunk/py/_plugin/pytest_pytester.py Tue May 25 16:58:25 2010 @@ -185,6 +185,7 @@ return reports[0] def inline_run(self, *args): + args = ("-s", ) + args # otherwise FD leakage config = self.parseconfig(*args) config.pluginmanager.do_configure(config) session = config.initsession() Modified: pypy/trunk/py/_plugin/pytest_terminal.py ============================================================================== --- pypy/trunk/py/_plugin/pytest_terminal.py (original) +++ pypy/trunk/py/_plugin/pytest_terminal.py Tue May 25 16:58:25 2010 @@ -312,12 +312,14 @@ self._keyboardinterrupt_memo = excinfo.getrepr(funcargs=True) def _report_keyboardinterrupt(self): - self.write_sep("!", "KEYBOARD INTERRUPT") excrepr = self._keyboardinterrupt_memo - if self.config.option.verbose: - excrepr.toterminal(self._tw) - else: - excrepr.reprcrash.toterminal(self._tw) + msg = excrepr.reprcrash.message + self.write_sep("!", msg) + if "KeyboardInterrupt" in msg: + if self.config.getvalue("fulltrace"): + excrepr.toterminal(self._tw) + else: + excrepr.reprcrash.toterminal(self._tw) def _getcrashline(self, report): try: Modified: pypy/trunk/py/_test/session.py ============================================================================== --- pypy/trunk/py/_test/session.py (original) +++ pypy/trunk/py/_test/session.py Tue May 25 16:58:25 2010 @@ -20,11 +20,14 @@ class Session(object): nodeid = "" + class Interrupted(KeyboardInterrupt): + """ signals an interrupted test run. """ + def __init__(self, config): self.config = config self.pluginmanager = config.pluginmanager # shortcut self.pluginmanager.register(self) - self._testsfailed = False + self._testsfailed = 0 self._nomatch = False self.shouldstop = False @@ -52,7 +55,7 @@ yield x self.config.hook.pytest_collectreport(report=rep) if self.shouldstop: - break + raise self.Interrupted(self.shouldstop) def filteritems(self, colitems): """ return items to process (some may be deselected)""" @@ -86,9 +89,11 @@ def pytest_runtest_logreport(self, report): if report.failed: - self._testsfailed = True - if self.config.option.exitfirst: - self.shouldstop = True + self._testsfailed += 1 + maxfail = self.config.getvalue("maxfail") + if maxfail and self._testsfailed >= maxfail: + self.shouldstop = "stopping after %d failures" % ( + self._testsfailed) pytest_collectreport = pytest_runtest_logreport def sessionfinishes(self, exitstatus): @@ -122,7 +127,8 @@ def _mainloop(self, colitems): for item in self.collect(colitems): - if self.shouldstop: - break if not self.config.option.collectonly: item.config.hook.pytest_runtest_protocol(item=item) + if self.shouldstop: + raise self.Interrupted(self.shouldstop) + From jcreigh at codespeak.net Tue May 25 17:15:21 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Tue, 25 May 2010 17:15:21 +0200 (CEST) Subject: [pypy-svn] r74736 - in pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86: . test Message-ID: <20100525151521.37DA0282B9D@codespeak.net> Author: jcreigh Date: Tue May 25 17:15:19 2010 New Revision: 74736 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py Log: add SHL/SHR/SAR to rx86 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Tue May 25 17:15:19 2010 @@ -634,6 +634,9 @@ genop_int_and = _binaryop("AND", True) genop_int_or = _binaryop("OR", True) genop_int_xor = _binaryop("XOR", True) + genop_int_lshift = _binaryop("SHL") + genop_int_rshift = _binaryop("SAR") + genop_uint_rshift = _binaryop("SHR") genop_float_add = _binaryop("ADDSD", True) genop_float_sub = _binaryop('SUBSD') genop_float_mul = _binaryop('MULSD', True) @@ -733,24 +736,6 @@ def genop_cast_int_to_float(self, op, arglocs, resloc): self.mc.CVTSI2SD(resloc, arglocs[0]) - def genop_int_lshift(self, op, arglocs, resloc): - loc, loc2 = arglocs - if loc2 is ecx: - loc2 = cl - self.mc.SHL(loc, loc2) - - def genop_int_rshift(self, op, arglocs, resloc): - loc, loc2 = arglocs - if loc2 is ecx: - loc2 = cl - self.mc.SAR(loc, loc2) - - def genop_uint_rshift(self, op, arglocs, resloc): - loc, loc2 = arglocs - if loc2 is ecx: - loc2 = cl - self.mc.SHR(loc, loc2) - def genop_guard_int_is_true(self, op, guard_op, addr, arglocs, resloc): guard_opnum = guard_op.opnum self.mc.CMP(arglocs[0], imm(0)) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Tue May 25 17:15:19 2010 @@ -135,6 +135,9 @@ ADD = _binaryop('ADD') OR = _binaryop('OR') XOR = _binaryop('XOR') + SHL = _binaryop('SHL') + SHR = _binaryop('SHR') + SAR = _binaryop('SAR') TEST = _binaryop('TEST') AND = _binaryop('AND') Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py Tue May 25 17:15:19 2010 @@ -333,6 +333,23 @@ return INSN +def shifts(mod_field): + modrm = chr(0xC0 | (mod_field << 3)) + shift_once = insn(rex_w, '\xD1', register(1), modrm) + shift_r_by_cl = insn(rex_w, '\xD3', register(1), modrm) + shift_ri8 = insn(rex_w, '\xC1', register(1), modrm, immediate(2, 'b')) + + def shift_ri(mc, reg, immed): + if immed == 1: + shift_once(mc, reg) + else: + shift_ri8(mc, reg, immed) + + def shift_rr(mc, reg1, reg2): + assert reg2 == R.ecx + shift_r_by_cl(mc, reg1) + + return (shift_ri, shift_rr) # ____________________________________________________________ @@ -423,6 +440,10 @@ def IMUL_ri(self, reg, immed): self.IMUL_rri(reg, reg, immed) + SHL_ri, SHL_rr = shifts(4) + SHR_ri, SHR_rr = shifts(5) + SAR_ri, SAR_rr = shifts(7) + # ------------------------------ Misc stuff ------------------------------ NOP = insn('\x90') Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py Tue May 25 17:15:19 2010 @@ -16,6 +16,10 @@ def tell(self): return 0x76543210 + len(self.buffer) +def assert_encodes_as(code_builder_cls, insn_name, args, expected_encoding): + s = code_builder_cls() + getattr(s, insn_name)(*args) + assert s.getvalue() == expected_encoding class CodeBuilder32(CodeBuilderMixin, X86_32_CodeBuilder): pass @@ -155,6 +159,20 @@ s.IMUL_rri(ebx, ecx, 0x2A) assert s.getvalue() == '\x6B\xD9\x2A' +def test_shifts(): + cb = CodeBuilder32 + assert_encodes_as(cb, 'SHL_ri', (edx, 1), '\xD1\xE2') + assert_encodes_as(cb, 'SHL_ri', (edx, 5), '\xC1\xE2\x05') + assert_encodes_as(cb, 'SHL_rr', (edx, ecx), '\xD3\xE2') + + assert_encodes_as(cb, 'SHR_ri', (edx, 1), '\xD1\xEA') + assert_encodes_as(cb, 'SHR_ri', (edx, 5), '\xC1\xEA\x05') + assert_encodes_as(cb, 'SHR_rr', (edx, ecx), '\xD3\xEA') + + assert_encodes_as(cb, 'SAR_ri', (edx, 1), '\xD1\xFA') + assert_encodes_as(cb, 'SAR_ri', (edx, 5), '\xC1\xFA\x05') + assert_encodes_as(cb, 'SAR_rr', (edx, ecx), '\xD3\xFA') + class CodeBuilder64(CodeBuilderMixin, X86_64_CodeBuilder): pass Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py Tue May 25 17:15:19 2010 @@ -222,6 +222,9 @@ return [] # MOV [immediate], EAX: there is a special encoding if methname == 'SET_ir': py.test.skip("SET_ir: must be tested manually") + if methname.startswith('SHL') or methname.startswith('SAR') or methname.startswith('SHR'): + # XXX: Would be nice to test these automatically + py.test.skip('Shifts must be tested manually') return [args] def get_code_checker_class(self): From cfbolz at codespeak.net Tue May 25 17:37:31 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 25 May 2010 17:37:31 +0200 (CEST) Subject: [pypy-svn] r74737 - pypy/extradoc/talk/s3-2010 Message-ID: <20100525153731.0084C282B9D@codespeak.net> Author: cfbolz Date: Tue May 25 17:37:30 2010 New Revision: 74737 Modified: pypy/extradoc/talk/s3-2010/virtuals.txt Log: some quick notes Modified: pypy/extradoc/talk/s3-2010/virtuals.txt ============================================================================== --- pypy/extradoc/talk/s3-2010/virtuals.txt (original) +++ pypy/extradoc/talk/s3-2010/virtuals.txt Tue May 25 17:37:30 2010 @@ -1,6 +1,16 @@ -Escape analysis in Pypy's JIT +Escape analysis in PyPy's JIT ============================= +XXX + +- should probably start by describing the setting more: ie linear traces from + the tracing interpreter, either a loop or a bridge +- traces can have guards that leave the trace (so that the state reconstruction + explanation has context) +- introduce the running example interpreter and probably show some traces + before optimizations + + One of the major problems of Python's performance is costs of allocating objects. Even though PyPy's garbage collector is pretty good, especially for short lived objects, the costs of allocations are still relatively From afa at codespeak.net Tue May 25 18:10:02 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 25 May 2010 18:10:02 +0200 (CEST) Subject: [pypy-svn] r74738 - in pypy/trunk/pypy/rpython/module: . test Message-ID: <20100525161002.32DAF282B9D@codespeak.net> Author: afa Date: Tue May 25 18:10:00 2010 New Revision: 74738 Modified: pypy/trunk/pypy/rpython/module/ll_os_stat.py pypy/trunk/pypy/rpython/module/test/test_ll_os_stat.py Log: On Windows, make os.fstat() work on non-disk files, stdout for example. Modified: pypy/trunk/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/trunk/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/trunk/pypy/rpython/module/ll_os_stat.py Tue May 25 18:10:00 2010 @@ -317,6 +317,11 @@ 'ERROR_SHARING_VIOLATION') _S_IFDIR = platform.ConstantInteger('_S_IFDIR') _S_IFREG = platform.ConstantInteger('_S_IFREG') + _S_IFCHR = platform.ConstantInteger('_S_IFCHR') + _S_IFIFO = platform.ConstantInteger('_S_IFIFO') + FILE_TYPE_UNKNOWN = platform.ConstantInteger('FILE_TYPE_UNKNOWN') + FILE_TYPE_CHAR = platform.ConstantInteger('FILE_TYPE_CHAR') + FILE_TYPE_PIPE = platform.ConstantInteger('FILE_TYPE_PIPE') WIN32_FILE_ATTRIBUTE_DATA = platform.Struct( 'WIN32_FILE_ATTRIBUTE_DATA', @@ -365,6 +370,12 @@ rwin32.BOOL, calling_conv='win') + GetFileType = rffi.llexternal( + 'GetFileType', + [rwin32.HANDLE], + rwin32.DWORD, + calling_conv='win') + FindFirstFile = rffi.llexternal( 'FindFirstFileA', [rffi.CCHARP, lltype.Ptr(WIN32_FIND_DATA)], @@ -477,9 +488,29 @@ win32_lstat_llimpl = win32_stat_llimpl def win32_fstat_llimpl(fd): - info = lltype.malloc(BY_HANDLE_FILE_INFORMATION, flavor='raw') + handle = rwin32._get_osfhandle(fd) + + filetype = GetFileType(handle) + if filetype == FILE_TYPE_CHAR: + # console or LPT device + return make_stat_result((_S_IFCHR, + 0, 0, 0, 0, 0, + 0, 0, 0, 0)) + elif filetype == FILE_TYPE_PIPE: + # socket or named pipe + return make_stat_result((_S_IFIFO, + 0, 0, 0, 0, 0, + 0, 0, 0, 0)) + elif filetype == FILE_TYPE_UNKNOWN: + error = rwin32.GetLastError() + if error != 0: + raise WindowsError(error, "os_fstat failed") + # else: unknown but valid file + + # normal disk file (FILE_TYPE_DISK) + info = lltype.malloc(BY_HANDLE_FILE_INFORMATION, flavor='raw', + zero=True) try: - handle = rwin32._get_osfhandle(fd) res = GetFileInformationByHandle(handle, info) if res == 0: raise WindowsError(rwin32.GetLastError(), "os_fstat failed") Modified: pypy/trunk/pypy/rpython/module/test/test_ll_os_stat.py ============================================================================== --- pypy/trunk/pypy/rpython/module/test/test_ll_os_stat.py (original) +++ pypy/trunk/pypy/rpython/module/test/test_ll_os_stat.py Tue May 25 18:10:00 2010 @@ -15,3 +15,7 @@ check('c:/') check('c:/temp') check('c:/pagefile.sys') + + def test_fstat(self): + stat = ll_os_stat.win32_fstat_llimpl(0) # stdout + assert stat.st_mode != 0 From arigo at codespeak.net Tue May 25 18:39:33 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 25 May 2010 18:39:33 +0200 (CEST) Subject: [pypy-svn] r74739 - pypy/trunk/pypy/jit/metainterp Message-ID: <20100525163933.E3178282B9D@codespeak.net> Author: arigo Date: Tue May 25 18:39:32 2010 New Revision: 74739 Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py Log: Add an assert. Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Tue May 25 18:39:32 2010 @@ -860,6 +860,11 @@ self.debug_merge_point() if self.metainterp.seen_can_enter_jit: self.metainterp.seen_can_enter_jit = False + assert not self.metainterp.in_recursion + # ^^^ it's impossible to arrive here with in_recursion set + # to a non-zero value: seen_can_enter_jit can only be set + # to True by opimpl_can_enter_jit, which should be executed + # just before opimpl_jit_merge_point (no recursion inbetween). try: self.metainterp.reached_can_enter_jit(self.env) except GiveUp: From arigo at codespeak.net Tue May 25 19:27:34 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 25 May 2010 19:27:34 +0200 (CEST) Subject: [pypy-svn] r74741 - pypy/trunk/pypy/jit/metainterp Message-ID: <20100525172734.DE8BB282B9D@codespeak.net> Author: arigo Date: Tue May 25 19:27:33 2010 New Revision: 74741 Modified: pypy/trunk/pypy/jit/metainterp/compile.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/warmstate.py Log: Attempt to simplify the logic about catching ContinueRunningNormally, resetting some counters, and re-raising the exception. I think that the logic was subtly broken -- there are cases in which we would actually exit the metainterp with another exception, like DoneWithThisFrame, but we still want to reset counters. Did not add tests for this case because it is a rare case needing a completely custom test, and the test would arguably not be most useful because it makes little sense in the new way the logic is written. Modified: pypy/trunk/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/compile.py (original) +++ pypy/trunk/pypy/jit/metainterp/compile.py Tue May 25 19:27:33 2010 @@ -255,12 +255,10 @@ counter = self._counters.see(box) return counter >= trace_eagerness - def reset_counter_from_failure(self, metainterp): + def reset_counter_from_failure(self): if self._counter >= 0: self._counter = 0 self._counters = None - warmrunnerstate = metainterp.staticdata.state - warmrunnerstate.disable_noninlinable_function(metainterp) def compile_and_attach(self, metainterp, new_loop): # We managed to create a bridge. Attach the new operations @@ -423,6 +421,9 @@ # general loop token old_loop_tokens.append(new_loop_token) + def reset_counter_from_failure(self): + pass + def compile_new_bridge(metainterp, old_loop_tokens, resumekey): """Try to compile a new bridge leading from the beginning of the history Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Tue May 25 19:27:33 2010 @@ -1304,7 +1304,6 @@ self.staticdata = staticdata self.cpu = staticdata.cpu self.portal_trace_positions = [] - self.greenkey_of_huge_function = None def is_blackholing(self): return self.history is None @@ -1503,15 +1502,20 @@ self.staticdata.stats.aborted() self.staticdata.profiler.end_tracing() self.staticdata.profiler.start_blackhole() + self.resumekey.reset_counter_from_failure() switch_to_blackhole._dont_inline_ = True def switch_to_blackhole_if_trace_too_long(self): if not self.is_blackholing(): warmrunnerstate = self.staticdata.state if len(self.history.operations) > warmrunnerstate.trace_limit: - self.greenkey_of_huge_function = self.find_biggest_function() + greenkey_of_huge_function = self.find_biggest_function() self.portal_trace_positions = None self.switch_to_blackhole(ABORT_TOO_LONG) + if greenkey_of_huge_function is not None: + warmrunnerstate = self.staticdata.state + warmrunnerstate.disable_noninlinable_function( + greenkey_of_huge_function) def _interpret(self): # Execute the frames forward until we raise a DoneWithThisFrame, @@ -1580,7 +1584,6 @@ debug_stop('jit-tracing') def _handle_guard_failure(self, key): - from pypy.jit.metainterp.warmspot import ContinueRunningNormallyBase original_greenkey = key.original_greenkey # notice that here we just put the greenkey # use -1 to mark that we will have to give up @@ -1588,17 +1591,12 @@ self.current_merge_points = [(original_greenkey, -1)] self.resumekey = key self.seen_can_enter_jit = False - started_as_blackhole = self.is_blackholing() try: self.prepare_resume_from_failure(key.guard_opnum) self.interpret() assert False, "should always raise" except GenerateMergePoint, gmp: return self.designate_target_loop(gmp) - except ContinueRunningNormallyBase: - if not started_as_blackhole: - key.reset_counter_from_failure(self) - raise def remove_consts_and_duplicates(self, boxes, startindex, endindex, duplicates): Modified: pypy/trunk/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/warmstate.py (original) +++ pypy/trunk/pypy/jit/metainterp/warmstate.py Tue May 25 19:27:33 2010 @@ -157,16 +157,14 @@ if self.profiler is not None: self.profiler.set_printing(value >= DEBUG_PROFILE) - def disable_noninlinable_function(self, metainterp): - greenkey = metainterp.greenkey_of_huge_function - if greenkey is not None: - cell = self.jit_cell_at_key(greenkey) - cell.dont_trace_here = True - debug_start("jit-disableinlining") - sd = self.warmrunnerdesc.metainterp_sd - loc = sd.state.get_location_str(greenkey) - debug_print("disabled inlining", loc) - debug_stop("jit-disableinlining") + def disable_noninlinable_function(self, greenkey): + cell = self.jit_cell_at_key(greenkey) + cell.dont_trace_here = True + debug_start("jit-disableinlining") + sd = self.warmrunnerdesc.metainterp_sd + loc = sd.state.get_location_str(greenkey) + debug_print("disabled inlining", loc) + debug_stop("jit-disableinlining") def attach_unoptimized_bridge_from_interp(self, greenkey, entry_loop_token): @@ -227,11 +225,6 @@ cell.counter = -2 try: loop_token = metainterp.compile_and_run_once(*args) - except ContinueRunningNormally: - # the trace got too long, reset the counter - cell.counter = 0 - self.disable_noninlinable_function(metainterp) - raise finally: if cell.counter == -2: cell.counter = 0 From arigo at codespeak.net Tue May 25 19:33:20 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 25 May 2010 19:33:20 +0200 (CEST) Subject: [pypy-svn] r74742 - pypy/trunk/pypy/jit/metainterp Message-ID: <20100525173320.B9982282B9D@codespeak.net> Author: arigo Date: Tue May 25 19:33:19 2010 New Revision: 74742 Modified: pypy/trunk/pypy/jit/metainterp/warmstate.py Log: Forgot to kill this. Modified: pypy/trunk/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/warmstate.py (original) +++ pypy/trunk/pypy/jit/metainterp/warmstate.py Tue May 25 19:33:19 2010 @@ -181,7 +181,6 @@ metainterp_sd = self.warmrunnerdesc.metainterp_sd vinfo = metainterp_sd.virtualizable_info - ContinueRunningNormally = self.warmrunnerdesc.ContinueRunningNormally num_green_args = self.warmrunnerdesc.num_green_args get_jitcell = self.make_jitcell_getter() set_future_values = self.make_set_future_values() From agaynor at codespeak.net Tue May 25 19:39:21 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Tue, 25 May 2010 19:39:21 +0200 (CEST) Subject: [pypy-svn] r74743 - pypy/trunk/pypy/translator/goal Message-ID: <20100525173921.373F2282B9D@codespeak.net> Author: agaynor Date: Tue May 25 19:39:19 2010 New Revision: 74743 Added: pypy/trunk/pypy/translator/goal/targetosreadbench.py Log: Added target for os.read benchmarking Added: pypy/trunk/pypy/translator/goal/targetosreadbench.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/translator/goal/targetosreadbench.py Tue May 25 19:39:19 2010 @@ -0,0 +1,25 @@ +""" +A benchmark for read() +""" + +import os + +# __________ Entry point __________ + +def entry_point(argv): + length = 0 + if len(argv) > 1: + length = int(argv[1]) + else: + length = 100 + for i in xrange(100000): + f = os.open(__file__, 0666, os.O_RDONLY) + os.read(f, length) + os.close(f) + return 0 + +# _____ Define and setup target ___ + +def target(*args): + return entry_point, None + From fijal at codespeak.net Tue May 25 20:15:20 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 25 May 2010 20:15:20 +0200 (CEST) Subject: [pypy-svn] r74744 - pypy/trunk/pypy/doc Message-ID: <20100525181520.93044282B9D@codespeak.net> Author: fijal Date: Tue May 25 20:15:18 2010 New Revision: 74744 Modified: pypy/trunk/pypy/doc/faq.txt Log: update FAQ Modified: pypy/trunk/pypy/doc/faq.txt ============================================================================== --- pypy/trunk/pypy/doc/faq.txt (original) +++ pypy/trunk/pypy/doc/faq.txt Tue May 25 20:15:18 2010 @@ -45,6 +45,11 @@ We list the differences we know about in `cpython_differences`_. +There is also an experimental support for CPython extension modules, so +they'll run without change (from current observation, rather with little +change) on trunk. It has not been released yet, although it should be a major +point of the next pypy release. + .. _`extension modules`: cpython_differences.html#extension-modules .. _`cpython_differences`: cpython_differences.html @@ -61,7 +66,9 @@ PyPy also basically works in a 64-bit Linux environment, but *it requires a 32-bit Intel CPU* for the JIT right now. (It works -fine in a 32-bit chroot of an Intel 64.) +fine in a 32-bit chroot of an Intel 64.). There is also an ongoing +summero of code project to provide 64bit support for JIT. It's scheduled +to be finished by end of summer 2010. ------------------------------------------------ Which Python version (2.x?) does PyPy implement? @@ -82,20 +89,11 @@ Note that PyPy also fully supports `stackless-like microthreads`_ (although both cannot be mixed yet). -As for other modules: The general rule of thumb is that pure-Python modules -work, C extension modules don't. However, many of the C extension modules -of the standard -library have been re-implemented in pure Python or as a mixed module (for some -there were also older pure-Python versions available). The list of supported -modules can be found as follows: - - * pure Python implementations: all the modules found in the `pypy/lib/`_ - directory - - * mixed module implementations: all the modules listed in - ``essential_modules``, ``default_modules`` and ``working_modules`` - in `pypy/config/pypyoption.py`_. - +All pure-python modules should work, unless they rely on ugly +cpython implementation details, in which case it's their fault. +There is an increasing number of compatible CPython extensions working, +including things like wxPython or PIL. This is an ongoing development effort +to bring as many CPython extension modules working as possible. .. _`stackless-like microthreads`: stackless.html @@ -104,16 +102,13 @@ Can I use CPython extension modules? ------------------------------------ -No and there are no short-term plans to support this. CPython extension modules -rely heavily on CPython's C API which contains a lot of implementation details -like reference counting, exact C-level object implementation and layout etc. - -A module based on ctypes is a different matter -- we support these nowadays. - -The long-term answer might be different. In principle, it should be possible -for PyPy to support the CPython C API (or at least a large subset of its -official part). It means that "in the fullness of time" you might be able to -simply recompile existing CPython extension modules and use them with PyPy. +Yes, but the feature is in alpha state and is available only on trunk +(not in the 1.2 release). However, we'll only ever support well-behaving +CPython extensions. Please consult PyPy developers on IRC or mailing list +for explanations if your favorite module works and how you can help to make +it happen in case it does not. + +We fully support ctypes-based extensions, however. ------------------------------------------ How do I write extension modules for PyPy? From arigo at codespeak.net Tue May 25 20:28:37 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 25 May 2010 20:28:37 +0200 (CEST) Subject: [pypy-svn] r74746 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100525182837.A3BF1282B9D@codespeak.net> Author: arigo Date: Tue May 25 20:28:36 2010 New Revision: 74746 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitcode.py Log: Clean up format.py. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py Tue May 25 20:28:36 2010 @@ -54,12 +54,14 @@ insns = insns[:-1] for i, asm in enumerate(insns): if ssarepr._insns_pos: - print >> output, '%4d ' % ssarepr._insns_pos[i], + prefix = '%4d ' % ssarepr._insns_pos[i] + else: + prefix = '' if isinstance(asm[0], Label): if asm[0].name in seenlabels: - print >> output, '%s:' % getlabelname(asm[0]) + print >> output, prefix + '%s:' % getlabelname(asm[0]) else: - print >> output, asm[0], + print >> output, prefix + asm[0], if len(asm) > 1: if asm[-2] == '->': if len(asm) == 3: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py Tue May 25 20:28:36 2010 @@ -59,11 +59,11 @@ # for testing only info = self.get_live_vars_info(pc) lst_i = ['%%i%d' % info.get_register_index_i(index) - for index in range(info.get_register_count_i())] + for index in range(info.get_register_count_i()-1, -1, -1)] lst_r = ['%%r%d' % info.get_register_index_r(index) - for index in range(info.get_register_count_r())] + for index in range(info.get_register_count_r()-1, -1, -1)] lst_f = ['%%f%d' % info.get_register_index_f(index) - for index in range(info.get_register_count_f())] + for index in range(info.get_register_count_f()-1, -1, -1)] return ' '.join(lst_i + lst_r + lst_f) def _missing_liveness(self, pc): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitcode.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitcode.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitcode.py Tue May 25 20:28:36 2010 @@ -21,4 +21,4 @@ j.setup(liveness={5: (" A", "b", "CD")}) assert j.has_liveness_info(5) assert not j.has_liveness_info(4) - assert j._live_vars(5) == "%i65 %i32 %r98 %f68 %f67" + assert j._live_vars(5) == "%i32 %i65 %r98 %f67 %f68" From jcreigh at codespeak.net Tue May 25 20:54:20 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Tue, 25 May 2010 20:54:20 +0200 (CEST) Subject: [pypy-svn] r74747 - in pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86: . test Message-ID: <20100525185420.277CD282B9D@codespeak.net> Author: jcreigh Date: Tue May 25 20:54:18 2010 New Revision: 74747 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py Log: add concept of "byte registers" to rx86 to try to avoid confusion Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Tue May 25 20:54:18 2010 @@ -543,14 +543,14 @@ self.mc.UCOMISD(arglocs[0], arglocs[1]) rl = result_loc.lowest8bits() rh = result_loc.higher8bits() - getattr(self.mc, 'SET' + cond)(rl) + self.mc.SET_ir(rx86.Conditions[cond], rl.value) if is_ne: - self.mc.SETP(rh) - self.mc.OR(rl, rh) + self.mc.SET_ir(rx86.Conditions['P'], rh.value) + self.mc.OR8_rr(rl.value, rh.value) else: - self.mc.SETNP(rh) - self.mc.AND(rl, rh) - self.mc.MOVZX(result_loc, rl) + self.mc.SET_ir(rx86.Conditions['NP'], rh.value) + self.mc.AND8_rr(rl.value, rh.value) + self.mc.MOVZX8_rr(result_loc.value, rl.value) return genop_cmp def _cmpop_guard(cond, rev_cond, false_cond, false_rev_cond): @@ -720,15 +720,8 @@ return self.implement_guard(addr) def genop_float_is_true(self, op, arglocs, resloc): - loc0, loc1 = arglocs - self.mc.XORPD(loc0, loc0) - self.mc.UCOMISD(loc0, loc1) - rl = resloc.lowest8bits() - rh = resloc.higher8bits() - self.mc.SETNE(rl) - self.mc.SETP(rh) - self.mc.OR(rl, rh) - self.mc.MOVZX(resloc, rl) + self.mc.XORPD(arglocs[0], arglocs[0]) + self.genop_float_ne(op, arglocs, resloc) def genop_cast_float_to_int(self, op, arglocs, resloc): self.mc.CVTTSD2SI(resloc, arglocs[0]) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Tue May 25 20:54:18 2010 @@ -51,10 +51,12 @@ return rx86.R.names[self.value] def lowest8bits(self): - # XXX: Only handling i386 al, cl, dl, bl for now - assert self.value < 4 assert not self.is_xmm - return self + return RegLoc(rx86.low_byte(self.value), False) + + def higher8bits(self): + assert not self.is_xmm + return RegLoc(rx86.high_byte(self.value), False) def location_code(self): return 'r' Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py Tue May 25 20:54:18 2010 @@ -4,10 +4,15 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rpython.lltypesystem import rffi +BYTE_REG_FLAG = 0x20 + class R(object): # the following are synonyms for rax, rcx, etc. on 64 bits eax, ecx, edx, ebx, esp, ebp, esi, edi = range(8) + # 8-bit registers + al, cl, dl, bl, ah, ch, dh, bh = [reg | BYTE_REG_FLAG for reg in range(8)] + # xmm registers xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 = range(8) @@ -15,10 +20,23 @@ r8, r9, r10, r11, r12, r13, r14, r15 = range(8, 16) xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15 = range(8, 16) + # These replace ah, ch, dh, bh when the REX-prefix is used + spl, bpl, sil, dil = ah, ch, dh, bh + + # Low-byte of extra registers + r8l, r9l, r10l, r11l, r12l, r13l, r14l, r15l = [reg | BYTE_REG_FLAG for reg in range(8, 16)] + names = ['eax', 'ecx', 'edx', 'ebx', 'esp', 'ebp', 'esi', 'edi', 'r8', 'r9', 'r10', 'r11', 'r12', 'r13', 'r14', 'r15'] xmmnames = ['xmm%d' % i for i in range(16)] +def low_byte(reg): + assert 0 <= reg < 4 + return reg | BYTE_REG_FLAG + +def high_byte(reg): + assert 0 <= reg < 4 + return (reg + 4) | BYTE_REG_FLAG def single_byte(value): return -128 <= value < 128 @@ -63,6 +81,16 @@ assert factor in (1, 8) return encode_register, argnum, factor, rex_register + at specialize.arg(2) +def encode_byte_register(mc, reg, factor, orbyte): + assert reg & BYTE_REG_FLAG + return encode_register(mc, reg & ~BYTE_REG_FLAG, factor, orbyte) + +def byte_register(argnum, factor=1): + assert factor in (1, 8) + return encode_byte_register, argnum, factor, rex_register + + # ____________________________________________________________ # Encode a constant in the orbyte @@ -400,8 +428,9 @@ MOV_jr = insn(rex_w, '\x89', register(2,8), '\x05', immediate(1)) MOV_ji = insn(rex_w, '\xC7', '\x05', immediate(1), immediate(2)) - MOV8_mr = insn(rex_w, '\x88', register(2, 8), mem_reg_plus_const(1)) + MOV8_mr = insn(rex_w, '\x88', byte_register(2, 8), mem_reg_plus_const(1)) + MOVZX8_rr = insn(rex_w, '\x0F\xB6', register(1,8), byte_register(2)) MOVZX8_rm = insn(rex_w, '\x0F\xB6', register(1,8), mem_reg_plus_const(2)) MOVZX8_ra = insn(rex_w, '\x0F\xB6', register(1,8), mem_reg_plus_scaled_reg_plus_const(2)) @@ -425,6 +454,10 @@ CMP_ji32 = insn(rex_w, '\x81', '\x3D', immediate(1), immediate(2)) CMP_ji = select_8_or_32_bit_immed(CMP_ji8, CMP_ji32) + AND8_rr = insn(rex_w, '\x20', byte_register(1), byte_register(2,8), '\xC0') + + OR8_rr = insn(rex_w, '\x08', byte_register(1), byte_register(2,8), '\xC0') + NEG_r = insn(rex_w, '\xF7', register(1), '\xD8') DIV_r = insn(rex_w, '\xF7', register(1), '\xF0') @@ -469,7 +502,8 @@ # We need to be consistent. J_il8 = insn(immediate(1, 'o'), '\x70', immediate(2, 'b')) J_il = insn('\x0F', immediate(1,'o'), '\x80', relative(2)) - SET_ir = insn('\x0F', immediate(1,'o'),'\x90', register(2), '\xC0') + + SET_ir = insn('\x0F', immediate(1,'o'),'\x90', byte_register(2), '\xC0') # The 64-bit version of this, CQO, is defined in X86_64_CodeBuilder CDQ = insn(rex_nw, '\x99') @@ -590,15 +624,15 @@ py.test.skip("MOVSD_rj unsupported") def MOVSD_jr(self, xmm_reg, mem_immed): py.test.skip("MOVSD_jr unsupported") - def ADDSD_rj(self, xxm_reg, mem_immed): + def ADDSD_rj(self, xmm_reg, mem_immed): py.test.skip("ADDSD_rj unsupported") - def SUBSD_rj(self, xxm_reg, mem_immed): + def SUBSD_rj(self, xmm_reg, mem_immed): py.test.skip("SUBSD_rj unsupported") - def MULSD_rj(self, xxm_reg, mem_immed): + def MULSD_rj(self, xmm_reg, mem_immed): py.test.skip("MULSD_rj unsupported") - def DIVSD_rj(self, xxm_reg, mem_immed): + def DIVSD_rj(self, xmm_reg, mem_immed): py.test.skip("DIVSD_rj unsupported") - def UCOMISD_rj(self, xxm_reg, mem_immed): + def UCOMISD_rj(self, xmm_reg, mem_immed): py.test.skip("UCOMISD_rj unsupported") Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py Tue May 25 20:54:18 2010 @@ -121,7 +121,7 @@ def test_set_ir(): s = CodeBuilder32() - s.SET_ir(5, 2) + s.SET_ir(5, dl) assert s.getvalue() == '\x0F\x95\xC2' def test_xchg_rj(): @@ -173,6 +173,12 @@ assert_encodes_as(cb, 'SAR_ri', (edx, 5), '\xC1\xFA\x05') assert_encodes_as(cb, 'SAR_rr', (edx, ecx), '\xD3\xFA') +def test_and8_rr(): + assert_encodes_as(CodeBuilder32, 'AND8_rr', (bl, bh), '\x20\xFB') + +def test_or8_rr(): + assert_encodes_as(CodeBuilder32, 'OR8_rr', (bl, bh), '\x08\xFB') + class CodeBuilder64(CodeBuilderMixin, X86_64_CodeBuilder): pass From arigo at codespeak.net Tue May 25 20:59:37 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 25 May 2010 20:59:37 +0200 (CEST) Subject: [pypy-svn] r74748 - in pypy/branch/blackhole-improvement/pypy/jit/metainterp: . test Message-ID: <20100525185937.7ACE7282BDB@codespeak.net> Author: arigo Date: Tue May 25 20:59:35 2010 New Revision: 74748 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmstate.py Log: In-progress. Link the blackhole interp to pyjitpl directly, called when the new exception SwitchToBlackhole is raised. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Tue May 25 20:59:35 2010 @@ -789,8 +789,16 @@ @arguments("self", "I", "R", "F", "I", "R", "F") def bhimpl_jit_merge_point(self, *results): - CRN = self.builder.metainterp_sd.ContinueRunningNormally - raise CRN(*results) + if self.nextblackholeinterp is None: # we are the last level + CRN = self.builder.metainterp_sd.ContinueRunningNormally + raise CRN(*results) + else: + # This occurs when we reach 'jit_merge_point' in the portal + # function called by recursion. In this case, we can directly + # call the interpreter main loop from here, and just return its + # result. + XXX + raise LeaveFrame # ---------- # list operations @@ -1160,8 +1168,29 @@ e = lltype.cast_opaque_ptr(llmemory.GCREF, e) raise sd.ExitFrameWithExceptionRef(self.cpu, e) + def _copy_data_from_miframe(self, miframe): + self.setposition(miframe.jitcode, miframe.pc) + for i in range(self.jitcode.num_regs_i()): + box = miframe.registers_i[i] + if box is not None: + self.setarg_i(i, box.getint()) + for i in range(self.jitcode.num_regs_r()): + box = miframe.registers_r[i] + if box is not None: + self.setarg_r(i, box.getref_base()) + for i in range(self.jitcode.num_regs_f()): + box = miframe.registers_f[i] + if box is not None: + self.setarg_f(i, box.getfloat()) + # ____________________________________________________________ +def _run_forever(blackholeinterp, current_exc): + current_exc = lltype.cast_opaque_ptr(rclass.OBJECTPTR, current_exc) + while True: + current_exc = blackholeinterp._resume_mainloop(current_exc) + blackholeinterp = blackholeinterp.nextblackholeinterp + def resume_in_blackhole(metainterp_sd, resumedescr): from pypy.jit.metainterp.resume import blackhole_from_resumedata debug_start('jit-blackhole') @@ -1175,10 +1204,29 @@ current_exc = blackholeinterp._prepare_resume_from_failure( resumedescr.guard_opnum) try: - current_exc = lltype.cast_opaque_ptr(rclass.OBJECTPTR, current_exc) - while True: - current_exc = blackholeinterp._resume_mainloop(current_exc) - blackholeinterp = blackholeinterp.nextblackholeinterp + _run_forever(blackholeinterp, current_exc) + finally: + metainterp_sd.profiler.end_blackhole() + debug_stop('jit-blackhole') + +def convert_and_run_from_pyjitpl(metainterp, current_exc=NULL): + # Get a chain of blackhole interpreters and fill them by copying + # 'metainterp.framestack'. Note that the order is important: the + # first one we get must be the bottom one, in order to make + # the comment in BlackholeInterpreter.setposition() valid. + debug_start('jit-blackhole') + metainterp_sd = metainterp.staticdata + metainterp_sd.profiler.start_blackhole() + nextbh = None + for frame in metainterp.framestack: + curbh = metainterp_sd.blackholeinterpbuilder.acquire_interp() + curbh._copy_data_from_miframe(frame) + curbh.nextblackholeinterp = nextbh + nextbh = curbh + firstbh = nextbh + # + try: + _run_forever(firstbh, current_exc) finally: metainterp_sd.profiler.end_blackhole() debug_stop('jit-blackhole') Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py Tue May 25 20:59:35 2010 @@ -294,12 +294,10 @@ assert 0, typetag return counter >= trace_eagerness - def reset_counter_from_failure(self, metainterp): + def reset_counter_from_failure(self): if self._counter >= 0: self._counter = 0 self._counters = None - warmrunnerstate = metainterp.staticdata.state - warmrunnerstate.disable_noninlinable_function(metainterp) def compile_and_attach(self, metainterp, new_loop): # We managed to create a bridge. Attach the new operations @@ -476,6 +474,9 @@ # general loop token old_loop_tokens.append(new_loop_token) + def reset_counter_from_failure(self): + pass + def compile_new_bridge(metainterp, old_loop_tokens, resumekey): """Try to compile a new bridge leading from the beginning of the history Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Tue May 25 20:59:35 2010 @@ -1,6 +1,5 @@ import py, os -from pypy.rpython.lltypesystem import llmemory -from pypy.rpython.ootypesystem import ootype +from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.llinterp import LLException from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable @@ -319,47 +318,13 @@ if value: self.pc = target -## def follow_jump(self): -## _op_goto_if_not = self.metainterp.staticdata._op_goto_if_not -## assert ord(self.bytecode[self.pc]) == _op_goto_if_not -## self.pc += 1 # past the bytecode for 'goto_if_not' -## target = self.load_3byte() # load the 'target' argument -## self.pc = target # jump - -## def ignore_next_guard_nullness(self, opnum): -## _op_ooisnull = self.metainterp.staticdata._op_ooisnull -## _op_oononnull = self.metainterp.staticdata._op_oononnull -## bc = ord(self.bytecode[self.pc]) -## if bc == _op_ooisnull: -## if opnum == rop.GUARD_ISNULL: -## res = ConstInt(0) -## else: -## res = ConstInt(1) -## else: -## assert bc == _op_oononnull -## if opnum == rop.GUARD_ISNULL: -## res = ConstInt(1) -## else: -## res = ConstInt(0) -## self.pc += 1 # past the bytecode for ptr_iszero/ptr_nonzero -## self.load_int() # past the 'box' argument -## self.make_result_box(res) - -## def dont_follow_jump(self): -## _op_goto_if_not = self.metainterp.staticdata._op_goto_if_not -## assert ord(self.bytecode[self.pc]) == _op_goto_if_not -## self.pc += 1 # past the bytecode for 'goto_if_not' -## self.load_3byte() # past the 'target' argument -## self.load_int() # past the 'box' argument -## self.ignore_varargs() # past the 'livelist' argument - @arguments("box", "descr", "orgpc") def opimpl_switch(self, valuebox, switchdict, orgpc): box = self.implement_guard_value(orgpc, valuebox) - switchvalue = box.getint() + search_value = box.getint() assert isinstance(switchdict, SwitchDictDescr) try: - self.pc = switchdict.dict[switchvalue] + self.pc = switchdict.dict[search_value] except KeyError: pass @@ -800,26 +765,26 @@ for i in range(num_green_args): assert isinstance(varargs[i], Const) - def blackhole_reached_merge_point(self, varargs): - if self.metainterp.in_recursion: - portal_code = self.metainterp.staticdata.portal_code - # small hack: fish for the result box - lenenv = len(self.env) - raised = self.perform_call(portal_code, varargs) - # in general this cannot be assumed, but when blackholing, - # perform_call returns True only if an exception is called. In - # this case perform_call has called finishframe_exception - # already, so we need to return. - if raised: - return - if lenenv == len(self.env): - res = None - else: - assert lenenv == len(self.env) - 1 - res = self.env.pop() - self.metainterp.finishframe(res) - else: - raise self.metainterp.staticdata.ContinueRunningNormally(varargs) +## def blackhole_reached_merge_point(self, varargs): +## if self.metainterp.in_recursion: +## portal_code = self.metainterp.staticdata.portal_code +## # small hack: fish for the result box +## lenenv = len(self.env) +## raised = self.perform_call(portal_code, varargs) +## # in general this cannot be assumed, but when blackholing, +## # perform_call returns True only if an exception is called. In +## # this case perform_call has called finishframe_exception +## # already, so we need to return. +## if raised: +## return +## if lenenv == len(self.env): +## res = None +## else: +## assert lenenv == len(self.env) - 1 +## res = self.env.pop() +## self.metainterp.finishframe(res) +## else: +## raise self.metainterp.staticdata.ContinueRunningNormally(varargs) @arguments() def opimpl_can_enter_jit(self): @@ -828,18 +793,21 @@ raise CannotInlineCanEnterJit() self.metainterp.seen_can_enter_jit = True - @arguments("boxes3", "boxes3") - def opimpl_jit_merge_point(self, greenboxes, redboxes): + @arguments("orgpc", "boxes3", "boxes3") + def opimpl_jit_merge_point(self, orgpc, greenboxes, redboxes): self.verify_green_args(greenboxes) # xxx we may disable the following line in some context later self.debug_merge_point(greenboxes) if self.metainterp.seen_can_enter_jit: self.metainterp.seen_can_enter_jit = False - try: - self.metainterp.reached_can_enter_jit(greenboxes, redboxes) - except GiveUp: - XXX - self.metainterp.switch_to_blackhole(ABORT_BRIDGE) + # Set self.pc to point to jit_merge_point instead of just after: + # if reached_can_enter_jit() raises SwitchToBlackhole, then the + # pc is still at the jit_merge_point, which is a point that is + # much less expensive to blackhole out of. + saved_pc = self.pc + self.pc = orgpc + self.metainterp.reached_can_enter_jit(greenboxes, redboxes) + self.pc = saved_pc def debug_merge_point(self, greenkey): # debugging: produce a DEBUG_MERGE_POINT operation @@ -1054,6 +1022,7 @@ effectinfo = descr.get_extra_info() if (effectinfo is None or effectinfo.extraeffect == effectinfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE): + XXX # residual calls require attention to keep virtualizables in-sync self.metainterp.vable_and_vrefs_before_residual_call() # xxx do something about code duplication @@ -1273,7 +1242,6 @@ self.staticdata = staticdata self.cpu = staticdata.cpu self.portal_trace_positions = [] - self.greenkey_of_huge_function = None self.free_frames_list = [] self.last_exc_value_box = None @@ -1319,11 +1287,10 @@ self.framestack[-1].make_result_of_lastop(resultbox) raise ChangeFrame else: - if not self.is_blackholing(): - try: - self.compile_done_with_this_frame(resultbox) - except GiveUp: - self.switch_to_blackhole(ABORT_BRIDGE) + try: + self.compile_done_with_this_frame(resultbox) + except GiveUp: + self.aborted_tracing(ABORT_BRIDGE) sd = self.staticdata if sd.result_type == 'void': assert resultbox is None @@ -1361,11 +1328,10 @@ frame.pc = target raise ChangeFrame self.popframe() - if not self.is_blackholing(): - try: - self.compile_exit_frame_with_exception(excvaluebox) - except GiveUp: - self.switch_to_blackhole(ABORT_BRIDGE) + try: + self.compile_exit_frame_with_exception(excvaluebox) + except GiveUp: + self.aborted_tracing(ABORT_BRIDGE) raise self.staticdata.ExitFrameWithExceptionRef(self.cpu, excvaluebox.getref_base()) def check_recursion_invariant(self): @@ -1488,41 +1454,32 @@ def execute_did_not_raise(self): self.last_exc_value_box = None - def switch_to_blackhole(self, reason): - XXX + def aborted_tracing(self, reason): self.staticdata.profiler.count(reason) debug_print('~~~ ABORTING TRACING') - debug_stop('jit-tracing') - debug_start('jit-blackhole') - self.history = None # start blackholing self.staticdata.stats.aborted() - self.staticdata.profiler.end_tracing() - self.staticdata.profiler.start_blackhole() - switch_to_blackhole._dont_inline_ = True - - def switch_to_blackhole_if_trace_too_long(self): - if not self.is_blackholing(): - warmrunnerstate = self.staticdata.state - if len(self.history.operations) > warmrunnerstate.trace_limit: - self.greenkey_of_huge_function = self.find_biggest_function() - self.portal_trace_positions = None - self.switch_to_blackhole(ABORT_TOO_LONG) + self.resumekey.reset_counter_from_failure() + + def blackhole_if_trace_too_long(self): + warmrunnerstate = self.staticdata.state + if len(self.history.operations) > warmrunnerstate.trace_limit: + greenkey_of_huge_function = self.find_biggest_function() + self.portal_trace_positions = None + if greenkey_of_huge_function is not None: + warmrunnerstate = self.staticdata.state + warmrunnerstate.disable_noninlinable_function( + greenkey_of_huge_function) + raise SwitchToBlackhole(ABORT_TOO_LONG) def _interpret(self): # Execute the frames forward until we raise a DoneWithThisFrame, - # a ContinueRunningNormally, or a GenerateMergePoint exception. + # a ExitFrameWithException, or a GenerateMergePoint exception. self.staticdata.stats.entered() - try: - while True: - self.framestack[-1].run_one_step() - self.switch_to_blackhole_if_trace_too_long() - if not we_are_translated(): - self.check_recursion_invariant() - finally: - if self.is_blackholing(): - self.staticdata.profiler.end_blackhole() - else: - self.staticdata.profiler.end_tracing() + while True: + self.framestack[-1].run_one_step() + self.blackhole_if_trace_too_long() + if not we_are_translated(): + self.check_recursion_invariant() def interpret(self): if we_are_translated(): @@ -1538,11 +1495,13 @@ def compile_and_run_once(self, *args): debug_start('jit-tracing') + self.staticdata.profiler.start_tracing() self.staticdata._setup_once() self.create_empty_history() try: return self._compile_and_run_once(*args) finally: + self.staticdata.profiler.end_tracing() debug_stop('jit-tracing') def _compile_and_run_once(self, *args): @@ -1559,17 +1518,21 @@ assert False, "should always raise" except GenerateMergePoint, gmp: return self.designate_target_loop(gmp) + except SwitchToBlackhole, stb: + self.run_blackhole_interp_to_cancel_tracing(stb) def handle_guard_failure(self, key): + debug_start('jit-tracing') + self.staticdata.profiler.start_tracing() assert isinstance(key, compile.ResumeGuardDescr) self.initialize_state_from_guard_failure(key) try: return self._handle_guard_failure(key) finally: + self.staticdata.profiler.end_tracing() debug_stop('jit-tracing') def _handle_guard_failure(self, key): - from pypy.jit.metainterp.warmspot import ContinueRunningNormallyBase original_greenkey = key.original_greenkey # notice that here we just put the greenkey # use -1 to mark that we will have to give up @@ -1583,9 +1546,17 @@ assert False, "should always raise" except GenerateMergePoint, gmp: return self.designate_target_loop(gmp) - except ContinueRunningNormallyBase: - key.reset_counter_from_failure(self) - raise + except SwitchToBlackhole, stb: + self.run_blackhole_interp_to_cancel_tracing(stb) + + def run_blackhole_interp_to_cancel_tracing(self, stb): + # We got a SwitchToBlackhole exception. Convert the framestack into + # a stack of blackhole interpreters filled with the same values, and + # run it. + from pypy.jit.metainterp.blackhole import convert_and_run_from_pyjitpl + self.aborted_tracing(stb.reason) + convert_and_run_from_pyjitpl(self, stb.current_exc) + assert False # ^^^ must raise def remove_consts_and_duplicates(self, boxes, endindex, duplicates): for i in range(endindex): @@ -1639,7 +1610,7 @@ # Found! Compile it as a loop. if start < 0: # we cannot reconstruct the beginning of the proper loop - raise GiveUp + raise SwitchToBlackhole(ABORT_BRIDGE) # raises in case it works -- which is the common case self.compile(original_boxes, live_arg_boxes, start) @@ -1777,7 +1748,6 @@ def initialize_state_from_start(self, *args): self.in_recursion = -1 # always one portal around - self.staticdata.profiler.start_tracing() num_green_args = self.staticdata.num_green_args original_boxes = [] self._initialize_from_start(original_boxes, num_green_args, *args) @@ -1791,8 +1761,6 @@ def initialize_state_from_guard_failure(self, resumedescr): # guard failure: rebuild a complete MIFrame stack - debug_start('jit-tracing') - self.staticdata.profiler.start_tracing() self.in_recursion = -1 # always one portal around self.history = history.History() inputargs_and_holes = self.rebuild_state_after_failure(resumedescr) @@ -1842,36 +1810,25 @@ None, descr=vinfo.vable_token_descr) def vable_and_vrefs_after_residual_call(self): - if self.is_blackholing(): - escapes = True - else: - escapes = False - # - vrefinfo = self.staticdata.virtualref_info - for i in range(0, len(self.virtualref_boxes), 2): - virtualbox = self.virtualref_boxes[i] - vrefbox = self.virtualref_boxes[i+1] - vref = vrefbox.getref_base() - if vrefinfo.tracing_after_residual_call(vref): - # this vref was really a virtual_ref, but it escaped - # during this CALL_MAY_FORCE. Mark this fact by - # generating a VIRTUAL_REF_FINISH on it and replacing - # it by ConstPtr(NULL). - self.stop_tracking_virtualref(i) - # - vinfo = self.staticdata.virtualizable_info - if vinfo is not None: - virtualizable_box = self.virtualizable_boxes[-1] - virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) - if vinfo.tracing_after_residual_call(virtualizable): - # the virtualizable escaped during CALL_MAY_FORCE. - escapes = True - # - if escapes: - self.switch_to_blackhole(ABORT_ESCAPE) + vrefinfo = self.staticdata.virtualref_info + for i in range(0, len(self.virtualref_boxes), 2): + virtualbox = self.virtualref_boxes[i] + vrefbox = self.virtualref_boxes[i+1] + vref = vrefbox.getref_base() + if vrefinfo.tracing_after_residual_call(vref): + # this vref was really a virtual_ref, but it escaped + # during this CALL_MAY_FORCE. Mark this fact by + # generating a VIRTUAL_REF_FINISH on it and replacing + # it by ConstPtr(NULL). + self.stop_tracking_virtualref(i) # - if escapes: - self.load_fields_from_virtualizable() + vinfo = self.staticdata.virtualizable_info + if vinfo is not None: + virtualizable_box = self.virtualizable_boxes[-1] + virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) + if vinfo.tracing_after_residual_call(virtualizable): + # the virtualizable escaped during CALL_MAY_FORCE. + raise XXX-SwitchToBlackhole(ABORT_ESCAPE, yyy) def stop_tracking_virtualref(self, i): virtualbox = self.virtualref_boxes[i] @@ -2072,18 +2029,25 @@ self.history.record(rop.CALL_ASSEMBLER, args[:], resbox, descr=token) self.history.operations += rest +# ____________________________________________________________ + class GenerateMergePoint(Exception): def __init__(self, args, target_loop_token): assert target_loop_token is not None self.argboxes = args self.target_loop_token = target_loop_token -# ____________________________________________________________ - class ChangeFrame(Exception): """Raised after we mutated metainterp.framestack, in order to force it to reload the current top-of-stack frame that gets interpreted.""" +class SwitchToBlackhole(Exception): + def __init__(self, reason, current_exc=lltype.nullptr(llmemory.GCREF.TO)): + self.reason = reason + self.current_exc = current_exc + +# ____________________________________________________________ + def _get_opimpl_method(name, argcodes): from pypy.jit.metainterp.blackhole import signedord # Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py Tue May 25 20:59:35 2010 @@ -1,6 +1,9 @@ +import py from pypy.rlib.jit import JitDriver from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin from pypy.jit.metainterp.blackhole import BlackholeInterpBuilder +from pypy.jit.metainterp.blackhole import convert_and_run_from_pyjitpl +from pypy.jit.metainterp import history from pypy.jit.codewriter.assembler import JitCode from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.llinterp import LLException @@ -106,3 +109,33 @@ blackholeinterp.setarg_i(0x9, -100) blackholeinterp.run() assert blackholeinterp.final_result_i() == 42 + +def test_convert_and_run_from_pyjitpl(): + class MyMIFrame: + jitcode = JitCode("test") + jitcode.setup("\xFF" # illegal instruction + "\x00\x00\x01\x02" # int_add/ii>i + "\x01\x02", # int_return/i + [], + num_regs_i=3, num_regs_r=0, num_regs_f=0) + pc = 1 + registers_i = [history.BoxInt(40), history.ConstInt(2), None] + class MyMetaInterp: + class staticdata: + result_type = 'int' + class profiler: + @staticmethod + def start_blackhole(): pass + @staticmethod + def end_blackhole(): pass + class DoneWithThisFrameInt(Exception): + pass + framestack = [MyMIFrame()] + MyMetaInterp.staticdata.blackholeinterpbuilder = getblackholeinterp( + {'int_add/ii>i': 0, 'int_return/i': 1}).builder + MyMetaInterp.staticdata.blackholeinterpbuilder.metainterp_sd = \ + MyMetaInterp.staticdata + # + d = py.test.raises(MyMetaInterp.staticdata.DoneWithThisFrameInt, + convert_and_run_from_pyjitpl, MyMetaInterp()) + assert d.value.args == (42,) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmstate.py Tue May 25 20:59:35 2010 @@ -172,16 +172,14 @@ if self.profiler is not None: self.profiler.set_printing(value >= DEBUG_PROFILE) - def disable_noninlinable_function(self, metainterp): - greenkey = metainterp.greenkey_of_huge_function - if greenkey is not None: - cell = self.jit_cell_at_key(greenkey) - cell.dont_trace_here = True - debug_start("jit-disableinlining") - sd = self.warmrunnerdesc.metainterp_sd - loc = sd.state.get_location_str(greenkey) - debug_print("disabled inlining", loc) - debug_stop("jit-disableinlining") + def disable_noninlinable_function(self, greenkey): + cell = self.jit_cell_at_key(greenkey) + cell.dont_trace_here = True + debug_start("jit-disableinlining") + sd = self.warmrunnerdesc.metainterp_sd + loc = sd.state.get_location_str(greenkey) + debug_print("disabled inlining", loc) + debug_stop("jit-disableinlining") def attach_unoptimized_bridge_from_interp(self, greenkey, entry_loop_token): @@ -198,7 +196,6 @@ metainterp_sd = self.warmrunnerdesc.metainterp_sd vinfo = metainterp_sd.virtualizable_info - ContinueRunningNormally = self.warmrunnerdesc.ContinueRunningNormally num_green_args = self.warmrunnerdesc.num_green_args get_jitcell = self.make_jitcell_getter() set_future_values = self.make_set_future_values() @@ -242,11 +239,6 @@ cell.counter = -2 try: loop_token = metainterp.compile_and_run_once(*args) - except ContinueRunningNormally: - # the trace got too long, reset the counter - cell.counter = 0 - self.disable_noninlinable_function(metainterp) - raise finally: if cell.counter == -2: cell.counter = 0 From arigo at codespeak.net Tue May 25 21:13:48 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 25 May 2010 21:13:48 +0200 (CEST) Subject: [pypy-svn] r74749 - pypy/branch/blackhole-improvement/pypy/jit/metainterp/test Message-ID: <20100525191348.52FCD36C227@codespeak.net> Author: arigo Date: Tue May 25 21:13:07 2010 New Revision: 74749 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_send.py Log: Fix (or kill) some tests. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_optimizeopt.py Tue May 25 21:13:07 2010 @@ -35,7 +35,7 @@ def test_store_final_boxes_in_guard(): from pypy.jit.metainterp.compile import ResumeGuardDescr - from pypy.jit.metainterp.resume import tag, TAGBOX, TYPEBARRIER + from pypy.jit.metainterp.resume import tag, TAGBOX b0 = BoxInt() b1 = BoxInt() opt = optimizeopt.Optimizer(FakeMetaInterpStaticData(LLtypeMixin.cpu), @@ -50,12 +50,12 @@ # opt.store_final_boxes_in_guard(op) if op.fail_args == [b0, b1]: - assert fdescr.rd_numb.nums == [tag(1, TAGBOX), TYPEBARRIER] - assert fdescr.rd_numb.prev.nums == [tag(0, TAGBOX), TYPEBARRIER] + assert fdescr.rd_numb.nums == [tag(1, TAGBOX)] + assert fdescr.rd_numb.prev.nums == [tag(0, TAGBOX)] else: assert op.fail_args == [b1, b0] - assert fdescr.rd_numb.nums == [tag(0, TAGBOX), TYPEBARRIER] - assert fdescr.rd_numb.prev.nums == [tag(1, TAGBOX), TYPEBARRIER] + assert fdescr.rd_numb.nums == [tag(0, TAGBOX)] + assert fdescr.rd_numb.prev.nums == [tag(1, TAGBOX)] assert fdescr.rd_virtuals is None assert fdescr.rd_consts == [] Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_pyjitpl.py Tue May 25 21:13:07 2010 @@ -1,49 +1,20 @@ # some unit tests for the bytecode decoding -from pypy.jit.metainterp import pyjitpl, codewriter, resoperation, history +from pypy.jit.metainterp import pyjitpl from pypy.jit.metainterp import jitprof -from pypy.jit.metainterp.history import AbstractFailDescr, BoxInt, ConstInt +from pypy.jit.metainterp.history import BoxInt, ConstInt from pypy.jit.metainterp.history import History from pypy.jit.metainterp.resoperation import ResOperation, rop from pypy.jit.metainterp.test.test_optimizeopt import equaloplists +from pypy.jit.codewriter.jitcode import JitCode -def make_frame(code): - bytecode = codewriter.JitCode("hello") - bytecode.code = code - bytecode.constants = None - frame = pyjitpl.MIFrame(None, bytecode) - frame.pc = 0 - return frame - - -def test_decode_big_int(): - for code, value in [("\x80\x01", 128), ("\x81\x81\x01", 1 + (1 << 7) + (1 << 14))]: - frame = make_frame(code) - val = frame.load_int() - assert val == value - -def test_decode_bool(): - frame = make_frame("\x00") - assert not frame.load_bool() - - frame = make_frame("\x01") - assert frame.load_bool() - -def test_simple_opimpl_exist(): - rop = resoperation.rop - for opnum, opname in resoperation.opname.items(): - if opnum in (rop.SAME_AS, rop.CALL_PURE, rop.OOSEND_PURE, - rop.FORCE_TOKEN): - continue - if rop._NOSIDEEFFECT_FIRST <= opnum <= rop._NOSIDEEFFECT_LAST: - assert hasattr(pyjitpl.MIFrame, 'opimpl_' + opname.lower()), opname def test_portal_trace_positions(): - jitcode = codewriter.JitCode("f") - jitcode.code = jitcode.constants = None - portal = codewriter.JitCode("portal") - portal.code = portal.constants = None + jitcode = JitCode("f") + jitcode.setup(None) + portal = JitCode("portal") + portal.setup(None) class FakeStaticData: cpu = None portal_code = portal @@ -95,7 +66,7 @@ c3 = ConstInt(3) boxes = [b1, b2, b1, c3] dup = {} - metainterp.remove_consts_and_duplicates(boxes, 0, 4, dup) + metainterp.remove_consts_and_duplicates(boxes, 4, dup) assert boxes[0] is b1 assert boxes[1] is b2 assert is_another_box_like(boxes[2], b1) @@ -108,55 +79,10 @@ # del metainterp.history.operations[:] b4 = BoxInt(4) - boxes = ["something random", b2, b4, "something else"] - metainterp.remove_consts_and_duplicates(boxes, 1, 3, dup) - assert is_another_box_like(boxes[1], b2) - assert boxes[2] is b4 + boxes = [b2, b4, "something random"] + metainterp.remove_consts_and_duplicates(boxes, 2, dup) + assert is_another_box_like(boxes[0], b2) + assert boxes[1] is b4 assert equaloplists(metainterp.history.operations, [ - ResOperation(rop.SAME_AS, [b2], boxes[1]), + ResOperation(rop.SAME_AS, [b2], boxes[0]), ]) - -def test_get_name_from_address(): - class FakeMetaInterpSd(pyjitpl.MetaInterpStaticData): - def __init__(self): - pass - metainterp_sd = FakeMetaInterpSd() - metainterp_sd.info_from_codewriter(None, None, - [(123, "a"), (456, "b")], - None) - assert metainterp_sd.get_name_from_address(123) == 'a' - assert metainterp_sd.get_name_from_address(456) == 'b' - assert metainterp_sd.get_name_from_address(789) == '' - -def test_initialize_state_from_guard_failure(): - from pypy.jit.metainterp.typesystem import llhelper - calls = [] - - class FakeCPU: - ts = llhelper - - def make_boxes_from_latest_values(self, faildescr): - return [BoxInt(0), None, BoxInt(2), None, BoxInt(4)] - - class FakeStaticData: - cpu = FakeCPU() - profiler = jitprof.EmptyProfiler() - - metainterp = pyjitpl.MetaInterp(FakeStaticData()) - - def rebuild_state_after_failure(descr, newboxes): - calls.append(newboxes) - metainterp.rebuild_state_after_failure = rebuild_state_after_failure - - class FakeResumeDescr: - def must_compile(self, *args): - return True - resumedescr = FakeResumeDescr() - metainterp.initialize_state_from_guard_failure(resumedescr) - - inp = metainterp.history.inputargs - assert len(inp) == 3 - assert [box.value for box in inp] == [0, 2, 4] - b0, b2, b4 = inp - assert len(calls) == 1 - assert calls[0] == [b0, None, b2, None, b4] Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_send.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_send.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_send.py Tue May 25 21:13:07 2010 @@ -374,7 +374,7 @@ self.check_loop_count(2) def test_behavior_change_after_a_while(self): - myjitdriver = JitDriver(greens = [], reds = ['x', 'y']) + myjitdriver = JitDriver(greens = [], reds = ['y', 'x']) class Base: def __init__(self, value): self.value = value @@ -454,7 +454,7 @@ self.check_tree_loop_count(2) def test_bug1(self): - myjitdriver = JitDriver(greens = [], reds = ['node', 'n']) + myjitdriver = JitDriver(greens = [], reds = ['n', 'node']) class Base: pass class A(Base): From arigo at codespeak.net Tue May 25 21:15:30 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 25 May 2010 21:15:30 +0200 (CEST) Subject: [pypy-svn] r74750 - pypy/branch/blackhole-improvement/pypy/jit/metainterp/test Message-ID: <20100525191530.884CE36C227@codespeak.net> Author: arigo Date: Tue May 25 21:15:27 2010 New Revision: 74750 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtual.py Log: Fix test. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtual.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtual.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtual.py Tue May 25 21:15:27 2010 @@ -1,12 +1,12 @@ import py from pypy.rlib.jit import JitDriver, hint from pypy.rlib.objectmodel import compute_unique_id -from pypy.jit.metainterp.policy import StopAtXPolicy +from pypy.jit.codewriter.policy import StopAtXPolicy from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin from pypy.rpython.lltypesystem import lltype, rclass from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.ootypesystem import ootype -from pypy.jit.metainterp import heaptracker +from pypy.jit.codewriter import heaptracker class VirtualTests: def _freeze_(self): @@ -299,10 +299,9 @@ self.check_tree_loop_count(2) # the loop and the entry path # we get: # ENTER - compile the new loop - # ENTER (BlackHole) - leave # ENTER - compile the entry bridge # ENTER - compile the leaving path - self.check_enter_count(4) + self.check_enter_count(3) class VirtualMiscTests: From jcreigh at codespeak.net Tue May 25 21:35:18 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Tue, 25 May 2010 21:35:18 +0200 (CEST) Subject: [pypy-svn] r74751 - in pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86: . test Message-ID: <20100525193518.63D1D282B9C@codespeak.net> Author: jcreigh Date: Tue May 25 21:35:16 2010 New Revision: 74751 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py Log: add a few more float operations Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Tue May 25 21:35:16 2010 @@ -698,16 +698,16 @@ def genop_float_neg(self, op, arglocs, resloc): # Following what gcc does: res = x ^ 0x8000000000000000 - self.mc.XORPD(arglocs[0], self.loc_float_const_neg) + self.mc.XORPD_rj(arglocs[0].value, self.loc_float_const_neg) def genop_float_abs(self, op, arglocs, resloc): # Following what gcc does: res = x & 0x7FFFFFFFFFFFFFFF - self.mc.ANDPD(arglocs[0], self.loc_float_const_abs) + self.mc.ANDPD_rj(arglocs[0].value, self.loc_float_const_abs) def genop_guard_float_is_true(self, op, guard_op, addr, arglocs, resloc): guard_opnum = guard_op.opnum loc0, loc1 = arglocs - self.mc.XORPD(loc0, loc0) + self.mc.XORPD_rr(loc0.value, loc0.value) self.mc.UCOMISD(loc0, loc1) mc = self.mc._mc if guard_opnum == rop.GUARD_TRUE: @@ -720,7 +720,7 @@ return self.implement_guard(addr) def genop_float_is_true(self, op, arglocs, resloc): - self.mc.XORPD(arglocs[0], arglocs[0]) + self.mc.XORPD_rr(arglocs[0].value, arglocs[0].value) self.genop_float_ne(op, arglocs, resloc) def genop_cast_float_to_int(self, op, arglocs, resloc): Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Tue May 25 21:35:16 2010 @@ -161,6 +161,7 @@ MULSD = _binaryop('MULSD') DIVSD = _binaryop('DIVSD') UCOMISD = _binaryop('UCOMISD') + CVTSI2SD = _binaryop('CVTSI2SD') def CALL(self, loc): Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py Tue May 25 21:35:16 2010 @@ -548,6 +548,17 @@ UCOMISD_rb = xmminsn('\x66', rex_nw, '\x0F\x2E', register(1, 8), stack_bp(2)) UCOMISD_rj = xmminsn('\x66', rex_nw, '\x0F\x2E', register(1, 8), '\x05', immediate(2)) + # Conversion + # FIXME: Super confusing! The source is a GPR/mem, the destination is an xmm register + CVTSI2SD_rr = xmminsn('\xF2', rex_nw, '\x0F\x2A', register(1, 8), register(2), '\xC0') + CVTSI2SD_rb = xmminsn('\xF2', rex_nw, '\x0F\x2A', register(1, 8), stack_bp(2)) + + # Bitwise + ANDPD_rj = xmminsn('\x66', rex_nw, '\x0F\x54', register(1, 8), '\x05', immediate(2)) + + XORPD_rr = xmminsn('\x66', rex_nw, '\x0F\x57', register(1, 8), register(2), '\xC0') + XORPD_rj = xmminsn('\x66', rex_nw, '\x0F\x57', register(1, 8), '\x05', immediate(2)) + # ------------------------------------------------------------ Conditions = { @@ -634,6 +645,10 @@ py.test.skip("DIVSD_rj unsupported") def UCOMISD_rj(self, xmm_reg, mem_immed): py.test.skip("UCOMISD_rj unsupported") + def ANDPD_rj(self, xmm_reg, mem_immed): + py.test.skip("ANDPD_rj unsupported") + def XORPD_rj(self, xmm_reg, mem_immed): + py.test.skip("XORPD_rj unsupported") # ____________________________________________________________ Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py Tue May 25 21:35:16 2010 @@ -225,6 +225,11 @@ if methname.startswith('SHL') or methname.startswith('SAR') or methname.startswith('SHR'): # XXX: Would be nice to test these automatically py.test.skip('Shifts must be tested manually') + if methname.startswith('CVT'): + # Can't test automatically right now, we don't know + # which register types to use + py.test.skip('Skipping CVT instructions for now') + return [args] def get_code_checker_class(self): From hpk at codespeak.net Tue May 25 21:48:10 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 25 May 2010 21:48:10 +0200 (CEST) Subject: [pypy-svn] r74752 - pypy/trunk/py/_test Message-ID: <20100525194810.D1F0936C230@codespeak.net> Author: hpk Date: Tue May 25 21:48:09 2010 New Revision: 74752 Modified: pypy/trunk/py/_test/session.py Log: this is now released, a last fix for python3 Modified: pypy/trunk/py/_test/session.py ============================================================================== --- pypy/trunk/py/_test/session.py (original) +++ pypy/trunk/py/_test/session.py Tue May 25 21:48:09 2010 @@ -22,6 +22,7 @@ nodeid = "" class Interrupted(KeyboardInterrupt): """ signals an interrupted test run. """ + __module__ = 'builtins' # for py3 def __init__(self, config): self.config = config From jcreigh at codespeak.net Tue May 25 22:57:00 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Tue, 25 May 2010 22:57:00 +0200 (CEST) Subject: [pypy-svn] r74753 - in pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86: . test Message-ID: <20100525205700.D346D282B9C@codespeak.net> Author: jcreigh Date: Tue May 25 22:56:58 2010 New Revision: 74753 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regalloc.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_gc_integration.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_jump.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_regalloc.py Log: fix some more test failures Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Tue May 25 22:56:58 2010 @@ -202,16 +202,16 @@ self.malloc_fixedsize_slowpath1 = mc.tell() if self.cpu.supports_floats: # save the XMM registers in for i in range(8): # the *caller* frame, from esp+8 - mc.MOVSD(mem64(esp, 8+8*i), xmm_registers[i]) + mc.MOVSD_sr(8+8*i, i) mc.SUB(edx, eax) # compute the size we want - mc.MOV(mem(esp, 4), edx) # save it as the new argument + mc.MOV_sr(4, edx.value) # save it as the new argument addr = self.cpu.gc_ll_descr.get_malloc_fixedsize_slowpath_addr() mc.JMP_l(addr) # tail call to the real malloc # ---------- second helper for the slow path of malloc ---------- self.malloc_fixedsize_slowpath2 = mc.tell() if self.cpu.supports_floats: # restore the XMM registers for i in range(8): # from where they were saved - mc.MOVSD(xmm_registers[i], mem64(esp, 8+8*i)) + mc.MOVSD_rs(i, 8+8*i) nursery_free_adr = self.cpu.gc_ll_descr.get_nursery_free_addr() mc.MOV(edx, heap(nursery_free_adr)) # load this in EDX mc.RET() @@ -467,19 +467,19 @@ self.mc.MOVSD_sr(0, loc.value) elif isinstance(loc, StackLoc) and loc.width == 8: # XXX evil trick - self.mc.PUSH(mem(ebp, get_ebp_ofs(loc.position))) - self.mc.PUSH(mem(ebp, get_ebp_ofs(loc.position + 1))) + self.mc.PUSH_b(get_ebp_ofs(loc.position)) + self.mc.PUSH_b(get_ebp_ofs(loc.position + 1)) else: self.mc.PUSH(loc) def regalloc_pop(self, loc): - if isinstance(loc, XMMREG): - self.mc.MOVSD(loc, mem64(esp, 0)) + if isinstance(loc, RegLoc) and loc.is_xmm: + self.mc.MOVSD_rs(loc.value, 0) self.mc.ADD(esp, imm(2*WORD)) - elif isinstance(loc, MODRM64): + elif isinstance(loc, StackLoc) and loc.width == 8: # XXX evil trick - self.mc.POP(mem(ebp, get_ebp_ofs(loc.position + 1))) - self.mc.POP(mem(ebp, get_ebp_ofs(loc.position))) + self.mc.POP_b(get_ebp_ofs(loc.position + 1)) + self.mc.POP_b(get_ebp_ofs(loc.position)) else: self.mc.POP(loc) @@ -527,8 +527,6 @@ def _cmpop(cond, rev_cond): def genop_cmp(self, op, arglocs, result_loc): - # Clear high bits - self.mc.MOV_ri(result_loc.value, 0) rl = result_loc.lowest8bits() if isinstance(op.args[0], Const): self.mc.CMP(arglocs[1], arglocs[0]) @@ -536,6 +534,7 @@ else: self.mc.CMP(arglocs[0], arglocs[1]) self.mc.SET_ir(rx86.Conditions[cond], rl.value) + self.mc.MOVZX8_rr(result_loc.value, rl.value) return genop_cmp def _cmpop_float(cond, is_ne=False): @@ -740,8 +739,8 @@ def genop_int_is_true(self, op, arglocs, resloc): self.mc.CMP(arglocs[0], imm(0)) rl = resloc.lowest8bits() - self.mc.SETNE(rl) - self.mc.MOVZX(resloc, rl) + self.mc.SET_ir(rx86.Conditions['NE'], rl.value) + self.mc.MOVZX8(resloc, rl) def genop_guard_bool_not(self, op, guard_op, addr, arglocs, resloc): guard_opnum = guard_op.opnum @@ -1415,16 +1414,16 @@ arglocs, result_loc): faildescr = guard_op.descr fail_index = self.cpu.get_fail_descr_number(faildescr) - self.mc.MOV(mem(ebp, FORCE_INDEX_OFS), imm(fail_index)) + self.mc.MOV_bi(FORCE_INDEX_OFS, fail_index) self.genop_call(op, arglocs, result_loc) - self.mc.CMP(mem(ebp, FORCE_INDEX_OFS), imm(0)) + self.mc.CMP_bi(FORCE_INDEX_OFS, 0) return self.implement_guard(addr, 'L') def genop_guard_call_assembler(self, op, guard_op, addr, arglocs, result_loc): faildescr = guard_op.descr fail_index = self.cpu.get_fail_descr_number(faildescr) - self.mc.MOV(mem(ebp, FORCE_INDEX_OFS), imm(fail_index)) + self.mc.MOV_bi(FORCE_INDEX_OFS, fail_index) descr = op.descr assert isinstance(descr, LoopToken) assert len(arglocs) - 2 == len(descr._x86_arglocs[0]) @@ -1432,11 +1431,11 @@ tmp=eax) mc = self._start_block() mc.CMP(eax, imm(self.cpu.done_with_this_frame_int_v)) - mc.JE(rel8_patched_later) + mc.J_il8(rx86.Conditions['E'], 0) # patched later je_location = mc.get_relative_pos() self._emit_call(rel32(self.assembler_helper_adr), [eax, arglocs[1]], 0, tmp=ecx, force_mc=True, mc=mc) - mc.JMP(rel8_patched_later) + mc.JMP_l8(0) # patched later jmp_location = mc.get_relative_pos() offset = jmp_location - je_location assert 0 < offset <= 127 @@ -1446,11 +1445,11 @@ assert 0 < offset <= 127 mc.overwrite(jmp_location - 1, [chr(offset)]) self._stop_block() - if isinstance(result_loc, MODRM64): + if isinstance(result_loc, StackLoc) and result_loc.width == 8: self.mc.FSTP(result_loc) else: assert result_loc is eax or result_loc is None - self.mc.CMP(mem(ebp, FORCE_INDEX_OFS), imm(0)) + self.mc.CMP_bi(FORCE_INDEX_OFS, 0) return self.implement_guard(addr, 'L') def genop_discard_cond_call_gc_wb(self, op, arglocs): @@ -1464,7 +1463,7 @@ mc = self._start_block() mc.TEST(mem8(loc_base, descr.jit_wb_if_flag_byteofs), imm8(descr.jit_wb_if_flag_singlebyte)) - mc.JZ(rel8_patched_later) + mc.J_il8(rx86.Conditions['Z'], 0) # patched later jz_location = mc.get_relative_pos() # the following is supposed to be the slow path, so whenever possible # we choose the most compact encoding over the most efficient one. @@ -1524,7 +1523,7 @@ mc.MOV(eax, heap(nursery_free_adr)) mc.LEA(edx, addr_add(eax, imm(size))) mc.CMP(edx, heap(nursery_top_adr)) - mc.JNA(rel8_patched_later) + mc.J_il8(rx86.Conditions['NA'], 0) # patched later jmp_adr = mc.get_relative_pos() # See comments in _build_malloc_fixedsize_slowpath for the Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regalloc.py Tue May 25 22:56:58 2010 @@ -968,7 +968,7 @@ shape = gcrootmap.get_basic_shape() for v, val in self.fm.frame_bindings.items(): if (isinstance(v, BoxPtr) and self.rm.stays_alive(v)): - assert isinstance(val, MODRM) + assert isinstance(val, StackLoc) gcrootmap.add_ebp_offset(shape, get_ebp_ofs(val.position)) for v, reg in self.rm.reg_bindings.items(): if (isinstance(v, BoxPtr) and self.rm.stays_alive(v)): Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py Tue May 25 22:56:58 2010 @@ -430,7 +430,7 @@ MOV8_mr = insn(rex_w, '\x88', byte_register(2, 8), mem_reg_plus_const(1)) - MOVZX8_rr = insn(rex_w, '\x0F\xB6', register(1,8), byte_register(2)) + MOVZX8_rr = insn(rex_w, '\x0F\xB6', register(1,8), byte_register(2), '\xC0') MOVZX8_rm = insn(rex_w, '\x0F\xB6', register(1,8), mem_reg_plus_const(2)) MOVZX8_ra = insn(rex_w, '\x0F\xB6', register(1,8), mem_reg_plus_scaled_reg_plus_const(2)) @@ -454,6 +454,8 @@ CMP_ji32 = insn(rex_w, '\x81', '\x3D', immediate(1), immediate(2)) CMP_ji = select_8_or_32_bit_immed(CMP_ji8, CMP_ji32) + CMP_rj = insn(rex_w, '\x3B', register(1, 8), '\x05', immediate(2)) + AND8_rr = insn(rex_w, '\x20', byte_register(1), byte_register(2,8), '\xC0') OR8_rr = insn(rex_w, '\x08', byte_register(1), byte_register(2,8), '\xC0') @@ -483,11 +485,15 @@ RET = insn('\xC3') PUSH_r = insn(rex_nw, register(1), '\x50') + PUSH_b = insn(rex_nw, '\xFF', orbyte(6<<3), stack_bp(1)) + POP_r = insn(rex_nw, register(1), '\x58') + POP_b = insn(rex_nw, '\x8F', orbyte(0<<3), stack_bp(1)) LEA_rb = insn(rex_w, '\x8D', register(1,8), stack_bp(2)) LEA32_rb = insn(rex_w, '\x8D', register(1,8),stack_bp(2,force_32bits=True)) LEA_ra = insn(rex_w, '\x8D', register(1, 8), mem_reg_plus_scaled_reg_plus_const(2)) + LEA_rm = insn(rex_w, '\x8D', register(1, 8), mem_reg_plus_const(2)) CALL_l = insn('\xE8', relative(1)) CALL_r = insn(rex_nw, '\xFF', register(1), chr(0xC0 | (2<<3))) @@ -497,9 +503,10 @@ XCHG_rj = insn(rex_w, '\x87', register(1,8), '\x05', immediate(2)) JMP_l = insn('\xE9', relative(1)) - # FIXME: J_il8 assume the caller will do the appropriate calculation - # to find the displacement, but J_il does it for the caller. + # FIXME: J_il8 and JMP_l8 assume the caller will do the appropriate + # calculation to find the displacement, but J_il does it for the caller. # We need to be consistent. + JMP_l8 = insn('\xEB', immediate(1, 'b')) J_il8 = insn(immediate(1, 'o'), '\x70', immediate(2, 'b')) J_il = insn('\x0F', immediate(1,'o'), '\x80', relative(2)) @@ -631,6 +638,8 @@ py.test.skip("XCGH_rj unsupported") def CMP_ji(self, addr, immed): py.test.skip("CMP_ji unsupported") + def CMP_rj(self, reg, immed): + py.test.skip("CMP_rj unsupported") def MOVSD_rj(self, xmm_reg, mem_immed): py.test.skip("MOVSD_rj unsupported") def MOVSD_jr(self, xmm_reg, mem_immed): Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_gc_integration.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_gc_integration.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_gc_integration.py Tue May 25 22:56:58 2010 @@ -14,7 +14,6 @@ from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.annlowlevel import llhelper from pypy.rpython.lltypesystem import rclass, rstr -from pypy.jit.backend.x86.ri386 import * from pypy.jit.backend.llsupport.gc import GcLLDescr_framework, GcRefList, GcPtrFieldDescr from pypy.jit.backend.x86.test.test_regalloc import MockAssembler Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_jump.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_jump.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_jump.py Tue May 25 22:56:58 2010 @@ -1,4 +1,4 @@ -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.regloc import * from pypy.jit.backend.x86.regalloc import X86FrameManager from pypy.jit.backend.x86.jump import remap_frame_layout @@ -25,7 +25,7 @@ continue assert len(op1) == len(op2) for x, y in zip(op1, op2): - if isinstance(x, MODRM) and isinstance(y, MODRM): + if isinstance(x, StackLoc) and isinstance(y, MODRM): assert x.byte == y.byte assert x.extradata == y.extradata else: Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_regalloc.py Tue May 25 22:56:58 2010 @@ -14,7 +14,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.annlowlevel import llhelper from pypy.rpython.lltypesystem import rclass, rstr -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.rx86 import * class MockGcDescr(GcCache): def get_funcptr_for_new(self): From fijal at codespeak.net Tue May 25 23:28:44 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 25 May 2010 23:28:44 +0200 (CEST) Subject: [pypy-svn] r74754 - in pypy/extradoc/pypy.org: . source Message-ID: <20100525212844.0CE5B36C227@codespeak.net> Author: fijal Date: Tue May 25 23:28:42 2010 New Revision: 74754 Added: pypy/extradoc/pypy.org/source/yatiblog.conf - copied unchanged from r74744, pypy/extradoc/pypy.org/source/_config.yml Removed: pypy/extradoc/pypy.org/source/_config.yml Modified: pypy/extradoc/pypy.org/compat.html pypy/extradoc/pypy.org/source/compat.txt Log: * Update compat.txt * Move _config.yaml to newer yatiblog.conf Modified: pypy/extradoc/pypy.org/compat.html ============================================================================== --- pypy/extradoc/pypy.org/compat.html (original) +++ pypy/extradoc/pypy.org/compat.html Tue May 25 23:28:42 2010 @@ -53,6 +53,9 @@ of the commonly used Python standard library modules; details below.

PyPy does not support the CPython C API, which means that third party libraries for python, written in C, will not work.

+

Note: PyPy trunk has experimental support for CPython extension modules +in the trunk, however it was not released yet so downloads below won't have +it.

Standard library modules supported by PyPy, in alphabetical order:

  • __builtin__ __pypy__ _codecs _lsprof _minimal_curses _random _rawffi _socket _sre _weakref bz2 cStringIO crypt errno exceptions fcntl gc itertools marshal math md5 mmap operator parser posix pyexpat select sha signal struct symbol sys termios thread time token unicodedata zipimport zlib
    • Modified: pypy/extradoc/pypy.org/source/compat.txt ============================================================================== --- pypy/extradoc/pypy.org/source/compat.txt (original) +++ pypy/extradoc/pypy.org/source/compat.txt Tue May 25 23:28:42 2010 @@ -11,6 +11,12 @@ PyPy does not support the `CPython C API`_, which means that third party libraries for python, written in C, will not work. +.. class:: download_menu + + *Note*: PyPy trunk has *experimental* support for CPython extension modules + in the trunk, however it was not released yet so downloads below won't have + it. + Standard library modules supported by PyPy, in alphabetical order: * ``__builtin__ __pypy__ _codecs _lsprof _minimal_curses _random _rawffi _socket _sre _weakref bz2 cStringIO crypt errno exceptions fcntl gc itertools marshal math md5 mmap operator parser posix pyexpat select sha signal struct symbol sys termios thread time token unicodedata zipimport zlib`` From fijal at codespeak.net Wed May 26 00:14:55 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 26 May 2010 00:14:55 +0200 (CEST) Subject: [pypy-svn] r74755 - pypy/trunk/pypy/config Message-ID: <20100525221455.BE00A282B9C@codespeak.net> Author: fijal Date: Wed May 26 00:14:54 2010 New Revision: 74755 Modified: pypy/trunk/pypy/config/pypyoption.py Log: Enable cpyext by default Modified: pypy/trunk/pypy/config/pypyoption.py ============================================================================== --- pypy/trunk/pypy/config/pypyoption.py (original) +++ pypy/trunk/pypy/config/pypyoption.py Wed May 26 00:14:54 2010 @@ -29,8 +29,7 @@ "rctime" , "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios", "zlib", "struct", "md5", "sha", "bz2", "_minimal_curses", "cStringIO", - "thread", "itertools", "pyexpat", "_ssl"] # "cpyext"] commented out until - # it stops adding _pyolifeline on W_Root + "thread", "itertools", "pyexpat", "_ssl", "cpyext"] )) working_oo_modules = default_modules.copy() From fijal at codespeak.net Wed May 26 00:23:20 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 26 May 2010 00:23:20 +0200 (CEST) Subject: [pypy-svn] r74756 - pypy/trunk/pypy/translator/goal Message-ID: <20100525222320.78C63282B9C@codespeak.net> Author: fijal Date: Wed May 26 00:23:19 2010 New Revision: 74756 Modified: pypy/trunk/pypy/translator/goal/targetosreadbench.py Log: update & cleanup Modified: pypy/trunk/pypy/translator/goal/targetosreadbench.py ============================================================================== --- pypy/trunk/pypy/translator/goal/targetosreadbench.py (original) +++ pypy/trunk/pypy/translator/goal/targetosreadbench.py Wed May 26 00:23:19 2010 @@ -7,13 +7,13 @@ # __________ Entry point __________ def entry_point(argv): - length = 0 - if len(argv) > 1: + if len(argv) > 2: length = int(argv[1]) else: length = 100 + fname = argv[1] for i in xrange(100000): - f = os.open(__file__, 0666, os.O_RDONLY) + f = os.open(fname, 0666, os.O_RDONLY) os.read(f, length) os.close(f) return 0 From fijal at codespeak.net Wed May 26 00:23:57 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 26 May 2010 00:23:57 +0200 (CEST) Subject: [pypy-svn] r74757 - pypy/trunk/pypy/translator/goal Message-ID: <20100525222357.22C82282B9C@codespeak.net> Author: fijal Date: Wed May 26 00:23:55 2010 New Revision: 74757 Modified: pypy/trunk/pypy/translator/goal/targetosreadbench.py Log: bummer Modified: pypy/trunk/pypy/translator/goal/targetosreadbench.py ============================================================================== --- pypy/trunk/pypy/translator/goal/targetosreadbench.py (original) +++ pypy/trunk/pypy/translator/goal/targetosreadbench.py Wed May 26 00:23:55 2010 @@ -8,7 +8,7 @@ def entry_point(argv): if len(argv) > 2: - length = int(argv[1]) + length = int(argv[2]) else: length = 100 fname = argv[1] From fijal at codespeak.net Wed May 26 00:25:22 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 26 May 2010 00:25:22 +0200 (CEST) Subject: [pypy-svn] r74758 - pypy/trunk/pypy/translator/goal Message-ID: <20100525222522.439A0282B9C@codespeak.net> Author: fijal Date: Wed May 26 00:25:20 2010 New Revision: 74758 Modified: pypy/trunk/pypy/translator/goal/targetosreadbench.py Log: Store the result somewhere Modified: pypy/trunk/pypy/translator/goal/targetosreadbench.py ============================================================================== --- pypy/trunk/pypy/translator/goal/targetosreadbench.py (original) +++ pypy/trunk/pypy/translator/goal/targetosreadbench.py Wed May 26 00:25:20 2010 @@ -12,9 +12,10 @@ else: length = 100 fname = argv[1] + l = [] for i in xrange(100000): f = os.open(fname, 0666, os.O_RDONLY) - os.read(f, length) + l.append(os.read(f, length)) os.close(f) return 0 From fijal at codespeak.net Wed May 26 00:55:44 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 26 May 2010 00:55:44 +0200 (CEST) Subject: [pypy-svn] r74759 - pypy/trunk/pypy/rpython/lltypesystem Message-ID: <20100525225544.ACE96282B9C@codespeak.net> Author: fijal Date: Wed May 26 00:55:41 2010 New Revision: 74759 Modified: pypy/trunk/pypy/rpython/lltypesystem/rffi.py Log: Never run rgc.malloc_nonmovable here. It seems to be a bad idea in some places and does not help in other places. I'll run benchmarks and look if to disable it permanently or to revive it. Modified: pypy/trunk/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rffi.py Wed May 26 00:55:41 2010 @@ -645,7 +645,7 @@ """ str_chars_offset = (offsetof(STRTYPE, 'chars') + \ itemoffsetof(STRTYPE.chars, 0)) - gc_buf = rgc.malloc_nonmovable(STRTYPE, count) + gc_buf = lltype.nullptr(STRTYPE) # rgc.malloc_nonmovable(STRTYPE, count) if gc_buf: realbuf = cast_ptr_to_adr(gc_buf) + str_chars_offset raw_buf = cast(TYPEP, realbuf) From afa at codespeak.net Wed May 26 10:49:14 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 26 May 2010 10:49:14 +0200 (CEST) Subject: [pypy-svn] r74765 - pypy/trunk/pypy/module/imp Message-ID: <20100526084914.A937836C22F@codespeak.net> Author: afa Date: Wed May 26 10:49:13 2010 New Revision: 74765 Modified: pypy/trunk/pypy/module/imp/importing.py Log: The flow object space executes all 'import' statements in a function it processes, regardless of the values in the "config" object. Move the actual import of the cpyext module in another function, so that --withoutmod-cpyext doesn't try the import. Modified: pypy/trunk/pypy/module/imp/importing.py ============================================================================== --- pypy/trunk/pypy/module/imp/importing.py (original) +++ pypy/trunk/pypy/module/imp/importing.py Wed May 26 10:49:13 2010 @@ -356,6 +356,12 @@ if pkgdir is not None: space.setattr(w_mod, w('__path__'), space.newlist([w(pkgdir)])) +def load_c_extension(space, filename, modulename): + # the next line is mandatory to init cpyext + space.getbuiltinmodule("cpyext") + from pypy.module.cpyext.api import load_extension_module + load_extension_module(space, filename, modulename) + @jit.dont_look_inside def load_module(space, w_modulename, find_info, reuse=False): if find_info is None: @@ -408,10 +414,7 @@ w_mod = check_sys_modules(space, w_modulename) return w_mod elif find_info.modtype == C_EXTENSION and space.config.objspace.usemodules.cpyext: - # the next line is mandantory to init cpyext - space.getbuiltinmodule("cpyext") - from pypy.module.cpyext.api import load_extension_module - load_extension_module(space, find_info.filename, space.str_w(w_modulename)) + load_c_extension(space, find_info.filename, space.str_w(w_modulename)) return check_sys_modules(space, w_modulename) except OperationError: w_mods = space.sys.get('modules') From afa at codespeak.net Wed May 26 11:46:39 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 26 May 2010 11:46:39 +0200 (CEST) Subject: [pypy-svn] r74766 - pypy/trunk/pypy/module/cpyext/include Message-ID: <20100526094639.0BEEC282B90@codespeak.net> Author: afa Date: Wed May 26 11:46:38 2010 New Revision: 74766 Modified: pypy/trunk/pypy/module/cpyext/include/Python.h pypy/trunk/pypy/module/cpyext/include/pyport.h Log: Move code from Python.h to pyport.h, avoids duplicate definitions Modified: pypy/trunk/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/trunk/pypy/module/cpyext/include/Python.h (original) +++ pypy/trunk/pypy/module/cpyext/include/Python.h Wed May 26 11:46:38 2010 @@ -35,37 +35,6 @@ #define Py_USING_UNICODE -// from pyport.h -#ifdef SIZE_MAX -#define PY_SIZE_MAX SIZE_MAX -#else -#define PY_SIZE_MAX ((size_t)-1) -#endif -/* uintptr_t is the C9X name for an unsigned integral type such that a - * legitimate void* can be cast to uintptr_t and then back to void* again - * without loss of information. Similarly for intptr_t, wrt a signed - * integral type. - */ -#ifdef HAVE_UINTPTR_T -typedef uintptr_t Py_uintptr_t; -typedef intptr_t Py_intptr_t; - -#elif SIZEOF_VOID_P <= SIZEOF_INT -typedef unsigned int Py_uintptr_t; -typedef int Py_intptr_t; - -#elif SIZEOF_VOID_P <= SIZEOF_LONG -typedef unsigned long Py_uintptr_t; -typedef long Py_intptr_t; - -#elif defined(HAVE_LONG_LONG) && (SIZEOF_VOID_P <= SIZEOF_LONG_LONG) -typedef unsigned PY_LONG_LONG Py_uintptr_t; -typedef PY_LONG_LONG Py_intptr_t; - -#else -# error "Python needs a typedef for Py_uintptr_t in pyport.h." -#endif /* HAVE_UINTPTR_T */ - #include /* Convert a possibly signed character to a nonnegative int */ Modified: pypy/trunk/pypy/module/cpyext/include/pyport.h ============================================================================== --- pypy/trunk/pypy/module/cpyext/include/pyport.h (original) +++ pypy/trunk/pypy/module/cpyext/include/pyport.h Wed May 26 11:46:38 2010 @@ -16,4 +16,29 @@ #define PY_SIZE_MAX ((size_t)-1) #endif +/* uintptr_t is the C9X name for an unsigned integral type such that a + * legitimate void* can be cast to uintptr_t and then back to void* again + * without loss of information. Similarly for intptr_t, wrt a signed + * integral type. + */ +#ifdef HAVE_UINTPTR_T +typedef uintptr_t Py_uintptr_t; +typedef intptr_t Py_intptr_t; + +#elif SIZEOF_VOID_P <= SIZEOF_INT +typedef unsigned int Py_uintptr_t; +typedef int Py_intptr_t; + +#elif SIZEOF_VOID_P <= SIZEOF_LONG +typedef unsigned long Py_uintptr_t; +typedef long Py_intptr_t; + +#elif defined(HAVE_LONG_LONG) && (SIZEOF_VOID_P <= SIZEOF_LONG_LONG) +typedef unsigned PY_LONG_LONG Py_uintptr_t; +typedef PY_LONG_LONG Py_intptr_t; + +#else +# error "Python needs a typedef for Py_uintptr_t in pyport.h." +#endif /* HAVE_UINTPTR_T */ + #endif /* Py_PYPORT_H */ From arigo at codespeak.net Wed May 26 11:49:50 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 26 May 2010 11:49:50 +0200 (CEST) Subject: [pypy-svn] r74767 - pypy/branch/blackhole-improvement/pypy/jit/codewriter/test Message-ID: <20100526094950.3BDEA282B90@codespeak.net> Author: arigo Date: Wed May 26 11:49:48 2010 New Revision: 74767 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Log: Implement int_abs using local control flow. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Wed May 26 11:49:48 2010 @@ -682,3 +682,14 @@ int_add %i1, %i2 -> %i3 int_return %i3 """, transform=True) + + def test_int_abs(self): + def f(n): + return abs(n) + self.encoding_test(f, [5], """ + int_copy %i0 -> %i1 + goto_if_not_int_lt %i0, $0, L1 + int_neg %i0 -> %i1 + L1: + int_return %i1 + """, transform=True) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Wed May 26 11:49:48 2010 @@ -6,6 +6,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr from pypy.translator.unsimplify import varoftype from pypy.jit.codewriter import heaptracker +from pypy.jit.codewriter.flatten import Label, TLabel class FakeRTyper: class type_system: name = 'lltypesystem' @@ -641,3 +642,22 @@ assert block.operations[1].args == [v1] assert block.operations[1].result is None assert block.exits[0].args == [v1] + +def test_int_abs(): + v1 = varoftype(lltype.Signed) + v2 = varoftype(lltype.Signed) + op = SpaceOperation('int_abs', [v1], v2) + oplist = Transformer().rewrite_operation(op) + assert len(oplist) == 4 + assert oplist[0].opname == 'int_copy' + assert oplist[0].args == [v1] + assert oplist[0].result == v2 + # + assert oplist[1].opname == 'goto_if_not_int_lt' + assert oplist[1].args == [v1, Constant(0, lltype.Signed), TLabel(v2)] + # + assert oplist[2].opname == 'int_neg' + assert oplist[2].args == [v1] + assert oplist[2].result == v2 + # + assert oplist[3] == Label(v2) From afa at codespeak.net Wed May 26 11:54:19 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 26 May 2010 11:54:19 +0200 (CEST) Subject: [pypy-svn] r74768 - in pypy/trunk/pypy/module/cpyext: . include test Message-ID: <20100526095419.C686436C22F@codespeak.net> Author: afa Date: Wed May 26 11:54:17 2010 New Revision: 74768 Modified: pypy/trunk/pypy/module/cpyext/include/sliceobject.h pypy/trunk/pypy/module/cpyext/sliceobject.py pypy/trunk/pypy/module/cpyext/stubs.py pypy/trunk/pypy/module/cpyext/test/test_sliceobject.py Log: add PySlice_New, and expose at C level the (readonly!) slice attributes. Modified: pypy/trunk/pypy/module/cpyext/include/sliceobject.h ============================================================================== --- pypy/trunk/pypy/module/cpyext/include/sliceobject.h (original) +++ pypy/trunk/pypy/module/cpyext/include/sliceobject.h Wed May 26 11:54:17 2010 @@ -12,6 +12,9 @@ typedef struct { PyObject_HEAD + PyObject *start; + PyObject *stop; + PyObject *step; } PySliceObject; #ifdef __cplusplus Modified: pypy/trunk/pypy/module/cpyext/sliceobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/sliceobject.py (original) +++ pypy/trunk/pypy/module/cpyext/sliceobject.py Wed May 26 11:54:17 2010 @@ -1,13 +1,63 @@ from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, Py_ssize_t,\ - Py_ssize_tP, build_type_checkers -from pypy.module.cpyext.pyobject import Py_DecRef, PyObject +from pypy.module.cpyext.api import ( + cpython_api, cpython_struct, bootstrap_function, build_type_checkers, + CANNOT_FAIL, Py_ssize_t, Py_ssize_tP, PyObjectFields) +from pypy.module.cpyext.pyobject import ( + Py_DecRef, PyObject, make_ref, make_typedescr) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.interpreter.error import OperationError from pypy.objspace.std.sliceobject import W_SliceObject +# Slice objects directly expose their members as PyObject. +# Don't change them! + +PySliceObjectStruct = lltype.ForwardReference() +PySliceObject = lltype.Ptr(PySliceObjectStruct) +PySliceObjectFields = PyObjectFields + \ + (("start", PyObject), ("step", PyObject), ("stop", PyObject), ) +cpython_struct("PySliceObject", PySliceObjectFields, PySliceObjectStruct) + + at bootstrap_function +def init_sliceobject(space): + "Type description of PySliceObject" + make_typedescr(W_SliceObject.typedef, + basestruct=PySliceObject.TO, + attach=slice_attach, + dealloc=slice_dealloc) + +def slice_attach(space, py_obj, w_obj): + """ + Fills a newly allocated PySliceObject with the given slice object. The + fields must not be modified. + """ + py_slice = rffi.cast(PySliceObject, py_obj) + assert isinstance(w_obj, W_SliceObject) + py_slice.c_start = make_ref(space, w_obj.w_start) + py_slice.c_stop = make_ref(space, w_obj.w_stop) + py_slice.c_step = make_ref(space, w_obj.w_step) + + at cpython_api([PyObject], lltype.Void, external=False) +def slice_dealloc(space, py_obj): + """Frees allocated PyStringObject resources. + """ + py_slice = rffi.cast(PySliceObject, py_obj) + Py_DecRef(space, py_slice.c_start) + Py_DecRef(space, py_slice.c_stop) + Py_DecRef(space, py_slice.c_step) + from pypy.module.cpyext.object import PyObject_dealloc + PyObject_dealloc(space, py_obj) + PySlice_Check, PySlice_CheckExact = build_type_checkers("Slice") + at cpython_api([PyObject, PyObject, PyObject], PyObject) +def PySlice_New(space, w_start, w_stop, w_step): + """Return a new slice object with the given values. The start, stop, and + step parameters are used as the values of the slice object attributes of + the same names. Any of the values may be NULL, in which case the + None will be used for the corresponding attribute. Return NULL if + the new object could not be allocated.""" + return W_SliceObject(w_start, w_stop, w_step) + @cpython_api([PyObject, Py_ssize_t, Py_ssize_tP, Py_ssize_tP, Py_ssize_tP, Py_ssize_tP], rffi.INT_real, error=-1) def PySlice_GetIndicesEx(space, w_slice, length, start_p, stop_p, Modified: pypy/trunk/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/stubs.py (original) +++ pypy/trunk/pypy/module/cpyext/stubs.py Wed May 26 11:54:17 2010 @@ -2631,15 +2631,6 @@ """Empty an existing set of all elements.""" raise NotImplementedError - at cpython_api([PyObject, PyObject, PyObject], PyObject) -def PySlice_New(space, start, stop, step): - """Return a new slice object with the given values. The start, stop, and - step parameters are used as the values of the slice object attributes of - the same names. Any of the values may be NULL, in which case the - None will be used for the corresponding attribute. Return NULL if - the new object could not be allocated.""" - raise NotImplementedError - @cpython_api([PyObject, Py_ssize_t, Py_ssize_t, Py_ssize_t, Py_ssize_t], rffi.INT_real, error=-1) def PySlice_GetIndices(space, slice, length, start, stop, step): """Retrieve the start, stop and step indices from the slice object slice, Modified: pypy/trunk/pypy/module/cpyext/test/test_sliceobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_sliceobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_sliceobject.py Wed May 26 11:54:17 2010 @@ -25,3 +25,17 @@ lltype.free(values, flavor='raw') return rv assert get_indices(w(10), w(20), w(1), 200) == (10, 20, 1, 10) + +class AppTestSliceMembers(AppTestCpythonExtensionBase): + def test_members(self): + module = self.import_extension('foo', [ + ("clone", "METH_O", + """ + PySliceObject *slice = (PySliceObject *)args; + return PySlice_New(slice->start, + slice->stop, + slice->step); + """), + ]) + s = slice(10, 20, 30) + assert module.clone(s) == s From afa at codespeak.net Wed May 26 12:00:10 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 26 May 2010 12:00:10 +0200 (CEST) Subject: [pypy-svn] r74769 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100526100010.C035636C22F@codespeak.net> Author: afa Date: Wed May 26 12:00:09 2010 New Revision: 74769 Modified: pypy/trunk/pypy/module/cpyext/api.py pypy/trunk/pypy/module/cpyext/test/test_sliceobject.py Log: Expose PySlice_Type Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Wed May 26 12:00:09 2010 @@ -19,6 +19,7 @@ from pypy.interpreter.gateway import ObjSpace, unwrap_spec from pypy.interpreter.nestedscope import Cell from pypy.interpreter.module import Module +from pypy.objspace.std.sliceobject import W_SliceObject from pypy.module.__builtin__.descriptor import W_Property from pypy.rlib.entrypoint import entrypoint from pypy.rlib.unroll import unrolling_iterable @@ -335,6 +336,7 @@ 'Cell': 'space.gettypeobject(Cell.typedef)', 'Module': 'space.gettypeobject(Module.typedef)', 'Property': 'space.gettypeobject(W_Property.typedef)', + 'Slice': 'space.gettypeobject(W_SliceObject.typedef)', }.items(): GLOBALS['Py%s_Type#' % (cpyname, )] = ('PyTypeObject*', pypyexpr) Modified: pypy/trunk/pypy/module/cpyext/test/test_sliceobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_sliceobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_sliceobject.py Wed May 26 12:00:09 2010 @@ -32,6 +32,10 @@ ("clone", "METH_O", """ PySliceObject *slice = (PySliceObject *)args; + if (slice->ob_type != &PySlice_Type) { + PyErr_SetNone(PyExc_ValueError); + return NULL; + } return PySlice_New(slice->start, slice->stop, slice->step); From arigo at codespeak.net Wed May 26 12:16:34 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 26 May 2010 12:16:34 +0200 (CEST) Subject: [pypy-svn] r74770 - pypy/branch/blackhole-improvement/pypy/jit/codewriter/test Message-ID: <20100526101634.8C34536C22F@codespeak.net> Author: arigo Date: Wed May 26 12:16:33 2010 New Revision: 74770 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Log: Revert r74767; does not work. (Also, I forgot to check in the parts in codewriter/*.py....) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Wed May 26 12:16:33 2010 @@ -682,14 +682,3 @@ int_add %i1, %i2 -> %i3 int_return %i3 """, transform=True) - - def test_int_abs(self): - def f(n): - return abs(n) - self.encoding_test(f, [5], """ - int_copy %i0 -> %i1 - goto_if_not_int_lt %i0, $0, L1 - int_neg %i0 -> %i1 - L1: - int_return %i1 - """, transform=True) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Wed May 26 12:16:33 2010 @@ -6,7 +6,6 @@ from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr from pypy.translator.unsimplify import varoftype from pypy.jit.codewriter import heaptracker -from pypy.jit.codewriter.flatten import Label, TLabel class FakeRTyper: class type_system: name = 'lltypesystem' @@ -642,22 +641,3 @@ assert block.operations[1].args == [v1] assert block.operations[1].result is None assert block.exits[0].args == [v1] - -def test_int_abs(): - v1 = varoftype(lltype.Signed) - v2 = varoftype(lltype.Signed) - op = SpaceOperation('int_abs', [v1], v2) - oplist = Transformer().rewrite_operation(op) - assert len(oplist) == 4 - assert oplist[0].opname == 'int_copy' - assert oplist[0].args == [v1] - assert oplist[0].result == v2 - # - assert oplist[1].opname == 'goto_if_not_int_lt' - assert oplist[1].args == [v1, Constant(0, lltype.Signed), TLabel(v2)] - # - assert oplist[2].opname == 'int_neg' - assert oplist[2].args == [v1] - assert oplist[2].result == v2 - # - assert oplist[3] == Label(v2) From arigo at codespeak.net Wed May 26 12:30:37 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 26 May 2010 12:30:37 +0200 (CEST) Subject: [pypy-svn] r74771 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100526103037.C33DF36C22F@codespeak.net> Author: arigo Date: Wed May 26 12:30:36 2010 New Revision: 74771 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: Fix for test_jitprof.py. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Wed May 26 12:30:36 2010 @@ -1495,8 +1495,8 @@ def compile_and_run_once(self, *args): debug_start('jit-tracing') - self.staticdata.profiler.start_tracing() self.staticdata._setup_once() + self.staticdata.profiler.start_tracing() self.create_empty_history() try: return self._compile_and_run_once(*args) From afa at codespeak.net Wed May 26 13:55:02 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 26 May 2010 13:55:02 +0200 (CEST) Subject: [pypy-svn] r74772 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100526115502.1675836C22F@codespeak.net> Author: afa Date: Wed May 26 13:55:00 2010 New Revision: 74772 Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py pypy/trunk/pypy/module/cpyext/typeobject.py Log: Add _PyType_Lookup, not documented but used by Boost.Python... Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Wed May 26 13:55:00 2010 @@ -173,3 +173,12 @@ """) ref = make_ref(space, w_class) api.Py_DecRef(ref) + + def test_lookup(self, space, api): + w_type = space.w_str + w_obj = api._PyType_Lookup(w_type, space.wrap("upper")) + assert space.is_w(w_obj, space.w_str.getdictvalue(space, "upper")) + + w_obj = api._PyType_Lookup(w_type, space.wrap("__invalid")) + assert w_obj is None + assert api.PyErr_Occurred() is None Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Wed May 26 13:55:00 2010 @@ -17,7 +17,7 @@ PyBufferProcs, build_type_checkers) from pypy.module.cpyext.pyobject import ( PyObject, make_ref, create_ref, from_ref, get_typedescr, make_typedescr, - track_reference, RefcountState) + track_reference, RefcountState, borrow_from) from pypy.interpreter.module import Module from pypy.interpreter.function import FunctionWithFixedCode, StaticMethod from pypy.module.cpyext import structmemberdefs @@ -675,4 +675,14 @@ return generic_cpy_call( space, type.c_tp_alloc, type, 0) + at cpython_api([PyTypeObjectPtr, PyObject], PyObject, error=CANNOT_FAIL) +def _PyType_Lookup(space, type, w_name): + """Internal API to look for a name through the MRO. + This returns a borrowed reference, and doesn't set an exception!""" + py_type = rffi.cast(PyObject, type) + w_type = from_ref(space, py_type) + assert isinstance(w_type, W_TypeObject) + name = space.str_w(w_name) + w_obj = w_type.lookup(name) + return borrow_from(w_type, w_obj) From afa at codespeak.net Wed May 26 14:15:33 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 26 May 2010 14:15:33 +0200 (CEST) Subject: [pypy-svn] r74773 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100526121533.221EA36C22F@codespeak.net> Author: afa Date: Wed May 26 14:15:31 2010 New Revision: 74773 Modified: pypy/trunk/pypy/module/cpyext/methodobject.py pypy/trunk/pypy/module/cpyext/modsupport.py pypy/trunk/pypy/module/cpyext/test/foo.c pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Log: Support METH_STATIC, add PyStaticMethod_New Modified: pypy/trunk/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/methodobject.py (original) +++ pypy/trunk/pypy/module/cpyext/methodobject.py Wed May 26 14:15:31 2010 @@ -5,7 +5,7 @@ from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.interpreter.function import BuiltinFunction, Method +from pypy.interpreter.function import BuiltinFunction, Method, StaticMethod from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.pyobject import (PyObject, from_ref, make_ref, make_typedescr, Py_DecRef) @@ -228,6 +228,10 @@ def PyCFunction_NewEx(space, ml, w_self, w_name): return space.wrap(W_PyCFunctionObject(space, ml, w_self, w_name)) + at cpython_api([PyObject], PyObject) +def PyStaticMethod_New(space, w_func): + return space.wrap(StaticMethod(w_func)) + def PyDescr_NewMethod(space, w_type, method): return space.wrap(W_PyCMethodObject(space, method, w_type)) Modified: pypy/trunk/pypy/module/cpyext/modsupport.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/modsupport.py (original) +++ pypy/trunk/pypy/module/cpyext/modsupport.py Wed May 26 14:15:31 2010 @@ -3,7 +3,9 @@ METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, CONST_STRING from pypy.module.cpyext.pyobject import PyObject, borrow_from from pypy.interpreter.module import Module -from pypy.module.cpyext.methodobject import W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod, PyMethodDef, PyCFunction +from pypy.module.cpyext.methodobject import ( + W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod, + PyMethodDef, PyCFunction, PyStaticMethod_New) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.module.cpyext.state import State from pypy.interpreter.error import OperationError @@ -90,8 +92,7 @@ w_obj = space.w_Ellipsis # XXX elif flags & METH_STATIC: w_func = PyCFunction_NewEx(space, method, None, None) - w_obj = space.w_Ellipsis # XXX - #w_obj = PyStaticMethod_New(space, w_func) + w_obj = PyStaticMethod_New(space, w_func) else: w_obj = PyDescr_NewMethod(space, w_type, method) Modified: pypy/trunk/pypy/module/cpyext/test/foo.c ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/foo.c (original) +++ pypy/trunk/pypy/module/cpyext/test/foo.c Wed May 26 14:15:31 2010 @@ -53,6 +53,12 @@ } static PyObject * +foo_create(fooobject *self) +{ + return newfooobject(); +} + +static PyObject * foo_unset(fooobject *self) { self->foo_string = NULL; @@ -62,6 +68,7 @@ static PyMethodDef foo_methods[] = { {"copy", (PyCFunction)foo_copy, METH_NOARGS, NULL}, + {"create", (PyCFunction)foo_create, METH_NOARGS|METH_STATIC, NULL}, {"unset_string_member", (PyCFunction)foo_unset, METH_NOARGS, NULL}, {NULL, NULL} /* sentinel */ }; Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Wed May 26 14:15:31 2010 @@ -89,6 +89,13 @@ raises(TypeError, "obj.char_member = 'spam'") raises(TypeError, "obj.char_member = 42") + def test_staticmethod(self): + module = self.import_module(name="foo") + obj = module.fooType.create() + assert obj.foo == 42 + obj2 = obj.create() + assert obj2.foo == 42 + def test_new(self): module = self.import_module(name='foo') obj = module.new() From afa at codespeak.net Wed May 26 16:15:33 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 26 May 2010 16:15:33 +0200 (CEST) Subject: [pypy-svn] r74774 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100526141533.27622282B9C@codespeak.net> Author: afa Date: Wed May 26 16:15:31 2010 New Revision: 74774 Modified: pypy/trunk/pypy/module/cpyext/stringobject.py pypy/trunk/pypy/module/cpyext/test/test_stringobject.py Log: add PyString_InternFromString Modified: pypy/trunk/pypy/module/cpyext/stringobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/stringobject.py (original) +++ pypy/trunk/pypy/module/cpyext/stringobject.py Wed May 26 16:15:31 2010 @@ -238,3 +238,12 @@ args. The args argument must be a tuple.""" return space.mod(w_format, w_args) + at cpython_api([CONST_STRING], PyObject) +def PyString_InternFromString(space, string): + """A combination of PyString_FromString() and + PyString_InternInPlace(), returning either a new string object that has + been interned, or a new ("owned") reference to an earlier interned string + object with the same value.""" + s = rffi.charp2str(string) + return space.new_interned_str(s) + Modified: pypy/trunk/pypy/module/cpyext/test/test_stringobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_stringobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_stringobject.py Wed May 26 16:15:31 2010 @@ -246,3 +246,10 @@ lltype.free(bufp, flavor='raw') lltype.free(lenp, flavor='raw') + + def test_intern(self, space, api): + buf = rffi.str2charp("test") + w_s1 = api.PyString_InternFromString(buf) + w_s2 = api.PyString_InternFromString(buf) + rffi.free_charp(buf) + assert w_s1 is w_s2 From afa at codespeak.net Wed May 26 16:20:40 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 26 May 2010 16:20:40 +0200 (CEST) Subject: [pypy-svn] r74775 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100526142040.62723282B9C@codespeak.net> Author: afa Date: Wed May 26 16:20:38 2010 New Revision: 74775 Modified: pypy/trunk/pypy/module/cpyext/object.py pypy/trunk/pypy/module/cpyext/test/test_object.py Log: add PyFile_FromString() Modified: pypy/trunk/pypy/module/cpyext/object.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/object.py (original) +++ pypy/trunk/pypy/module/cpyext/object.py Wed May 26 16:20:38 2010 @@ -413,3 +413,13 @@ rffi.free_nonmovingbuffer(data, buf) return 0 + at cpython_api([CONST_STRING, CONST_STRING], PyObject) +def PyFile_FromString(space, filename, mode): + """ + On success, return a new file object that is opened on the file given by + filename, with a file mode given by mode, where mode has the same + semantics as the standard C routine fopen(). On failure, return NULL.""" + w_filename = space.wrap(rffi.charp2str(filename)) + w_mode = space.wrap(rffi.charp2str(mode)) + return space.call_method(space.builtin, 'file', w_filename, w_mode) + Modified: pypy/trunk/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_object.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_object.py Wed May 26 16:20:38 2010 @@ -192,6 +192,17 @@ assert api.PyObject_Unicode(space.wrap("\xe9")) is None api.PyErr_Clear() + def test_file_fromstring(self, space, api): + filename = rffi.str2charp(str(udir / "_test_file")) + mode = rffi.str2charp("wb") + w_file = api.PyFile_FromString(filename, mode) + rffi.free_charp(filename) + rffi.free_charp(mode) + + space.call_method(w_file, "write", space.wrap("text")) + space.call_method(w_file, "close") + assert (udir / "_test_file").read() == "text" + class AppTestObjectPrint(AppTestCpythonExtensionBase): def setup_class(cls): AppTestCpythonExtensionBase.setup_class.im_func(cls) From afa at codespeak.net Wed May 26 16:24:43 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 26 May 2010 16:24:43 +0200 (CEST) Subject: [pypy-svn] r74776 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100526142443.A2C7436C394@codespeak.net> Author: afa Date: Wed May 26 16:24:42 2010 New Revision: 74776 Modified: pypy/trunk/pypy/module/cpyext/api.py pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Log: Expose PyStaticMethod_Type and PyCFunction_Type Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Wed May 26 16:24:42 2010 @@ -19,6 +19,7 @@ from pypy.interpreter.gateway import ObjSpace, unwrap_spec from pypy.interpreter.nestedscope import Cell from pypy.interpreter.module import Module +from pypy.interpreter.function import StaticMethod from pypy.objspace.std.sliceobject import W_SliceObject from pypy.module.__builtin__.descriptor import W_Property from pypy.rlib.entrypoint import entrypoint @@ -337,10 +338,12 @@ 'Module': 'space.gettypeobject(Module.typedef)', 'Property': 'space.gettypeobject(W_Property.typedef)', 'Slice': 'space.gettypeobject(W_SliceObject.typedef)', + 'StaticMethod': 'space.gettypeobject(StaticMethod.typedef)', + 'CFunction': 'space.gettypeobject(cpyext.methodobject.W_PyCFunctionObject.typedef)', }.items(): GLOBALS['Py%s_Type#' % (cpyname, )] = ('PyTypeObject*', pypyexpr) - for cpyname in 'Method List Int Long Dict Tuple'.split(): + for cpyname in 'Method List Int Long Dict Tuple Class'.split(): FORWARD_DECLS.append('typedef struct { PyObject_HEAD } ' 'Py%sObject' % (cpyname, )) build_exported_objects() @@ -604,6 +607,7 @@ # populate static data for name, (type, expr) in GLOBALS.iteritems(): + from pypy.module import cpyext w_obj = eval(expr) name = name.replace("#", "") INTERPLEVEL_API[name] = w_obj @@ -776,6 +780,7 @@ # populate static data for name, (type, expr) in GLOBALS.iteritems(): name = name.replace("#", "") + from pypy.module import cpyext w_obj = eval(expr) struct_ptr = make_ref(space, w_obj) struct = rffi.cast(get_structtype_for_ctype(type), struct_ptr)._obj Modified: pypy/trunk/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Wed May 26 16:24:42 2010 @@ -656,12 +656,16 @@ ''' /* XXX in tests, the C type is not correct */ #define NAME(type) ((PyTypeObject*)&type)->tp_name - return Py_BuildValue("sss", + return Py_BuildValue("sssss", NAME(PyCell_Type), NAME(PyModule_Type), - NAME(PyProperty_Type) + NAME(PyProperty_Type), + NAME(PyStaticMethod_Type), + NAME(PyCFunction_Type) ); ''' ), ]) - assert mod.get_names() == ('cell', 'module', 'property') + assert mod.get_names() == ('cell', 'module', 'property', + 'staticmethod', + 'builtin_function_or_method') From afa at codespeak.net Wed May 26 16:26:46 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 26 May 2010 16:26:46 +0200 (CEST) Subject: [pypy-svn] r74777 - pypy/trunk/pypy/module/cpyext Message-ID: <20100526142646.C04E636C394@codespeak.net> Author: afa Date: Wed May 26 16:26:45 2010 New Revision: 74777 Modified: pypy/trunk/pypy/module/cpyext/stubs.py Log: Remove stubs that are now implemented. Modified: pypy/trunk/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/stubs.py (original) +++ pypy/trunk/pypy/module/cpyext/stubs.py Wed May 26 16:26:45 2010 @@ -982,17 +982,6 @@ """ raise NotImplementedError - at cpython_api([rffi.CCHARP, rffi.CCHARP], PyObject) -def PyFile_FromString(space, filename, mode): - """ - - - - On success, return a new file object that is opened on the file given by - filename, with a file mode given by mode, where mode has the same - semantics as the standard C routine fopen(). On failure, return NULL.""" - raise NotImplementedError - @cpython_api([FILE, rffi.CCHARP, rffi.CCHARP, rffi.INT_real], PyObject) def PyFile_FromFile(space, fp, name, mode, close): """Create a new PyFileObject from the already-open standard C file @@ -2680,16 +2669,6 @@ This function is not available in 3.x and does not have a PyBytes alias.""" raise NotImplementedError - at cpython_api([rffi.CCHARP], PyObject) -def PyString_InternFromString(space, v): - """A combination of PyString_FromString() and - PyString_InternInPlace(), returning either a new string object that has - been interned, or a new ("owned") reference to an earlier interned string object - with the same value. - - This function is not available in 3.x and does not have a PyBytes alias.""" - raise NotImplementedError - @cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.CCHARP], PyObject) def PyString_Decode(space, s, size, encoding, errors): """Create an object by decoding size bytes of the encoded buffer s using the @@ -3602,12 +3581,6 @@ exception was raised.""" raise NotImplementedError - at cpython_api([FILE, rffi.CCHARP, rffi.INT_real, PyObject, PyObject], PyObject) -def PyRun_File(space, fp, filename, start, globals, locals): - """This is a simplified interface to PyRun_FileExFlags() below, leaving - closeit set to 0 and flags set to NULL.""" - raise NotImplementedError - @cpython_api([FILE, rffi.CCHARP, rffi.INT_real, PyObject, PyObject, rffi.INT_real], PyObject) def PyRun_FileEx(space, fp, filename, start, globals, locals, closeit): """This is a simplified interface to PyRun_FileExFlags() below, leaving From arigo at codespeak.net Wed May 26 16:33:21 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 26 May 2010 16:33:21 +0200 (CEST) Subject: [pypy-svn] r74778 - pypy/branch/blackhole-improvement/pypy/jit/codewriter/test Message-ID: <20100526143321.864E936C394@codespeak.net> Author: arigo Date: Wed May 26 16:33:20 2010 New Revision: 74778 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Log: Mfph. Found another regalloc bug. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Wed May 26 16:33:20 2010 @@ -222,3 +222,21 @@ rescall I[%i0, %i1] -> %i0 int_return %i0 """) + + def test_regalloc_bug_1(self): + def _ll_2_int_lshift_ovf(x, y): + result = x << y + if (result >> y) != x: + raise OverflowError + return result + graph = self.make_graphs(_ll_2_int_lshift_ovf, [5, 6])[0] + self.check_assembler(graph, """ + int_lshift %i0, %i1 -> %i2 + int_rshift %i0, %i1 -> %i1 + goto_if_not_int_ne %i1, %i2, L1 + -live- L1 + raise $<* struct object> + --- + L1: + int_return %i2 + """, transform=True) From arigo at codespeak.net Wed May 26 16:35:18 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 26 May 2010 16:35:18 +0200 (CEST) Subject: [pypy-svn] r74779 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100526143518.AD99636C394@codespeak.net> Author: arigo Date: Wed May 26 16:35:17 2010 New Revision: 74779 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Log: Fix the bug. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py Wed May 26 16:35:17 2010 @@ -37,7 +37,11 @@ die_at[v1] = i if op.result is not None: die_at[op.result] = i - die_at.pop(block.exitswitch, None) + if isinstance(block.exitswitch, tuple): + for x in block.exitswitch: + die_at.pop(x, None) + else: + die_at.pop(block.exitswitch, None) for link in block.exits: for v in link.args: die_at.pop(v, None) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Wed May 26 16:35:17 2010 @@ -232,8 +232,8 @@ graph = self.make_graphs(_ll_2_int_lshift_ovf, [5, 6])[0] self.check_assembler(graph, """ int_lshift %i0, %i1 -> %i2 - int_rshift %i0, %i1 -> %i1 - goto_if_not_int_ne %i1, %i2, L1 + int_rshift %i2, %i1 -> %i1 + goto_if_not_int_ne %i1, %i0, L1 -live- L1 raise $<* struct object> --- From arigo at codespeak.net Wed May 26 16:36:39 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 26 May 2010 16:36:39 +0200 (CEST) Subject: [pypy-svn] r74780 - pypy/branch/blackhole-improvement/pypy/jit/codewriter Message-ID: <20100526143639.AA62236C394@codespeak.net> Author: arigo Date: Wed May 26 16:36:38 2010 New Revision: 74780 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Log: Small simplification. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Wed May 26 16:36:38 2010 @@ -1,5 +1,5 @@ import py, sys -from pypy.rpython.lltypesystem import lltype, rstr +from pypy.rpython.lltypesystem import lltype, rstr, rclass from pypy.rpython import rlist from pypy.jit.metainterp.history import getkind from pypy.objspace.flow.model import SpaceOperation, Variable, Constant @@ -498,6 +498,9 @@ op.args[0].concretetype.TO._hints.get('typeptr')) def handle_getfield_typeptr(self, op): + if isinstance(op.args[0], Constant): + cls = op.args[0].value.typeptr + return Constant(cls, concretetype=rclass.CLASSTYPE) op0 = SpaceOperation('-live-', [], None) op1 = SpaceOperation('guard_class', [op.args[0]], op.result) return [op0, op1] From arigo at codespeak.net Wed May 26 16:44:53 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 26 May 2010 16:44:53 +0200 (CEST) Subject: [pypy-svn] r74781 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100526144453.4EE34282B9C@codespeak.net> Author: arigo Date: Wed May 26 16:44:51 2010 New Revision: 74781 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Log: Found out how to write it so that int_abs and int_lshift_ovf are seen by the JIT. We don't care too much that they are not inlined by the codewriter. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py Wed May 26 16:44:51 2010 @@ -41,6 +41,11 @@ assert self.portal_graph is not None todo = [self.portal_graph] + for oopspec_name, ll_args, ll_res in support.inline_calls_to: + c_func, _ = support.builtin_func_for_spec(self.rtyper, + oopspec_name, + ll_args, ll_res) + todo.append(c_func.value._obj.graph) candidate_graphs = set(todo) def callers(): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py Wed May 26 16:44:51 2010 @@ -185,10 +185,22 @@ return llop.int_mod_zer(lltype.Signed, x, y) def _ll_2_int_lshift_ovf(x, y): - return llop.int_lshift_ovf(lltype.Signed, x, y) + result = x << y + if (result >> y) != x: + raise OverflowError + return result def _ll_1_int_abs(x): - return abs(x) + if x < 0: + return -x + else: + return x + +# in the following calls to builtins, the JIT is allowed to look inside: +inline_calls_to = [ + ('int_lshift_ovf', [lltype.Signed, lltype.Signed], lltype.Signed), + ('int_abs', [lltype.Signed], lltype.Signed), + ] class LLtypeHelpers: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py Wed May 26 16:44:51 2010 @@ -127,3 +127,15 @@ else: assert 0, "missing instantiate_*_%s in:\n%r" % (expected, names) + +def test_int_abs(): + def f(n): + return abs(n) + rtyper = support.annotate(f, [35]) + maingraph = rtyper.annotator.translator.graphs[0] + cw = CodeWriter(FakeCPU(rtyper), maingraph) + cw.find_all_graphs(FakePolicy()) + cw.make_jitcodes(verbose=True) + # + s = cw.mainjitcode.dump() + assert "inline_call_ir_i " in s Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Wed May 26 16:44:51 2010 @@ -641,3 +641,13 @@ assert block.operations[1].args == [v1] assert block.operations[1].result is None assert block.exits[0].args == [v1] + +def test_int_abs(): + v1 = varoftype(lltype.Signed) + v2 = varoftype(lltype.Signed) + op = SpaceOperation('int_abs', [v1], v2) + tr = Transformer(FakeCPU(), FakeRegularCallControl()) + tr.graph = "somemaingraph" + oplist = tr.rewrite_operation(op) + assert oplist[0].opname == 'inline_call_ir_i' + assert oplist[0].args[0] == 'somejitcode' From arigo at codespeak.net Wed May 26 16:56:55 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 26 May 2010 16:56:55 +0200 (CEST) Subject: [pypy-svn] r74782 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter metainterp/test Message-ID: <20100526145655.53C46282B9C@codespeak.net> Author: arigo Date: Wed May 26 16:56:53 2010 New Revision: 74782 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Log: Use the same trick for int_floordiv_xxx and int_mod_xxx. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py Wed May 26 16:56:53 2010 @@ -1,3 +1,4 @@ +import sys from pypy.rpython.lltypesystem import lltype, rclass from pypy.rpython.ootypesystem import ootype from pypy.rpython import rlist @@ -170,19 +171,40 @@ def _ll_1_jit_force_virtual(inst): return llop.jit_force_virtual(lltype.typeOf(inst), inst) + def _ll_2_int_floordiv_ovf_zer(x, y): - return llop.int_floordiv_ovf_zer(lltype.Signed, x, y) + if y == 0: + raise ZeroDivisionError + if ((x + sys.maxint) & y) == -1: # detect "x = -sys.maxint-1, y = -1". + raise OverflowError + return llop.int_floordiv(lltype.Signed, x, y) + def _ll_2_int_floordiv_ovf(x, y): - return llop.int_floordiv_ovf(lltype.Signed, x, y) + if ((x + sys.maxint) & y) == -1: # detect "x = -sys.maxint-1, y = -1". + raise OverflowError + return llop.int_floordiv(lltype.Signed, x, y) + def _ll_2_int_floordiv_zer(x, y): + if y == 0: + raise ZeroDivisionError return llop.int_floordiv_zer(lltype.Signed, x, y) def _ll_2_int_mod_ovf_zer(x, y): - return llop.int_mod_ovf_zer(lltype.Signed, x, y) + if y == 0: + raise ZeroDivisionError + if ((x + sys.maxint) & y) == -1: # detect "x = -sys.maxint-1, y = -1". + raise OverflowError + return llop.int_mod(lltype.Signed, x, y) + def _ll_2_int_mod_ovf(x, y): - return llop.int_mod_ovf(lltype.Signed, x, y) + if ((x + sys.maxint) & y) == -1: # detect "x = -sys.maxint-1, y = -1". + raise OverflowError + return llop.int_mod(lltype.Signed, x, y) + def _ll_2_int_mod_zer(x, y): - return llop.int_mod_zer(lltype.Signed, x, y) + if y == 0: + raise ZeroDivisionError + return llop.int_mod(lltype.Signed, x, y) def _ll_2_int_lshift_ovf(x, y): result = x << y @@ -198,8 +220,14 @@ # in the following calls to builtins, the JIT is allowed to look inside: inline_calls_to = [ - ('int_lshift_ovf', [lltype.Signed, lltype.Signed], lltype.Signed), - ('int_abs', [lltype.Signed], lltype.Signed), + ('int_floordiv_ovf_zer', [lltype.Signed, lltype.Signed], lltype.Signed), + ('int_floordiv_ovf', [lltype.Signed, lltype.Signed], lltype.Signed), + ('int_floordiv_zer', [lltype.Signed, lltype.Signed], lltype.Signed), + ('int_mod_ovf_zer', [lltype.Signed, lltype.Signed], lltype.Signed), + ('int_mod_ovf', [lltype.Signed, lltype.Signed], lltype.Signed), + ('int_mod_zer', [lltype.Signed, lltype.Signed], lltype.Signed), + ('int_lshift_ovf', [lltype.Signed, lltype.Signed], lltype.Signed), + ('int_abs', [lltype.Signed], lltype.Signed), ] Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Wed May 26 16:56:53 2010 @@ -578,6 +578,7 @@ return n res = self.meta_interp(f, [20, 1, 2]) assert res == 0 + self.check_loops(call=0) def test_abs(self): myjitdriver = JitDriver(greens = [], reds = ['i', 't']) From arigo at codespeak.net Wed May 26 18:12:21 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 26 May 2010 18:12:21 +0200 (CEST) Subject: [pypy-svn] r74783 - pypy/trunk/pypy/module/posix/test Message-ID: <20100526161221.5948C282B9C@codespeak.net> Author: arigo Date: Wed May 26 18:12:19 2010 New Revision: 74783 Modified: pypy/trunk/pypy/module/posix/test/test_posix2.py Log: First clean up the zombie processes left behind by some other reason. This is needed because otherwise the following os.wait() might return one of them instead of the expected one. Modified: pypy/trunk/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/trunk/pypy/module/posix/test/test_posix2.py (original) +++ pypy/trunk/pypy/module/posix/test/test_posix2.py Wed May 26 18:12:19 2010 @@ -490,6 +490,12 @@ if not hasattr(os, "fork"): skip("Need fork() to test wait()") + if hasattr(os, "waitpid") and hasattr(os, "WNOHANG"): + try: + while os.waitpid(-1, os.WNOHANG)[0]: + pass + except OSError: # until we get "No child processes", hopefully + pass child = os.fork() if child == 0: # in child os._exit(exit_status) From arigo at codespeak.net Wed May 26 18:32:11 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 26 May 2010 18:32:11 +0200 (CEST) Subject: [pypy-svn] r74784 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter metainterp metainterp/test Message-ID: <20100526163211.F090E282BDE@codespeak.net> Author: arigo Date: Wed May 26 18:32:09 2010 New Revision: 74784 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_recursive.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py Log: Some progress towards getting test_recursive to pass again. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py Wed May 26 18:32:09 2010 @@ -15,7 +15,8 @@ class CallControl(object): - virtualref_info = None # optionally set from outside to a VirtualRefInfo() + virtualref_info = None # optionally set from outside + portal_runner_ptr = None # optionally set from outside def __init__(self, cpu=None, portal_graph=None): self.cpu = cpu @@ -119,12 +120,12 @@ def guess_call_kind(self, op, is_candidate=None): if op.opname == 'direct_call': funcptr = op.args[0].value + if funcptr is self.portal_runner_ptr: + return 'recursive' funcobj = get_funcobj(funcptr) if getattr(funcobj, 'graph', None) is None: return 'residual' targetgraph = funcobj.graph - if targetgraph is self.portal_graph: - return 'recursive' if (hasattr(targetgraph, 'func') and hasattr(targetgraph.func, 'oopspec')): return 'builtin' @@ -164,8 +165,9 @@ because it is not needed there; it is only used by the blackhole interp to really do the call corresponding to 'inline_call' ops. """ - fnptr = self.rtyper.getcallable(graph) + fnptr = self.rtyper.type_system.getcallable(graph) FUNC = get_functype(lltype.typeOf(fnptr)) + assert lltype.Ptr(lltype.PyObject) not in FUNC.ARGS if self.rtyper.type_system.name == 'ootypesystem': XXX else: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py Wed May 26 18:32:09 2010 @@ -71,6 +71,9 @@ def setup_vrefinfo(self, vrefinfo): self.callcontrol.virtualref_info = vrefinfo + def setup_portal_runner_ptr(self, portal_runner_ptr): + self.callcontrol.portal_runner_ptr = portal_runner_ptr + def find_all_graphs(self, policy): return self.callcontrol.find_all_graphs(policy) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Wed May 26 18:32:09 2010 @@ -283,11 +283,12 @@ op1 = [op1, SpaceOperation('-live-', [], None)] return op1 - def handle_regular_call(self, op): + def handle_regular_call(self, op, targetgraph=None): """A direct_call turns into the operation 'inline_call_xxx' if it is calling a function that we want to JIT. The initial arg of 'inline_call_xxx' is the JitCode of the called function.""" - [targetgraph] = self.callcontrol.graphs_from(op) + if targetgraph is None: + [targetgraph] = self.callcontrol.graphs_from(op) jitcode = self.callcontrol.get_jitcode(targetgraph, called_from=self.graph) op0 = self.rewrite_call(op, 'inline_call', [jitcode]) @@ -313,6 +314,11 @@ op1 = self.handle_residual_call(op1 or op) return op1 + def handle_recursive_call(self, op): + ops = self.promote_greens(op.args[1:]) + targetgraph = self.callcontrol.portal_graph + return ops + self.handle_regular_call(op, targetgraph) + handle_residual_indirect_call = handle_residual_call def handle_regular_indirect_call(self, op): @@ -682,28 +688,34 @@ return Constant(value, lltype.Bool) return op - def rewrite_op_jit_marker(self, op): - jitdriver = op.args[1].value - self.callcontrol.found_jitdriver(jitdriver) - key = op.args[0].value - return getattr(self, 'handle_jit_marker__%s' % key)(op, jitdriver) - - def handle_jit_marker__jit_merge_point(self, op, jitdriver): - assert self.portal, "jit_merge_point in non-main graph!" + def promote_greens(self, args): + jitdriver = self.callcontrol.jitdriver + assert jitdriver is not None, "order dependency issue?" ops = [] num_green_args = len(jitdriver.greens) - for v in op.args[2:2+num_green_args]: + for v in args[:num_green_args]: if isinstance(v, Variable) and v.concretetype is not lltype.Void: kind = getkind(v.concretetype) ops.append(SpaceOperation('-live-', [], None)) ops.append(SpaceOperation('%s_guard_value' % kind, [v], None)) + return ops + + def rewrite_op_jit_marker(self, op): + self.callcontrol.found_jitdriver(op.args[1].value) + key = op.args[0].value + return getattr(self, 'handle_jit_marker__%s' % key)(op) + + def handle_jit_marker__jit_merge_point(self, op): + assert self.portal, "jit_merge_point in non-main graph!" + ops = self.promote_greens(op.args[2:]) + num_green_args = len(self.callcontrol.jitdriver.greens) args = (self.make_three_lists(op.args[2:2+num_green_args]) + self.make_three_lists(op.args[2+num_green_args:])) - ops.append(SpaceOperation('jit_merge_point', args, None)) - return ops + op1 = SpaceOperation('jit_merge_point', args, None) + return ops + [op1] - def handle_jit_marker__can_enter_jit(self, op, jitdriver): + def handle_jit_marker__can_enter_jit(self, op): return SpaceOperation('can_enter_jit', [], None) def rewrite_op_debug_assert(self, op): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py Wed May 26 18:32:09 2010 @@ -86,6 +86,7 @@ do_call_pure = do_call do_call_loopinvariant = do_call +do_call_may_force = do_call def do_getarrayitem_gc(cpu, _, arraybox, indexbox, arraydescr): array = arraybox.getref_base() Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Wed May 26 18:32:09 2010 @@ -603,13 +603,13 @@ @arguments("jitcode", "boxes") def _opimpl_inline_call1(self, jitcode, argboxes): - self.metainterp.perform_call(jitcode, argboxes) + return self.metainterp.perform_call(jitcode, argboxes) @arguments("jitcode", "boxes2") def _opimpl_inline_call2(self, jitcode, argboxes): - self.metainterp.perform_call(jitcode, argboxes) + return self.metainterp.perform_call(jitcode, argboxes) @arguments("jitcode", "boxes3") def _opimpl_inline_call3(self, jitcode, argboxes): - self.metainterp.perform_call(jitcode, argboxes) + return self.metainterp.perform_call(jitcode, argboxes) opimpl_inline_call_r_i = _opimpl_inline_call1 opimpl_inline_call_r_r = _opimpl_inline_call1 @@ -760,32 +760,6 @@ self.generate_guard(rop.GUARD_CLASS, box, [clsbox], resumepc=orgpc) return clsbox - def verify_green_args(self, varargs): - num_green_args = self.metainterp.staticdata.num_green_args - for i in range(num_green_args): - assert isinstance(varargs[i], Const) - -## def blackhole_reached_merge_point(self, varargs): -## if self.metainterp.in_recursion: -## portal_code = self.metainterp.staticdata.portal_code -## # small hack: fish for the result box -## lenenv = len(self.env) -## raised = self.perform_call(portal_code, varargs) -## # in general this cannot be assumed, but when blackholing, -## # perform_call returns True only if an exception is called. In -## # this case perform_call has called finishframe_exception -## # already, so we need to return. -## if raised: -## return -## if lenenv == len(self.env): -## res = None -## else: -## assert lenenv == len(self.env) - 1 -## res = self.env.pop() -## self.metainterp.finishframe(res) -## else: -## raise self.metainterp.staticdata.ContinueRunningNormally(varargs) - @arguments() def opimpl_can_enter_jit(self): if self.metainterp.in_recursion: @@ -795,7 +769,7 @@ @arguments("orgpc", "boxes3", "boxes3") def opimpl_jit_merge_point(self, orgpc, greenboxes, redboxes): - self.verify_green_args(greenboxes) + self.metainterp.verify_green_args(greenboxes) # xxx we may disable the following line in some context later self.debug_merge_point(greenboxes) if self.metainterp.seen_can_enter_jit: @@ -1022,7 +996,6 @@ effectinfo = descr.get_extra_info() if (effectinfo is None or effectinfo.extraeffect == effectinfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE): - XXX # residual calls require attention to keep virtualizables in-sync self.metainterp.vable_and_vrefs_before_residual_call() # xxx do something about code duplication @@ -1030,12 +1003,10 @@ rop.CALL_MAY_FORCE, allboxes, descr=descr) self.metainterp.vable_and_vrefs_after_residual_call() if resbox is not None: - self.make_result_box(resbox) - self.generate_guard(self.pc, rop.GUARD_NOT_FORCED, None, []) - if exc: - return self.metainterp.handle_exception() - else: - return self.metainterp.assert_no_exception() + self.make_result_of_lastop(resbox) + self.generate_guard(rop.GUARD_NOT_FORCED, None) + self.metainterp.handle_possible_exception() + return resbox else: effect = effectinfo.extraeffect if effect == effectinfo.EF_CANNOT_RAISE: @@ -1248,12 +1219,25 @@ def is_blackholing(self): return False # XXX get rid of this method - def perform_call(self, jitcode, boxes, greenkey=None): + def perform_call(self, jitcode, boxes): # causes the metainterp to enter the given subfunction + # with a special case for recursive portal calls + if jitcode is self.staticdata.portal_code: + return self.perform_recursive_call_to_portal(boxes) + else: + self._perform_call(jitcode, boxes) + # ^^^ always raises + + def _perform_call(self, jitcode, boxes, greenkey=None): f = self.newframe(jitcode, greenkey) f.setup_call(boxes) raise ChangeFrame + def verify_green_args(self, varargs): + num_green_args = self.staticdata.num_green_args + for i in range(num_green_args): + assert isinstance(varargs[i], Const) + def newframe(self, jitcode, greenkey=None): if jitcode is self.staticdata.portal_code: self.in_recursion += 1 @@ -2014,17 +1998,48 @@ max_key = key return max_key - def direct_assembler_call(self, pc, varargs, token, call_position): + def perform_recursive_call_to_portal(self, boxes): + warmrunnerstate = self.staticdata.state + portal_code = self.staticdata.portal_code + token = None + if not self.is_blackholing() and warmrunnerstate.inlining: + num_green_args = self.staticdata.num_green_args + greenkey = boxes[:num_green_args] + if warmrunnerstate.can_inline_callable(greenkey): + return self._perform_call(portal_code, boxes, greenkey) + token = warmrunnerstate.get_assembler_token(greenkey) + call_position = 0 + if token is not None: + call_position = len(self.history.operations) + # verify that we have all green args, needed to make sure + # that assembler that we call is still correct + self.verify_green_args(boxes) + funcbox = ConstInt(llmemory.cast_adr_to_int(portal_code.fnaddr)) + frame = self.framestack[-1] + resbox = frame.do_residual_call(funcbox, portal_code.calldescr, boxes) + if token is not None: + # XXX fix the call position, + while True: + op = self.history.operations[call_position] + if op.opnum == rop.CALL or op.opnum == rop.CALL_MAY_FORCE: + break + call_position += 1 + # + # this will substitute the residual call with assembler call + self.direct_assembler_call(boxes, token, call_position) + return resbox + + def direct_assembler_call(self, boxes, token, call_position): """ Generate a direct call to assembler for portal entry point. """ assert not self.is_blackholing() # XXX num_green_args = self.staticdata.num_green_args - args = varargs[num_green_args + 1:] + args = boxes[num_green_args:] resbox = self.history.operations[call_position].result rest = self.history.slice_history_at(call_position) if self.staticdata.virtualizable_info is not None: vindex = self.staticdata.virtualizable_info.index_of_virtualizable - vbox = args[vindex - num_green_args] + vbox = boxes[vindex] args += self.gen_load_from_other_virtualizable(vbox) self.history.record(rop.CALL_ASSEMBLER, args[:], resbox, descr=token) self.history.operations += rest Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_recursive.py Wed May 26 18:32:09 2010 @@ -24,6 +24,7 @@ return 1 res = self.meta_interp(main, [20], optimizer=OPTIMIZER_SIMPLE) assert res == main(20) + self.check_history(call=0) def test_simple_recursion_with_exc(self): myjitdriver = JitDriver(greens=[], reds=['n', 'm']) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py Wed May 26 18:32:09 2010 @@ -554,6 +554,7 @@ self.PTR_PORTAL_FUNCTYPE.TO, self.PTR_PORTAL_FUNCTYPE.TO.ARGS, self.PTR_PORTAL_FUNCTYPE.TO.RESULT) + self.codewriter.setup_portal_runner_ptr(self.portal_runner_ptr) vinfo = self.metainterp_sd.virtualizable_info From arigo at codespeak.net Wed May 26 18:49:28 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 26 May 2010 18:49:28 +0200 (CEST) Subject: [pypy-svn] r74785 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter metainterp Message-ID: <20100526164928.5D21C282BDE@codespeak.net> Author: arigo Date: Wed May 26 18:49:26 2010 New Revision: 74785 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: Fix test_recursion.test_recursion_three_times. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py Wed May 26 18:49:26 2010 @@ -72,6 +72,7 @@ self.callcontrol.virtualref_info = vrefinfo def setup_portal_runner_ptr(self, portal_runner_ptr): + self.portal_runner_ptr = portal_runner_ptr self.callcontrol.portal_runner_ptr = portal_runner_ptr def find_all_graphs(self, policy): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Wed May 26 18:49:26 2010 @@ -1094,6 +1094,7 @@ self.setup_indirectcalltargets(asm.indirectcalltargets) # self.portal_code = codewriter.mainjitcode + self._portal_runner_ptr = codewriter.portal_runner_ptr RESULT = codewriter.portal_graph.getreturnvar().concretetype self.result_type = history.getkind(RESULT) # @@ -2014,9 +2015,12 @@ # verify that we have all green args, needed to make sure # that assembler that we call is still correct self.verify_green_args(boxes) - funcbox = ConstInt(llmemory.cast_adr_to_int(portal_code.fnaddr)) + # + k = llmemory.cast_ptr_to_adr(self.staticdata._portal_runner_ptr) + funcbox = ConstInt(llmemory.cast_adr_to_int(k)) frame = self.framestack[-1] resbox = frame.do_residual_call(funcbox, portal_code.calldescr, boxes) + # if token is not None: # XXX fix the call position, while True: From jcreigh at codespeak.net Wed May 26 19:06:49 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Wed, 26 May 2010 19:06:49 +0200 (CEST) Subject: [pypy-svn] r74786 - in pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86: . test Message-ID: <20100526170649.8C0DE282BDE@codespeak.net> Author: jcreigh Date: Wed May 26 19:06:47 2010 New Revision: 74786 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_runner.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py Log: fix some more test failures Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Wed May 26 19:06:47 2010 @@ -576,16 +576,15 @@ self.mc.UCOMISD(arglocs[0], arglocs[1]) if guard_opnum == rop.GUARD_FALSE: mc = self.mc._mc - name = 'J' + cond if need_jp: - mc.JP(rel8(6)) - getattr(mc, name)(rel32(addr)) + mc.J_il8(rx86.Conditions['P'], 6) + mc.J_il(rx86.Conditions[cond], addr) return mc.tell() - 4 else: if need_jp: mc = self.mc._mc - mc.JP(rel8(2)) - getattr(mc, 'J' + cond)(rel8(5)) + mc.J_il8(rx86.Conditions['P'], 2) + mc.J_il8(rx86.Conditions[cond], 5) return self.implement_guard(addr) return self.implement_guard(addr, false_cond) return genop_cmp_guard_float @@ -687,12 +686,12 @@ self.mc.UCOMISD(arglocs[0], arglocs[1]) mc = self.mc._mc if guard_opnum == rop.GUARD_TRUE: - mc.JP(rel8(6)) - mc.JE(rel32(addr)) + mc.J_il8(rx86.Conditions['P'], 6) + mc.J_il(rx86.Conditions['E'], addr) return mc.tell() - 4 else: - mc.JP(rel8(2)) - mc.JE(rel8(5)) + mc.J_il8(rx86.Conditions['P'], 2) + mc.J_il8(rx86.Conditions['E'], 5) return self.implement_guard(addr) def genop_float_neg(self, op, arglocs, resloc): @@ -710,12 +709,12 @@ self.mc.UCOMISD(loc0, loc1) mc = self.mc._mc if guard_opnum == rop.GUARD_TRUE: - mc.JP(rel8(6)) - mc.JZ(rel32(addr)) + mc.J_il8(rx86.Conditions['P'], 6) + mc.J_il(rx86.Conditions['Z'], addr) return mc.tell() - 4 else: - mc.JP(rel8(2)) - mc.JZ(rel8(5)) + mc.J_il8(rx86.Conditions['P'], 2) + mc.J_il8(rx86.Conditions['Z'], 5) return self.implement_guard(addr) def genop_float_is_true(self, op, arglocs, resloc): @@ -832,7 +831,7 @@ scale.value)) else: if scale.value == 0: - self.mc.MOVZX(resloc, addr8_add(base_loc, ofs_loc, ofs.value, + self.mc.MOVZX8(resloc, addr8_add(base_loc, ofs_loc, ofs.value, scale.value)) elif scale.value == 2: self.mc.MOV(resloc, addr_add(base_loc, ofs_loc, ofs.value, @@ -1402,7 +1401,7 @@ self._emit_call(x, arglocs, 2, tmp=tmp) if isinstance(resloc, StackLoc) and resloc.width == 8: - self.mc.FSTP(resloc) + self.mc.FSTP_b(resloc.value) elif size == 1: self.mc.AND(eax, imm(0xff)) elif size == 2: @@ -1446,7 +1445,7 @@ mc.overwrite(jmp_location - 1, [chr(offset)]) self._stop_block() if isinstance(result_loc, StackLoc) and result_loc.width == 8: - self.mc.FSTP(result_loc) + self.mc.FSTP_b(result_loc.value) else: assert result_loc is eax or result_loc is None self.mc.CMP_bi(FORCE_INDEX_OFS, 0) @@ -1461,8 +1460,8 @@ assert cls is not None and isinstance(descr, cls) loc_base = arglocs[0] mc = self._start_block() - mc.TEST(mem8(loc_base, descr.jit_wb_if_flag_byteofs), - imm8(descr.jit_wb_if_flag_singlebyte)) + mc.TEST8_mi((loc_base.value, descr.jit_wb_if_flag_byteofs), + descr.jit_wb_if_flag_singlebyte) mc.J_il8(rx86.Conditions['Z'], 0) # patched later jz_location = mc.get_relative_pos() # the following is supposed to be the slow path, so whenever possible @@ -1475,7 +1474,7 @@ mc.CALL_l(descr.get_write_barrier_fn(self.cpu)) for i in range(len(arglocs)): loc = arglocs[i] - assert isinstance(loc, REG) + assert isinstance(loc, RegLoc) mc.POP(loc) # patch the JZ above offset = mc.get_relative_pos() - jz_location @@ -1579,6 +1578,8 @@ def addr_add(reg_or_imm1, reg_or_imm2, offset=0, scale=0): return AddressLoc(reg_or_imm1, reg_or_imm2, scale, offset) +addr64_add = addr_add +addr8_add = addr_add def addr_add_const(reg_or_imm1, offset): return AddressLoc(reg_or_imm1, ImmedLoc(0), 0, offset) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Wed May 26 19:06:47 2010 @@ -81,6 +81,12 @@ def __repr__(self): return "ImmedLoc(%d)" % (self.value) + def lowest8bits(self): + # XXX: Maybe we could just truncate? But I'm not sure when that + # would be the correct behavior. + assert rx86.single_byte(self.value) + return self + class AddressLoc(AssemblerLocation): _immutable_ = True @@ -96,7 +102,7 @@ self.value = base_loc.value + (scaled_loc.value << scale) + static_offset else: self._location_code = 'a' - self.value = (None, scaled_loc.value, scale, static_offset) + self.value = (None, scaled_loc.value, scale, base_loc.value + static_offset) else: if isinstance(scaled_loc, ImmedLoc): # FIXME: What if base_loc is ebp or esp? @@ -134,15 +140,16 @@ getattr(self, name + '_' + loc.location_code())(loc.value) return INSN - ADD = _binaryop('ADD') + AND = _binaryop('AND') OR = _binaryop('OR') XOR = _binaryop('XOR') + NOT = _unaryop('NOT') SHL = _binaryop('SHL') SHR = _binaryop('SHR') SAR = _binaryop('SAR') TEST = _binaryop('TEST') - AND = _binaryop('AND') + ADD = _binaryop('ADD') SUB = _binaryop('SUB') IMUL = _binaryop('IMUL') NEG = _unaryop('NEG') @@ -153,6 +160,9 @@ MOVZX8 = _binaryop('MOVZX8') MOVZX16 = _binaryop('MOVZX16') + PUSH = _unaryop("PUSH") + POP = _unaryop("POP") + LEA = _binaryop('LEA') MOVSD = _binaryop('MOVSD') @@ -162,6 +172,7 @@ DIVSD = _binaryop('DIVSD') UCOMISD = _binaryop('UCOMISD') CVTSI2SD = _binaryop('CVTSI2SD') + CVTTSD2SI = _binaryop('CVTTSD2SI') def CALL(self, loc): @@ -177,14 +188,6 @@ self.writechar('\x66') self.MOV(dest_loc, src_loc) - def PUSH(self, loc): - assert isinstance(loc, RegLoc) - self.PUSH_r(loc.value) - - def POP(self, loc): - assert isinstance(loc, RegLoc) - self.POP_r(loc.value) - def CMPi(self, loc0, loc1): # like CMP, but optimized for the case of loc1 being a Const assert isinstance(loc1, Const) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py Wed May 26 19:06:47 2010 @@ -429,6 +429,9 @@ MOV_ji = insn(rex_w, '\xC7', '\x05', immediate(1), immediate(2)) MOV8_mr = insn(rex_w, '\x88', byte_register(2, 8), mem_reg_plus_const(1)) + MOV8_ar = insn(rex_w, '\x88', byte_register(2, 8), mem_reg_plus_scaled_reg_plus_const(1)) + MOV8_mi = insn(rex_w, '\xC6', orbyte(0<<3), mem_reg_plus_const(1), immediate(2, 'b')) + MOV8_ai = insn(rex_w, '\xC6', orbyte(0<<3), mem_reg_plus_scaled_reg_plus_const(1), immediate(2, 'b')) MOVZX8_rr = insn(rex_w, '\x0F\xB6', register(1,8), byte_register(2), '\xC0') MOVZX8_rm = insn(rex_w, '\x0F\xB6', register(1,8), mem_reg_plus_const(2)) @@ -479,6 +482,9 @@ SHR_ri, SHR_rr = shifts(5) SAR_ri, SAR_rr = shifts(7) + NOT_r = insn(rex_w, '\xF7', register(1), '\xD0') + NOT_b = insn(rex_w, '\xF7', orbyte(2<<3), stack_bp(1)) + # ------------------------------ Misc stuff ------------------------------ NOP = insn('\x90') @@ -515,8 +521,12 @@ # The 64-bit version of this, CQO, is defined in X86_64_CodeBuilder CDQ = insn(rex_nw, '\x99') + TEST8_mi = insn(rex_w, '\xF6', orbyte(0<<3), mem_reg_plus_const(1), immediate(2, 'b')) TEST_rr = insn(rex_w, '\x85', register(2,8), register(1), '\xC0') + # x87 instructions + FSTP_b = insn('\xDD', orbyte(3<<3), stack_bp(1)) + # ------------------------------ SSE2 ------------------------------ MOVSD_rr = xmminsn('\xF2', rex_nw, '\x0F\x10', register(1,8), register(2), @@ -527,8 +537,10 @@ MOVSD_sr = xmminsn('\xF2', rex_nw, '\x0F\x11', register(2,8), stack_sp(1)) MOVSD_rm = xmminsn('\xF2', rex_nw, '\x0F\x10', register(1,8), mem_reg_plus_const(2)) + MOVSD_ra = xmminsn('\xF2', rex_nw, '\x0F\x10', register(1,8), mem_reg_plus_scaled_reg_plus_const(2)) MOVSD_mr = xmminsn('\xF2', rex_nw, '\x0F\x11', register(2,8), mem_reg_plus_const(1)) + MOVSD_ar = xmminsn('\xF2', rex_nw, '\x0F\x11', register(2,8), mem_reg_plus_scaled_reg_plus_const(1)) MOVSD_rj = xmminsn('\xF2', rex_nw, '\x0F\x10', register(1, 8), '\x05', immediate(2)) MOVSD_jr = xmminsn('\xF2', rex_nw, '\x0F\x11', register(2, 8), '\x05', immediate(1)) @@ -557,9 +569,13 @@ # Conversion # FIXME: Super confusing! The source is a GPR/mem, the destination is an xmm register + # (and the same goes for SD2SI too) CVTSI2SD_rr = xmminsn('\xF2', rex_nw, '\x0F\x2A', register(1, 8), register(2), '\xC0') CVTSI2SD_rb = xmminsn('\xF2', rex_nw, '\x0F\x2A', register(1, 8), stack_bp(2)) + CVTTSD2SI_rr = xmminsn('\xF2', rex_nw, '\x0F\x2C', register(1, 8), register(2), '\xC0') + CVTTSD2SI_rb = xmminsn('\xF2', rex_nw, '\x0F\x2C', register(1, 8), stack_bp(2)) + # Bitwise ANDPD_rj = xmminsn('\x66', rex_nw, '\x0F\x54', register(1, 8), '\x05', immediate(2)) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_runner.py Wed May 26 19:06:47 2010 @@ -71,6 +71,9 @@ return ctypes.cast(buf, ctypes.c_void_p).value func = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int)(f) addr = ctypes.cast(func, ctypes.c_void_p).value + # ctypes produces an unsigned value. We need it to be signed for, eg, + # relative addressing to work properly. + addr = rffi.cast(lltype.Signed, addr) self.cpu.assembler.make_sure_mc_exists() self.cpu.assembler.malloc_func_addr = addr Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py Wed May 26 19:06:47 2010 @@ -179,6 +179,14 @@ def test_or8_rr(): assert_encodes_as(CodeBuilder32, 'OR8_rr', (bl, bh), '\x08\xFB') +def test_test8_mi(): + assert_encodes_as(CodeBuilder32, 'TEST8_mi', ((edx, 16), 99), '\xF6\x42\x10\x63') + +def test_mov8(): + cb = CodeBuilder32 + assert_encodes_as(cb, 'MOV8_mi', ((edx, 16), 99), '\xC6\x42\x10\x63') + assert_encodes_as(cb, 'MOV8_ai', ((ebx, ecx, 2, 16), 99), '\xC6\x44\x8B\x10\x63') + class CodeBuilder64(CodeBuilderMixin, X86_64_CodeBuilder): pass Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py Wed May 26 19:06:47 2010 @@ -229,6 +229,9 @@ # Can't test automatically right now, we don't know # which register types to use py.test.skip('Skipping CVT instructions for now') + if methname == 'FSTP_b': + # Doesn't work on 64-bit, skipping for now + py.test.skip('Skipping FSTP') return [args] From arigo at codespeak.net Wed May 26 19:14:21 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 26 May 2010 19:14:21 +0200 (CEST) Subject: [pypy-svn] r74787 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter codewriter/test metainterp metainterp/test Message-ID: <20100526171421.7C09A282BDE@codespeak.net> Author: arigo Date: Wed May 26 19:14:19 2010 New Revision: 74787 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_call.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_pyjitpl.py Log: Progress. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Wed May 26 19:14:19 2010 @@ -4,7 +4,7 @@ from pypy.jit.codewriter.format import format_assembler from pypy.jit.codewriter.jitcode import SwitchDictDescr, JitCode from pypy.objspace.flow.model import Constant -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem import lltype, llmemory, rclass class Assembler(object): @@ -13,8 +13,10 @@ self.insns = {} self.descrs = [] self.indirectcalltargets = set() # set of JitCodes + self.list_of_addr2name = [] self._descr_dict = {} self._count_jitcodes = 0 + self._seen_raw_objects = set() def assemble(self, ssarepr, jitcode=None): """Take the 'ssarepr' representation of the code and assemble @@ -62,6 +64,7 @@ if kind == 'int': if isinstance(TYPE, lltype.Ptr): assert TYPE.TO._gckind == 'raw' + self.see_raw_object(value) value = llmemory.cast_ptr_to_adr(value) TYPE = llmemory.Address if TYPE == llmemory.Address: @@ -207,3 +210,16 @@ liveness=self.liveness, startpoints=self.startpoints, alllabels=self.alllabels) + + def see_raw_object(self, value): + if value._obj not in self._seen_raw_objects: + self._seen_raw_objects.add(value._obj) + TYPE = lltype.typeOf(value).TO + if isinstance(TYPE, lltype.FuncType): + name = value._obj._name + elif TYPE == rclass.OBJECT_VTABLE: + name = ''.join(value.name).rstrip('\x00') + else: + return + addr = llmemory.cast_ptr_to_adr(value) + self.list_of_addr2name.append((addr, name)) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py Wed May 26 19:14:19 2010 @@ -42,11 +42,12 @@ assert self.portal_graph is not None todo = [self.portal_graph] - for oopspec_name, ll_args, ll_res in support.inline_calls_to: - c_func, _ = support.builtin_func_for_spec(self.rtyper, - oopspec_name, - ll_args, ll_res) - todo.append(c_func.value._obj.graph) + if hasattr(self, 'rtyper'): + for oopspec_name, ll_args, ll_res in support.inline_calls_to: + c_func, _ = support.builtin_func_for_spec(self.rtyper, + oopspec_name, + ll_args, ll_res) + todo.append(c_func.value._obj.graph) candidate_graphs = set(todo) def callers(): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py Wed May 26 19:14:19 2010 @@ -72,7 +72,6 @@ self.callcontrol.virtualref_info = vrefinfo def setup_portal_runner_ptr(self, portal_runner_ptr): - self.portal_runner_ptr = portal_runner_ptr self.callcontrol.portal_runner_ptr = portal_runner_ptr def find_all_graphs(self, policy): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_call.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_call.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_call.py Wed May 26 19:14:19 2010 @@ -85,7 +85,8 @@ graph = object() g = object() g1 = object() - cc = CallControl(portal_graph=portal_runner_obj.graph) + cc = CallControl() + cc.portal_runner_ptr = portal_runner_obj cc.candidate_graphs = [g, g1] op = SpaceOperation('direct_call', [Constant(portal_runner_obj)], @@ -144,9 +145,10 @@ translator = None class type_system: name = 'lltypesystem' - def getcallable(self, graph): - F = lltype.FuncType([], lltype.Signed) - return lltype.functionptr(F, 'bar') + @staticmethod + def getcallable(graph): + F = lltype.FuncType([], lltype.Signed) + return lltype.functionptr(F, 'bar') # cc = CallControl(FakeCPU(FakeRTyper())) class somegraph: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py Wed May 26 19:14:19 2010 @@ -31,7 +31,7 @@ class FakePolicy: def look_inside_graph(self, graph): - return True + return graph.name != 'dont_look' def test_loop(): @@ -105,13 +105,14 @@ class A2(A1): id = 652 class B1: id = 661 class B2(B1): id = 662 + def dont_look(n): + return n + 1 def f(n): if n > 5: x, y = A1, B1 else: x, y = A2, B2 - n += 1 - return x().id + y().id + n + return x().id + y().id + dont_look(n) rtyper = support.annotate(f, [35]) maingraph = rtyper.annotator.translator.graphs[0] cw = CodeWriter(FakeCPU(rtyper), maingraph) @@ -127,6 +128,15 @@ else: assert 0, "missing instantiate_*_%s in:\n%r" % (expected, names) + # + print cw.assembler.list_of_addr2name + names = dict.fromkeys([value + for key, value in cw.assembler.list_of_addr2name]) + assert 'A1' in names + assert 'B1' in names + assert 'A2' in names + assert 'B2' in names + assert 'dont_look' in names def test_int_abs(): def f(n): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Wed May 26 19:14:19 2010 @@ -68,6 +68,7 @@ return calldescr is not self._descr_cannot_raise def found_jitdriver(self, jitdriver): assert isinstance(jitdriver, JitDriver) + self.jitdriver = jitdriver class FakeCallControlWithVRefInfo: class virtualref_info: Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Wed May 26 19:14:19 2010 @@ -1084,6 +1084,10 @@ def setup_indirectcalltargets(self, indirectcalltargets): self.indirectcalltargets = list(indirectcalltargets) + def setup_list_of_addr2name(self, list_of_addr2name): + self._addr2name_keys = [key for key, value in list_of_addr2name] + self._addr2name_values = [value for key, value in list_of_addr2name] + def finish_setup(self, codewriter, optimizer=None): from pypy.jit.metainterp.blackhole import BlackholeInterpBuilder self.blackholeinterpbuilder = BlackholeInterpBuilder(codewriter, self) @@ -1092,9 +1096,10 @@ self.setup_insns(asm.insns) self.setup_descrs(asm.descrs) self.setup_indirectcalltargets(asm.indirectcalltargets) + self.setup_list_of_addr2name(asm.list_of_addr2name) # self.portal_code = codewriter.mainjitcode - self._portal_runner_ptr = codewriter.portal_runner_ptr + self._portal_runner_ptr = codewriter.callcontrol.portal_runner_ptr RESULT = codewriter.portal_graph.getreturnvar().concretetype self.result_type = history.getkind(RESULT) # Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_pyjitpl.py Wed May 26 19:14:19 2010 @@ -86,3 +86,13 @@ assert equaloplists(metainterp.history.operations, [ ResOperation(rop.SAME_AS, [b2], boxes[0]), ]) + +def test_get_name_from_address(): + class FakeMetaInterpSd(pyjitpl.MetaInterpStaticData): + def __init__(self): + pass + metainterp_sd = FakeMetaInterpSd() + metainterp_sd.setup_list_of_addr2name([(123, 'a'), (456, 'b')]) + assert metainterp_sd.get_name_from_address(123) == 'a' + assert metainterp_sd.get_name_from_address(456) == 'b' + assert metainterp_sd.get_name_from_address(789) == '' From arigo at codespeak.net Wed May 26 19:17:15 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 26 May 2010 19:17:15 +0200 (CEST) Subject: [pypy-svn] r74788 - pypy/branch/blackhole-improvement/pypy/jit/codewriter Message-ID: <20100526171715.4AEC6282BDE@codespeak.net> Author: arigo Date: Wed May 26 19:17:13 2010 New Revision: 74788 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Log: Filter out NULL pointers. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Wed May 26 19:17:13 2010 @@ -214,6 +214,8 @@ def see_raw_object(self, value): if value._obj not in self._seen_raw_objects: self._seen_raw_objects.add(value._obj) + if not value: # filter out NULL pointers + return TYPE = lltype.typeOf(value).TO if isinstance(TYPE, lltype.FuncType): name = value._obj._name From jcreigh at codespeak.net Wed May 26 19:34:28 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Wed, 26 May 2010 19:34:28 +0200 (CEST) Subject: [pypy-svn] r74789 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86 Message-ID: <20100526173428.BA8E6282BDE@codespeak.net> Author: jcreigh Date: Wed May 26 19:34:27 2010 New Revision: 74789 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Log: define lowest8bits on ImmedLoc to behave the same as ri386 IMM32 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Wed May 26 19:34:27 2010 @@ -82,10 +82,10 @@ return "ImmedLoc(%d)" % (self.value) def lowest8bits(self): - # XXX: Maybe we could just truncate? But I'm not sure when that - # would be the correct behavior. - assert rx86.single_byte(self.value) - return self + val = self.value & 0xFF + if val > 0x7F: + val -= 0x100 + return ImmedLoc(val) class AddressLoc(AssemblerLocation): _immutable_ = True From afa at codespeak.net Wed May 26 19:46:34 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 26 May 2010 19:46:34 +0200 (CEST) Subject: [pypy-svn] r74790 - in pypy/trunk/pypy/module/cpyext: . include test Message-ID: <20100526174634.4E776282BDE@codespeak.net> Author: afa Date: Wed May 26 19:46:33 2010 New Revision: 74790 Modified: pypy/trunk/pypy/module/cpyext/include/pymem.h pypy/trunk/pypy/module/cpyext/object.py pypy/trunk/pypy/module/cpyext/pyobject.py pypy/trunk/pypy/module/cpyext/test/test_bufferobject.py Log: Really implement PyObject_Init(), independently from PyObject_New(). PyObject_Init() initializes an allocated structure PyObject_New() allocates memory and calls PyObject_Init(). Some modules allocate memory themselves and use PyObject_Init() directly. Modified: pypy/trunk/pypy/module/cpyext/include/pymem.h ============================================================================== --- pypy/trunk/pypy/module/cpyext/include/pymem.h (original) +++ pypy/trunk/pypy/module/cpyext/include/pymem.h Wed May 26 19:46:33 2010 @@ -5,12 +5,6 @@ #define PyMem_REALLOC(p, n) realloc((p), (n) ? (n) : 1) #define PyMem_FREE free -/* XXX use obmalloc like cpython and pypy do, otherwise we might get segfaults */ -#define PyObject_MALLOC PyMem_MALLOC -// we won't support this -// #define PyObject_REALLOC PyMem_REALLOC -#define PyObject_FREE PyMem_FREE - #define PyMem_Malloc PyMem_MALLOC #define PyMem_Free PyMem_FREE #define PyMem_Realloc PyMem_REALLOC Modified: pypy/trunk/pypy/module/cpyext/object.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/object.py (original) +++ pypy/trunk/pypy/module/cpyext/object.py Wed May 26 19:46:33 2010 @@ -5,7 +5,7 @@ Py_GE, CONST_STRING, FILEP, fwrite) from pypy.module.cpyext.pyobject import ( PyObject, PyObjectP, create_ref, from_ref, Py_IncRef, Py_DecRef, - track_reference) + track_reference, get_typedescr) from pypy.module.cpyext.typeobject import PyTypeObjectPtr, W_PyCTypeObject from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall from pypy.objspace.std.objectobject import W_ObjectObject @@ -14,19 +14,32 @@ import pypy.module.__builtin__.operation as operation + at cpython_api([Py_ssize_t], rffi.VOIDP, error=lltype.nullptr(rffi.VOIDP.TO)) +def PyObject_MALLOC(space, size): + return lltype.malloc(rffi.VOIDP.TO, size, + flavor='raw', zero=True) + + at cpython_api([rffi.VOIDP], lltype.Void) +def PyObject_FREE(space, ptr): + return lltype.free(ptr, flavor='raw') + @cpython_api([PyTypeObjectPtr], PyObject) def _PyObject_New(space, type): return _PyObject_NewVar(space, type, 0) @cpython_api([PyTypeObjectPtr, Py_ssize_t], PyObject) -def _PyObject_NewVar(space, type, size): +def _PyObject_NewVar(space, type, itemcount): w_type = from_ref(space, rffi.cast(PyObject, type)) - if isinstance(w_type, W_PyCTypeObject): - w_obj = space.allocate_instance(W_ObjectObject, w_type) - py_obj = create_ref(space, w_obj, items=size) - track_reference(space, py_obj, w_obj) - return py_obj - assert False, "Please add more cases in _PyObject_New" + assert isinstance(w_type, W_TypeObject) + typedescr = get_typedescr(w_type.instancetypedef) + py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) + py_obj.c_ob_refcnt = 0 + if type.c_tp_itemsize == 0: + w_obj = PyObject_Init(space, py_obj, type) + else: + py_objvar = rffi.cast(PyVarObject, py_obj) + w_obj = PyObject_InitVar(space, py_objvar, type, itemcount) + return py_obj @cpython_api([rffi.VOIDP_real], lltype.Void) def PyObject_Del(space, obj): @@ -171,28 +184,34 @@ return 0 @cpython_api([PyObject, PyTypeObjectPtr], PyObject) -def PyObject_Init(space, op, type): +def PyObject_Init(space, py_obj, type): """Initialize a newly-allocated object op with its type and initial reference. Returns the initialized object. If type indicates that the object participates in the cyclic garbage detector, it is added to the detector's set of observed objects. Other fields of the object are not affected.""" - if not op: + if not py_obj: PyErr_NoMemory(space) - op.c_ob_type = type - op.c_ob_refcnt = 1 - return from_ref(space, op) # XXX will give an exception + py_obj.c_ob_type = type + py_obj.c_ob_refcnt = 1 + w_type = from_ref(space, rffi.cast(PyObject, type)) + if isinstance(w_type, W_PyCTypeObject): + w_obj = space.allocate_instance(W_ObjectObject, w_type) + track_reference(space, py_obj, w_obj) + from pypy.module.cpyext.typeobject import lifeline_dict, PyOLifeline + lifeline_dict.set(w_obj, PyOLifeline(space, py_obj)) + else: + assert False, "Please add more cases in PyObject_Init" + return py_obj @cpython_api([PyVarObject, PyTypeObjectPtr, Py_ssize_t], PyObject) -def PyObject_InitVar(space, op, type, size): +def PyObject_InitVar(space, py_obj, type, size): """This does everything PyObject_Init() does, and also initializes the length information for a variable-size object.""" - if not op: + if not py_obj: PyErr_NoMemory(space) - op.c_ob_size = size - op.c_ob_type = type - op.c_ob_refcnt = 1 - return from_ref(space, rffi.cast(PyObject, op)) # XXX likewise + py_obj.c_ob_size = size + return PyObject_Init(space, rffi.cast(PyObject, py_obj), type) @cpython_api([PyObject], PyObject) def PyObject_Type(space, w_obj): Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Wed May 26 19:46:33 2010 @@ -342,11 +342,9 @@ state = space.fromcache(RefcountState) ptr = rffi.cast(ADDR, obj) if ptr not in state.py_objects_r2w: - w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type)) - if space.is_w(w_type, space.w_str) or space.is_w(w_type, space.w_unicode): - # this is a half-allocated string, lets call the deallocator - # without modifying the r2w/w2r dicts - _Py_Dealloc(space, obj) + # this is a half-allocated object, lets call the deallocator + # without modifying the r2w/w2r dicts + _Py_Dealloc(space, obj) else: w_obj = state.py_objects_r2w[ptr] del state.py_objects_r2w[ptr] Modified: pypy/trunk/pypy/module/cpyext/test/test_bufferobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_bufferobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_bufferobject.py Wed May 26 19:46:33 2010 @@ -38,3 +38,12 @@ assert module.check_ascharbuffer(buf) == 'abc\0' module.free_buffer() + + def test_Buffer_New(self): + module = self.import_extension('foo', [ + ("buffer_new", "METH_NOARGS", + """ + return PyBuffer_New(150); + """), + ]) + module.buffer_new() From afa at codespeak.net Wed May 26 20:02:42 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 26 May 2010 20:02:42 +0200 (CEST) Subject: [pypy-svn] r74791 - pypy/trunk/pypy/module/cpyext Message-ID: <20100526180242.50F18282BE0@codespeak.net> Author: afa Date: Wed May 26 20:02:40 2010 New Revision: 74791 Modified: pypy/trunk/pypy/module/cpyext/object.py Log: Fix translation, maybe Modified: pypy/trunk/pypy/module/cpyext/object.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/object.py (original) +++ pypy/trunk/pypy/module/cpyext/object.py Wed May 26 20:02:40 2010 @@ -21,7 +21,7 @@ @cpython_api([rffi.VOIDP], lltype.Void) def PyObject_FREE(space, ptr): - return lltype.free(ptr, flavor='raw') + lltype.free(ptr, flavor='raw') @cpython_api([PyTypeObjectPtr], PyObject) def _PyObject_New(space, type): From jcreigh at codespeak.net Wed May 26 20:29:52 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Wed, 26 May 2010 20:29:52 +0200 (CEST) Subject: [pypy-svn] r74792 - in pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86: . test Message-ID: <20100526182952.7587F282BE0@codespeak.net> Author: jcreigh Date: Wed May 26 20:29:49 2010 New Revision: 74792 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py Log: add new 'x' code to rx86 to indicate an xmm register instead of using 'r'. This makes automated testing easier and makes instructions like CVTSI2SD_xr clearer Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Wed May 26 20:29:49 2010 @@ -202,7 +202,7 @@ self.malloc_fixedsize_slowpath1 = mc.tell() if self.cpu.supports_floats: # save the XMM registers in for i in range(8): # the *caller* frame, from esp+8 - mc.MOVSD_sr(8+8*i, i) + mc.MOVSD_sx(8+8*i, i) mc.SUB(edx, eax) # compute the size we want mc.MOV_sr(4, edx.value) # save it as the new argument addr = self.cpu.gc_ll_descr.get_malloc_fixedsize_slowpath_addr() @@ -211,7 +211,7 @@ self.malloc_fixedsize_slowpath2 = mc.tell() if self.cpu.supports_floats: # restore the XMM registers for i in range(8): # from where they were saved - mc.MOVSD_rs(i, 8+8*i) + mc.MOVSD_xs(i, 8+8*i) nursery_free_adr = self.cpu.gc_ll_descr.get_nursery_free_addr() mc.MOV(edx, heap(nursery_free_adr)) # load this in EDX mc.RET() @@ -370,7 +370,7 @@ loc = floatlocs[i] if isinstance(loc, RegLoc): assert loc.is_xmm - self.mc.MOVSD_rb(loc.value, (1 + i) * 2 * WORD) + self.mc.MOVSD_xb(loc.value, (1 + i) * 2 * WORD) tmp = eax xmmtmp = xmm0 for i in range(len(nonfloatlocs)): @@ -380,9 +380,9 @@ self.mc.MOV(loc, tmp) loc = floatlocs[i] if loc is not None and not isinstance(loc, RegLoc): - self.mc.MOVSD_rb(xmmtmp.value, (1 + i) * 2 * WORD) + self.mc.MOVSD_xb(xmmtmp.value, (1 + i) * 2 * WORD) assert isinstance(loc, StackLoc) - self.mc.MOVSD_br(loc.value, xmmtmp.value) + self.mc.MOVSD_bx(loc.value, xmmtmp.value) self.mc.JMP_l(jmpadr) return adr_stackadjust @@ -417,11 +417,11 @@ continue adr = self.fail_boxes_float.get_addr_for_num(i) if isinstance(loc, RegLoc): - self.mc.MOVSD_rj(loc.value, adr) + self.mc.MOVSD_xj(loc.value, adr) else: - self.mc.MOVSD_rj(xmmtmp.value, adr) + self.mc.MOVSD_xj(xmmtmp.value, adr) assert isinstance(loc, StackLoc) - self.mc.MOVSD_br(loc.value, xmmtmp.value) + self.mc.MOVSD_bx(loc.value, xmmtmp.value) return adr_stackadjust def dump(self, text): @@ -464,7 +464,7 @@ def regalloc_push(self, loc): if isinstance(loc, RegLoc) and loc.is_xmm: self.mc.SUB_ri(esp.value, 2*WORD) - self.mc.MOVSD_sr(0, loc.value) + self.mc.MOVSD_sx(0, loc.value) elif isinstance(loc, StackLoc) and loc.width == 8: # XXX evil trick self.mc.PUSH_b(get_ebp_ofs(loc.position)) @@ -474,7 +474,7 @@ def regalloc_pop(self, loc): if isinstance(loc, RegLoc) and loc.is_xmm: - self.mc.MOVSD_rs(loc.value, 0) + self.mc.MOVSD_xs(loc.value, 0) self.mc.ADD(esp, imm(2*WORD)) elif isinstance(loc, StackLoc) and loc.width == 8: # XXX evil trick @@ -600,7 +600,7 @@ loc = arglocs[i] if isinstance(loc, RegLoc): if loc.is_xmm: - mc.MOVSD_sr(p, loc.value) + mc.MOVSD_sx(p, loc.value) else: mc.MOV_sr(p, loc.value) p += round_up_to_4(loc.width) @@ -610,7 +610,7 @@ if not isinstance(loc, RegLoc): if loc.width == 8: mc.MOVSD(xmm0, loc) - mc.MOVSD_sr(p, xmm0.value) + mc.MOVSD_sx(p, xmm0.value) else: mc.MOV(tmp, loc) mc.MOV_sr(p, tmp.value) @@ -696,16 +696,16 @@ def genop_float_neg(self, op, arglocs, resloc): # Following what gcc does: res = x ^ 0x8000000000000000 - self.mc.XORPD_rj(arglocs[0].value, self.loc_float_const_neg) + self.mc.XORPD_xj(arglocs[0].value, self.loc_float_const_neg) def genop_float_abs(self, op, arglocs, resloc): # Following what gcc does: res = x & 0x7FFFFFFFFFFFFFFF - self.mc.ANDPD_rj(arglocs[0].value, self.loc_float_const_abs) + self.mc.ANDPD_xj(arglocs[0].value, self.loc_float_const_abs) def genop_guard_float_is_true(self, op, guard_op, addr, arglocs, resloc): guard_opnum = guard_op.opnum loc0, loc1 = arglocs - self.mc.XORPD_rr(loc0.value, loc0.value) + self.mc.XORPD_xx(loc0.value, loc0.value) self.mc.UCOMISD(loc0, loc1) mc = self.mc._mc if guard_opnum == rop.GUARD_TRUE: @@ -718,7 +718,7 @@ return self.implement_guard(addr) def genop_float_is_true(self, op, arglocs, resloc): - self.mc.XORPD_rr(arglocs[0].value, arglocs[0].value) + self.mc.XORPD_xx(arglocs[0].value, arglocs[0].value) self.genop_float_ne(op, arglocs, resloc) def genop_cast_float_to_int(self, op, arglocs, resloc): @@ -1294,7 +1294,7 @@ if withfloats: mc.SUB_ri(esp.value, 8*8) for i in range(8): - mc.MOVSD_sr(8*i, i) + mc.MOVSD_sx(8*i, i) # we call a provided function that will # - call our on_leave_jitted_hook which will mark @@ -1335,7 +1335,7 @@ if isinstance(loc, RegLoc): if loc.width == 8: adr = self.fail_boxes_float.get_addr_for_num(i) - mc.MOVSD_jr(adr, loc.value) + mc.MOVSD_jx(adr, loc.value) else: if locs_are_ref[i]: adr = self.fail_boxes_ptr.get_addr_for_num(i) @@ -1347,9 +1347,9 @@ if not isinstance(loc, RegLoc): if loc.width == 8: assert isinstance(loc, StackLoc) - mc.MOVSD_rb(xmm0.value, loc.value) + mc.MOVSD_xb(xmm0.value, loc.value) adr = self.fail_boxes_float.get_addr_for_num(i) - mc.MOVSD_jr(adr, xmm0.value) + mc.MOVSD_jx(adr, xmm0.value) else: if locs_are_ref[i]: adr = self.fail_boxes_ptr.get_addr_for_num(i) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Wed May 26 20:29:49 2010 @@ -59,7 +59,10 @@ return RegLoc(rx86.high_byte(self.value), False) def location_code(self): - return 'r' + if self.is_xmm: + return 'x' + else: + return 'r' # FIXME: This definition of assembler sufficient? def assembler(self): Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py Wed May 26 20:29:49 2010 @@ -529,58 +529,56 @@ # ------------------------------ SSE2 ------------------------------ - MOVSD_rr = xmminsn('\xF2', rex_nw, '\x0F\x10', register(1,8), register(2), + MOVSD_xx = xmminsn('\xF2', rex_nw, '\x0F\x10', register(1,8), register(2), '\xC0') - MOVSD_rb = xmminsn('\xF2', rex_nw, '\x0F\x10', register(1,8), stack_bp(2)) - MOVSD_br = xmminsn('\xF2', rex_nw, '\x0F\x11', register(2,8), stack_bp(1)) - MOVSD_rs = xmminsn('\xF2', rex_nw, '\x0F\x10', register(1,8), stack_sp(2)) - MOVSD_sr = xmminsn('\xF2', rex_nw, '\x0F\x11', register(2,8), stack_sp(1)) - MOVSD_rm = xmminsn('\xF2', rex_nw, '\x0F\x10', register(1,8), + MOVSD_xb = xmminsn('\xF2', rex_nw, '\x0F\x10', register(1,8), stack_bp(2)) + MOVSD_bx = xmminsn('\xF2', rex_nw, '\x0F\x11', register(2,8), stack_bp(1)) + MOVSD_xs = xmminsn('\xF2', rex_nw, '\x0F\x10', register(1,8), stack_sp(2)) + MOVSD_sx = xmminsn('\xF2', rex_nw, '\x0F\x11', register(2,8), stack_sp(1)) + MOVSD_xm = xmminsn('\xF2', rex_nw, '\x0F\x10', register(1,8), mem_reg_plus_const(2)) - MOVSD_ra = xmminsn('\xF2', rex_nw, '\x0F\x10', register(1,8), mem_reg_plus_scaled_reg_plus_const(2)) - MOVSD_mr = xmminsn('\xF2', rex_nw, '\x0F\x11', register(2,8), + MOVSD_xa = xmminsn('\xF2', rex_nw, '\x0F\x10', register(1,8), mem_reg_plus_scaled_reg_plus_const(2)) + MOVSD_mx = xmminsn('\xF2', rex_nw, '\x0F\x11', register(2,8), mem_reg_plus_const(1)) - MOVSD_ar = xmminsn('\xF2', rex_nw, '\x0F\x11', register(2,8), mem_reg_plus_scaled_reg_plus_const(1)) + MOVSD_ax = xmminsn('\xF2', rex_nw, '\x0F\x11', register(2,8), mem_reg_plus_scaled_reg_plus_const(1)) - MOVSD_rj = xmminsn('\xF2', rex_nw, '\x0F\x10', register(1, 8), '\x05', immediate(2)) - MOVSD_jr = xmminsn('\xF2', rex_nw, '\x0F\x11', register(2, 8), '\x05', immediate(1)) + MOVSD_xj = xmminsn('\xF2', rex_nw, '\x0F\x10', register(1, 8), '\x05', immediate(2)) + MOVSD_jx = xmminsn('\xF2', rex_nw, '\x0F\x11', register(2, 8), '\x05', immediate(1)) # Arithmetic - ADDSD_rr = xmminsn('\xF2', rex_nw, '\x0F\x58', register(1, 8), register(2), '\xC0') - ADDSD_rb = xmminsn('\xF2', rex_nw, '\x0F\x58', register(1, 8), stack_bp(2)) - ADDSD_rj = xmminsn('\xF2', rex_nw, '\x0F\x58', register(1, 8), '\x05', immediate(2)) - - SUBSD_rr = xmminsn('\xF2', rex_nw, '\x0F\x5C', register(1, 8), register(2), '\xC0') - SUBSD_rb = xmminsn('\xF2', rex_nw, '\x0F\x5C', register(1, 8), stack_bp(2)) - SUBSD_rj = xmminsn('\xF2', rex_nw, '\x0F\x5C', register(1, 8), '\x05', immediate(2)) - - MULSD_rr = xmminsn('\xF2', rex_nw, '\x0F\x59', register(1, 8), register(2), '\xC0') - MULSD_rb = xmminsn('\xF2', rex_nw, '\x0F\x59', register(1, 8), stack_bp(2)) - MULSD_rj = xmminsn('\xF2', rex_nw, '\x0F\x59', register(1, 8), '\x05', immediate(2)) - - DIVSD_rr = xmminsn('\xF2', rex_nw, '\x0F\x5E', register(1, 8), register(2), '\xC0') - DIVSD_rb = xmminsn('\xF2', rex_nw, '\x0F\x5E', register(1, 8), stack_bp(2)) - DIVSD_rj = xmminsn('\xF2', rex_nw, '\x0F\x5E', register(1, 8), '\x05', immediate(2)) + ADDSD_xx = xmminsn('\xF2', rex_nw, '\x0F\x58', register(1, 8), register(2), '\xC0') + ADDSD_xb = xmminsn('\xF2', rex_nw, '\x0F\x58', register(1, 8), stack_bp(2)) + ADDSD_xj = xmminsn('\xF2', rex_nw, '\x0F\x58', register(1, 8), '\x05', immediate(2)) + + SUBSD_xx = xmminsn('\xF2', rex_nw, '\x0F\x5C', register(1, 8), register(2), '\xC0') + SUBSD_xb = xmminsn('\xF2', rex_nw, '\x0F\x5C', register(1, 8), stack_bp(2)) + SUBSD_xj = xmminsn('\xF2', rex_nw, '\x0F\x5C', register(1, 8), '\x05', immediate(2)) + + MULSD_xx = xmminsn('\xF2', rex_nw, '\x0F\x59', register(1, 8), register(2), '\xC0') + MULSD_xb = xmminsn('\xF2', rex_nw, '\x0F\x59', register(1, 8), stack_bp(2)) + MULSD_xj = xmminsn('\xF2', rex_nw, '\x0F\x59', register(1, 8), '\x05', immediate(2)) + + DIVSD_xx = xmminsn('\xF2', rex_nw, '\x0F\x5E', register(1, 8), register(2), '\xC0') + DIVSD_xb = xmminsn('\xF2', rex_nw, '\x0F\x5E', register(1, 8), stack_bp(2)) + DIVSD_xj = xmminsn('\xF2', rex_nw, '\x0F\x5E', register(1, 8), '\x05', immediate(2)) # Comparision - UCOMISD_rr = xmminsn('\x66', rex_nw, '\x0F\x2E', register(1, 8), register(2), '\xC0') - UCOMISD_rb = xmminsn('\x66', rex_nw, '\x0F\x2E', register(1, 8), stack_bp(2)) - UCOMISD_rj = xmminsn('\x66', rex_nw, '\x0F\x2E', register(1, 8), '\x05', immediate(2)) + UCOMISD_xx = xmminsn('\x66', rex_nw, '\x0F\x2E', register(1, 8), register(2), '\xC0') + UCOMISD_xb = xmminsn('\x66', rex_nw, '\x0F\x2E', register(1, 8), stack_bp(2)) + UCOMISD_xj = xmminsn('\x66', rex_nw, '\x0F\x2E', register(1, 8), '\x05', immediate(2)) # Conversion - # FIXME: Super confusing! The source is a GPR/mem, the destination is an xmm register - # (and the same goes for SD2SI too) - CVTSI2SD_rr = xmminsn('\xF2', rex_nw, '\x0F\x2A', register(1, 8), register(2), '\xC0') - CVTSI2SD_rb = xmminsn('\xF2', rex_nw, '\x0F\x2A', register(1, 8), stack_bp(2)) + CVTSI2SD_xr = xmminsn('\xF2', rex_w, '\x0F\x2A', register(1, 8), register(2), '\xC0') + CVTSI2SD_xb = xmminsn('\xF2', rex_w, '\x0F\x2A', register(1, 8), stack_bp(2)) - CVTTSD2SI_rr = xmminsn('\xF2', rex_nw, '\x0F\x2C', register(1, 8), register(2), '\xC0') - CVTTSD2SI_rb = xmminsn('\xF2', rex_nw, '\x0F\x2C', register(1, 8), stack_bp(2)) + CVTTSD2SI_rx = xmminsn('\xF2', rex_w, '\x0F\x2C', register(1, 8), register(2), '\xC0') + CVTTSD2SI_rb = xmminsn('\xF2', rex_w, '\x0F\x2C', register(1, 8), stack_bp(2)) # Bitwise - ANDPD_rj = xmminsn('\x66', rex_nw, '\x0F\x54', register(1, 8), '\x05', immediate(2)) + ANDPD_xj = xmminsn('\x66', rex_nw, '\x0F\x54', register(1, 8), '\x05', immediate(2)) - XORPD_rr = xmminsn('\x66', rex_nw, '\x0F\x57', register(1, 8), register(2), '\xC0') - XORPD_rj = xmminsn('\x66', rex_nw, '\x0F\x57', register(1, 8), '\x05', immediate(2)) + XORPD_xx = xmminsn('\x66', rex_nw, '\x0F\x57', register(1, 8), register(2), '\xC0') + XORPD_xj = xmminsn('\x66', rex_nw, '\x0F\x57', register(1, 8), '\x05', immediate(2)) # ------------------------------------------------------------ @@ -656,23 +654,23 @@ py.test.skip("CMP_ji unsupported") def CMP_rj(self, reg, immed): py.test.skip("CMP_rj unsupported") - def MOVSD_rj(self, xmm_reg, mem_immed): + def MOVSD_xj(self, xmm_reg, mem_immed): py.test.skip("MOVSD_rj unsupported") - def MOVSD_jr(self, xmm_reg, mem_immed): + def MOVSD_jx(self, xmm_reg, mem_immed): py.test.skip("MOVSD_jr unsupported") - def ADDSD_rj(self, xmm_reg, mem_immed): + def ADDSD_xj(self, xmm_reg, mem_immed): py.test.skip("ADDSD_rj unsupported") - def SUBSD_rj(self, xmm_reg, mem_immed): + def SUBSD_xj(self, xmm_reg, mem_immed): py.test.skip("SUBSD_rj unsupported") - def MULSD_rj(self, xmm_reg, mem_immed): + def MULSD_xj(self, xmm_reg, mem_immed): py.test.skip("MULSD_rj unsupported") - def DIVSD_rj(self, xmm_reg, mem_immed): + def DIVSD_xj(self, xmm_reg, mem_immed): py.test.skip("DIVSD_rj unsupported") - def UCOMISD_rj(self, xmm_reg, mem_immed): + def UCOMISD_xj(self, xmm_reg, mem_immed): py.test.skip("UCOMISD_rj unsupported") - def ANDPD_rj(self, xmm_reg, mem_immed): + def ANDPD_xj(self, xmm_reg, mem_immed): py.test.skip("ANDPD_rj unsupported") - def XORPD_rj(self, xmm_reg, mem_immed): + def XORPD_xj(self, xmm_reg, mem_immed): py.test.skip("XORPD_rj unsupported") Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py Wed May 26 20:29:49 2010 @@ -131,7 +131,7 @@ def test_movsd_rj(): s = CodeBuilder32() - s.MOVSD_rj(xmm2, 0x01234567) + s.MOVSD_xj(xmm2, 0x01234567) assert s.getvalue() == '\xF2\x0F\x10\x15\x67\x45\x23\x01' def test_movzx8_rm(): Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py Wed May 26 20:29:49 2010 @@ -52,6 +52,9 @@ def reg_tests(self): return self.REGS + def xmm_reg_tests(self): + return self.reg_tests() + def stack_bp_tests(self, count=COUNT1): return ([0, 4, -4, 124, 128, -128, -132] + [random.randrange(-0x20000000, 0x20000000) * 4 @@ -94,6 +97,7 @@ def get_all_tests(self): return { 'r': self.reg_tests, + 'x': self.xmm_reg_tests, 'b': self.stack_bp_tests, 's': self.stack_sp_tests, 'm': self.memory_tests, @@ -104,10 +108,10 @@ } def assembler_operand_reg(self, regnum): - if self.is_xmm_insn: - return self.XMMREGNAMES[regnum] - else: - return self.REGNAMES[regnum] + return self.REGNAMES[regnum] + + def assembler_operand_xmm_reg(self, regnum): + return self.XMMREGNAMES[regnum] def assembler_operand_stack_bp(self, position): return '%d(%s)' % (position, self.REGNAMES[5]) @@ -133,6 +137,7 @@ def get_all_assembler_operands(self): return { 'r': self.assembler_operand_reg, + 'x': self.assembler_operand_xmm_reg, 'b': self.assembler_operand_stack_bp, 's': self.assembler_operand_stack_sp, 'm': self.assembler_operand_memory, @@ -158,6 +163,11 @@ ## suffix = suffixes[sizes[m]] + suffix if argmodes and not self.is_xmm_insn: suffix = suffixes[self.WORD] + # Special case: On 64-bit CPUs, rx86 assumes 64-bit integer + # operands when converting to/from floating point, so we need to + # indicate that with a suffix + if (self.WORD == 8) and instrname.startswith('CVT'): + suffix = suffixes[self.WORD] following = "" if False: # instr.indirect: @@ -225,10 +235,6 @@ if methname.startswith('SHL') or methname.startswith('SAR') or methname.startswith('SHR'): # XXX: Would be nice to test these automatically py.test.skip('Shifts must be tested manually') - if methname.startswith('CVT'): - # Can't test automatically right now, we don't know - # which register types to use - py.test.skip('Skipping CVT instructions for now') if methname == 'FSTP_b': # Doesn't work on 64-bit, skipping for now py.test.skip('Skipping FSTP') From fijal at codespeak.net Wed May 26 21:31:17 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 26 May 2010 21:31:17 +0200 (CEST) Subject: [pypy-svn] r74793 - in pypy/trunk/pypy/objspace/std: . test Message-ID: <20100526193117.246C8282BE0@codespeak.net> Author: fijal Date: Wed May 26 21:31:15 2010 New Revision: 74793 Modified: pypy/trunk/pypy/objspace/std/floattype.py pypy/trunk/pypy/objspace/std/strutil.py pypy/trunk/pypy/objspace/std/test/test_floatobject.py pypy/trunk/pypy/objspace/std/test/test_strutil.py Log: * Fix issue 534 by special casing -inf inf nan and -nan (not sure if that's enough though) when doing float(string) * Remove unused code (the comment said "for reference", but I don't see a point) * Run tests over code that we use and not test unused functions Modified: pypy/trunk/pypy/objspace/std/floattype.py ============================================================================== --- pypy/trunk/pypy/objspace/std/floattype.py (original) +++ pypy/trunk/pypy/objspace/std/floattype.py Wed May 26 21:31:15 2010 @@ -1,21 +1,16 @@ from pypy.interpreter import gateway from pypy.interpreter.error import OperationError from pypy.objspace.std.stdtypedef import StdTypeDef -from pypy.objspace.std.strutil import string_to_float, ParseStringError +from pypy.objspace.std.strutil import ParseStringError from pypy.objspace.std.strutil import interp_string_to_float -USE_NEW_S2F = True - def descr__new__(space, w_floattype, w_x=0.0): from pypy.objspace.std.floatobject import W_FloatObject w_value = w_x # 'x' is the keyword argument name in CPython if space.is_true(space.isinstance(w_value, space.w_str)): strvalue = space.str_w(w_value) try: - if USE_NEW_S2F: - value = interp_string_to_float(space, strvalue) - else: - value = string_to_float(strvalue) + value = interp_string_to_float(space, strvalue) except ParseStringError, e: raise OperationError(space.w_ValueError, space.wrap(e.msg)) @@ -26,10 +21,7 @@ from unicodeobject import unicode_to_decimal_w strvalue = unicode_to_decimal_w(space, w_value) try: - if USE_NEW_S2F: - value = interp_string_to_float(space, strvalue) - else: - value = string_to_float(strvalue) + value = interp_string_to_float(space, strvalue) except ParseStringError, e: raise OperationError(space.w_ValueError, space.wrap(e.msg)) Modified: pypy/trunk/pypy/objspace/std/strutil.py ============================================================================== --- pypy/trunk/pypy/objspace/std/strutil.py (original) +++ pypy/trunk/pypy/objspace/std/strutil.py Wed May 26 21:31:15 2010 @@ -2,7 +2,8 @@ Pure Python implementation of string utilities. """ -from pypy.rlib.rarithmetic import ovfcheck, break_up_float, parts_to_float +from pypy.rlib.rarithmetic import ovfcheck, break_up_float, parts_to_float,\ + INFINITY, NAN from pypy.interpreter.error import OperationError import math @@ -138,33 +139,6 @@ return w_result w_result = space.add(space.mul(w_result,w_base), space.newlong(digit)) -def string_to_float(s): - """ - Conversion of string to float. - This version tries to only raise on invalid literals. - Overflows should be converted to infinity whenever possible. - """ - - s = strip_spaces(s) - - if not s: - raise ParseStringError("empty string for float()") - - # 1) parse the string into pieces. - try: - sign, before_point, after_point, exponent = break_up_float(s) - except ValueError: - raise ParseStringError("invalid literal for float()") - - if not before_point and not after_point: - raise ParseStringError("invalid literal for float()") - - try: - return parts_to_float(sign, before_point, after_point, exponent) - except ValueError: - raise ParseStringError("invalid literal for float()") - - # Tim's comment: # 57 bits are more than needed in any case. # to allow for some rounding, we take one @@ -187,140 +161,6 @@ del calc_mantissa_bits MANTISSA_DIGITS = len(str( (1L << MANTISSA_BITS)-1 )) + 1 -# we keep this version for reference. -def applevel_string_to_float(s): - """ - Conversion of string to float. - This version tries to only raise on invalid literals. - Overflows should be converted to infinity whenever possible. - """ - # this version was triggered by Python 2.4 which adds - # a test that breaks on overflow. - # XXX The test still breaks for a different reason: - # float must implement rich comparisons, where comparison - # between infinity and a too large long does not overflow! - - # The problem: - # there can be extreme notations of floats which are not - # infinity. - # For instance, this works in CPython: - # float('1' + '0'*1000 + 'e-1000') - # should evaluate to 1.0. - # note: float('1' + '0'*10000 + 'e-10000') - # does not work in CPython, but PyPy can do it, now. - - # The idea: - # in order to compensate between very long digit strings - # and extreme exponent numbers, we try to avoid overflows - # by adjusting the exponent by the number of mantissa - # digits. For simplicity, all computations are done in - # long math. - - # The plan: - # 1) parse the string into pieces. - # 2) pre-calculate digit exponent dexp. - # 3) truncate and adjust dexp. - # 4) compute the exponent and truncate to +-400. - # 5) compute the value using long math and proper rounding. - - # Positive results: - # The algorithm appears appears to produce correct round-trip - # values for the perfect input of _float_formatting. - # Note: - # XXX: the builtin rounding of long->float does not work, correctly. - # Ask Tim Peters for the reasons why no correct rounding is done. - # XXX: limitations: - # - It is possibly not too efficient. - # - Really optimum results need a more sophisticated algorithm - # like Bellerophon from William D. Clinger, cf. - # http://citeseer.csail.mit.edu/clinger90how.html - - s = strip_spaces(s) - - if not s: - raise ParseStringError("empty string for float()") - - # 1) parse the string into pieces. - try: - sign, before_point, after_point, exponent = break_up_float(s) - except ValueError: - raise ParseStringError("invalid literal for float()") - - digits = before_point + after_point - if digits: - raise ParseStringError("invalid literal for float()") - - # 2) pre-calculate digit exponent dexp. - dexp = len(before_point) - - # 3) truncate and adjust dexp. - p = 0 - plim = dexp + len(after_point) - while p < plim and digits[p] == '0': - p += 1 - dexp -= 1 - digits = digits[p : p + MANTISSA_DIGITS] - p = len(digits) - 1 - while p >= 0 and digits[p] == '0': - p -= 1 - dexp -= p + 1 - digits = digits[:p+1] - if len(digits) == 0: - digits = '0' - - # 4) compute the exponent and truncate to +-400 - if not exponent: - exponent = '0' - e = long(exponent) + dexp - if e >= 400: - e = 400 - elif e <= -400: - e = -400 - - # 5) compute the value using long math and proper rounding. - lr = long(digits) - if e >= 0: - bits = 0 - m = lr * 10L ** e - else: - # compute a sufficiently large scale - prec = MANTISSA_DIGITS * 2 + 22 # 128, maybe - bits = - (int(math.ceil(-e / math.log10(2.0) - 1e-10)) + prec) - scale = 2L ** -bits - pten = 10L ** -e - m = (lr * scale) // pten - - # we now have a fairly large mantissa. - # Shift it and round the last bit. - - # first estimate the bits and do a big shift - if m: - mbits = int(math.ceil(math.log(m, 2) - 1e-10)) - needed = MANTISSA_BITS - if mbits > needed: - if mbits > needed+1: - shifted = mbits - (needed+1) - m >>= shifted - bits += shifted - # do the rounding - bits += 1 - m = (m >> 1) + (m & 1) - - try: - r = math.ldexp(m, bits) - except OverflowError: - r = 1e200 * 1e200 # produce inf, hopefully - - if sign == '-': - r = -r - - return r - - -# the "real" implementation. -# for comments, see above. -# XXX probably this very specific thing should go into longobject? - def interp_string_to_float(space, s): """ Conversion of string to float. @@ -336,6 +176,15 @@ raise OperationError(space.w_ValueError, space.wrap( "empty string for float()")) + + low = s.lower() + if low == "-inf": + return -INFINITY + elif low == "inf": + return INFINITY + elif low == "nan" or low == "-nan": + return NAN + # 1) parse the string into pieces. try: sign, before_point, after_point, exponent = break_up_float(s) @@ -432,7 +281,7 @@ if r == 2*r and r != 0.0: raise OverflowError except OverflowError: - r = 1e200 * 1e200 # produce inf, hopefully + r = INFINITY if sign == '-': r = -r Modified: pypy/trunk/pypy/objspace/std/test/test_floatobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_floatobject.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_floatobject.py Wed May 26 21:31:15 2010 @@ -92,6 +92,9 @@ def test_float_string(self): assert 42 == float("42") assert 42.25 == float("42.25") + assert str(float("inf")).startswith("inf") + assert str(float("-INf")).startswith("-inf") + assert str(float("-nAn")).startswith("nan") def test_float_unicode(self): # u00A0 and u2000 are some kind of spaces Modified: pypy/trunk/pypy/objspace/std/test/test_strutil.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_strutil.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_strutil.py Wed May 26 21:31:15 2010 @@ -1,5 +1,6 @@ import py, random from pypy.objspace.std.strutil import * +from pypy.interpreter.error import OperationError import py @@ -132,50 +133,53 @@ assert string_to_w_long(space, '123L', 21).longval() == 441 + 42 + 3 assert string_to_w_long(space, '1891234174197319').longval() == 1891234174197319 -def test_string_to_float(): - assert string_to_float('0') == 0.0 - assert string_to_float('1') == 1.0 - assert string_to_float('-1.5') == -1.5 - assert string_to_float('1.5E2') == 150.0 - assert string_to_float('2.5E-1') == 0.25 - assert string_to_float('1e1111111111111') == float('1e1111111111111') - assert string_to_float('1e-1111111111111') == float('1e-1111111111111') - assert string_to_float('-1e1111111111111') == float('-1e1111111111111') - assert string_to_float('-1e-1111111111111') == float('-1e-1111111111111') - assert string_to_float('1e111111111111111111111') == float('1e111111111111111111111') - assert string_to_float('1e-111111111111111111111') == float('1e-111111111111111111111') - assert string_to_float('-1e111111111111111111111') == float('-1e111111111111111111111') - assert string_to_float('-1e-111111111111111111111') == float('-1e-111111111111111111111') - - valid_parts = [['', ' ', ' \f\n\r\t\v'], - ['', '+', '-'], - ['00', '90', '.5', '2.4', '3.', '0.07', - '12.3489749871982471987198371293717398256187563298638726' - '2187362820947193247129871083561249818451804287437824015' - '013816418758104762348932657836583048761487632840726386'], - ['', 'e0', 'E+1', 'E-01', 'E42'], - ['', ' ', ' \f\n\r\t\v'], - ] - invalid_parts = [['#'], - ['++', '+-', '-+', '--'], - ['', '1.2.3', '.', '5..6'], - ['E+', 'E-', 'e', 'e++', 'E++2'], - ['#'], - ] - for part0 in valid_parts[0]: - for part1 in valid_parts[1]: - for part2 in valid_parts[2]: - for part3 in valid_parts[3]: - for part4 in valid_parts[4]: - s = part0+part1+part2+part3+part4 - assert (abs(string_to_float(s) - float(s)) <= - 1E-13 * abs(float(s))) - - for j in range(len(invalid_parts)): - for invalid in invalid_parts[j]: - for i in range(20): - parts = [random.choice(lst) for lst in valid_parts] - parts[j] = invalid - s = ''.join(parts) - print repr(s) - py.test.raises(ParseStringError, string_to_float, s) + def test_string_to_float(self): + def string_to_float(x): + return interp_string_to_float(self.space, x) + assert string_to_float('0') == 0.0 + assert string_to_float('1') == 1.0 + assert string_to_float('-1.5') == -1.5 + assert string_to_float('1.5E2') == 150.0 + assert string_to_float('2.5E-1') == 0.25 + assert string_to_float('1e1111111111111') == float('1e1111111111111') + assert string_to_float('1e-1111111111111') == float('1e-1111111111111') + assert string_to_float('-1e1111111111111') == float('-1e1111111111111') + assert string_to_float('-1e-1111111111111') == float('-1e-1111111111111') + assert string_to_float('1e111111111111111111111') == float('1e111111111111111111111') + assert string_to_float('1e-111111111111111111111') == float('1e-111111111111111111111') + assert string_to_float('-1e111111111111111111111') == float('-1e111111111111111111111') + assert string_to_float('-1e-111111111111111111111') == float('-1e-111111111111111111111') + + valid_parts = [['', ' ', ' \f\n\r\t\v'], + ['', '+', '-'], + ['00', '90', '.5', '2.4', '3.', '0.07', + '12.3489749871982471987198371293717398256187563298638726' + '2187362820947193247129871083561249818451804287437824015' + '013816418758104762348932657836583048761487632840726386'], + ['', 'e0', 'E+1', 'E-01', 'E42'], + ['', ' ', ' \f\n\r\t\v'], + ] + invalid_parts = [['#'], + ['++', '+-', '-+', '--'], + ['', '1.2.3', '.', '5..6'], + ['E+', 'E-', 'e', 'e++', 'E++2'], + ['#'], + ] + for part0 in valid_parts[0]: + for part1 in valid_parts[1]: + for part2 in valid_parts[2]: + for part3 in valid_parts[3]: + for part4 in valid_parts[4]: + s = part0+part1+part2+part3+part4 + assert (abs(string_to_float(s) - float(s)) <= + 1E-13 * abs(float(s))) + + for j in range(len(invalid_parts)): + for invalid in invalid_parts[j]: + for i in range(20): + parts = [random.choice(lst) for lst in valid_parts] + parts[j] = invalid + s = ''.join(parts) + print repr(s) + if s.strip(): # empty s raises OperationError directly + py.test.raises(ParseStringError, string_to_float, s) From jcreigh at codespeak.net Wed May 26 21:55:58 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Wed, 26 May 2010 21:55:58 +0200 (CEST) Subject: [pypy-svn] r74794 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86 Message-ID: <20100526195558.AC0CA282BE0@codespeak.net> Author: jcreigh Date: Wed May 26 21:55:57 2010 New Revision: 74794 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py Log: add a few more instruction encodings Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py Wed May 26 21:55:57 2010 @@ -422,6 +422,7 @@ mem_reg_plus_scaled_reg_plus_const(2)) MOV_ar = insn(rex_w, '\x89', register(2,8), mem_reg_plus_scaled_reg_plus_const(1)) + MOV_ai = insn(rex_w, '\xC7', orbyte(0<<3), mem_reg_plus_scaled_reg_plus_const(1), immediate(2)) # "MOV reg1, [immediate2]" and the opposite direction MOV_rj = insn(rex_w, '\x8B', register(1,8), '\x05', immediate(2)) @@ -432,10 +433,13 @@ MOV8_ar = insn(rex_w, '\x88', byte_register(2, 8), mem_reg_plus_scaled_reg_plus_const(1)) MOV8_mi = insn(rex_w, '\xC6', orbyte(0<<3), mem_reg_plus_const(1), immediate(2, 'b')) MOV8_ai = insn(rex_w, '\xC6', orbyte(0<<3), mem_reg_plus_scaled_reg_plus_const(1), immediate(2, 'b')) + MOV8_ji = insn(rex_w, '\xC6', orbyte(0<<3), '\x05', immediate(1), immediate(2, 'b')) + MOV8_jr = insn(rex_w, '\x88', byte_register(2, 8), '\x05', immediate(1)) MOVZX8_rr = insn(rex_w, '\x0F\xB6', register(1,8), byte_register(2), '\xC0') MOVZX8_rm = insn(rex_w, '\x0F\xB6', register(1,8), mem_reg_plus_const(2)) MOVZX8_ra = insn(rex_w, '\x0F\xB6', register(1,8), mem_reg_plus_scaled_reg_plus_const(2)) + MOVZX8_rj = insn(rex_w, '\x0F\xB6', register(1,8), '\x05', immediate(2)) MOVZX16_rm = insn(rex_w, '\x0F\xB7', register(1,8), mem_reg_plus_const(2)) MOVZX16_ra = insn(rex_w, '\x0F\xB7', register(1,8), mem_reg_plus_scaled_reg_plus_const(2)) From jcreigh at codespeak.net Wed May 26 23:01:24 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Wed, 26 May 2010 23:01:24 +0200 (CEST) Subject: [pypy-svn] r74795 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86 Message-ID: <20100526210124.7FF36282BDE@codespeak.net> Author: jcreigh Date: Wed May 26 23:01:22 2010 New Revision: 74795 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Log: fix some annotation errors Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Wed May 26 23:01:22 2010 @@ -124,8 +124,8 @@ self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) self.fail_boxes_float = values_array(lltype.Float, failargs_limit) self.fail_ebp = 0 - self.loc_float_const_neg = None - self.loc_float_const_abs = None + self.float_const_neg_addr = 0 + self.float_const_abs_addr = 0 self.malloc_fixedsize_slowpath1 = 0 self.malloc_fixedsize_slowpath2 = 0 self.setup_failure_recovery() @@ -193,8 +193,8 @@ addr[5] = 2147483647 # / for abs addr[6] = 0 # addr[7] = 0 # - self.loc_float_const_neg = float_constants - self.loc_float_const_abs = float_constants + 16 + self.float_const_neg_addr = float_constants + self.float_const_abs_addr = float_constants + 16 def _build_malloc_fixedsize_slowpath(self): mc = self.mc2._mc @@ -696,11 +696,11 @@ def genop_float_neg(self, op, arglocs, resloc): # Following what gcc does: res = x ^ 0x8000000000000000 - self.mc.XORPD_xj(arglocs[0].value, self.loc_float_const_neg) + self.mc.XORPD_xj(arglocs[0].value, self.float_const_neg_addr) def genop_float_abs(self, op, arglocs, resloc): # Following what gcc does: res = x & 0x7FFFFFFFFFFFFFFF - self.mc.ANDPD_xj(arglocs[0].value, self.loc_float_const_abs) + self.mc.ANDPD_xj(arglocs[0].value, self.float_const_abs_addr) def genop_guard_float_is_true(self, op, guard_op, addr, arglocs, resloc): guard_opnum = guard_op.opnum Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Wed May 26 23:01:22 2010 @@ -134,13 +134,163 @@ def INSN(self, loc1, loc2): code1 = loc1.location_code() code2 = loc2.location_code() - # XXX: We just hope that the right method exists - getattr(self, name + '_' + code1 + code2)(loc1.value, loc2.value) + # XXX: All possible combinations are given, even those that are + # impossible + if code1 == 'r' and code2 == 'r': + getattr(self, name + "_rr")(loc1.value, loc2.value) + elif code1 == 'r' and code2 == 'b': + getattr(self, name + "_rb")(loc1.value, loc2.value) + elif code1 == 'r' and code2 == 's': + getattr(self, name + "_rs")(loc1.value, loc2.value) + elif code1 == 'r' and code2 == 'm': + getattr(self, name + "_rm")(loc1.value, loc2.value) + elif code1 == 'r' and code2 == 'a': + getattr(self, name + "_ra")(loc1.value, loc2.value) + elif code1 == 'r' and code2 == 'j': + getattr(self, name + "_rj")(loc1.value, loc2.value) + elif code1 == 'r' and code2 == 'i': + getattr(self, name + "_ri")(loc1.value, loc2.value) + elif code1 == 'r' and code2 == 'x': + getattr(self, name + "_rx")(loc1.value, loc2.value) + elif code1 == 'b' and code2 == 'r': + getattr(self, name + "_br")(loc1.value, loc2.value) + elif code1 == 'b' and code2 == 'b': + getattr(self, name + "_bb")(loc1.value, loc2.value) + elif code1 == 'b' and code2 == 's': + getattr(self, name + "_bs")(loc1.value, loc2.value) + elif code1 == 'b' and code2 == 'm': + getattr(self, name + "_bm")(loc1.value, loc2.value) + elif code1 == 'b' and code2 == 'a': + getattr(self, name + "_ba")(loc1.value, loc2.value) + elif code1 == 'b' and code2 == 'j': + getattr(self, name + "_bj")(loc1.value, loc2.value) + elif code1 == 'b' and code2 == 'i': + getattr(self, name + "_bi")(loc1.value, loc2.value) + elif code1 == 'b' and code2 == 'x': + getattr(self, name + "_bx")(loc1.value, loc2.value) + elif code1 == 's' and code2 == 'r': + getattr(self, name + "_sr")(loc1.value, loc2.value) + elif code1 == 's' and code2 == 'b': + getattr(self, name + "_sb")(loc1.value, loc2.value) + elif code1 == 's' and code2 == 's': + getattr(self, name + "_ss")(loc1.value, loc2.value) + elif code1 == 's' and code2 == 'm': + getattr(self, name + "_sm")(loc1.value, loc2.value) + elif code1 == 's' and code2 == 'a': + getattr(self, name + "_sa")(loc1.value, loc2.value) + elif code1 == 's' and code2 == 'j': + getattr(self, name + "_sj")(loc1.value, loc2.value) + elif code1 == 's' and code2 == 'i': + getattr(self, name + "_si")(loc1.value, loc2.value) + elif code1 == 's' and code2 == 'x': + getattr(self, name + "_sx")(loc1.value, loc2.value) + elif code1 == 'm' and code2 == 'r': + getattr(self, name + "_mr")(loc1.value, loc2.value) + elif code1 == 'm' and code2 == 'b': + getattr(self, name + "_mb")(loc1.value, loc2.value) + elif code1 == 'm' and code2 == 's': + getattr(self, name + "_ms")(loc1.value, loc2.value) + elif code1 == 'm' and code2 == 'm': + getattr(self, name + "_mm")(loc1.value, loc2.value) + elif code1 == 'm' and code2 == 'a': + getattr(self, name + "_ma")(loc1.value, loc2.value) + elif code1 == 'm' and code2 == 'j': + getattr(self, name + "_mj")(loc1.value, loc2.value) + elif code1 == 'm' and code2 == 'i': + getattr(self, name + "_mi")(loc1.value, loc2.value) + elif code1 == 'm' and code2 == 'x': + getattr(self, name + "_mx")(loc1.value, loc2.value) + elif code1 == 'a' and code2 == 'r': + getattr(self, name + "_ar")(loc1.value, loc2.value) + elif code1 == 'a' and code2 == 'b': + getattr(self, name + "_ab")(loc1.value, loc2.value) + elif code1 == 'a' and code2 == 's': + getattr(self, name + "_as")(loc1.value, loc2.value) + elif code1 == 'a' and code2 == 'm': + getattr(self, name + "_am")(loc1.value, loc2.value) + elif code1 == 'a' and code2 == 'a': + getattr(self, name + "_aa")(loc1.value, loc2.value) + elif code1 == 'a' and code2 == 'j': + getattr(self, name + "_aj")(loc1.value, loc2.value) + elif code1 == 'a' and code2 == 'i': + getattr(self, name + "_ai")(loc1.value, loc2.value) + elif code1 == 'a' and code2 == 'x': + getattr(self, name + "_ax")(loc1.value, loc2.value) + elif code1 == 'j' and code2 == 'r': + getattr(self, name + "_jr")(loc1.value, loc2.value) + elif code1 == 'j' and code2 == 'b': + getattr(self, name + "_jb")(loc1.value, loc2.value) + elif code1 == 'j' and code2 == 's': + getattr(self, name + "_js")(loc1.value, loc2.value) + elif code1 == 'j' and code2 == 'm': + getattr(self, name + "_jm")(loc1.value, loc2.value) + elif code1 == 'j' and code2 == 'a': + getattr(self, name + "_ja")(loc1.value, loc2.value) + elif code1 == 'j' and code2 == 'j': + getattr(self, name + "_jj")(loc1.value, loc2.value) + elif code1 == 'j' and code2 == 'i': + getattr(self, name + "_ji")(loc1.value, loc2.value) + elif code1 == 'j' and code2 == 'x': + getattr(self, name + "_jx")(loc1.value, loc2.value) + elif code1 == 'i' and code2 == 'r': + getattr(self, name + "_ir")(loc1.value, loc2.value) + elif code1 == 'i' and code2 == 'b': + getattr(self, name + "_ib")(loc1.value, loc2.value) + elif code1 == 'i' and code2 == 's': + getattr(self, name + "_is")(loc1.value, loc2.value) + elif code1 == 'i' and code2 == 'm': + getattr(self, name + "_im")(loc1.value, loc2.value) + elif code1 == 'i' and code2 == 'a': + getattr(self, name + "_ia")(loc1.value, loc2.value) + elif code1 == 'i' and code2 == 'j': + getattr(self, name + "_ij")(loc1.value, loc2.value) + elif code1 == 'i' and code2 == 'i': + getattr(self, name + "_ii")(loc1.value, loc2.value) + elif code1 == 'i' and code2 == 'x': + getattr(self, name + "_ix")(loc1.value, loc2.value) + elif code1 == 'x' and code2 == 'r': + getattr(self, name + "_xr")(loc1.value, loc2.value) + elif code1 == 'x' and code2 == 'b': + getattr(self, name + "_xb")(loc1.value, loc2.value) + elif code1 == 'x' and code2 == 's': + getattr(self, name + "_xs")(loc1.value, loc2.value) + elif code1 == 'x' and code2 == 'm': + getattr(self, name + "_xm")(loc1.value, loc2.value) + elif code1 == 'x' and code2 == 'a': + getattr(self, name + "_xa")(loc1.value, loc2.value) + elif code1 == 'x' and code2 == 'j': + getattr(self, name + "_xj")(loc1.value, loc2.value) + elif code1 == 'x' and code2 == 'i': + getattr(self, name + "_xi")(loc1.value, loc2.value) + elif code1 == 'x' and code2 == 'x': + getattr(self, name + "_xx")(loc1.value, loc2.value) + else: + raise AssertionError("Invalid location codes") + return INSN def _unaryop(name): def INSN(self, loc): - getattr(self, name + '_' + loc.location_code())(loc.value) + code = loc.location_code() + # "if" is unrolled for RPython + if code == 'r': + getattr(self, name + '_r')(loc.value) + elif code == 'b': + getattr(self, name + '_b')(loc.value) + elif code == 's': + getattr(self, name + '_s')(loc.value) + elif code == 'm': + getattr(self, name + '_m')(loc.value) + elif code == 'a': + getattr(self, name + '_a')(loc.value) + elif code == 'j': + getattr(self, name + '_j')(loc.value) + elif code == 'i': + getattr(self, name + '_i')(loc.value) + elif code == 'x': + getattr(self, name + '_x')(loc.value) + else: + raise AssertionError("Unknown code") return INSN AND = _binaryop('AND') From fijal at codespeak.net Thu May 27 07:30:43 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 27 May 2010 07:30:43 +0200 (CEST) Subject: [pypy-svn] r74796 - pypy/trunk/pypy/config Message-ID: <20100527053043.DF144282BDE@codespeak.net> Author: fijal Date: Thu May 27 07:30:41 2010 New Revision: 74796 Modified: pypy/trunk/pypy/config/translationoption.py Log: Mark -Ojit --stackless as incompatible Modified: pypy/trunk/pypy/config/translationoption.py ============================================================================== --- pypy/trunk/pypy/config/translationoption.py (original) +++ pypy/trunk/pypy/config/translationoption.py Thu May 27 07:30:41 2010 @@ -376,6 +376,8 @@ config.translation.suggest(withsmallfuncsets=5) elif word == 'jit': config.translation.suggest(jit=True) + if config.translation.stackless: + raise NotImplementedError("JIT conflicts with stackless for now") elif word == 'removetypeptr': config.translation.suggest(gcremovetypeptr=True) else: From afa at codespeak.net Thu May 27 11:01:13 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 27 May 2010 11:01:13 +0200 (CEST) Subject: [pypy-svn] r74797 - pypy/trunk/pypy/module/cpyext Message-ID: <20100527090113.333CF282BF3@codespeak.net> Author: afa Date: Thu May 27 11:01:10 2010 New Revision: 74797 Modified: pypy/trunk/pypy/module/cpyext/object.py Log: suppress a compilation warning. Modified: pypy/trunk/pypy/module/cpyext/object.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/object.py (original) +++ pypy/trunk/pypy/module/cpyext/object.py Thu May 27 11:01:10 2010 @@ -19,7 +19,7 @@ return lltype.malloc(rffi.VOIDP.TO, size, flavor='raw', zero=True) - at cpython_api([rffi.VOIDP], lltype.Void) + at cpython_api([rffi.VOIDP_real], lltype.Void) def PyObject_FREE(space, ptr): lltype.free(ptr, flavor='raw') From afa at codespeak.net Thu May 27 11:06:03 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 27 May 2010 11:06:03 +0200 (CEST) Subject: [pypy-svn] r74798 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100527090603.9FE49282BF3@codespeak.net> Author: afa Date: Thu May 27 11:06:02 2010 New Revision: 74798 Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py pypy/trunk/pypy/module/cpyext/test/test_typeobject.py pypy/trunk/pypy/module/cpyext/typeobject.py Log: Fill the slot tp_as_number.nb_int, when the type defines a __init__ method. Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/slotdefs.py (original) +++ pypy/trunk/pypy/module/cpyext/slotdefs.py Thu May 27 11:06:02 2010 @@ -124,6 +124,10 @@ space.get_and_call_args(w_descr, w_self, args) return 0 + at cpython_api([PyObject], PyObject) +def slot_nb_int(space, w_self): + return space.int(w_self) + PyWrapperFlag_KEYWORDS = 1 # adopted from typeobject.c Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Thu May 27 11:06:02 2010 @@ -189,3 +189,22 @@ w_obj = api._PyType_Lookup(w_type, space.wrap("__invalid")) assert w_obj is None assert api.PyErr_Occurred() is None + +class AppTestSlots(AppTestCpythonExtensionBase): + def test_nb_int(self): + module = self.import_extension('foo', [ + ("nb_int", "METH_O", + ''' + if (!args->ob_type->tp_as_number || + !args->ob_type->tp_as_number->nb_int) + { + PyErr_SetNone(PyExc_ValueError); + return NULL; + } + return args->ob_type->tp_as_number->nb_int(args); + ''' + ) + ]) + assert module.nb_int(10) == 10 + assert module.nb_int(-12.3) == -12 + raises(ValueError, module.nb_int, "123") Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Thu May 27 11:06:02 2010 @@ -27,8 +27,9 @@ PyDescr_NewWrapper, PyCFunction_NewEx) from pypy.module.cpyext.pyobject import Py_IncRef, Py_DecRef, _Py_Dealloc from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne -from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr, PyTypeObject, \ - PyGetSetDef, PyMemberDef, newfunc +from pypy.module.cpyext.typeobjectdefs import ( + PyTypeObjectPtr, PyTypeObject, PyGetSetDef, PyMemberDef, newfunc, + PyNumberMethods) from pypy.module.cpyext.slotdefs import slotdefs from pypy.interpreter.error import OperationError, operationerrfmt from pypy.rlib.rstring import rsplit @@ -104,10 +105,10 @@ dict_w[name] = w_descr i += 1 -def update_all_slots(space, w_obj, pto): +def update_all_slots(space, w_type, pto): # XXX fill slots in pto for method_name, slot_name, slot_func, _, _, _ in slotdefs: - w_descr = space.lookup(w_obj, method_name) + w_descr = w_type.lookup(method_name) if w_descr is None: # XXX special case iternext continue @@ -129,7 +130,14 @@ assert len(slot_name) == 2 struct = getattr(pto, slot_name[0]) if not struct: - continue + if slot_name[0] == 'c_tp_as_number': + STRUCT_TYPE = PyNumberMethods + else: + raise AssertionError( + "Structure not allocated: %s" % (slot_name[0],)) + struct = lltype.malloc(STRUCT_TYPE, flavor='raw', zero=True) + setattr(pto, slot_name[0], struct) + setattr(struct, slot_name[1], slot_func_helper) def add_operators(space, dict_w, pto): @@ -472,6 +480,8 @@ if obj_pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE: if obj_pto.c_tp_as_buffer: lltype.free(obj_pto.c_tp_as_buffer, flavor='raw') + if obj_pto.c_tp_as_number: + lltype.free(obj_pto.c_tp_as_number, flavor='raw') Py_DecRef(space, base_pyo) rffi.free_charp(obj_pto.c_tp_name) obj_pto_voidp = rffi.cast(rffi.VOIDP_real, obj_pto) From afa at codespeak.net Thu May 27 11:58:53 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 27 May 2010 11:58:53 +0200 (CEST) Subject: [pypy-svn] r74803 - pypy/trunk/pypy/module/cpyext Message-ID: <20100527095853.0DE9A282BEC@codespeak.net> Author: afa Date: Thu May 27 11:58:51 2010 New Revision: 74803 Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py Log: Add comments to regexps Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/slotdefs.py (original) +++ pypy/trunk/pypy/module/cpyext/slotdefs.py Thu May 27 11:58:51 2010 @@ -184,18 +184,6 @@ return ETSLOT(NAME, "tp_as_number.c_" + SLOT, FUNCTION, "wrap_binaryfunc_r", \ "x." + NAME + "(y) <==> " + DOC) -slotdef_replacements = ( - ("\s+", " "), - ("static [^{]*{", "("), - ("};", ")"), - (r"(?P +..SLOT\([^,]*, )(?P[^,]*), (?P[^,]*), (?P[^,]*)", r"\g'\g', '\g', '\g'"), - (r"(?P *R?[^ ]{3}SLOT(NOTINFIX)?\([^,]*, )(?P[^,]*), (?P[^,]*)", r"\g'\g', '\g'"), - ("'NULL'", "None"), - ("{NULL}", ""), - ("\(wrapperfunc\)", ""), - ("\),", "),\n"), -) - """ /* Heap types defining __add__/__mul__ have sq_concat/sq_repeat == NULL. The logic in abstract.c always falls back to nb_add/nb_multiply in @@ -207,7 +195,7 @@ # Copy new slotdefs from typeobject.c # Remove comments and tabs # Done. -slotdefs_str = """ +slotdefs_str = r""" static slotdef slotdefs[] = { SQSLOT("__len__", sq_length, slot_sq_length, wrap_lenfunc, "x.__len__() <==> len(x)"), @@ -399,6 +387,21 @@ {NULL} }; """ + +# Convert the above string into python code +slotdef_replacements = ( + ("\s+", " "), # all on one line + ("static [^{]*{", "("), # remove first line... + ("};", ")"), # ...last line... + ("{NULL}", ""), # ...and sentinel + # add quotes around function name, slot name, and wrapper name + (r"(?P +..SLOT\([^,]*, )(?P[^,]*), (?P[^,]*), (?P[^,]*)", r"\g'\g', '\g', '\g'"), + (r"(?P *R?[^ ]{3}SLOT(NOTINFIX)?\([^,]*, )(?P[^,]*), (?P[^,]*)", r"\g'\g', '\g'"), + ("'NULL'", "None"), # but NULL becomes None + ("\(wrapperfunc\)", ""), # casts are not needed in python tuples + ("\),", "),\n"), # add newlines again +) + for regex, repl in slotdef_replacements: slotdefs_str = re.sub(regex, repl, slotdefs_str) From afa at codespeak.net Thu May 27 13:21:52 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 27 May 2010 13:21:52 +0200 (CEST) Subject: [pypy-svn] r74804 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100527112152.059C0282BE0@codespeak.net> Author: afa Date: Thu May 27 13:21:51 2010 New Revision: 74804 Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Log: Fill "tp_call" slot Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/slotdefs.py (original) +++ pypy/trunk/pypy/module/cpyext/slotdefs.py Thu May 27 13:21:51 2010 @@ -124,6 +124,10 @@ space.get_and_call_args(w_descr, w_self, args) return 0 + at cpython_api([PyObject, PyObject, PyObject], PyObject) +def slot_tp_call(space, w_self, w_args, w_kwds): + return space.call(w_self, w_args, w_kwds) + @cpython_api([PyObject], PyObject) def slot_nb_int(space, w_self): return space.int(w_self) Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Thu May 27 13:21:51 2010 @@ -208,3 +208,23 @@ assert module.nb_int(10) == 10 assert module.nb_int(-12.3) == -12 raises(ValueError, module.nb_int, "123") + + def test_tp_call(self): + module = self.import_extension('foo', [ + ("tp_call", "METH_VARARGS", + ''' + PyObject *obj = PyTuple_GET_ITEM(args, 0); + PyObject *c_args = PyTuple_GET_ITEM(args, 1); + if (!obj->ob_type->tp_call) + { + PyErr_SetNone(PyExc_ValueError); + return NULL; + } + return obj->ob_type->tp_call(obj, c_args, NULL); + ''' + ) + ]) + class C: + def __call__(self, *args): + return args + assert module.tp_call(C(), ('x', 2)) == ('x', 2) From afa at codespeak.net Thu May 27 13:24:14 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 27 May 2010 13:24:14 +0200 (CEST) Subject: [pypy-svn] r74805 - pypy/trunk/pypy/module/cpyext Message-ID: <20100527112414.2DAE9282BE0@codespeak.net> Author: afa Date: Thu May 27 13:24:12 2010 New Revision: 74805 Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py Log: Don't "export" these functions, they are internal function pointers Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/slotdefs.py (original) +++ pypy/trunk/pypy/module/cpyext/slotdefs.py Thu May 27 13:24:12 2010 @@ -106,7 +106,7 @@ raise OperationError(space.w_StopIteration, space.w_None) return w_res - at cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject, external=True) + at cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject, external=False) def slot_tp_new(space, type, w_args, w_kwds): from pypy.module.cpyext.tupleobject import PyTuple_Check pyo = rffi.cast(PyObject, type) @@ -117,18 +117,18 @@ w_args_new = space.newtuple(args_w) return space.call(w_func, w_args_new, w_kwds) - at cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) + at cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1, external=False) def slot_tp_init(space, w_self, w_args, w_kwds): w_descr = space.lookup(w_self, '__init__') args = Arguments.frompacked(space, w_args, w_kwds) space.get_and_call_args(w_descr, w_self, args) return 0 - at cpython_api([PyObject, PyObject, PyObject], PyObject) + at cpython_api([PyObject, PyObject, PyObject], PyObject, external=False) def slot_tp_call(space, w_self, w_args, w_kwds): return space.call(w_self, w_args, w_kwds) - at cpython_api([PyObject], PyObject) + at cpython_api([PyObject], PyObject, external=False) def slot_nb_int(space, w_self): return space.int(w_self) From arigo at codespeak.net Thu May 27 15:22:36 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 May 2010 15:22:36 +0200 (CEST) Subject: [pypy-svn] r74809 - in pypy/trunk/pypy: jit/metainterp jit/metainterp/test rlib rlib/test Message-ID: <20100527132236.EA66C282BE0@codespeak.net> Author: arigo Date: Thu May 27 15:22:34 2010 New Revision: 74809 Modified: pypy/trunk/pypy/jit/metainterp/codewriter.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/test/test_basic.py pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py pypy/trunk/pypy/jit/metainterp/test/test_recursive.py pypy/trunk/pypy/jit/metainterp/warmspot.py pypy/trunk/pypy/rlib/jit.py pypy/trunk/pypy/rlib/test/test_jit.py Log: Kill the deprecated "leave" hook. Preparation for trying to clean up pyjitpl's production of CALL_ASSEMBLER. Modified: pypy/trunk/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/codewriter.py (original) +++ pypy/trunk/pypy/jit/metainterp/codewriter.py Thu May 27 15:22:34 2010 @@ -76,16 +76,13 @@ self.raise_analyzer = None self.jitdriver = None - def find_all_graphs(self, portal_graph, leave_graph, - policy, supports_floats): + def find_all_graphs(self, portal_graph, policy, supports_floats): from pypy.translator.simplify import get_graph def is_candidate(graph): return policy.look_inside_graph(graph) policy.set_supports_floats(supports_floats) todo = [portal_graph] - if leave_graph is not None: - todo.append(leave_graph) self.candidate_graphs = seen = set(todo) def callers(): @@ -178,15 +175,12 @@ return graph in self.candidate_graphs - def generate_bytecode(self, metainterp_sd, portal_graph, leave_graph, + def generate_bytecode(self, metainterp_sd, portal_graph, portal_runner_ptr): self._start(metainterp_sd, portal_runner_ptr) - leave_code = None - if leave_graph: - leave_code = self.make_one_bytecode((leave_graph, None), False) portal_code = self.make_portal_bytecode(portal_graph) - self.metainterp_sd.info_from_codewriter(portal_code, leave_code, + self.metainterp_sd.info_from_codewriter(portal_code, self.class_sizes, self.list_of_addr2name, portal_runner_ptr) @@ -1285,8 +1279,6 @@ calldescr, non_void_args = self.codewriter.getcalldescr(op.args[0], args, op.result) - self.emit('recursion_leave_prep') - self.emit_varargs(non_void_args) self.emit('recursive_call') self.emit(self.get_position(calldescr)) self.emit_varargs([op.args[0]] + non_void_args) Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Thu May 27 15:22:34 2010 @@ -647,19 +647,6 @@ @arguments("descr", "varargs") def opimpl_residual_call_loopinvariant(self, calldescr, varargs): return self.execute_varargs(rop.CALL_LOOPINVARIANT, varargs, calldescr, exc=True) - - @arguments("varargs") - def opimpl_recursion_leave_prep(self, varargs): - warmrunnerstate = self.metainterp.staticdata.state - if warmrunnerstate.inlining: - num_green_args = self.metainterp.staticdata.num_green_args - greenkey = varargs[:num_green_args] - if warmrunnerstate.can_inline_callable(greenkey): - return False - leave_code = self.metainterp.staticdata.leave_code - if leave_code is None: - return False - return self.perform_call(leave_code, varargs) @arguments("orgpc", "descr", "varargs") def opimpl_recursive_call(self, pc, calldescr, varargs): @@ -1133,7 +1120,6 @@ self.jit_starting_line = 'JIT starting (%s)' % backendmodule self.portal_code = None - self.leave_code = None self._class_sizes = None self._addr2name_keys = [] self._addr2name_values = [] @@ -1146,10 +1132,9 @@ def _freeze_(self): return True - def info_from_codewriter(self, portal_code, leave_code, class_sizes, + def info_from_codewriter(self, portal_code, class_sizes, list_of_addr2name, portal_runner_ptr): self.portal_code = portal_code - self.leave_code = leave_code self._class_sizes = class_sizes self._addr2name_keys = [key for key, value in list_of_addr2name] self._addr2name_values = [value for key, value in list_of_addr2name] Modified: pypy/trunk/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_basic.py Thu May 27 15:22:34 2010 @@ -93,7 +93,7 @@ portal_graph = rtyper.annotator.translator.graphs[0] cw = codewriter.CodeWriter(rtyper) - graphs = cw.find_all_graphs(portal_graph, None, JitPolicy(), + graphs = cw.find_all_graphs(portal_graph, JitPolicy(), self.CPUClass.supports_floats) cw._start(metainterp.staticdata, None) portal_graph.func._jit_unroll_safe_ = True Modified: pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py Thu May 27 15:22:34 2010 @@ -25,7 +25,7 @@ rtyper = support.annotate(i, [7]) cw = CodeWriter(rtyper) jitpolicy = JitPolicy() - res = cw.find_all_graphs(rtyper.annotator.translator.graphs[0], None, + res = cw.find_all_graphs(rtyper.annotator.translator.graphs[0], jitpolicy, True) translator = rtyper.annotator.translator @@ -41,13 +41,13 @@ cw = CodeWriter(rtyper) jitpolicy = JitPolicy() translator = rtyper.annotator.translator - res = cw.find_all_graphs(translator.graphs[0], None, jitpolicy, + res = cw.find_all_graphs(translator.graphs[0], jitpolicy, supports_floats=True) funcs = set([graph.func for graph in res]) assert funcs == set([f, g]) cw = CodeWriter(rtyper) - res = cw.find_all_graphs(translator.graphs[0], None, jitpolicy, + res = cw.find_all_graphs(translator.graphs[0], jitpolicy, supports_floats=False) funcs = [graph.func for graph in res] assert funcs == [f] @@ -75,7 +75,7 @@ cw = CodeWriter(rtyper) jitpolicy = JitPolicy() translator = rtyper.annotator.translator - res = cw.find_all_graphs(translator.graphs[0], None, jitpolicy, + res = cw.find_all_graphs(translator.graphs[0], jitpolicy, supports_floats=True) funcs = set([graph.func for graph in res]) assert funcs == set([f, h]) @@ -96,7 +96,7 @@ cw = CodeWriter(rtyper) jitpolicy = JitPolicy() translator = rtyper.annotator.translator - res = cw.find_all_graphs(translator.graphs[0], None, jitpolicy, + res = cw.find_all_graphs(translator.graphs[0], jitpolicy, supports_floats=True) funcs = set([graph.func for graph in res]) assert funcs == set([g, h]) @@ -110,7 +110,7 @@ jitpolicy = JitPolicy() translator = rtyper.annotator.translator # does not explode - cw.find_all_graphs(translator.graphs[0], None, jitpolicy, True) + cw.find_all_graphs(translator.graphs[0], jitpolicy, True) class SomeLabel(object): def __eq__(self, other): Modified: pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py Thu May 27 15:22:34 2010 @@ -121,7 +121,7 @@ def __init__(self): pass metainterp_sd = FakeMetaInterpSd() - metainterp_sd.info_from_codewriter(None, None, None, + metainterp_sd.info_from_codewriter(None, None, [(123, "a"), (456, "b")], None) assert metainterp_sd.get_name_from_address(123) == 'a' Modified: pypy/trunk/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_recursive.py Thu May 27 15:22:34 2010 @@ -494,77 +494,6 @@ res = self.meta_interp(main, [1], optimizer=OPTIMIZER_SIMPLE, trace_limit=TRACE_LIMIT) self.check_loops(call_may_force=0, call=0) - def test_leave_jit_hook(self): - from pypy.rpython.annlowlevel import hlstr - def p(code, pc): - code = hlstr(code) - return "%s %d %s" % (code, pc, code[pc]) - def c(code, pc): - return "L" not in hlstr(code) - - def really_leave(frame): - frame.hookcalled = True - - def leave(code, pc, frame): - really_leave(frame) - - class ExpectedHook(Exception): - pass - class UnexpectedHook(Exception): - pass - - myjitdriver = JitDriver(greens=['code', 'pc'], reds=['self'], - get_printable_location=p, can_inline=c, - leave=leave) - class Frame(object): - hookcalled = True - - def __init__(self, n): - self.n = n - self.hookcalled = False - def f(self, code): - pc = 0 - while pc < len(code): - - myjitdriver.jit_merge_point(self=self, code=code, pc=pc) - op = code[pc] - if op == "-": - self.n -= 1 - elif op == "c": - frame = Frame(self.n) - self.n = frame.f("---i---") - if we_are_jitted(): - if frame.hookcalled: - raise UnexpectedHook - elif op == "C": - frame = Frame(self.n) - self.n = frame.f("cL") - if we_are_jitted(): - if not frame.hookcalled: - raise ExpectedHook - elif op == "i": - if self.n % 5 == 1: - return self.n - elif op == "l": - if self.n > 0: - myjitdriver.can_enter_jit(self=self, code=code, pc=0) - pc = 0 - continue - elif op == "L": - if self.n > 50: - myjitdriver.can_enter_jit(self=self, code=code, pc=0) - pc = 0 - continue - else: - assert 0 - pc += 1 - return self.n - def main(n): - frame = Frame(n) - return frame.f("C-l") - res = self.meta_interp(main, [100], optimizer=OPTIMIZER_SIMPLE, inline=True) - assert res == main(100) - def test_trace_from_start(self): def p(code, pc): code = hlstr(code) Modified: pypy/trunk/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/warmspot.py (original) +++ pypy/trunk/pypy/jit/metainterp/warmspot.py Thu May 27 15:22:34 2010 @@ -150,10 +150,8 @@ policy = JitPolicy() self.set_translator(translator) self.find_portal() - self.make_leave_jit_graph() self.codewriter = codewriter.CodeWriter(self.rtyper) graphs = self.codewriter.find_all_graphs(self.portal_graph, - self.leave_graph, policy, CPUClass.supports_floats) policy.dump_unsafe_loops() @@ -177,7 +175,6 @@ self.codewriter.generate_bytecode(self.metainterp_sd, self.portal_graph, - self.leave_graph, self.portal_runner_ptr ) self.rewrite_can_enter_jit() @@ -332,6 +329,7 @@ self.metainterp_sd.DoneWithThisFrameFloat = DoneWithThisFrameFloat self.metainterp_sd.ExitFrameWithExceptionRef = ExitFrameWithExceptionRef self.metainterp_sd.ContinueRunningNormally = ContinueRunningNormally + def make_enter_function(self): from pypy.jit.metainterp.warmstate import WarmEnterState state = WarmEnterState(self) @@ -369,18 +367,6 @@ maybe_compile_and_run(*args) maybe_enter_from_start._always_inline_ = True self.maybe_enter_from_start_fn = maybe_enter_from_start - - - def make_leave_jit_graph(self): - self.leave_graph = None - if self.jitdriver.leave: - args_s = self.portal_args_s - from pypy.annotation import model as annmodel - annhelper = MixLevelHelperAnnotator(self.translator.rtyper) - s_result = annmodel.s_None - self.leave_graph = annhelper.getgraph(self.jitdriver.leave, - args_s, s_result) - annhelper.finish() def make_driverhook_graphs(self): from pypy.rlib.jit import BaseJitCell Modified: pypy/trunk/pypy/rlib/jit.py ============================================================================== --- pypy/trunk/pypy/rlib/jit.py (original) +++ pypy/trunk/pypy/rlib/jit.py Thu May 27 15:22:34 2010 @@ -224,8 +224,7 @@ def __init__(self, greens=None, reds=None, virtualizables=None, get_jitcell_at=None, set_jitcell_at=None, can_inline=None, get_printable_location=None, - confirm_enter_jit=None, - leave=None): # XXX 'leave' is deprecated + confirm_enter_jit=None): if greens is not None: self.greens = greens if reds is not None: @@ -243,7 +242,6 @@ self.get_printable_location = get_printable_location self.can_inline = can_inline self.confirm_enter_jit = confirm_enter_jit - self.leave = leave def _freeze_(self): return True @@ -358,7 +356,6 @@ **kwds_s) self.annotate_hook(driver.can_inline, driver.greens, **kwds_s) self.annotate_hook(driver.get_printable_location, driver.greens, **kwds_s) - self.annotate_hook(driver.leave, driver.greens + driver.reds, **kwds_s) def annotate_hook(self, func, variables, args_s=[], **kwds_s): if func is None: Modified: pypy/trunk/pypy/rlib/test/test_jit.py ============================================================================== --- pypy/trunk/pypy/rlib/test/test_jit.py (original) +++ pypy/trunk/pypy/rlib/test/test_jit.py Thu May 27 15:22:34 2010 @@ -61,12 +61,10 @@ def can_inline(m): pass def get_printable_location(m): pass - def leave(m, n): pass myjitdriver = JitDriver(greens=['m'], reds=['n'], can_inline=can_inline, - get_printable_location=get_printable_location, - leave=leave) + get_printable_location=get_printable_location) def fn(n): m = 42.5 while n > 0: @@ -83,9 +81,6 @@ return [v.concretetype for v in graph.getargs()] raise Exception, 'function %r has not been annotated' % func - leave_args = getargs(leave) - assert leave_args == [lltype.Float, lltype.Signed] - can_inline_args = getargs(can_inline) get_printable_location_args = getargs(get_printable_location) assert can_inline_args == get_printable_location_args == [lltype.Float] From arigo at codespeak.net Thu May 27 16:27:09 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 May 2010 16:27:09 +0200 (CEST) Subject: [pypy-svn] r74812 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter metainterp metainterp/test Message-ID: <20100527142709.546DD282BE0@codespeak.net> Author: arigo Date: Thu May 27 16:27:07 2010 New Revision: 74812 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizeopt.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_recursive.py Log: Start to support recursive calls, but give up for now (with py.test.mark.xfail()). Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py Thu May 27 16:27:07 2010 @@ -245,3 +245,7 @@ self.jitdriver = jitdriver else: assert self.jitdriver is jitdriver + + def getjitdriver(self): + assert self.jitdriver is not None, "order dependency issue?" + return self.jitdriver Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Thu May 27 16:27:07 2010 @@ -283,12 +283,11 @@ op1 = [op1, SpaceOperation('-live-', [], None)] return op1 - def handle_regular_call(self, op, targetgraph=None): + def handle_regular_call(self, op): """A direct_call turns into the operation 'inline_call_xxx' if it is calling a function that we want to JIT. The initial arg of 'inline_call_xxx' is the JitCode of the called function.""" - if targetgraph is None: - [targetgraph] = self.callcontrol.graphs_from(op) + [targetgraph] = self.callcontrol.graphs_from(op) jitcode = self.callcontrol.get_jitcode(targetgraph, called_from=self.graph) op0 = self.rewrite_call(op, 'inline_call', [jitcode]) @@ -317,7 +316,13 @@ def handle_recursive_call(self, op): ops = self.promote_greens(op.args[1:]) targetgraph = self.callcontrol.portal_graph - return ops + self.handle_regular_call(op, targetgraph) + num_green_args = len(self.callcontrol.getjitdriver().greens) + args = (self.make_three_lists(op.args[1:1+num_green_args]) + + self.make_three_lists(op.args[1+num_green_args:])) + kind = getkind(op.result.concretetype)[0] + op0 = SpaceOperation('recursive_call_%s' % kind, args, op.result) + op1 = SpaceOperation('-live-', [], None) + return ops + [op0, op1] handle_residual_indirect_call = handle_residual_call @@ -689,10 +694,8 @@ return op def promote_greens(self, args): - jitdriver = self.callcontrol.jitdriver - assert jitdriver is not None, "order dependency issue?" ops = [] - num_green_args = len(jitdriver.greens) + num_green_args = len(self.callcontrol.getjitdriver().greens) for v in args[:num_green_args]: if isinstance(v, Variable) and v.concretetype is not lltype.Void: kind = getkind(v.concretetype) @@ -709,7 +712,7 @@ def handle_jit_marker__jit_merge_point(self, op): assert self.portal, "jit_merge_point in non-main graph!" ops = self.promote_greens(op.args[2:]) - num_green_args = len(self.callcontrol.jitdriver.greens) + num_green_args = len(self.callcontrol.getjitdriver().greens) args = (self.make_three_lists(op.args[2:2+num_green_args]) + self.make_three_lists(op.args[2+num_green_args:])) op1 = SpaceOperation('jit_merge_point', args, None) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Thu May 27 16:27:07 2010 @@ -800,6 +800,45 @@ XXX raise LeaveFrame + def get_portal_runner(self): + metainterp_sd = self.builder.metainterp_sd + fnptr = llmemory.cast_ptr_to_adr(metainterp_sd.portal_runner_ptr) + calldescr = metainterp_sd.portal_code.calldescr + return fnptr, calldescr + + @arguments("self", "I", "R", "F", "I", "R", "F", returns="i") + def bhimpl_recursive_call_i(self, greens_i, greens_r, greens_f, + reds_i, reds_r, reds_f): + fnptr, calldescr = self.get_portal_runner() + return self.cpu.bh_call_i(fnptr, calldescr, + greens_i + reds_i, + greens_r + reds_r, + greens_f + reds_f) + @arguments("self", "I", "R", "F", "I", "R", "F", returns="r") + def bhimpl_recursive_call_r(self, greens_i, greens_r, greens_f, + reds_i, reds_r, reds_f): + fnptr, calldescr = self.get_portal_runner() + return self.cpu.bh_call_r(fnptr, calldescr, + greens_i + reds_i, + greens_r + reds_r, + greens_f + reds_f) + @arguments("self", "I", "R", "F", "I", "R", "F", returns="f") + def bhimpl_recursive_call_f(self, greens_i, greens_r, greens_f, + reds_i, reds_r, reds_f): + fnptr, calldescr = self.get_portal_runner() + return self.cpu.bh_call_f(fnptr, calldescr, + greens_i + reds_i, + greens_r + reds_r, + greens_f + reds_f) + @arguments("self", "I", "R", "F", "I", "R", "F") + def bhimpl_recursive_call_v(self, greens_i, greens_r, greens_f, + reds_i, reds_r, reds_f): + fnptr, calldescr = self.get_portal_runner() + return self.cpu.bh_call_v(fnptr, calldescr, + greens_i + reds_i, + greens_r + reds_r, + greens_f + reds_f) + # ---------- # list operations @@ -861,7 +900,7 @@ return cpu.bh_call_r(func, calldescr, None, args_r, None) @arguments("cpu", "i", "d", "R") def bhimpl_residual_call_r_v(cpu, func, calldescr, args_r): - cpu.bh_call_v(func, calldescr, None, args_r, None) + return cpu.bh_call_v(func, calldescr, None, args_r, None) @arguments("cpu", "i", "d", "I", "R", returns="i") def bhimpl_residual_call_ir_i(cpu, func, calldescr, args_i, args_r): @@ -871,7 +910,7 @@ return cpu.bh_call_r(func, calldescr, args_i, args_r, None) @arguments("cpu", "i", "d", "I", "R") def bhimpl_residual_call_ir_v(cpu, func, calldescr, args_i, args_r): - cpu.bh_call_v(func, calldescr, args_i, args_r, None) + return cpu.bh_call_v(func, calldescr, args_i, args_r, None) @arguments("cpu", "i", "d", "I", "R", "F", returns="i") def bhimpl_residual_call_irf_i(cpu, func, calldescr,args_i,args_r,args_f): @@ -884,7 +923,7 @@ return cpu.bh_call_f(func, calldescr, args_i, args_r, args_f) @arguments("cpu", "i", "d", "I", "R", "F") def bhimpl_residual_call_irf_v(cpu, func, calldescr,args_i,args_r,args_f): - cpu.bh_call_v(func, calldescr, args_i, args_r, args_f) + return cpu.bh_call_v(func, calldescr, args_i, args_r, args_f) @arguments("cpu", "j", "R", returns="i") def bhimpl_inline_call_r_i(cpu, jitcode, args_r): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py Thu May 27 16:27:07 2010 @@ -15,8 +15,10 @@ from pypy.jit.metainterp.typesystem import llhelper, oohelper from pypy.jit.metainterp.optimizeutil import InvalidLoop -class GiveUp(Exception): - pass +def giveup(): + from pypy.jit.metainterp.pyjitpl import SwitchToBlackhole + from pypy.jit.metainterp.jitprof import ABORT_BRIDGE + raise SwitchToBlackhole(ABORT_BRIDGE) def show_loop(metainterp_sd, loop=None, error=None): # debugging Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizeopt.py Thu May 27 16:27:07 2010 @@ -573,7 +573,7 @@ modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo) newboxes = modifier.finish(self.values, pendingfields) if len(newboxes) > self.metainterp_sd.options.failargs_limit: # XXX be careful here - raise compile.GiveUp + compile.giveup() descr.store_final_boxes(op, newboxes) # if op.opnum == rop.GUARD_VALUE: Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Thu May 27 16:27:07 2010 @@ -19,7 +19,6 @@ from pypy.rlib.rarithmetic import intmask from pypy.rlib.objectmodel import specialize from pypy.rlib.jit import DEBUG_OFF, DEBUG_PROFILE, DEBUG_STEPS, DEBUG_DETAILED -from pypy.jit.metainterp.compile import GiveUp from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr # ____________________________________________________________ @@ -648,40 +647,46 @@ return self.execute_varargs(rop.CALL_LOOPINVARIANT, varargs, calldescr, exc=True) - @FixME #arguments("orgpc", "descr", "varargs") - def opimpl_recursive_call(self, pc, calldescr, varargs): - warmrunnerstate = self.metainterp.staticdata.state - token = None - if not self.metainterp.is_blackholing() and warmrunnerstate.inlining: - num_green_args = self.metainterp.staticdata.num_green_args - portal_code = self.metainterp.staticdata.portal_code - greenkey = varargs[1:num_green_args + 1] - if warmrunnerstate.can_inline_callable(greenkey): - return self.perform_call(portal_code, varargs[1:], greenkey) - token = warmrunnerstate.get_assembler_token(greenkey) - call_position = 0 - if token is not None: - call_position = len(self.metainterp.history.operations) - # verify that we have all green args, needed to make sure - # that assembler that we call is still correct - greenargs = varargs[1:num_green_args + 1] - self.verify_green_args(greenargs) - res = self.do_residual_call(varargs, descr=calldescr, exc=True) - if not self.metainterp.is_blackholing() and token is not None: - # XXX fix the call position, - found = False - while True: - op = self.metainterp.history.operations[call_position] - if op.opnum == rop.CALL or op.opnum == rop.CALL_MAY_FORCE: - found = True - break - call_position += 1 - assert found - # - # this will substitute the residual call with assembler call - self.metainterp.direct_assembler_call(pc, varargs, token, - call_position) - return res + @arguments("boxes3", "boxes3") + def _opimpl_recursive_call(self, greenboxes, redboxes): + allboxes = greenboxes + redboxes + metainterp_sd = self.metainterp.staticdata + portal_code = metainterp_sd.portal_code +## warmrunnerstate = metainterp_sd.state +## token = None +## if not self.is_blackholing() and warmrunnerstate.inlining: +## if warmrunnerstate.can_inline_callable(greenboxes): +## return self.metainterp.perform_call(portal_code, allboxes, +## greenkey=greenboxes) +## token = warmrunnerstate.get_assembler_token(greenboxes) +## call_position = 0 +## if token is not None: +## call_position = len(self.history.operations) +## # verify that we have all green args, needed to make sure +## # that assembler that we call is still correct +## self.verify_green_args(greenboxes) + # + k = llmemory.cast_ptr_to_adr(metainterp_sd._portal_runner_ptr) + funcbox = ConstInt(llmemory.cast_adr_to_int(k)) + resbox = self.do_residual_call(funcbox, portal_code.calldescr, + allboxes) + # +## if token is not None: +## # XXX fix the call position, +## while True: +## op = self.history.operations[call_position] +## if op.opnum == rop.CALL or op.opnum == rop.CALL_MAY_FORCE: +## break +## call_position += 1 +## # +## # this will substitute the residual call with assembler call +## self.direct_assembler_call(boxes, token, call_position) + return resbox + + opimpl_recursive_call_i = _opimpl_recursive_call + opimpl_recursive_call_r = _opimpl_recursive_call + opimpl_recursive_call_f = _opimpl_recursive_call + opimpl_recursive_call_v = _opimpl_recursive_call @FixME #arguments("orgpc", "methdescr", "varargs") def opimpl_oosend(self, pc, methdescr, varargs): @@ -767,9 +772,15 @@ raise CannotInlineCanEnterJit() self.metainterp.seen_can_enter_jit = True + def verify_green_args(self, varargs): + num_green_args = self.metainterp.staticdata.num_green_args + assert len(varargs) == num_green_args + for i in range(num_green_args): + assert isinstance(varargs[i], Const) + @arguments("orgpc", "boxes3", "boxes3") def opimpl_jit_merge_point(self, orgpc, greenboxes, redboxes): - self.metainterp.verify_green_args(greenboxes) + self.verify_green_args(greenboxes) # xxx we may disable the following line in some context later self.debug_merge_point(greenboxes) if self.metainterp.seen_can_enter_jit: @@ -1225,25 +1236,13 @@ def is_blackholing(self): return False # XXX get rid of this method - def perform_call(self, jitcode, boxes): + def perform_call(self, jitcode, boxes, greenkey=None): # causes the metainterp to enter the given subfunction # with a special case for recursive portal calls - if jitcode is self.staticdata.portal_code: - return self.perform_recursive_call_to_portal(boxes) - else: - self._perform_call(jitcode, boxes) - # ^^^ always raises - - def _perform_call(self, jitcode, boxes, greenkey=None): f = self.newframe(jitcode, greenkey) f.setup_call(boxes) raise ChangeFrame - def verify_green_args(self, varargs): - num_green_args = self.staticdata.num_green_args - for i in range(num_green_args): - assert isinstance(varargs[i], Const) - def newframe(self, jitcode, greenkey=None): if jitcode is self.staticdata.portal_code: self.in_recursion += 1 @@ -1279,8 +1278,8 @@ else: try: self.compile_done_with_this_frame(resultbox) - except GiveUp: - self.aborted_tracing(ABORT_BRIDGE) + except SwitchToBlackhole, stb: + self.aborted_tracing(stb.reason) sd = self.staticdata if sd.result_type == 'void': assert resultbox is None @@ -1320,8 +1319,8 @@ self.popframe() try: self.compile_exit_frame_with_exception(excvaluebox) - except GiveUp: - self.aborted_tracing(ABORT_BRIDGE) + except SwitchToBlackhole, stb: + self.aborted_tracing(stb.reason) raise self.staticdata.ExitFrameWithExceptionRef(self.cpu, excvaluebox.getref_base()) def check_recursion_invariant(self): @@ -2004,40 +2003,6 @@ max_key = key return max_key - def perform_recursive_call_to_portal(self, boxes): - warmrunnerstate = self.staticdata.state - portal_code = self.staticdata.portal_code - token = None - if not self.is_blackholing() and warmrunnerstate.inlining: - num_green_args = self.staticdata.num_green_args - greenkey = boxes[:num_green_args] - if warmrunnerstate.can_inline_callable(greenkey): - return self._perform_call(portal_code, boxes, greenkey) - token = warmrunnerstate.get_assembler_token(greenkey) - call_position = 0 - if token is not None: - call_position = len(self.history.operations) - # verify that we have all green args, needed to make sure - # that assembler that we call is still correct - self.verify_green_args(boxes) - # - k = llmemory.cast_ptr_to_adr(self.staticdata._portal_runner_ptr) - funcbox = ConstInt(llmemory.cast_adr_to_int(k)) - frame = self.framestack[-1] - resbox = frame.do_residual_call(funcbox, portal_code.calldescr, boxes) - # - if token is not None: - # XXX fix the call position, - while True: - op = self.history.operations[call_position] - if op.opnum == rop.CALL or op.opnum == rop.CALL_MAY_FORCE: - break - call_position += 1 - # - # this will substitute the residual call with assembler call - self.direct_assembler_call(boxes, token, call_position) - return resbox - def direct_assembler_call(self, boxes, token, call_position): """ Generate a direct call to assembler for portal entry point. """ Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_recursive.py Thu May 27 16:27:07 2010 @@ -137,6 +137,7 @@ return interpret + @py.test.mark.xfail() def test_inline(self): code = "021" subcode = "00" @@ -162,6 +163,7 @@ inline=True) == 42 self.check_loops(call_may_force = 1, call = 0) + @py.test.mark.xfail() def test_inline_faulty_can_inline(self): code = "021" subcode = "301" @@ -179,12 +181,12 @@ def test_guard_failure_in_inlined_function(self): from pypy.rpython.annlowlevel import hlstr - def p(code, pc): + def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(code, pc): + def c(pc, code): return "l" not in hlstr(code) - myjitdriver = JitDriver(greens=['code', 'pc'], reds=['n'], + myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], get_printable_location=p, can_inline=c) def f(code, n): pc = 0 @@ -216,12 +218,12 @@ def test_guard_failure_and_then_exception_in_inlined_function(self): from pypy.rpython.annlowlevel import hlstr - def p(code, pc): + def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(code, pc): + def c(pc, code): return "l" not in hlstr(code) - myjitdriver = JitDriver(greens=['code', 'pc'], reds=['n', 'flag'], + myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n', 'flag'], get_printable_location=p, can_inline=c) def f(code, n): pc = 0 @@ -260,12 +262,12 @@ def test_exception_in_inlined_function(self): from pypy.rpython.annlowlevel import hlstr - def p(code, pc): + def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(code, pc): + def c(pc, code): return "l" not in hlstr(code) - myjitdriver = JitDriver(greens=['code', 'pc'], reds=['n'], + myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], get_printable_location=p, can_inline=c) class Exc(Exception): @@ -306,12 +308,12 @@ # it fails, it is very delicate in terms of parameters, # bridge/loop creation order from pypy.rpython.annlowlevel import hlstr - def p(code, pc): + def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(code, pc): + def c(pc, code): return "l" not in hlstr(code) - myjitdriver = JitDriver(greens=['code', 'pc'], reds=['n'], + myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], get_printable_location=p, can_inline=c) def f(code, n): @@ -397,7 +399,7 @@ def test_max_failure_args(self): FAILARGS_LIMIT = 10 - jitdriver = JitDriver(greens = [], reds = ['o', 'i', 'n']) + jitdriver = JitDriver(greens = [], reds = ['i', 'n', 'o']) class A(object): def __init__(self, i0, i1, i2, i3, i4, i5, i6, i7, i8, i9): @@ -430,7 +432,7 @@ def test_max_failure_args_exc(self): FAILARGS_LIMIT = 10 - jitdriver = JitDriver(greens = [], reds = ['o', 'i', 'n']) + jitdriver = JitDriver(greens = [], reds = ['i', 'n', 'o']) class A(object): def __init__(self, i0, i1, i2, i3, i4, i5, i6, i7, i8, i9): @@ -469,6 +471,7 @@ assert not res self.check_aborted_count(5) + @py.test.mark.xfail() def test_set_param_inlining(self): myjitdriver = JitDriver(greens=[], reds=['n', 'recurse']) def loop(n, recurse=False): @@ -496,12 +499,12 @@ self.check_loops(call_may_force=0, call=0) def test_trace_from_start(self): - def p(code, pc): + def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(code, pc): + def c(pc, code): return "l" not in hlstr(code) - myjitdriver = JitDriver(greens=['code', 'pc'], reds=['n'], + myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], get_printable_location=p, can_inline=c) def f(code, n): @@ -535,13 +538,14 @@ self.check_tree_loop_count(3) self.check_history(int_add=1) + @py.test.mark.xfail() def test_dont_inline_huge_stuff(self): - def p(code, pc): + def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(code, pc): + def c(pc, code): return "l" not in hlstr(code) - myjitdriver = JitDriver(greens=['code', 'pc'], reds=['n'], + myjitdriver = JitDriver(greens=['pc', 'code'], reds=['n'], get_printable_location=p, can_inline=c) def f(code, n): @@ -579,6 +583,7 @@ self.check_history(call_assembler=1, call=0) self.check_tree_loop_count(3) + @py.test.mark.xfail() def test_directly_call_assembler(self): driver = JitDriver(greens = ['codeno'], reds = ['i'], get_printable_location = lambda codeno : str(codeno), @@ -616,6 +621,7 @@ self.check_history(call_assembler=0, call_may_force=1) self.check_enter_count_at_most(1) + @py.test.mark.xfail() def test_directly_call_assembler_return(self): driver = JitDriver(greens = ['codeno'], reds = ['i', 'k'], get_printable_location = lambda codeno : str(codeno), @@ -635,6 +641,7 @@ self.meta_interp(portal, [2], inline=True) self.check_history(call_assembler=1) + @py.test.mark.xfail() def test_directly_call_assembler_raise(self): class MyException(Exception): @@ -694,7 +701,7 @@ class Frame(object): _virtualizable2_ = ['thing'] - driver = JitDriver(greens = ['codeno'], reds = ['frame', 'i'], + driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'], virtualizables = ['frame'], get_printable_location = lambda codeno : str(codeno), can_inline = lambda codeno : False) @@ -730,7 +737,7 @@ class Frame(object): _virtualizable2_ = ['thing'] - driver = JitDriver(greens = ['codeno'], reds = ['frame', 'i'], + driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'], virtualizables = ['frame'], get_printable_location = lambda codeno : str(codeno), can_inline = lambda codeno : False) @@ -771,7 +778,7 @@ assert res == main(0) def test_directly_call_assembler_virtualizable_with_array(self): - myjitdriver = JitDriver(greens = ['codeno'], reds = ['n', 'frame', 'x'], + myjitdriver = JitDriver(greens = ['codeno'], reds = ['n', 'x', 'frame'], virtualizables = ['frame'], can_inline = lambda codeno : False) @@ -817,7 +824,7 @@ class Frame(object): _virtualizable2_ = ['thing'] - driver = JitDriver(greens = ['codeno'], reds = ['frame', 'i'], + driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'], virtualizables = ['frame'], get_printable_location = lambda codeno : str(codeno), can_inline = lambda codeno : False) @@ -860,6 +867,7 @@ policy=StopAtXPolicy(change)) assert res == main(0) + @py.test.mark.xfail() def test_assembler_call_red_args(self): driver = JitDriver(greens = ['codeno'], reds = ['i', 'k'], get_printable_location = lambda codeno : str(codeno), From arigo at codespeak.net Thu May 27 16:30:36 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 May 2010 16:30:36 +0200 (CEST) Subject: [pypy-svn] r74813 - pypy/branch/blackhole-improvement/pypy/jit/metainterp/test Message-ID: <20100527143036.ACBCB282BE0@codespeak.net> Author: arigo Date: Thu May 27 16:30:35 2010 New Revision: 74813 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_slist.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_string.py Log: Fix tests. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_slist.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_slist.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_slist.py Thu May 27 16:30:35 2010 @@ -64,7 +64,7 @@ assert res == 0 def test_getitem(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 'lst', 'i']) + myjitdriver = JitDriver(greens = [], reds = ['n', 'i', 'lst']) def f(n): lst = [] for i in range(n): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_string.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_string.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_string.py Thu May 27 16:30:35 2010 @@ -1,13 +1,13 @@ import py from pypy.rlib.jit import JitDriver -from pypy.jit.metainterp.policy import StopAtXPolicy +from pypy.jit.codewriter.policy import StopAtXPolicy from pypy.rpython.ootypesystem import ootype from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin class StringTests: def test_eq_residual(self): - jitdriver = JitDriver(greens = [], reds = ['s', 'n', 'i']) + jitdriver = JitDriver(greens = [], reds = ['n', 'i', 's']) global_s = "hello" def f(n, b, s): if b: From cfbolz at codespeak.net Thu May 27 16:32:48 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 27 May 2010 16:32:48 +0200 (CEST) Subject: [pypy-svn] r74814 - pypy/trunk/pypy/doc Message-ID: <20100527143248.D9C4E282BEC@codespeak.net> Author: cfbolz Date: Thu May 27 16:32:47 2010 New Revision: 74814 Modified: pypy/trunk/pypy/doc/faq.txt Log: typo Modified: pypy/trunk/pypy/doc/faq.txt ============================================================================== --- pypy/trunk/pypy/doc/faq.txt (original) +++ pypy/trunk/pypy/doc/faq.txt Thu May 27 16:32:47 2010 @@ -67,7 +67,7 @@ PyPy also basically works in a 64-bit Linux environment, but *it requires a 32-bit Intel CPU* for the JIT right now. (It works fine in a 32-bit chroot of an Intel 64.). There is also an ongoing -summero of code project to provide 64bit support for JIT. It's scheduled +summer of code project to provide 64bit support for JIT. It's scheduled to be finished by end of summer 2010. ------------------------------------------------ From arigo at codespeak.net Thu May 27 16:34:37 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 May 2010 16:34:37 +0200 (CEST) Subject: [pypy-svn] r74815 - in pypy/branch/blackhole-improvement/pypy/jit: metainterp/test tl Message-ID: <20100527143437.C8989282BF5@codespeak.net> Author: arigo Date: Thu May 27 16:34:35 2010 New Revision: 74815 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_tl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_tlc.py pypy/branch/blackhole-improvement/pypy/jit/tl/tl.py pypy/branch/blackhole-improvement/pypy/jit/tl/tlc.py pypy/branch/blackhole-improvement/pypy/jit/tl/tlr.py Log: Fix tests. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_tl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_tl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_tl.py Thu May 27 16:34:35 2010 @@ -1,6 +1,6 @@ import py -from pypy.rlib.jit import JitDriver, OPTIMIZER_SIMPLE -from pypy.jit.metainterp.policy import StopAtXPolicy +from pypy.rlib.jit import OPTIMIZER_SIMPLE +from pypy.jit.codewriter.policy import StopAtXPolicy from pypy.jit.metainterp.test.test_basic import OOJitMixin, LLJitMixin Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_tlc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_tlc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_tlc.py Thu May 27 16:34:35 2010 @@ -21,7 +21,7 @@ pool = tlc.ConstantPool() bytecode = tlc.compile(src, pool) interp = self._get_interp(bytecode, pool) - return self.meta_interp(interp, [0, inputarg], view=False) + return self.meta_interp(interp, [0, inputarg]) def test_method(self): code = """ Modified: pypy/branch/blackhole-improvement/pypy/jit/tl/tl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/tl/tl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/tl/tl.py Thu May 27 16:34:35 2010 @@ -56,7 +56,7 @@ def make_interp(supports_call): myjitdriver = JitDriver(greens = ['pc', 'code'], - reds = ['stack', 'inputarg'], + reds = ['inputarg', 'stack'], virtualizables = ['stack']) def interp(code='', pc=0, inputarg=0): if not isinstance(code,str): Modified: pypy/branch/blackhole-improvement/pypy/jit/tl/tlc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/tl/tlc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/tl/tlc.py Thu May 27 16:34:35 2010 @@ -226,7 +226,7 @@ def make_interp(supports_call, jitted=True): - myjitdriver = JitDriver(greens = ['code', 'pc'], + myjitdriver = JitDriver(greens = ['pc', 'code'], reds = ['frame', 'pool']) def interp(code='', pc=0, inputarg=0, pool=None): Modified: pypy/branch/blackhole-improvement/pypy/jit/tl/tlr.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/tl/tlr.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/tl/tlr.py Thu May 27 16:34:35 2010 @@ -12,7 +12,7 @@ NEG_A = 8 class TLRJitDriver(JitDriver): - greens = ['bytecode', 'pc'] + greens = ['pc', 'bytecode'] reds = ['a', 'regs'] tlrjitdriver = TLRJitDriver() From arigo at codespeak.net Thu May 27 17:10:05 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 May 2010 17:10:05 +0200 (CEST) Subject: [pypy-svn] r74816 - in pypy/branch/blackhole-improvement/pypy/jit/metainterp: . test Message-ID: <20100527151005.A2C0C282BEC@codespeak.net> Author: arigo Date: Thu May 27 17:10:03 2010 New Revision: 74816 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualref.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py Log: Start working on virtualrefs. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Thu May 27 17:10:03 2010 @@ -732,6 +732,9 @@ except KeyError: return pc + # ---------- + # exception handling operations + @arguments("L") def bhimpl_catch_exception(target): """This is a no-op when run normally. When an exception occurs @@ -783,6 +786,9 @@ etype = rclass.ll_type(e) raise LLException(etype, e) + # ---------- + # the main hints and recursive calls + @arguments() def bhimpl_can_enter_jit(): pass @@ -840,6 +846,17 @@ greens_f + reds_f) # ---------- + # virtual refs + + @arguments("r", returns="r") + def bhimpl_virtual_ref(a): + return a + + @arguments("r") + def bhimpl_virtual_ref_finish(a): + pass + + # ---------- # list operations @arguments("cpu", "r", "d", "i", returns="i") Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Thu May 27 17:10:03 2010 @@ -843,7 +843,7 @@ assert exc_value_box is not None return exc_value_box - @FixME #arguments("box") + @arguments("box") def opimpl_virtual_ref(self, box): # Details on the content of metainterp.virtualref_boxes: # @@ -852,7 +852,7 @@ # the 'virtual_ref(frame)'). # # * if we detect that the virtual box escapes during tracing - # already (by generating a CALl_MAY_FORCE that marks the flags + # already (by generating a CALL_MAY_FORCE that marks the flags # in the vref), then we replace the vref in the list with # ConstPtr(NULL). # @@ -874,9 +874,9 @@ # SETFIELD_GCs. metainterp.virtualref_boxes.append(box) metainterp.virtualref_boxes.append(resbox) - self.make_result_box(resbox) + return resbox - @FixME #arguments("box") + @arguments("box") def opimpl_virtual_ref_finish(self, box): # virtual_ref_finish() assumes that we have a stack-like, last-in # first-out order. @@ -1111,6 +1111,7 @@ # self.portal_code = codewriter.mainjitcode self._portal_runner_ptr = codewriter.callcontrol.portal_runner_ptr + self.virtualref_info = codewriter.callcontrol.virtualref_info RESULT = codewriter.portal_graph.getreturnvar().concretetype self.result_type = history.getkind(RESULT) # Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Thu May 27 17:10:03 2010 @@ -12,8 +12,20 @@ from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.ootypesystem import ootype -def _get_jitcodes(CPUClass, func, values, type_system): +def _get_jitcodes(testself, CPUClass, func, values, type_system): from pypy.jit.codewriter import support, codewriter + from pypy.jit.metainterp import simple_optimize + + class FakeWarmRunnerState: + def attach_unoptimized_bridge_from_interp(self, greenkey, newloop): + pass + + # pick the optimizer this way + optimize_loop = staticmethod(simple_optimize.optimize_loop) + optimize_bridge = staticmethod(simple_optimize.optimize_bridge) + + trace_limit = sys.maxint + debug_level = 2 func._jit_unroll_safe_ = True rtyper = support.annotate(func, values, type_system=type_system) @@ -21,12 +33,19 @@ stats = history.Stats() cpu = CPUClass(rtyper, stats, None, False) cw = codewriter.CodeWriter(cpu, graphs[0]) + testself.cw = cw cw.find_all_graphs(JitPolicy()) + # + testself.warmrunnerstate = FakeWarmRunnerState() + testself.warmrunnerstate.cpu = cpu + if hasattr(testself, 'finish_setup_for_interp_operations'): + testself.finish_setup_for_interp_operations() + # cw.make_jitcodes(verbose=True) - return cw -def _run_with_blackhole(cw, args): +def _run_with_blackhole(testself, args): from pypy.jit.metainterp.blackhole import BlackholeInterpBuilder + cw = testself.cw blackholeinterpbuilder = BlackholeInterpBuilder(cw) blackholeinterp = blackholeinterpbuilder.acquire_interp() count_i = count_r = count_f = 0 @@ -47,8 +66,7 @@ blackholeinterp.run() return blackholeinterp._final_result_anytype() -def _run_with_pyjitpl(cw, args, testself): - from pypy.jit.metainterp import simple_optimize +def _run_with_pyjitpl(testself, args): class DoneWithThisFrame(Exception): pass @@ -57,26 +75,13 @@ def __init__(self, cpu, *args): DoneWithThisFrame.__init__(self, *args) - class FakeWarmRunnerState: - def attach_unoptimized_bridge_from_interp(self, greenkey, newloop): - pass - - # pick the optimizer this way - optimize_loop = staticmethod(simple_optimize.optimize_loop) - optimize_bridge = staticmethod(simple_optimize.optimize_bridge) - - trace_limit = sys.maxint - debug_level = 2 - + cw = testself.cw opt = history.Options(listops=True) metainterp_sd = pyjitpl.MetaInterpStaticData(cw.cpu, opt) metainterp_sd.finish_setup(cw, optimizer="bogus") - metainterp_sd.state = FakeWarmRunnerState() + metainterp_sd.state = testself.warmrunnerstate metainterp_sd.state.cpu = metainterp_sd.cpu metainterp = pyjitpl.MetaInterp(metainterp_sd) - if hasattr(testself, 'finish_metainterp_for_interp_operations'): - testself.finish_metainterp_for_interp_operations(metainterp) - metainterp_sd.DoneWithThisFrameInt = DoneWithThisFrame metainterp_sd.DoneWithThisFrameRef = DoneWithThisFrameRef metainterp_sd.DoneWithThisFrameFloat = DoneWithThisFrame @@ -126,11 +131,11 @@ def interp_operations(self, f, args, **kwds): # get the JitCodes for the function f - cw = _get_jitcodes(self.CPUClass, f, args, self.type_system) + _get_jitcodes(self, self.CPUClass, f, args, self.type_system) # try to run it with blackhole.py - result1 = _run_with_blackhole(cw, args) + result1 = _run_with_blackhole(self, args) # try to run it with pyjitpl.py - result2 = _run_with_pyjitpl(cw, args, self) + result2 = _run_with_pyjitpl(self, args) assert result1 == result2 return result1 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualref.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualref.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualref.py Thu May 27 17:10:03 2010 @@ -12,9 +12,9 @@ class VRefTests: - def finish_metainterp_for_interp_operations(self, metainterp): - self.vrefinfo = VirtualRefInfo(metainterp.staticdata.state) - metainterp.staticdata.virtualref_info = self.vrefinfo + def finish_setup_for_interp_operations(self): + self.vrefinfo = VirtualRefInfo(self.warmrunnerstate) + self.cw.setup_vrefinfo(self.vrefinfo) def test_make_vref_simple(self): class X: Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py Thu May 27 17:10:03 2010 @@ -163,8 +163,8 @@ self.make_args_specification() # from pypy.jit.metainterp.virtualref import VirtualRefInfo - self.metainterp_sd.virtualref_info = VirtualRefInfo(self) - self.codewriter.setup_vrefinfo(self.metainterp_sd.virtualref_info) + vrefinfo = VirtualRefInfo(self) + self.codewriter.setup_vrefinfo(vrefinfo) if self.jitdriver.virtualizables: from pypy.jit.metainterp.virtualizable import VirtualizableInfo self.metainterp_sd.virtualizable_info = VirtualizableInfo(self) @@ -178,7 +178,7 @@ self.codewriter.make_jitcodes(verbose=verbose) self.rewrite_can_enter_jit() self.rewrite_set_param() - self.rewrite_force_virtual() + self.rewrite_force_virtual(vrefinfo) self.add_finish() self.metainterp_sd.finish_setup(self.codewriter, optimizer=optimizer) @@ -657,9 +657,8 @@ op.opname = 'direct_call' op.args[:3] = [closures[funcname]] - def rewrite_force_virtual(self): + def rewrite_force_virtual(self, vrefinfo): if self.cpu.ts.name != 'lltype': py.test.skip("rewrite_force_virtual: port it to ootype") all_graphs = self.translator.graphs - vrefinfo = self.metainterp_sd.virtualref_info vrefinfo.replace_force_virtual_with_call(all_graphs) From antocuni at codespeak.net Thu May 27 17:17:57 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 27 May 2010 17:17:57 +0200 (CEST) Subject: [pypy-svn] r74817 - pypy/branch/sys-prefix Message-ID: <20100527151757.733E9282BEC@codespeak.net> Author: antocuni Date: Thu May 27 17:17:56 2010 New Revision: 74817 Added: pypy/branch/sys-prefix/ (props changed) - copied from r74816, pypy/trunk/ Log: a branch where to try to reorganize the stdlib directories in order to get a sensible sys.prefix (and make e.g. virtualenv happy) From jcreigh at codespeak.net Thu May 27 17:22:37 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Thu, 27 May 2010 17:22:37 +0200 (CEST) Subject: [pypy-svn] r74818 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86 Message-ID: <20100527152237.61AA2282BEC@codespeak.net> Author: jcreigh Date: Thu May 27 17:22:35 2010 New Revision: 74818 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Log: try to replace mess of if statements with unrolling_iterable Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Thu May 27 17:22:35 2010 @@ -1,5 +1,6 @@ from pypy.jit.metainterp.history import AbstractValue, ConstInt from pypy.jit.backend.x86 import rx86 +from pypy.rlib.unroll import unrolling_iterable # # This module adds support for "locations", which can be either in a Const, @@ -126,6 +127,7 @@ eax, ecx, edx, ebx, esp, ebp, esi, edi = REGLOCS xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 = XMMREGLOCS +unrolling_possible_location_codes = unrolling_iterable(list("rbsmajix")) class LocationCodeBuilder(object): _mixin_ = True @@ -134,163 +136,28 @@ def INSN(self, loc1, loc2): code1 = loc1.location_code() code2 = loc2.location_code() - # XXX: All possible combinations are given, even those that are - # impossible - if code1 == 'r' and code2 == 'r': - getattr(self, name + "_rr")(loc1.value, loc2.value) - elif code1 == 'r' and code2 == 'b': - getattr(self, name + "_rb")(loc1.value, loc2.value) - elif code1 == 'r' and code2 == 's': - getattr(self, name + "_rs")(loc1.value, loc2.value) - elif code1 == 'r' and code2 == 'm': - getattr(self, name + "_rm")(loc1.value, loc2.value) - elif code1 == 'r' and code2 == 'a': - getattr(self, name + "_ra")(loc1.value, loc2.value) - elif code1 == 'r' and code2 == 'j': - getattr(self, name + "_rj")(loc1.value, loc2.value) - elif code1 == 'r' and code2 == 'i': - getattr(self, name + "_ri")(loc1.value, loc2.value) - elif code1 == 'r' and code2 == 'x': - getattr(self, name + "_rx")(loc1.value, loc2.value) - elif code1 == 'b' and code2 == 'r': - getattr(self, name + "_br")(loc1.value, loc2.value) - elif code1 == 'b' and code2 == 'b': - getattr(self, name + "_bb")(loc1.value, loc2.value) - elif code1 == 'b' and code2 == 's': - getattr(self, name + "_bs")(loc1.value, loc2.value) - elif code1 == 'b' and code2 == 'm': - getattr(self, name + "_bm")(loc1.value, loc2.value) - elif code1 == 'b' and code2 == 'a': - getattr(self, name + "_ba")(loc1.value, loc2.value) - elif code1 == 'b' and code2 == 'j': - getattr(self, name + "_bj")(loc1.value, loc2.value) - elif code1 == 'b' and code2 == 'i': - getattr(self, name + "_bi")(loc1.value, loc2.value) - elif code1 == 'b' and code2 == 'x': - getattr(self, name + "_bx")(loc1.value, loc2.value) - elif code1 == 's' and code2 == 'r': - getattr(self, name + "_sr")(loc1.value, loc2.value) - elif code1 == 's' and code2 == 'b': - getattr(self, name + "_sb")(loc1.value, loc2.value) - elif code1 == 's' and code2 == 's': - getattr(self, name + "_ss")(loc1.value, loc2.value) - elif code1 == 's' and code2 == 'm': - getattr(self, name + "_sm")(loc1.value, loc2.value) - elif code1 == 's' and code2 == 'a': - getattr(self, name + "_sa")(loc1.value, loc2.value) - elif code1 == 's' and code2 == 'j': - getattr(self, name + "_sj")(loc1.value, loc2.value) - elif code1 == 's' and code2 == 'i': - getattr(self, name + "_si")(loc1.value, loc2.value) - elif code1 == 's' and code2 == 'x': - getattr(self, name + "_sx")(loc1.value, loc2.value) - elif code1 == 'm' and code2 == 'r': - getattr(self, name + "_mr")(loc1.value, loc2.value) - elif code1 == 'm' and code2 == 'b': - getattr(self, name + "_mb")(loc1.value, loc2.value) - elif code1 == 'm' and code2 == 's': - getattr(self, name + "_ms")(loc1.value, loc2.value) - elif code1 == 'm' and code2 == 'm': - getattr(self, name + "_mm")(loc1.value, loc2.value) - elif code1 == 'm' and code2 == 'a': - getattr(self, name + "_ma")(loc1.value, loc2.value) - elif code1 == 'm' and code2 == 'j': - getattr(self, name + "_mj")(loc1.value, loc2.value) - elif code1 == 'm' and code2 == 'i': - getattr(self, name + "_mi")(loc1.value, loc2.value) - elif code1 == 'm' and code2 == 'x': - getattr(self, name + "_mx")(loc1.value, loc2.value) - elif code1 == 'a' and code2 == 'r': - getattr(self, name + "_ar")(loc1.value, loc2.value) - elif code1 == 'a' and code2 == 'b': - getattr(self, name + "_ab")(loc1.value, loc2.value) - elif code1 == 'a' and code2 == 's': - getattr(self, name + "_as")(loc1.value, loc2.value) - elif code1 == 'a' and code2 == 'm': - getattr(self, name + "_am")(loc1.value, loc2.value) - elif code1 == 'a' and code2 == 'a': - getattr(self, name + "_aa")(loc1.value, loc2.value) - elif code1 == 'a' and code2 == 'j': - getattr(self, name + "_aj")(loc1.value, loc2.value) - elif code1 == 'a' and code2 == 'i': - getattr(self, name + "_ai")(loc1.value, loc2.value) - elif code1 == 'a' and code2 == 'x': - getattr(self, name + "_ax")(loc1.value, loc2.value) - elif code1 == 'j' and code2 == 'r': - getattr(self, name + "_jr")(loc1.value, loc2.value) - elif code1 == 'j' and code2 == 'b': - getattr(self, name + "_jb")(loc1.value, loc2.value) - elif code1 == 'j' and code2 == 's': - getattr(self, name + "_js")(loc1.value, loc2.value) - elif code1 == 'j' and code2 == 'm': - getattr(self, name + "_jm")(loc1.value, loc2.value) - elif code1 == 'j' and code2 == 'a': - getattr(self, name + "_ja")(loc1.value, loc2.value) - elif code1 == 'j' and code2 == 'j': - getattr(self, name + "_jj")(loc1.value, loc2.value) - elif code1 == 'j' and code2 == 'i': - getattr(self, name + "_ji")(loc1.value, loc2.value) - elif code1 == 'j' and code2 == 'x': - getattr(self, name + "_jx")(loc1.value, loc2.value) - elif code1 == 'i' and code2 == 'r': - getattr(self, name + "_ir")(loc1.value, loc2.value) - elif code1 == 'i' and code2 == 'b': - getattr(self, name + "_ib")(loc1.value, loc2.value) - elif code1 == 'i' and code2 == 's': - getattr(self, name + "_is")(loc1.value, loc2.value) - elif code1 == 'i' and code2 == 'm': - getattr(self, name + "_im")(loc1.value, loc2.value) - elif code1 == 'i' and code2 == 'a': - getattr(self, name + "_ia")(loc1.value, loc2.value) - elif code1 == 'i' and code2 == 'j': - getattr(self, name + "_ij")(loc1.value, loc2.value) - elif code1 == 'i' and code2 == 'i': - getattr(self, name + "_ii")(loc1.value, loc2.value) - elif code1 == 'i' and code2 == 'x': - getattr(self, name + "_ix")(loc1.value, loc2.value) - elif code1 == 'x' and code2 == 'r': - getattr(self, name + "_xr")(loc1.value, loc2.value) - elif code1 == 'x' and code2 == 'b': - getattr(self, name + "_xb")(loc1.value, loc2.value) - elif code1 == 'x' and code2 == 's': - getattr(self, name + "_xs")(loc1.value, loc2.value) - elif code1 == 'x' and code2 == 'm': - getattr(self, name + "_xm")(loc1.value, loc2.value) - elif code1 == 'x' and code2 == 'a': - getattr(self, name + "_xa")(loc1.value, loc2.value) - elif code1 == 'x' and code2 == 'j': - getattr(self, name + "_xj")(loc1.value, loc2.value) - elif code1 == 'x' and code2 == 'i': - getattr(self, name + "_xi")(loc1.value, loc2.value) - elif code1 == 'x' and code2 == 'x': - getattr(self, name + "_xx")(loc1.value, loc2.value) - else: - raise AssertionError("Invalid location codes") + for possible_code1 in unrolling_possible_location_codes: + for possible_code2 in unrolling_possible_location_codes: + if code1 == possible_code1 and code2 == possible_code2: + methname = name + "_" + code1 + code2 + if hasattr(rx86.AbstractX86CodeBuilder, methname): + getattr(self, methname)(loc1.value, loc2.value) + else: + raise AssertionError("Instruction not defined: " + methname) return INSN def _unaryop(name): def INSN(self, loc): code = loc.location_code() - # "if" is unrolled for RPython - if code == 'r': - getattr(self, name + '_r')(loc.value) - elif code == 'b': - getattr(self, name + '_b')(loc.value) - elif code == 's': - getattr(self, name + '_s')(loc.value) - elif code == 'm': - getattr(self, name + '_m')(loc.value) - elif code == 'a': - getattr(self, name + '_a')(loc.value) - elif code == 'j': - getattr(self, name + '_j')(loc.value) - elif code == 'i': - getattr(self, name + '_i')(loc.value) - elif code == 'x': - getattr(self, name + '_x')(loc.value) - else: - raise AssertionError("Unknown code") + for possible_code in unrolling_possible_location_codes: + if code == possible_code: + methname = name + "_" + code + if hasattr(rx86.AbstractX86CodeBuilder, methname): + getattr(self, methname)(loc.value) + else: + raise AssertionError("Instruction not defined: " + methname) + return INSN AND = _binaryop('AND') From jcreigh at codespeak.net Thu May 27 17:28:43 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Thu, 27 May 2010 17:28:43 +0200 (CEST) Subject: [pypy-svn] r74819 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86 Message-ID: <20100527152843.094F1282BEC@codespeak.net> Author: jcreigh Date: Thu May 27 17:28:41 2010 New Revision: 74819 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Log: use constants, not variables, to construct string for compile-time hasattr Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Thu May 27 17:28:41 2010 @@ -139,7 +139,7 @@ for possible_code1 in unrolling_possible_location_codes: for possible_code2 in unrolling_possible_location_codes: if code1 == possible_code1 and code2 == possible_code2: - methname = name + "_" + code1 + code2 + methname = name + "_" + possible_code1 + possible_code2 if hasattr(rx86.AbstractX86CodeBuilder, methname): getattr(self, methname)(loc1.value, loc2.value) else: @@ -152,7 +152,7 @@ code = loc.location_code() for possible_code in unrolling_possible_location_codes: if code == possible_code: - methname = name + "_" + code + methname = name + "_" + possible_code if hasattr(rx86.AbstractX86CodeBuilder, methname): getattr(self, methname)(loc.value) else: From antocuni at codespeak.net Thu May 27 18:36:05 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 27 May 2010 18:36:05 +0200 (CEST) Subject: [pypy-svn] r74821 - in pypy/branch/sys-prefix/pypy/translator/goal: . test2 Message-ID: <20100527163605.21C83282BE0@codespeak.net> Author: antocuni Date: Thu May 27 18:36:03 2010 New Revision: 74821 Modified: pypy/branch/sys-prefix/pypy/translator/goal/app_main.py pypy/branch/sys-prefix/pypy/translator/goal/test2/test_app_main.py Log: try to factor out get_library_path from setup_initial_paths, and write a test for it as there were none Modified: pypy/branch/sys-prefix/pypy/translator/goal/app_main.py ============================================================================== --- pypy/branch/sys-prefix/pypy/translator/goal/app_main.py (original) +++ pypy/branch/sys-prefix/pypy/translator/goal/app_main.py Thu May 27 18:36:03 2010 @@ -182,12 +182,32 @@ option) return argv[i], i +def get_library_path(executable): + AUTOSUBPATH = 'share' + os.sep + 'pypy-%d.%d' + # set up a sys.path that depends on the local machine + autosubpath = AUTOSUBPATH % sys.pypy_version_info[:2] + search = executable + while 1: + dirname = resolvedirof(search) + if dirname == search: + # not found! let's hope that the compiled-in path is ok + print >> sys.stderr, ('debug: WARNING: library path not found, ' + 'using compiled-in sys.path') + newpath = sys.path[:] + break + newpath = sys.pypy_initial_path(dirname) + if newpath is None: + newpath = sys.pypy_initial_path(os.path.join(dirname, autosubpath)) + if newpath is None: + search = dirname # walk to the parent directory + continue + break # found! + return newpath def setup_initial_paths(executable, nanos): # a substituted os if we are translated global os os = nanos - AUTOSUBPATH = 'share' + os.sep + 'pypy-%d.%d' # find the full path to the executable, assuming that if there is no '/' # in the provided one then we must look along the $PATH if we_are_translated() and IS_WINDOWS and not executable.lower().endswith('.exe'): @@ -204,24 +224,7 @@ break sys.executable = os.path.abspath(executable) - # set up a sys.path that depends on the local machine - autosubpath = AUTOSUBPATH % sys.pypy_version_info[:2] - search = executable - while 1: - dirname = resolvedirof(search) - if dirname == search: - # not found! let's hope that the compiled-in path is ok - print >> sys.stderr, ('debug: WARNING: library path not found, ' - 'using compiled-in sys.path') - newpath = sys.path[:] - break - newpath = sys.pypy_initial_path(dirname) - if newpath is None: - newpath = sys.pypy_initial_path(os.path.join(dirname, autosubpath)) - if newpath is None: - search = dirname # walk to the parent directory - continue - break # found! + newpath = get_library_path(executable) path = os.getenv('PYTHONPATH') if path: newpath = path.split(os.pathsep) + newpath Modified: pypy/branch/sys-prefix/pypy/translator/goal/test2/test_app_main.py ============================================================================== --- pypy/branch/sys-prefix/pypy/translator/goal/test2/test_app_main.py (original) +++ pypy/branch/sys-prefix/pypy/translator/goal/test2/test_app_main.py Thu May 27 18:36:03 2010 @@ -460,3 +460,43 @@ data = child_out_err.read(11) assert data == '\x00(STDOUT)\n\x00' # from stdout child_out_err.close() + + +class AppTestAppMain: + + def setup_class(self): + # ------------------------------------ + # setup code for test_get_library_path + # ------------------------------------ + from pypy.module.sys.version import CPYTHON_VERSION, PYPY_VERSION + libroot = 'share/pypy-%d.%d' % PYPY_VERSION[:2] + cpy_ver = '%d.%d.%d' % CPYTHON_VERSION[:3] + + goal_dir = os.path.dirname(app_main) + # build a directory hierarchy like which contains both bin/pypy-c and + # share/pypy-1.2/* + prefix = udir.join('pathtest') + fake_exe = prefix.join('bin/pypy-c').ensure(file=1) + share_pypy = prefix.join(libroot).ensure(dir=1) + expected_path = [str(share_pypy.join(subdir).ensure(dir=1)) + for subdir in ('pypy/lib', + 'lib-python/modified-%s' % cpy_ver, + 'lib-python/%s' % cpy_ver)] + + self.w_goal_dir = self.space.wrap(goal_dir) + self.w_fake_exe = self.space.wrap(str(fake_exe)) + self.w_expected_path = self.space.wrap(expected_path) + + def test_get_library_path(self): + import sys + import os + sys.path.append(self.goal_dir) + try: + import app_main + app_main.os = os + newpath = app_main.get_library_path('/tmp/pypy-c') # stdlib not found + assert newpath == sys.path + newpath = app_main.get_library_path(self.fake_exe) + assert newpath == self.expected_path + finally: + sys.path.pop() From afa at codespeak.net Thu May 27 18:43:52 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 27 May 2010 18:43:52 +0200 (CEST) Subject: [pypy-svn] r74822 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100527164352.386A1282BE0@codespeak.net> Author: afa Date: Thu May 27 18:43:50 2010 New Revision: 74822 Modified: pypy/trunk/pypy/module/cpyext/test/foo.c pypy/trunk/pypy/module/cpyext/test/test_cpyext.py pypy/trunk/pypy/module/cpyext/test/test_typeobject.py pypy/trunk/pypy/module/cpyext/typeobject.py Log: Implement metatypes for cpyext. The change seems simple, but was hard to get right... Modified: pypy/trunk/pypy/module/cpyext/test/foo.c ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/foo.c (original) +++ pypy/trunk/pypy/module/cpyext/test/foo.c Thu May 27 18:43:50 2010 @@ -166,6 +166,8 @@ foo_getseters, /*tp_getset*/ }; +/* A type that inherits from 'unicode */ + typedef struct { PyUnicodeObject HEAD; int val; @@ -312,6 +314,68 @@ }; +/* A Metatype */ + +PyTypeObject MetaType = { + PyObject_HEAD_INIT(NULL) + 0, + "foo.Meta", + sizeof(PyTypeObject), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + 0, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_CHECKTYPES, /*tp_flags*/ + 0, /*tp_doc*/ + + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + + /* Attribute descriptor and subclassing stuff */ + + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + + 0, /*tp_init*/ + 0, /*tp_alloc*/ + 0, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0 /*tp_weaklist*/ +}; + + /* foo functions */ static PyObject * @@ -344,6 +408,7 @@ FuuType.tp_base = &PyUnicode_Type; Fuu2Type.tp_base = &FuuType; + MetaType.tp_base = &PyType_Type; if (PyType_Ready(&footype) < 0) return; @@ -351,15 +416,20 @@ return; if (PyType_Ready(&Fuu2Type) < 0) return; + if (PyType_Ready(&MetaType) < 0) + return; m = Py_InitModule("foo", foo_functions); if (m == NULL) return; d = PyModule_GetDict(m); - if (d) { - if (PyDict_SetItemString(d, "fooType", (PyObject *)&footype) < 0) - return; - PyDict_SetItemString(d, "FuuType", (PyObject *) &FuuType); - PyDict_SetItemString(d, "Fuu2Type", (PyObject *) &Fuu2Type); - } - /* No need to check the error here, the caller will do that */ + if (d == NULL) + return; + if (PyDict_SetItemString(d, "fooType", (PyObject *)&footype) < 0) + return; + if (PyDict_SetItemString(d, "FuuType", (PyObject *) &FuuType) < 0) + return; + if(PyDict_SetItemString(d, "Fuu2Type", (PyObject *) &Fuu2Type) < 0) + return; + if (PyDict_SetItemString(d, "MetaType", (PyObject *) &MetaType) < 0) + return; } Modified: pypy/trunk/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Thu May 27 18:43:50 2010 @@ -90,6 +90,14 @@ gc.collect() lost_objects_w = identity_dict() lost_objects_w.update((key, None) for key in self.frozen_refcounts.keys()) + + # Clear all lifelines, objects won't resurrect + for w_obj, obj in typeobject.lifeline_dict._dict.items(): + if w_obj not in state.py_objects_w2r: + typeobject.lifeline_dict.set(w_obj, None) + del obj + gc.collect() + for w_obj, obj in state.py_objects_w2r.iteritems(): base_refcnt = self.frozen_refcounts.get(w_obj) delta = obj.c_ob_refcnt Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Thu May 27 18:43:50 2010 @@ -135,6 +135,13 @@ assert newobj.get_val() == 42 assert newobj.foobar == 32 + def test_metatype(self): + module = self.import_module(name='foo') + assert module.MetaType.__mro__ == (module.MetaType, type, object) + x = module.MetaType('name', (), {}) + assert isinstance(x, type) + x() + def test_sre(self): module = self.import_module(name='_sre') import sre_compile Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Thu May 27 18:43:50 2010 @@ -120,10 +120,6 @@ slot_func.api_func.get_wrapper(space)) # XXX special case wrapper-functions and use a "specific" slot func - # the special case of __new__ in CPython works a bit differently, hopefully - # this matches the semantics - if method_name == "__new__" and not pto.c_tp_new: - continue if len(slot_name) == 1: setattr(pto, slot_name[0], slot_func_helper) else: @@ -439,6 +435,7 @@ typedescr = get_typedescr(w_obj.typedef) py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) lifeline_dict.set(w_obj, PyOLifeline(space, py_obj)) + typedescr.attach(space, py_obj, w_obj) return py_obj @cpython_api([PyObject, rffi.INTP], lltype.Signed, external=False, @@ -486,8 +483,9 @@ rffi.free_charp(obj_pto.c_tp_name) obj_pto_voidp = rffi.cast(rffi.VOIDP_real, obj_pto) generic_cpy_call(space, type_pto.c_tp_free, obj_pto_voidp) - pto = rffi.cast(PyObject, type_pto) - Py_DecRef(space, pto) + if type_pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE: + pto = rffi.cast(PyObject, type_pto) + Py_DecRef(space, pto) def type_attach(space, py_obj, w_type): From afa at codespeak.net Thu May 27 19:31:01 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 27 May 2010 19:31:01 +0200 (CEST) Subject: [pypy-svn] r74823 - pypy/trunk/pypy/module/cpyext/test Message-ID: <20100527173101.9EEEA282BE0@codespeak.net> Author: afa Date: Thu May 27 19:31:00 2010 New Revision: 74823 Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Log: Next test to pass: compatibility between the metaclass and the base class. Now I get a TypeError: "the metaclass of a derived class must be a subclass of the metaclasses of all its bases" Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Thu May 27 19:31:00 2010 @@ -140,8 +140,18 @@ assert module.MetaType.__mro__ == (module.MetaType, type, object) x = module.MetaType('name', (), {}) assert isinstance(x, type) + assert isinstance(x, module.MetaType) x() + @py.test.mark.xfail + def test_metaclass_compatible(self): + # metaclasses should not conflict here + print module.MetaType.__mro__ # (foo.Meta, type, object) + print type(module.fooType).__mro__ # (C_Type, type, object) + y = module.MetaType('other', (module.fooType,), {}) + assert isinstance(y, module.MetaType) + y() + def test_sre(self): module = self.import_module(name='_sre') import sre_compile From afa at codespeak.net Thu May 27 20:06:07 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 27 May 2010 20:06:07 +0200 (CEST) Subject: [pypy-svn] r74824 - pypy/trunk/pypy/module/cpyext Message-ID: <20100527180607.045C2282BE0@codespeak.net> Author: afa Date: Thu May 27 20:06:06 2010 New Revision: 74824 Modified: pypy/trunk/pypy/module/cpyext/typeobject.py Log: Don't crash when _PyType_Lookup() is given an invalid attribute name Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Thu May 27 20:06:06 2010 @@ -687,10 +687,14 @@ def _PyType_Lookup(space, type, w_name): """Internal API to look for a name through the MRO. This returns a borrowed reference, and doesn't set an exception!""" - py_type = rffi.cast(PyObject, type) - w_type = from_ref(space, py_type) + w_type = from_ref(space, rffi.cast(PyObject, type)) assert isinstance(w_type, W_TypeObject) + + if not space.isinstance_w(w_name, space.w_str): + return None name = space.str_w(w_name) w_obj = w_type.lookup(name) return borrow_from(w_type, w_obj) + + From getxsick at codespeak.net Thu May 27 22:19:38 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Thu, 27 May 2010 22:19:38 +0200 (CEST) Subject: [pypy-svn] r74825 - pypy/trunk/pypy/doc Message-ID: <20100527201938.89A7C282BE0@codespeak.net> Author: getxsick Date: Thu May 27 22:19:35 2010 New Revision: 74825 Modified: pypy/trunk/pypy/doc/coding-guide.txt Log: typo Modified: pypy/trunk/pypy/doc/coding-guide.txt ============================================================================== --- pypy/trunk/pypy/doc/coding-guide.txt (original) +++ pypy/trunk/pypy/doc/coding-guide.txt Thu May 27 22:19:35 2010 @@ -121,8 +121,8 @@ In any case, it should be obvious that the application-level implementation is definitely more readable, more elegant and more maintainable than the -interpreter-level one (and indeed, dict.update is really implemented at appleve -in PyPy). +interpreter-level one (and indeed, dict.update is really implemented at +applevel in PyPy). In fact, in almost all parts of PyPy, you find application level code in the middle of interpreter-level code. Apart from some bootstrapping From jcreigh at codespeak.net Thu May 27 22:23:31 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Thu, 27 May 2010 22:23:31 +0200 (CEST) Subject: [pypy-svn] r74826 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86 Message-ID: <20100527202331.DEC46282BE0@codespeak.net> Author: jcreigh Date: Thu May 27 22:23:30 2010 New Revision: 74826 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py Log: rpython related fixes Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Thu May 27 22:23:30 2010 @@ -996,6 +996,8 @@ if offset is not None: mc.CMP(mem(locs[0], offset), locs[1]) else: + # We haven't implemented CMP16 yet... + raise AssertionError("FIXME") # XXX hard-coded assumption: to go from an object to its class # we use the following algorithm: # - read the typeid from mem(locs[0]), i.e. at offset 0 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Thu May 27 22:23:30 2010 @@ -10,11 +10,18 @@ # class AssemblerLocation(AbstractValue): - __slots__ = 'value' + # XXX: Is adding "width" here correct? + __slots__ = ('value', 'width') _immutable_ = True def _getregkey(self): return self.value + def value_r(self): return self.value + def value_b(self): return self.value + def value_s(self): return self.value + def value_j(self): return self.value + def value_i(self): return self.value + def value_x(self): return self.value class StackLoc(AssemblerLocation): _immutable_ = True @@ -94,6 +101,8 @@ class AddressLoc(AssemblerLocation): _immutable_ = True + # XXX + width = 4 # The address is base_loc + (scaled_loc << scale) + static_offset def __init__(self, base_loc, scaled_loc, scale=0, static_offset=0): assert 0 <= scale < 4 @@ -106,28 +115,36 @@ self.value = base_loc.value + (scaled_loc.value << scale) + static_offset else: self._location_code = 'a' - self.value = (None, scaled_loc.value, scale, base_loc.value + static_offset) + self.loc_a = (rx86.NO_BASE_REGISTER, scaled_loc.value, scale, base_loc.value + static_offset) else: if isinstance(scaled_loc, ImmedLoc): # FIXME: What if base_loc is ebp or esp? self._location_code = 'm' - self.value = (base_loc.value, (scaled_loc.value << scale) + static_offset) + self.loc_m = (base_loc.value, (scaled_loc.value << scale) + static_offset) else: self._location_code = 'a' - self.value = (base_loc.value, scaled_loc.value, scale, static_offset) + self.loc_a = (base_loc.value, scaled_loc.value, scale, static_offset) def location_code(self): return self._location_code - def value(self): - return self.value + def value_a(self): + return self.loc_a + + def value_m(self): + return self.loc_m REGLOCS = [RegLoc(i, is_xmm=False) for i in range(8)] XMMREGLOCS = [RegLoc(i, is_xmm=True) for i in range(8)] eax, ecx, edx, ebx, esp, ebp, esi, edi = REGLOCS xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 = XMMREGLOCS -unrolling_possible_location_codes = unrolling_iterable(list("rbsmajix")) +possible_location_codes = list("rbsmajix") +# Separate objects are required because you can't use the same +# unrolling_iterable instance in more than once place +_binop_pc_outer = unrolling_iterable(possible_location_codes) +_binop_pc_inner = unrolling_iterable(possible_location_codes) +_unaryop_pc = unrolling_iterable(possible_location_codes) class LocationCodeBuilder(object): _mixin_ = True @@ -136,25 +153,31 @@ def INSN(self, loc1, loc2): code1 = loc1.location_code() code2 = loc2.location_code() - for possible_code1 in unrolling_possible_location_codes: - for possible_code2 in unrolling_possible_location_codes: - if code1 == possible_code1 and code2 == possible_code2: - methname = name + "_" + possible_code1 + possible_code2 - if hasattr(rx86.AbstractX86CodeBuilder, methname): - getattr(self, methname)(loc1.value, loc2.value) - else: - raise AssertionError("Instruction not defined: " + methname) + for possible_code1 in _binop_pc_outer: + if code1 == possible_code1: + for possible_code2 in _binop_pc_inner: + if code2 == possible_code2: + methname = name + "_" + possible_code1 + possible_code2 + if hasattr(rx86.AbstractX86CodeBuilder, methname): + val1 = getattr(loc1, "value_" + possible_code1)() + val2 = getattr(loc2, "value_" + possible_code2)() + getattr(self, methname)(val1, val2) + return + else: + raise AssertionError("Instruction not defined: " + methname) return INSN def _unaryop(name): def INSN(self, loc): code = loc.location_code() - for possible_code in unrolling_possible_location_codes: + for possible_code in _unaryop_pc: if code == possible_code: methname = name + "_" + possible_code if hasattr(rx86.AbstractX86CodeBuilder, methname): - getattr(self, methname)(loc.value) + val = getattr(loc, "value_" + possible_code)() + getattr(self, methname)(val) + return else: raise AssertionError("Instruction not defined: " + methname) @@ -194,29 +217,13 @@ CVTSI2SD = _binaryop('CVTSI2SD') CVTTSD2SI = _binaryop('CVTTSD2SI') - - def CALL(self, loc): - # FIXME: Kludge that works in 32-bit because the "relative" CALL is - # actually absolute on i386 - if loc.location_code() == 'j': - self.CALL_l(loc.value) - else: - getattr(self, 'CALL_' + loc.location_code())(loc.value) + CALL = _unaryop('CALL') def MOV16(self, dest_loc, src_loc): # Select 16-bit operand mode self.writechar('\x66') self.MOV(dest_loc, src_loc) - def CMPi(self, loc0, loc1): - # like CMP, but optimized for the case of loc1 being a Const - assert isinstance(loc1, Const) - if isinstance(loc0, RegLoc): - self.CMP_ri(loc0.value, loc1.getint()) - else: - assert isinstance(loc0, StackLoc) - self.CMP_bi(loc0.value, loc1.getint()) - def imm(x): # XXX: ri386 migration shim if isinstance(x, ConstInt): Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/rx86.py Thu May 27 22:23:30 2010 @@ -5,6 +5,7 @@ from pypy.rpython.lltypesystem import rffi BYTE_REG_FLAG = 0x20 +NO_BASE_REGISTER = -1 class R(object): # the following are synonyms for rax, rcx, etc. on 64 bits @@ -220,7 +221,7 @@ reg2 = reg_number_3bits(mc, reg2) # Special case for no base register - if reg1 == None: + if reg1 == NO_BASE_REGISTER: # modrm mc.writechar(chr(0x04 | orbyte)) # SIB @@ -508,6 +509,9 @@ CALL_l = insn('\xE8', relative(1)) CALL_r = insn(rex_nw, '\xFF', register(1), chr(0xC0 | (2<<3))) CALL_b = insn('\xFF', orbyte(2<<3), stack_bp(1)) + # XXX: Bit of kludge, but works in 32-bit because the relative 32-bit + # displacement is always enough to encode any address + CALL_j = CALL_l XCHG_rm = insn(rex_w, '\x87', register(1,8), mem_reg_plus_const(2)) XCHG_rj = insn(rex_w, '\x87', register(1,8), '\x05', immediate(2)) From jcreigh at codespeak.net Thu May 27 22:57:11 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Thu, 27 May 2010 22:57:11 +0200 (CEST) Subject: [pypy-svn] r74827 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86 Message-ID: <20100527205711.D311C282BE0@codespeak.net> Author: jcreigh Date: Thu May 27 22:57:10 2010 New Revision: 74827 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Log: add CMP16 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/assembler.py Thu May 27 22:57:10 2010 @@ -996,8 +996,6 @@ if offset is not None: mc.CMP(mem(locs[0], offset), locs[1]) else: - # We haven't implemented CMP16 yet... - raise AssertionError("FIXME") # XXX hard-coded assumption: to go from an object to its class # we use the following algorithm: # - read the typeid from mem(locs[0]), i.e. at offset 0 @@ -1014,7 +1012,7 @@ type_info_group = llop.gc_get_type_info_group(llmemory.Address) type_info_group = rffi.cast(lltype.Signed, type_info_group) expected_typeid = (classptr - sizeof_ti - type_info_group) >> 2 - mc.CMP16(mem(locs[0], 0), imm32(expected_typeid)) + mc.CMP16(mem(locs[0], 0), ImmedLoc(expected_typeid)) def genop_guard_guard_class(self, ign_1, guard_op, addr, locs, ign_2): mc = self._start_block() Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Thu May 27 22:57:10 2010 @@ -224,6 +224,11 @@ self.writechar('\x66') self.MOV(dest_loc, src_loc) + def CMP16(self, a_loc, b_loc): + # Select 16-bit operand mode + self.writechar('\x66') + self.CMP(a_loc, b_loc) + def imm(x): # XXX: ri386 migration shim if isinstance(x, ConstInt): From jcreigh at codespeak.net Thu May 27 23:38:26 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Thu, 27 May 2010 23:38:26 +0200 (CEST) Subject: [pypy-svn] r74828 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test Message-ID: <20100527213826.F0FD0282BE0@codespeak.net> Author: jcreigh Date: Thu May 27 23:38:25 2010 New Revision: 74828 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py Log: fix some test failures Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86.py Thu May 27 23:38:25 2010 @@ -69,11 +69,11 @@ def test_mov_ra_no_base(): s = CodeBuilder32() - s.MOV_ra(edx, (None, edi, 2, 0)) + s.MOV_ra(edx, (NO_BASE_REGISTER, edi, 2, 0)) assert s.getvalue() == '\x8B\x14\xBD\x00\x00\x00\x00' s = CodeBuilder32() - s.MOV_ra(edx, (None, edi, 2, 0xCD)) + s.MOV_ra(edx, (NO_BASE_REGISTER, edi, 2, 0xCD)) assert s.getvalue() == '\x8B\x14\xBD\xCD\x00\x00\x00' def test_mov_ar(): Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py Thu May 27 23:38:25 2010 @@ -238,6 +238,8 @@ if methname == 'FSTP_b': # Doesn't work on 64-bit, skipping for now py.test.skip('Skipping FSTP') + if methname == 'CALL_j': + py.test.skip("CALL_j is actually relative") return [args] From arigo at codespeak.net Fri May 28 02:02:46 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 02:02:46 +0200 (CEST) Subject: [pypy-svn] r74829 - in pypy/trunk/pypy/objspace/flow: . test Message-ID: <20100528000246.1F6EF282BAD@codespeak.net> Author: arigo Date: Fri May 28 02:02:44 2010 New Revision: 74829 Modified: pypy/trunk/pypy/objspace/flow/flowcontext.py pypy/trunk/pypy/objspace/flow/test/test_unroll.py Log: Test and fix for using the same unrolling_iterable twice in two nested loops. Modified: pypy/trunk/pypy/objspace/flow/flowcontext.py ============================================================================== --- pypy/trunk/pypy/objspace/flow/flowcontext.py (original) +++ pypy/trunk/pypy/objspace/flow/flowcontext.py Fri May 28 02:02:44 2010 @@ -388,11 +388,14 @@ def replace_in_stack(self, oldvalue, newvalue): w_new = Constant(newvalue) stack_items_w = self.crnt_frame.valuestack_w - for i in range(self.crnt_frame.valuestackdepth): + for i in range(self.crnt_frame.valuestackdepth-1, -1, -1): w_v = stack_items_w[i] if isinstance(w_v, Constant): if w_v.value is oldvalue: + # replace the topmost item of the stack that is equal + # to 'oldvalue' with 'newvalue'. stack_items_w[i] = w_new + break class FlowSpaceFrame(pyframe.PyFrame): """ Modified: pypy/trunk/pypy/objspace/flow/test/test_unroll.py ============================================================================== --- pypy/trunk/pypy/objspace/flow/test/test_unroll.py (original) +++ pypy/trunk/pypy/objspace/flow/test/test_unroll.py Fri May 28 02:02:44 2010 @@ -64,3 +64,15 @@ 'ne': 1, 'gt': 1, 'ge': 1} + + def test_unroll_twice(self): + operations = unrolling_iterable([1, 2, 3]) + def f(x): + for num1 in operations: + for num2 in operations: + x = x + (num1 + num2) + return x + + graph = self.codetest(f) + ops = self.all_operations(graph) + assert ops['add'] == 9 From arigo at codespeak.net Fri May 28 02:24:12 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 02:24:12 +0200 (CEST) Subject: [pypy-svn] r74830 - in pypy/branch/blackhole-improvement/pypy/jit/metainterp: . test Message-ID: <20100528002412.9DF0E282BAD@codespeak.net> Author: arigo Date: Fri May 28 02:24:10 2010 New Revision: 74830 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualref.py Log: In-progress: virtualrefs. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Fri May 28 02:24:10 2010 @@ -1279,6 +1279,7 @@ curbh._copy_data_from_miframe(frame) curbh.nextblackholeinterp = nextbh nextbh = curbh + XXX # virtualrefs firstbh = nextbh # try: Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Fri May 28 02:24:10 2010 @@ -1864,9 +1864,7 @@ expect_virtualizable = vinfo is not None boxlists = resume.rebuild_from_resumedata(self, resumedescr, expect_virtualizable) - #inputargs_and_holes, virtualizable_boxes, virtualref_boxes = boxlists - inputargs_and_holes = boxlists # XXX - virtualref_boxes = [] # XXX + inputargs_and_holes, virtualizable_boxes, virtualref_boxes = boxlists # # virtual refs: make the vrefs point to the freshly allocated virtuals self.virtualref_boxes = virtualref_boxes Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py Fri May 28 02:24:10 2010 @@ -57,7 +57,7 @@ storage.rd_frame_info_list = frame_info_list snapshot = Snapshot(top.parent_resumedata_snapshot, top.get_list_of_active_boxes(False)) - #snapshot = Snapshot(snapshot, virtualref_boxes[:]) # xxx for now + snapshot = Snapshot(snapshot, virtualref_boxes[:]) # xxx for now if virtualizable_boxes is not None: snapshot = Snapshot(snapshot, virtualizable_boxes[:]) # xxx for now storage.rd_snapshot = snapshot @@ -552,7 +552,7 @@ virtualizable_boxes = None if expects_virtualizables: XXX # virtualizable_boxes = resumereader.consume_boxes() - #resumereader.consume_boxes([], [], []) # XXX virtualref + virtualref_boxes = resumereader.consume_virtualref_boxes() frameinfo = storage.rd_frame_info_list while True: f = metainterp.newframe(frameinfo.jitcode) @@ -564,7 +564,7 @@ break metainterp.framestack.reverse() resumereader.done() - return resumereader.liveboxes + return resumereader.liveboxes, None, virtualref_boxes def force_from_resumedata(metainterp, newboxes, storage, expects_virtualizables): @@ -572,8 +572,7 @@ virtualizable_boxes = None if expects_virtualizables: virtualizable_boxes = resumereader.consume_boxes() - #virtualref_boxes = resumereader.consume_boxes() - virtualref_boxes = [] # XXX virtualrefs + virtualref_boxes = resumereader.consume_boxes() return virtualizable_boxes, virtualref_boxes, resumereader.virtuals class ResumeDataBoxReader(AbstractResumeDataReader): @@ -589,6 +588,13 @@ self.boxes_f = boxes_f self._prepare_next_section(info) + def consume_virtualref_boxes(self): + # Returns a list of boxes, assumed to be all BoxPtrs. + # We leave up to the caller to call vrefinfo.continue_tracing(). + nums = self.cur_numb.nums + self.cur_numb = self.cur_numb.prev + return [self.decode_ref(num) for num in nums] + def allocate_with_vtable(self, known_class): return self.metainterp.execute_and_record(rop.NEW_WITH_VTABLE, None, known_class) @@ -683,7 +689,8 @@ resumereader = ResumeDataDirectReader(blackholeinterpbuilder.cpu, storage) if expects_virtualizables: XXX - #virtualref_boxes = resumereader.consume_boxes() # virtualref XXX + vrefinfo = blackholeinterpbuilder.metainterp_sd.virtualref_info + resumereader.consume_virtualref_info(vrefinfo) # # First get a chain of blackhole interpreters whose length is given # by the depth of rd_frame_info_list. The first one we get must be @@ -723,6 +730,17 @@ info = blackholeinterp.get_current_position_info() self._prepare_next_section(info) + def consume_virtualref_info(self, vrefinfo): + # we have to decode a list of references containing pairs + # [..., virtual, vref, ...] + nums = self.cur_numb.nums + self.cur_numb = self.cur_numb.prev + for i in range(0, len(nums), 2): + virtual = self.decode_ref(nums[i]) + vref = self.decode_ref(nums[i+1]) + # For each pair, we store the virtual inside the vref. + vrefinfo.continue_tracing(vref, virtual) + def allocate_with_vtable(self, known_class): from pypy.jit.metainterp.executor import exec_new_with_vtable return exec_new_with_vtable(self.cpu, known_class) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualref.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualref.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualref.py Fri May 28 02:24:10 2010 @@ -81,8 +81,26 @@ JIT_VIRTUAL_REF = self.vrefinfo.JIT_VIRTUAL_REF bxs2[0].getref(lltype.Ptr(JIT_VIRTUAL_REF)).virtual_token = 1234567 # - self.metainterp.rebuild_state_after_failure(guard_op.descr, - guard_op.fail_args[:]) + # try reloading from blackhole.py's point of view + from pypy.jit.metainterp.resume import ResumeDataDirectReader + cpu = self.metainterp.cpu + cpu.get_latest_value_count = lambda : len(guard_op.fail_args) + cpu.get_latest_value_int = lambda i:guard_op.fail_args[i].getint() + cpu.get_latest_value_ref = lambda i:guard_op.fail_args[i].getref_base() + cpu.clear_latest_values = lambda : None + resumereader = ResumeDataDirectReader(cpu, guard_op.descr) + vrefinfo = self.metainterp.staticdata.virtualref_info + lst = [] + vrefinfo.continue_tracing = lambda vref, virtual: \ + lst.append((vref, virtual)) + resumereader.consume_virtualref_info(vrefinfo) + del vrefinfo.continue_tracing + assert len(lst) == 1 + lltype.cast_opaque_ptr(lltype.Ptr(JIT_VIRTUAL_REF), + lst[0][0]) # assert correct type + # + # try reloading from pyjitpl's point of view + self.metainterp.rebuild_state_after_failure(guard_op.descr) assert len(self.metainterp.framestack) == 1 assert len(self.metainterp.virtualref_boxes) == 2 assert self.metainterp.virtualref_boxes[0].value == bxs1[0].value From arigo at codespeak.net Fri May 28 02:57:46 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 02:57:46 +0200 (CEST) Subject: [pypy-svn] r74831 - in pypy/branch/blackhole-improvement/pypy/jit: backend/llgraph metainterp metainterp/test Message-ID: <20100528005746.EAAA6282BAD@codespeak.net> Author: arigo Date: Fri May 28 02:57:45 2010 New Revision: 74831 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualref.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/virtualref.py Log: I think that's all for virtualrefs. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py Fri May 28 02:57:45 2010 @@ -245,7 +245,7 @@ def get_latest_force_token(self): token = llimpl.get_frame_forced_token(self.latest_frame) - return self.cast_adr_to_int(token) + return llmemory.cast_adr_to_int(token) def clear_latest_values(self): llimpl.frame_clear_latest_values(self.latest_frame) @@ -426,7 +426,7 @@ return llimpl.cast_to_int(ptr) def force(self, force_token): - token = self.cast_int_to_adr(force_token) + token = llmemory.cast_int_to_adr(force_token) frame = llimpl.get_forced_token_frame(token) fail_index = llimpl.force(frame) self.latest_frame = frame Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Fri May 28 02:57:45 2010 @@ -1247,14 +1247,15 @@ current_exc = blackholeinterp._resume_mainloop(current_exc) blackholeinterp = blackholeinterp.nextblackholeinterp -def resume_in_blackhole(metainterp_sd, resumedescr): +def resume_in_blackhole(metainterp_sd, resumedescr, all_virtuals=None): from pypy.jit.metainterp.resume import blackhole_from_resumedata debug_start('jit-blackhole') metainterp_sd.profiler.start_blackhole() blackholeinterp = blackhole_from_resumedata( metainterp_sd.blackholeinterpbuilder, resumedescr, - False) # XXX + False, + all_virtuals) # XXX # XXX virtualrefs # XXX virtualizable current_exc = blackholeinterp._prepare_resume_from_failure( Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py Fri May 28 02:57:45 2010 @@ -324,17 +324,16 @@ class ResumeGuardForcedDescr(ResumeGuardDescr): def handle_fail(self, metainterp_sd): - from pypy.jit.metainterp.pyjitpl import MetaInterp - metainterp = MetaInterp(metainterp_sd) + # Failures of a GUARD_NOT_FORCED are never compiled, but + # always just blackholed. First fish for the data saved when + # the virtualrefs and virtualizable have been forced by + # handle_async_forcing() just a moment ago. + from pypy.jit.metainterp.blackhole import resume_in_blackhole token = metainterp_sd.cpu.get_latest_force_token() all_virtuals = self.fetch_data(token) if all_virtuals is None: all_virtuals = [] - metainterp._already_allocated_resume_virtuals = all_virtuals - return metainterp.handle_guard_failure(self) - - def must_compile(self, metainterp_sd, inputargs_and_holes): - return False # never compile GUARD_NOT_FORCED failures + resume_in_blackhole(metainterp_sd, self, all_virtuals) @staticmethod def force_now(cpu, token): @@ -346,27 +345,11 @@ faildescr.handle_async_forcing(token) def handle_async_forcing(self, force_token): - from pypy.jit.metainterp.pyjitpl import MetaInterp from pypy.jit.metainterp.resume import force_from_resumedata - # To handle the forcing itself, we create a temporary MetaInterp - # as a convenience to move the various data to its proper place. metainterp_sd = self.metainterp_sd - metainterp = MetaInterp(metainterp_sd) - metainterp.history = None # blackholing - liveboxes = metainterp_sd.cpu.make_boxes_from_latest_values(self) - # expect_virtualizable = metainterp_sd.virtualizable_info is not None - forced_data = force_from_resumedata(metainterp, liveboxes, self, + all_virtuals = force_from_resumedata(metainterp_sd, self, expect_virtualizable) - virtualizable_boxes, virtualref_boxes, all_virtuals = forced_data - # - # Handle virtualref_boxes: mark each JIT_VIRTUAL_REF as forced - vrefinfo = metainterp_sd.virtualref_info - for i in range(0, len(virtualref_boxes), 2): - virtualbox = virtualref_boxes[i] - vrefbox = virtualref_boxes[i+1] - vrefinfo.forced_single_vref(vrefbox.getref_base(), - virtualbox.getref_base()) # Handle virtualizable_boxes: store them on the real virtualizable now if expect_virtualizable: metainterp_sd.virtualizable_info.forced_vable(virtualizable_boxes) @@ -376,14 +359,30 @@ def save_data(self, key, value): globaldata = self.metainterp_sd.globaldata - assert key not in globaldata.resume_virtuals - globaldata.resume_virtuals[key] = value + if we_are_translated(): + assert key not in globaldata.resume_virtuals + globaldata.resume_virtuals[key] = value + else: + rv = globaldata.resume_virtuals_not_translated + for key1, value1 in rv: + assert key1 != key + rv.append((key, value)) def fetch_data(self, key): globaldata = self.metainterp_sd.globaldata - assert key in globaldata.resume_virtuals - data = globaldata.resume_virtuals[key] - del globaldata.resume_virtuals[key] + if we_are_translated(): + assert key in globaldata.resume_virtuals + data = globaldata.resume_virtuals[key] + del globaldata.resume_virtuals[key] + else: + rv = globaldata.resume_virtuals_not_translated + for i in range(len(rv)): + if rv[i][0] == key: + data = rv[i][1] + del rv[i] + break + else: + assert 0, "not found: %r" % (key,) return data Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Fri May 28 02:57:45 2010 @@ -1202,6 +1202,7 @@ self.addr2name = None self.loopnumbering = 0 self.resume_virtuals = {} + self.resume_virtuals_not_translated = [] # state = staticdata.state if state is not None: Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py Fri May 28 02:57:45 2010 @@ -496,10 +496,12 @@ _mixin_ = True virtuals = None - def _prepare(self, cpu, storage): + def _init(self, cpu, storage): self.cpu = cpu self.cur_numb = storage.rd_numb self.consts = storage.rd_consts + + def _prepare(self, storage): self._prepare_virtuals(storage.rd_virtuals) self._prepare_pendingfields(storage.rd_pendingfields) @@ -566,21 +568,13 @@ resumereader.done() return resumereader.liveboxes, None, virtualref_boxes -def force_from_resumedata(metainterp, newboxes, storage, - expects_virtualizables): - resumereader = ResumeDataReader(storage, newboxes, metainterp) - virtualizable_boxes = None - if expects_virtualizables: - virtualizable_boxes = resumereader.consume_boxes() - virtualref_boxes = resumereader.consume_boxes() - return virtualizable_boxes, virtualref_boxes, resumereader.virtuals - class ResumeDataBoxReader(AbstractResumeDataReader): def __init__(self, storage, metainterp): + self._init(metainterp.cpu, storage) self.metainterp = metainterp self.liveboxes = [None] * metainterp.cpu.get_latest_value_count() - self._prepare(metainterp.cpu, storage) + self._prepare(storage) def consume_boxes(self, info, boxes_i, boxes_r, boxes_f): self.boxes_i = boxes_i @@ -685,8 +679,9 @@ # ---------- when resuming for blackholing, get direct values ---------- def blackhole_from_resumedata(blackholeinterpbuilder, storage, - expects_virtualizables): - resumereader = ResumeDataDirectReader(blackholeinterpbuilder.cpu, storage) + expects_virtualizables, all_virtuals=None): + resumereader = ResumeDataDirectReader(blackholeinterpbuilder.cpu, storage, + all_virtuals) if expects_virtualizables: XXX vrefinfo = blackholeinterpbuilder.metainterp_sd.virtualref_info @@ -720,10 +715,25 @@ resumereader.done() return firstbh +def force_from_resumedata(metainterp_sd, storage, expects_virtualizables): + resumereader = ResumeDataDirectReader(metainterp_sd.cpu, storage) + virtualizable_boxes = None + if expects_virtualizables: + XXX + vrefinfo = metainterp_sd.virtualref_info + resumereader.consume_virtualref_info(vrefinfo) + return resumereader.virtuals + class ResumeDataDirectReader(AbstractResumeDataReader): - def __init__(self, cpu, storage): - self._prepare(cpu, storage) + def __init__(self, cpu, storage, all_virtuals=None): + self._init(cpu, storage) + if all_virtuals is None: # common case + self._prepare(storage) + else: + # special case for resuming after a GUARD_NOT_FORCED: we already + # have the virtuals + self.virtuals = all_virtuals def consume_one_section(self, blackholeinterp): self.blackholeinterp = blackholeinterp Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualref.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualref.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualref.py Fri May 28 02:57:45 2010 @@ -442,7 +442,7 @@ self.check_aborted_count(0) def test_recursive_call_1(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 'frame', 'rec']) + myjitdriver = JitDriver(greens = [], reds = ['n', 'rec', 'frame']) # class XY: pass @@ -472,7 +472,7 @@ self.check_aborted_count(0) def test_recursive_call_2(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 'frame', 'rec']) + myjitdriver = JitDriver(greens = [], reds = ['n', 'rec', 'frame']) # class XY: n = 0 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/virtualref.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/virtualref.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/virtualref.py Fri May 28 02:57:45 2010 @@ -61,11 +61,11 @@ # The 'virtual_token' field has the same meaning as the 'vable_token' field # of a virtualizable. It is equal to: - # * -2 (TOKEN_NONE) when tracing, except as described below; + # * -3 (TOKEN_NONE) when tracing, except as described below; # * -1 (TOKEN_TRACING_RESCALL) during tracing when we do a residual call; # * addr in the CPU stack (set by FORCE_TOKEN) when running the assembler; - # * -2 (TOKEN_NONE) after the virtual is forced, if it is forced at all. - TOKEN_NONE = -2 + # * -3 (TOKEN_NONE) after the virtual is forced, if it is forced at all. + TOKEN_NONE = -3 TOKEN_TRACING_RESCALL = -1 def virtual_ref_during_tracing(self, real_object): @@ -105,16 +105,6 @@ # marker "modified during residual call" set. return True - def forced_single_vref(self, gcref, real_object): - if not self.is_virtual_ref(gcref): - return - assert real_object - vref = lltype.cast_opaque_ptr(lltype.Ptr(self.JIT_VIRTUAL_REF), gcref) - assert (vref.virtual_token != self.TOKEN_NONE and - vref.virtual_token != self.TOKEN_TRACING_RESCALL) - vref.virtual_token = self.TOKEN_NONE - vref.forced = lltype.cast_opaque_ptr(rclass.OBJECTPTR, real_object) - def continue_tracing(self, gcref, real_object): if not self.is_virtual_ref(gcref): return From arigo at codespeak.net Fri May 28 03:03:58 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 03:03:58 +0200 (CEST) Subject: [pypy-svn] r74832 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100528010358.300FB282BAD@codespeak.net> Author: arigo Date: Fri May 28 03:03:56 2010 New Revision: 74832 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py Log: - add an assert. - remove a comment (done). - remove an XXX in the source: I think nothing has to be done here after all. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Fri May 28 03:03:56 2010 @@ -1256,7 +1256,6 @@ resumedescr, False, all_virtuals) # XXX - # XXX virtualrefs # XXX virtualizable current_exc = blackholeinterp._prepare_resume_from_failure( resumedescr.guard_opnum) @@ -1280,7 +1279,6 @@ curbh._copy_data_from_miframe(frame) curbh.nextblackholeinterp = nextbh nextbh = curbh - XXX # virtualrefs firstbh = nextbh # try: Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py Fri May 28 03:03:56 2010 @@ -334,6 +334,7 @@ if all_virtuals is None: all_virtuals = [] resume_in_blackhole(metainterp_sd, self, all_virtuals) + assert 0, "unreachable" @staticmethod def force_now(cpu, token): From arigo at codespeak.net Fri May 28 09:55:31 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 09:55:31 +0200 (CEST) Subject: [pypy-svn] r74833 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter codewriter/test metainterp metainterp/test Message-ID: <20100528075531.1F93C282BD6@codespeak.net> Author: arigo Date: Fri May 28 09:55:28 2010 New Revision: 74833 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualizable.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/virtualizable.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py Log: Start working on virtualizables. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py Fri May 28 09:55:28 2010 @@ -16,6 +16,7 @@ class CallControl(object): virtualref_info = None # optionally set from outside + virtualizable_info = None # optionally set from outside portal_runner_ptr = None # optionally set from outside def __init__(self, cpu=None, portal_graph=None): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py Fri May 28 09:55:28 2010 @@ -62,15 +62,24 @@ self.assembler.assemble(ssarepr, jitcode) def make_jitcodes(self, verbose=False): + log.info("making JitCodes...") maingraph = self.portal_graph self.mainjitcode = self.callcontrol.get_jitcode(maingraph) + count = 0 for graph, jitcode in self.callcontrol.enum_pending_graphs(): self.transform_graph_to_jitcode(graph, jitcode, graph is maingraph, verbose) + count += 1 + if not count % 500: + log.info("Produced %d jitcodes" % count) + log.info("there are %d JitCode instances." % count) def setup_vrefinfo(self, vrefinfo): self.callcontrol.virtualref_info = vrefinfo + def setup_virtualizable_info(self, vinfo): + self.callcontrol.virtualizable_info = vinfo + def setup_portal_runner_ptr(self, portal_runner_ptr): self.callcontrol.portal_runner_ptr = portal_runner_ptr Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Fri May 28 09:55:28 2010 @@ -37,6 +37,8 @@ if block.operations == (): return self.immutable_arrays = {} + self.vable_array_vars = {} + self.vable_flags = {} renamings = {} renamings_constants = {} # subset of 'renamings', {Var:Const} only newoperations = [] @@ -177,6 +179,7 @@ def rewrite_op_cast_int_to_unichar(self, op): pass def rewrite_op_cast_int_to_uint(self, op): pass def rewrite_op_cast_uint_to_int(self, op): pass + def rewrite_op_resume_point(self, op): pass def _rewrite_symmetric(self, op): """Rewrite 'c1+v2' into 'v2+c1' in an attempt to avoid generating @@ -406,6 +409,15 @@ assert ARRAY._gckind == 'gc' if self._array_of_voids(ARRAY): return [] + if op.args[0] in self.vable_array_vars: # for virtualizables + vars = self.vable_array_vars[op.args[0]] + (v_base, arrayfielddescr, arraydescr) = vars + kind = getkind(op.result.concretetype) + return [SpaceOperation('-live-', [], None), + SpaceOperation('getarrayitem_vable_%s' % kind, + [v_base, arrayfielddescr, arraydescr, + op.args[1]], op.result)] + # normal case follows arraydescr = self.cpu.arraydescrof(ARRAY) kind = getkind(op.result.concretetype) return SpaceOperation('getarrayitem_gc_%s' % kind[0], @@ -417,6 +429,14 @@ assert ARRAY._gckind == 'gc' if self._array_of_voids(ARRAY): return [] + if op.args[0] in self.vable_array_vars: # for virtualizables + vars = self.vable_array_vars[op.args[0]] + (v_base, arrayfielddescr, arraydescr) = vars + kind = getkind(op.result.concretetype) + return [SpaceOperation('-live-', [], None), + SpaceOperation('setarrayitem_vable_%s' % kind, + [v_base, arrayfielddescr, arraydescr, + op.args[1], op.args[2]], None)] arraydescr = self.cpu.arraydescrof(ARRAY) kind = getkind(op.args[2].concretetype) return SpaceOperation('setarrayitem_gc_%s' % kind[0], @@ -426,6 +446,14 @@ def rewrite_op_getarraysize(self, op): ARRAY = op.args[0].concretetype.TO assert ARRAY._gckind == 'gc' + if op.args[0] in self.vable_array_vars: # for virtualizables + vars = self.vable_array_vars[op.args[0]] + (v_base, arrayfielddescr, arraydescr) = vars + return [SpaceOperation('-live-', [], None), + SpaceOperation('arraylen_vable', + [v_base, arrayfielddescr, arraydescr], + op.result)] + # normal case follows arraydescr = self.cpu.arraydescrof(ARRAY) return SpaceOperation('arraylen_gc', [op.args[0], arraydescr], op.result) @@ -445,21 +473,23 @@ if RESULT is lltype.Void: return # check for virtualizable - #try: - # if self.is_virtualizable_getset(op): - # vinfo = self.codewriter.metainterp_sd.virtualizable_info - # index = vinfo.static_field_to_extra_box[op.args[1].value] - # self.emit('getfield_vable', - # self.var_position(v_inst), - # index) - # self.register_var(op.result) - # return - #except VirtualizableArrayField: - # # xxx hack hack hack - # vinfo = self.codewriter.metainterp_sd.virtualizable_info - # arrayindex = vinfo.array_field_counter[op.args[1].value] - # self.vable_array_vars[op.result] = (op.args[0], arrayindex) - # return + try: + if self.is_virtualizable_getset(op): + descr = self.get_virtualizable_field_descr(op.args[1].value) + kind = getkind(RESULT)[0] + return [SpaceOperation('-live-', [], None), + SpaceOperation('getfield_vable_%s' % kind, + [v_inst, descr], op.result)] + except VirtualizableArrayField: + # xxx hack hack hack + vinfo = self.callcontrol.virtualizable_info + arrayindex = vinfo.array_field_counter[op.args[1].value] + arrayfielddescr = vinfo.array_field_descrs[arrayindex] + arraydescr = vinfo.array_descrs[arrayindex] + self.vable_array_vars[op.result] = (op.args[0], + arrayfielddescr, + arraydescr) + return [] # check for deepfrozen structures that force constant-folding hints = v_inst.concretetype.TO._hints accessor = hints.get("immutable_fields") @@ -488,14 +518,12 @@ if RESULT is lltype.Void: return # check for virtualizable - #if self.is_virtualizable_getset(op): - # vinfo = self.codewriter.metainterp_sd.virtualizable_info - # index = vinfo.static_field_to_extra_box[op.args[1].value] - # self.emit('setfield_vable', - # self.var_position(v_inst), - # index, - # self.var_position(v_value)) - # return + if self.is_virtualizable_getset(op): + descr = self.get_virtualizable_field_descr(op.args[1].value) + kind = getkind(RESULT)[0] + return [SpaceOperation('-live-', [], None), + SpaceOperation('setfield_vable_%s' % kind, + [v_inst, descr, v_value], None)] argname = getattr(v_inst.concretetype.TO, '_gckind', 'gc') descr = self.cpu.fielddescrof(v_inst.concretetype.TO, c_fieldname.value) @@ -508,6 +536,35 @@ return (op.args[1].value == 'typeptr' and op.args[0].concretetype.TO._hints.get('typeptr')) + def is_virtualizable_getset(self, op): + # every access of an object of exactly the type VTYPEPTR is + # likely to be a virtualizable access, but we still have to + # check it in pyjitpl.py. + try: + vinfo = self.callcontrol.virtualizable_info + except AttributeError: + return False + if vinfo is None or not vinfo.is_vtypeptr(op.args[0].concretetype): + return False + res = False + if op.args[1].value in vinfo.static_field_to_extra_box: + res = True + if op.args[1].value in vinfo.array_fields: + res = VirtualizableArrayField(self.graph) + + if res: + flags = self.vable_flags[op.args[0]] + if 'fresh_virtualizable' in flags: + return False + if isinstance(res, Exception): + raise res + return res + + def get_virtualizable_field_descr(self, fieldname): + vinfo = self.callcontrol.virtualizable_info + index = vinfo.static_field_to_extra_box[fieldname] + return vinfo.static_field_descrs[index] + def handle_getfield_typeptr(self, op): if isinstance(op.args[0], Constant): cls = op.args[0].value.typeptr @@ -892,11 +949,24 @@ def rewrite_op_jit_force_virtual(self, op): return self._do_builtin_call(op) + def rewrite_op_jit_force_virtualizable(self, op): + # this one is for virtualizables + vinfo = self.callcontrol.virtualizable_info + assert vinfo is not None + assert vinfo.is_vtypeptr(op.args[0].concretetype) + self.vable_flags[op.args[0]] = op.args[2].value + return [] + # ____________________________________________________________ class NotSupported(Exception): pass +class VirtualizableArrayField(Exception): + def __str__(self): + return "using virtualizable array in illegal way in %r" % ( + self.args[0],) + def _with_prefix(prefix): result = {} for name in dir(Transformer): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Fri May 28 09:55:28 2010 @@ -69,6 +69,8 @@ def found_jitdriver(self, jitdriver): assert isinstance(jitdriver, JitDriver) self.jitdriver = jitdriver + def getjitdriver(self): + return self.jitdriver class FakeCallControlWithVRefInfo: class virtualref_info: Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Fri May 28 09:55:28 2010 @@ -907,6 +907,11 @@ cpu.bh_setarrayitem_gc_f(arraydescr, items, index, nval) # ---------- + # virtualizables operations + + #xxx + + # ---------- # the following operations are directly implemented by the backend @arguments("cpu", "i", "d", "R", returns="i") @@ -1035,6 +1040,10 @@ bhimpl_getfield_gc_r_pure = bhimpl_getfield_gc_r bhimpl_getfield_gc_f_pure = bhimpl_getfield_gc_f + bhimpl_getfield_vable_i = bhimpl_getfield_gc_i + bhimpl_getfield_vable_r = bhimpl_getfield_gc_r + bhimpl_getfield_vable_f = bhimpl_getfield_gc_f + @arguments("cpu", "i", "d", returns="i") def bhimpl_getfield_raw_i(cpu, struct, fielddescr): return cpu.bh_getfield_raw_i(struct, fielddescr) @@ -1059,6 +1068,10 @@ def bhimpl_setfield_gc_f(cpu, struct, fielddescr, newvalue): cpu.bh_setfield_gc_f(struct, fielddescr, newvalue) + bhimpl_setfield_vable_i = bhimpl_setfield_gc_i + bhimpl_setfield_vable_r = bhimpl_setfield_gc_r + bhimpl_setfield_vable_f = bhimpl_setfield_gc_f + @arguments("cpu", "i", "d", "i") def bhimpl_setfield_raw_i(cpu, struct, fielddescr, newvalue): cpu.bh_setfield_raw_i(struct, fielddescr, newvalue) @@ -1254,8 +1267,7 @@ blackholeinterp = blackhole_from_resumedata( metainterp_sd.blackholeinterpbuilder, resumedescr, - False, - all_virtuals) # XXX + all_virtuals) # XXX virtualizable current_exc = blackholeinterp._prepare_resume_from_failure( resumedescr.guard_opnum) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py Fri May 28 09:55:28 2010 @@ -349,8 +349,7 @@ from pypy.jit.metainterp.resume import force_from_resumedata metainterp_sd = self.metainterp_sd expect_virtualizable = metainterp_sd.virtualizable_info is not None - all_virtuals = force_from_resumedata(metainterp_sd, self, - expect_virtualizable) + all_virtuals = force_from_resumedata(metainterp_sd, self) # Handle virtualizable_boxes: store them on the real virtualizable now if expect_virtualizable: metainterp_sd.virtualizable_info.forced_vable(virtualizable_boxes) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py Fri May 28 09:55:28 2010 @@ -40,6 +40,7 @@ return "ref" else: raise NotImplementedError("type %s not supported" % TYPE) +getkind._annspecialcase_ = 'specialize:memo' def repr_pointer(box): from pypy.rpython.lltypesystem import rstr Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Fri May 28 09:55:28 2010 @@ -516,9 +516,9 @@ self.metainterp.replace_box(box, standard_box) return not isstandard - def _get_virtualizable_field_descr(self, index): + def _get_virtualizable_field_index(self, fielddescr): vinfo = self.metainterp.staticdata.virtualizable_info - return vinfo.static_field_descrs[index] + return vinfo.static_field_by_descrs[fielddescr] def _get_virtualizable_array_field_descr(self, index): vinfo = self.metainterp.staticdata.virtualizable_info @@ -528,23 +528,32 @@ vinfo = self.metainterp.staticdata.virtualizable_info return vinfo.array_descrs[index] - @FixME #arguments("orgpc", "box", "int") - def opimpl_getfield_vable(self, pc, basebox, index): - if self._nonstandard_virtualizable(pc, basebox): - self.execute_with_descr(rop.GETFIELD_GC, self._get_virtualizable_field_descr(index), basebox) - return + @arguments("orgpc", "box", "descr") + def _opimpl_getfield_vable(self, pc, box, fielddescr): + if self._nonstandard_virtualizable(pc, box): + return self.execute_with_descr(rop.GETFIELD_GC, fielddescr, box) self.metainterp.check_synchronized_virtualizable() - resbox = self.metainterp.virtualizable_boxes[index] - self.make_result_box(resbox) - @FixME #arguments("orgpc", "box", "int", "box") - def opimpl_setfield_vable(self, pc, basebox, index, valuebox): - if self._nonstandard_virtualizable(pc, basebox): - self.execute_with_descr(rop.SETFIELD_GC, self._get_virtualizable_field_descr(index), basebox, valuebox) + index = self._get_virtualizable_field_index(fielddescr) + return self.metainterp.virtualizable_boxes[index] + + opimpl_getfield_vable_i = _opimpl_getfield_vable + opimpl_getfield_vable_r = _opimpl_getfield_vable + opimpl_getfield_vable_f = _opimpl_getfield_vable + + @arguments("orgpc", "box", "descr", "box") + def _opimpl_setfield_vable(self, pc, box, fielddescr, valuebox): + if self._nonstandard_virtualizable(pc, box): + self.execute_with_descr(rop.SETFIELD_GC, fielddescr, box, valuebox) return + index = self._get_virtualizable_field_index(fielddescr) self.metainterp.virtualizable_boxes[index] = valuebox self.metainterp.synchronize_virtualizable() # XXX only the index'th field needs to be synchronized, really + opimpl_setfield_vable_i = _opimpl_setfield_vable + opimpl_setfield_vable_r = _opimpl_setfield_vable + opimpl_setfield_vable_f = _opimpl_setfield_vable + def _get_arrayitem_vable_index(self, pc, arrayindex, indexbox): indexbox = self.implement_guard_value(pc, indexbox) vinfo = self.metainterp.staticdata.virtualizable_info @@ -1112,6 +1121,7 @@ self.portal_code = codewriter.mainjitcode self._portal_runner_ptr = codewriter.callcontrol.portal_runner_ptr self.virtualref_info = codewriter.callcontrol.virtualref_info + self.virtualizable_info = codewriter.callcontrol.virtualizable_info RESULT = codewriter.portal_graph.getreturnvar().concretetype self.result_type = history.getkind(RESULT) # @@ -1862,9 +1872,7 @@ def rebuild_state_after_failure(self, resumedescr): vinfo = self.staticdata.virtualizable_info self.framestack = [] - expect_virtualizable = vinfo is not None - boxlists = resume.rebuild_from_resumedata(self, resumedescr, - expect_virtualizable) + boxlists = resume.rebuild_from_resumedata(self, resumedescr, vinfo) inputargs_and_holes, virtualizable_boxes, virtualref_boxes = boxlists # # virtual refs: make the vrefs point to the freshly allocated virtuals @@ -1880,11 +1888,6 @@ # boxes, in whichever direction is appropriate if expect_virtualizable: self.virtualizable_boxes = virtualizable_boxes - if 0: ## self._already_allocated_resume_virtuals is not None: - # resuming from a ResumeGuardForcedDescr: load the new values - # currently stored on the virtualizable fields - self.load_fields_from_virtualizable() - return # just jumped away from assembler (case 4 in the comment in # virtualizable.py) into tracing (case 2); check that vable_token # is and stays 0. Note the call to reset_vable_token() in Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py Fri May 28 09:55:28 2010 @@ -1,10 +1,10 @@ import sys, os -from pypy.jit.metainterp.history import Box, Const, ConstInt +from pypy.jit.metainterp.history import Box, Const, ConstInt, getkind from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat from pypy.jit.metainterp.history import INT, REF, FLOAT, HOLE from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp import jitprof -from pypy.rpython.lltypesystem import rffi +from pypy.rpython.lltypesystem import lltype, rffi from pypy.rlib import rarithmetic from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.rlib.debug import have_debug_prints @@ -57,9 +57,11 @@ storage.rd_frame_info_list = frame_info_list snapshot = Snapshot(top.parent_resumedata_snapshot, top.get_list_of_active_boxes(False)) - snapshot = Snapshot(snapshot, virtualref_boxes[:]) # xxx for now if virtualizable_boxes is not None: - snapshot = Snapshot(snapshot, virtualizable_boxes[:]) # xxx for now + boxes = virtualref_boxes + virtualizable_boxes + else: + boxes = virtualref_boxes[:] + snapshot = Snapshot(snapshot, boxes) storage.rd_snapshot = snapshot class Numbering(object): @@ -548,11 +550,10 @@ # ---------- when resuming for pyjitpl.py, make boxes ---------- -def rebuild_from_resumedata(metainterp, storage, - expects_virtualizables): +def rebuild_from_resumedata(metainterp, storage, virtualizable_info): resumereader = ResumeDataBoxReader(storage, metainterp) virtualizable_boxes = None - if expects_virtualizables: + if virtualizable_info: XXX # virtualizable_boxes = resumereader.consume_boxes() virtualref_boxes = resumereader.consume_virtualref_boxes() frameinfo = storage.rd_frame_info_list @@ -679,13 +680,12 @@ # ---------- when resuming for blackholing, get direct values ---------- def blackhole_from_resumedata(blackholeinterpbuilder, storage, - expects_virtualizables, all_virtuals=None): + all_virtuals=None): resumereader = ResumeDataDirectReader(blackholeinterpbuilder.cpu, storage, all_virtuals) - if expects_virtualizables: - XXX + vinfo = blackholeinterpbuilder.metainterp_sd.virtualizable_info vrefinfo = blackholeinterpbuilder.metainterp_sd.virtualref_info - resumereader.consume_virtualref_info(vrefinfo) + resumereader.consume_vref_and_vable(vrefinfo, vinfo) # # First get a chain of blackhole interpreters whose length is given # by the depth of rd_frame_info_list. The first one we get must be @@ -715,9 +715,8 @@ resumereader.done() return firstbh -def force_from_resumedata(metainterp_sd, storage, expects_virtualizables): +def force_from_resumedata(metainterp_sd, storage): resumereader = ResumeDataDirectReader(metainterp_sd.cpu, storage) - virtualizable_boxes = None if expects_virtualizables: XXX vrefinfo = metainterp_sd.virtualref_info @@ -725,6 +724,7 @@ return resumereader.virtuals class ResumeDataDirectReader(AbstractResumeDataReader): + resume_after_guard_not_forced = False def __init__(self, cpu, storage, all_virtuals=None): self._init(cpu, storage) @@ -733,6 +733,7 @@ else: # special case for resuming after a GUARD_NOT_FORCED: we already # have the virtuals + self.resume_after_guard_not_forced = True self.virtuals = all_virtuals def consume_one_section(self, blackholeinterp): @@ -740,17 +741,51 @@ info = blackholeinterp.get_current_position_info() self._prepare_next_section(info) - def consume_virtualref_info(self, vrefinfo): + def consume_virtualref_info(self, vrefinfo, nums, end): # we have to decode a list of references containing pairs - # [..., virtual, vref, ...] - nums = self.cur_numb.nums - self.cur_numb = self.cur_numb.prev - for i in range(0, len(nums), 2): + # [..., virtual, vref, ...] stopping at 'end' + assert (end & 1) == 0 + for i in range(0, end, 2): virtual = self.decode_ref(nums[i]) vref = self.decode_ref(nums[i+1]) # For each pair, we store the virtual inside the vref. vrefinfo.continue_tracing(vref, virtual) + def consume_vable_info(self, vinfo, nums): + # we have to ignore the initial part of 'nums' (containing vrefs), + # find the virtualizable from nums[-1], load all other values + # from the CPU stack, and copy them into the virtualizable + if vinfo is None: + return len(nums) + virtualizable = self.decode_ref(nums[-1]) + virtualizable = vinfo.cast_gcref_to_vtype(virtualizable) + # just jumped away from assembler (case 4 in the comment in + # virtualizable.py) into tracing (case 2); check that vable_token + # is and stays 0. Note the call to reset_vable_token() in + # warmstate.py. + assert not virtualizable.vable_token + return vinfo.write_from_resume_data(virtualizable, self, nums) + + def load_value_of_type(self, TYPE, tagged): + kind = getkind(TYPE) + if kind == 'int': + x = self.decode_int(tagged) + elif kind == 'ref': + x = self.decode_ref(tagged) + elif kind == 'float': + x = self.decode_float(tagged) + else: + raise AssertionError(kind) + return lltype._cast_whatever(TYPE, x) + load_value_of_type._annspecialcase_ = 'specialize:arg(1)' + + def consume_vref_and_vable(self, vrefinfo, vinfo): + nums = self.cur_numb.nums + self.cur_numb = self.cur_numb.prev + if not self.resume_after_guard_not_forced: + end_vref = self.consume_vable_info(vinfo, nums) + self.consume_virtualref_info(vrefinfo, nums, end_vref) + def allocate_with_vtable(self, known_class): from pypy.jit.metainterp.executor import exec_new_with_vtable return exec_new_with_vtable(self.cpu, known_class) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualizable.py Fri May 28 09:55:28 2010 @@ -2,14 +2,15 @@ from pypy.rpython.extregistry import ExtRegistryEntry from pypy.rpython.lltypesystem import lltype, lloperation, rclass, llmemory from pypy.rpython.annlowlevel import llhelper -from pypy.jit.metainterp.policy import StopAtXPolicy +from pypy.jit.codewriter.policy import StopAtXPolicy +from pypy.jit.codewriter import heaptracker from pypy.rlib.jit import JitDriver, hint, dont_look_inside from pypy.rlib.jit import OPTIMIZER_SIMPLE, OPTIMIZER_FULL from pypy.rlib.rarithmetic import intmask from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin from pypy.rpython.rclass import FieldListAccessor from pypy.jit.metainterp.warmspot import get_stats, get_translator -from pypy.jit.metainterp import history, heaptracker +from pypy.jit.metainterp import history from pypy.jit.metainterp.test.test_optimizefindnode import LLtypeMixin def promote_virtualizable(*args): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/virtualizable.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/virtualizable.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/virtualizable.py Fri May 28 09:55:28 2010 @@ -72,6 +72,8 @@ for name in static_fields] self.array_field_descrs = [cpu.fielddescrof(VTYPE, name) for name in array_fields] + self.static_field_by_descrs = dict( + [(descr, i) for (i, descr) in enumerate(self.static_field_descrs)]) # getlength = cpu.ts.getlength getarrayitem = cpu.ts.getarrayitem @@ -102,6 +104,21 @@ i = i + 1 assert len(boxes) == i + 1 # + def write_from_resume_data(virtualizable, reader, nums): + # similar to write_boxes(), but works from the end instead + i = len(nums) - 1 + for ARRAYITEMTYPE, fieldname in unroll_array_fields_rev: + lst = getattr(virtualizable, fieldname) + for j in range(getlength(lst)-1, -1, -1): + i -= 1 + x = reader.load_value_of_type(ARRAYITEMTYPE, nums[i]) + setarrayitem(lst, j, x) + for FIELDTYPE, fieldname in unroll_static_fields_rev: + i -= 1 + x = reader.load_value_of_type(FIELDTYPE, nums[i]) + setattr(virtualizable, fieldname, x) + return i + # def check_boxes(virtualizable, boxes): # for debugging i = 0 @@ -141,8 +158,13 @@ static_fields)) unroll_array_fields = unrolling_iterable(zip(ARRAYITEMTYPES, array_fields)) + unroll_static_fields_rev = unrolling_iterable( + reversed(list(unroll_static_fields))) + unroll_array_fields_rev = unrolling_iterable( + reversed(list(unroll_array_fields))) self.read_boxes = read_boxes self.write_boxes = write_boxes + self.write_from_resume_data = write_from_resume_data self.check_boxes = check_boxes self.get_index_in_array = get_index_in_array self.get_array_length = get_array_length @@ -172,6 +194,9 @@ return self.cpu.ts.cast_to_instance_maybe(self.VTYPEPTR, virtualizable) cast_to_vtype._annspecialcase_ = 'specialize:ll' + def cast_gcref_to_vtype(self, virtualizable): + return lltype.cast_opaque_ptr(self.VTYPEPTR, virtualizable) + def is_vtypeptr(self, TYPE): return rvirtualizable2.match_virtualizable_type(TYPE, self.VTYPEPTR) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py Fri May 28 09:55:28 2010 @@ -167,7 +167,8 @@ self.codewriter.setup_vrefinfo(vrefinfo) if self.jitdriver.virtualizables: from pypy.jit.metainterp.virtualizable import VirtualizableInfo - self.metainterp_sd.virtualizable_info = VirtualizableInfo(self) + self.virtualizable_info = VirtualizableInfo(self) + self.codewriter.setup_virtualizable_info(self.virtualizable_info) # self.make_exception_classes() self.make_driverhook_graphs() @@ -183,7 +184,7 @@ self.metainterp_sd.finish_setup(self.codewriter, optimizer=optimizer) def finish(self): - vinfo = self.metainterp_sd.virtualizable_info + vinfo = self.virtualizable_info if vinfo is not None: vinfo.finish() if self.cpu.translate_support_code: @@ -556,7 +557,7 @@ self.PTR_PORTAL_FUNCTYPE.TO.RESULT) self.codewriter.setup_portal_runner_ptr(self.portal_runner_ptr) - vinfo = self.metainterp_sd.virtualizable_info + vinfo = self.virtualizable_info def assembler_call_helper(failindex, virtualizableref): fail_descr = self.cpu.get_fail_descr_from_number(failindex) From arigo at codespeak.net Fri May 28 10:25:00 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 10:25:00 +0200 (CEST) Subject: [pypy-svn] r74835 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100528082500.B751A282BD6@codespeak.net> Author: arigo Date: Fri May 28 10:24:59 2010 New Revision: 74835 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/virtualizable.py Log: Progress, maybe. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Fri May 28 10:24:59 2010 @@ -1886,7 +1886,7 @@ # # virtualizable: synchronize the real virtualizable and the local # boxes, in whichever direction is appropriate - if expect_virtualizable: + if vinfo is not None: self.virtualizable_boxes = virtualizable_boxes # just jumped away from assembler (case 4 in the comment in # virtualizable.py) into tracing (case 2); check that vable_token Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py Fri May 28 10:24:59 2010 @@ -552,10 +552,8 @@ def rebuild_from_resumedata(metainterp, storage, virtualizable_info): resumereader = ResumeDataBoxReader(storage, metainterp) - virtualizable_boxes = None - if virtualizable_info: - XXX # virtualizable_boxes = resumereader.consume_boxes() - virtualref_boxes = resumereader.consume_virtualref_boxes() + boxes = resumereader.consume_vref_and_vable_boxes(virtualizable_info) + virtualizable_boxes, virtualref_boxes = boxes frameinfo = storage.rd_frame_info_list while True: f = metainterp.newframe(frameinfo.jitcode) @@ -567,7 +565,7 @@ break metainterp.framestack.reverse() resumereader.done() - return resumereader.liveboxes, None, virtualref_boxes + return resumereader.liveboxes, virtualizable_boxes, virtualref_boxes class ResumeDataBoxReader(AbstractResumeDataReader): @@ -583,12 +581,32 @@ self.boxes_f = boxes_f self._prepare_next_section(info) - def consume_virtualref_boxes(self): + def consume_virtualizable_boxes(self, vinfo, nums): + # we have to ignore the initial part of 'nums' (containing vrefs), + # find the virtualizable from nums[-1], and use it to know how many + # boxes of which type we have to return. This does not write + # anything into the virtualizable. + virtualizablebox = self.decode_ref(nums[-1]) + virtualizable = vinfo.unwrap_virtualizable_box(virtualizablebox) + return vinfo.load_list_of_boxes(virtualizable, self, nums) + + def consume_virtualref_boxes(self, nums, end): # Returns a list of boxes, assumed to be all BoxPtrs. # We leave up to the caller to call vrefinfo.continue_tracing(). + assert (end & 1) == 0 + return [self.decode_ref(nums[i]) for i in range(end)] + + def consume_vref_and_vable_boxes(self, vinfo): nums = self.cur_numb.nums self.cur_numb = self.cur_numb.prev - return [self.decode_ref(num) for num in nums] + if vinfo is None: + virtualizable_boxes = None + end = len(nums) + else: + virtualizable_boxes = self.consume_virtualizable_boxes(vinfo, nums) + end = len(nums) - len(virtualizable_boxes) + virtualref_boxes = self.consume_virtualref_boxes(nums, end) + return virtualizable_boxes, virtualref_boxes def allocate_with_vtable(self, known_class): return self.metainterp.execute_and_record(rop.NEW_WITH_VTABLE, @@ -670,6 +688,15 @@ self.liveboxes[num] = box return box + def decode_box_of_type(self, TYPE, tagged): + kind = getkind(TYPE) + if kind == 'int': kind = INT + elif kind == 'ref': kind = REF + elif kind == 'float': kind = FLOAT + else: raise AssertionError(kind) + return self.decode_box(tagged, kind) + decode_box_of_type._annspecialcase_ = 'specialize:arg(1)' + def write_an_int(self, index, box): self.boxes_i[index] = box def write_a_ref(self, index, box): @@ -717,10 +744,9 @@ def force_from_resumedata(metainterp_sd, storage): resumereader = ResumeDataDirectReader(metainterp_sd.cpu, storage) - if expects_virtualizables: - XXX + vinfo = metainterp_sd.virtualizable_info vrefinfo = metainterp_sd.virtualref_info - resumereader.consume_virtualref_info(vrefinfo) + resumereader.consume_vref_and_vable(vrefinfo, vinfo) return resumereader.virtuals class ResumeDataDirectReader(AbstractResumeDataReader): @@ -764,7 +790,7 @@ # is and stays 0. Note the call to reset_vable_token() in # warmstate.py. assert not virtualizable.vable_token - return vinfo.write_from_resume_data(virtualizable, self, nums) + return vinfo.write_from_resume_data_partial(virtualizable, self, nums) def load_value_of_type(self, TYPE, tagged): kind = getkind(TYPE) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/virtualizable.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/virtualizable.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/virtualizable.py Fri May 28 10:24:59 2010 @@ -104,21 +104,52 @@ i = i + 1 assert len(boxes) == i + 1 # - def write_from_resume_data(virtualizable, reader, nums): - # similar to write_boxes(), but works from the end instead + def write_from_resume_data_partial(virtualizable, reader, nums): + # Load values from the reader (see resume.py) described by + # the list of numbers 'nums', and write them in their proper + # place in the 'virtualizable'. This works from the end of + # the list and returns the index in 'nums' of the start of + # the virtualizable data found, allowing the caller to do + # further processing with the start of the list. i = len(nums) - 1 + assert i >= 0 for ARRAYITEMTYPE, fieldname in unroll_array_fields_rev: lst = getattr(virtualizable, fieldname) for j in range(getlength(lst)-1, -1, -1): i -= 1 + assert i >= 0 x = reader.load_value_of_type(ARRAYITEMTYPE, nums[i]) setarrayitem(lst, j, x) for FIELDTYPE, fieldname in unroll_static_fields_rev: i -= 1 + assert i >= 0 x = reader.load_value_of_type(FIELDTYPE, nums[i]) setattr(virtualizable, fieldname, x) return i # + def load_list_of_boxes(virtualizable, reader, nums): + # Uses 'virtualizable' only to know the length of the arrays; + # does not write anything into it. The returned list is in + # the format expected of virtualizable_boxes, so it ends in + # the virtualizable itself. + i = len(nums) - 1 + assert i >= 0 + boxes = [reader.decode_box_of_type(self.VTYPEPTR, nums[i])] + for ARRAYITEMTYPE, fieldname in unroll_array_fields_rev: + lst = getattr(virtualizable, fieldname) + for j in range(getlength(lst)-1, -1, -1): + i -= 1 + assert i >= 0 + box = reader.decode_box_of_type(ARRAYITEMTYPE, nums[i]) + boxes.append(box) + for FIELDTYPE, fieldname in unroll_static_fields_rev: + i -= 1 + assert i >= 0 + box = reader.decode_box_of_type(FIELDTYPE, nums[i]) + boxes.append(box) + boxes.reverse() + return boxes + # def check_boxes(virtualizable, boxes): # for debugging i = 0 @@ -164,7 +195,8 @@ reversed(list(unroll_array_fields))) self.read_boxes = read_boxes self.write_boxes = write_boxes - self.write_from_resume_data = write_from_resume_data + self.write_from_resume_data_partial = write_from_resume_data_partial + self.load_list_of_boxes = load_list_of_boxes self.check_boxes = check_boxes self.get_index_in_array = get_index_in_array self.get_array_length = get_array_length From arigo at codespeak.net Fri May 28 10:46:36 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 10:46:36 +0200 (CEST) Subject: [pypy-svn] r74836 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100528084636.060C7282BD6@codespeak.net> Author: arigo Date: Fri May 28 10:46:34 2010 New Revision: 74836 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmstate.py Log: Progress, really. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py Fri May 28 10:46:34 2010 @@ -262,6 +262,7 @@ else: from pypy.jit.metainterp.blackhole import resume_in_blackhole resume_in_blackhole(metainterp_sd, self) + assert 0, "unreachable" def _trace_and_compile_from_bridge(self, metainterp_sd): from pypy.jit.metainterp.pyjitpl import MetaInterp Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Fri May 28 10:46:34 2010 @@ -1057,7 +1057,6 @@ # ____________________________________________________________ class MetaInterpStaticData(object): - virtualizable_info = None logger_noopt = None logger_ops = None @@ -1516,11 +1515,11 @@ self.seen_can_enter_jit = False try: self.interpret() - assert False, "should always raise" except GenerateMergePoint, gmp: return self.designate_target_loop(gmp) except SwitchToBlackhole, stb: self.run_blackhole_interp_to_cancel_tracing(stb) + assert False, "should always raise" def handle_guard_failure(self, key): debug_start('jit-tracing') @@ -1544,11 +1543,11 @@ try: self.prepare_resume_from_failure(key.guard_opnum) self.interpret() - assert False, "should always raise" except GenerateMergePoint, gmp: return self.designate_target_loop(gmp) except SwitchToBlackhole, stb: self.run_blackhole_interp_to_cancel_tracing(stb) + assert False, "should always raise" def run_blackhole_interp_to_cancel_tracing(self, stb): # We got a SwitchToBlackhole exception. Convert the framestack into Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py Fri May 28 10:46:34 2010 @@ -793,6 +793,7 @@ return vinfo.write_from_resume_data_partial(virtualizable, self, nums) def load_value_of_type(self, TYPE, tagged): + from pypy.jit.metainterp.warmstate import specialize_value kind = getkind(TYPE) if kind == 'int': x = self.decode_int(tagged) @@ -802,7 +803,7 @@ x = self.decode_float(tagged) else: raise AssertionError(kind) - return lltype._cast_whatever(TYPE, x) + return specialize_value(TYPE, x) load_value_of_type._annspecialcase_ = 'specialize:arg(1)' def consume_vref_and_vable(self, vrefinfo, vinfo): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py Fri May 28 10:46:34 2010 @@ -169,6 +169,8 @@ from pypy.jit.metainterp.virtualizable import VirtualizableInfo self.virtualizable_info = VirtualizableInfo(self) self.codewriter.setup_virtualizable_info(self.virtualizable_info) + else: + self.virtualizable_info = None # self.make_exception_classes() self.make_driverhook_graphs() Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmstate.py Fri May 28 10:46:34 2010 @@ -195,7 +195,7 @@ return self.maybe_compile_and_run metainterp_sd = self.warmrunnerdesc.metainterp_sd - vinfo = metainterp_sd.virtualizable_info + vinfo = self.warmrunnerdesc.virtualizable_info num_green_args = self.warmrunnerdesc.num_green_args get_jitcell = self.make_jitcell_getter() set_future_values = self.make_set_future_values() @@ -416,7 +416,7 @@ warmrunnerdesc = self.warmrunnerdesc cpu = warmrunnerdesc.cpu - vinfo = warmrunnerdesc.metainterp_sd.virtualizable_info + vinfo = warmrunnerdesc.virtualizable_info red_args_types = unrolling_iterable(warmrunnerdesc.red_args_types) # def set_future_values(*redargs): From antocuni at codespeak.net Fri May 28 10:57:46 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 28 May 2010 10:57:46 +0200 (CEST) Subject: [pypy-svn] r74837 - in pypy/branch/sys-prefix/pypy: annotation lib lib/test2 module/cpyext module/cpyext/test objspace/flow rpython/lltypesystem rpython/lltypesystem/test rpython/memory rpython/ootypesystem tool tool/algo tool/test translator translator/c translator/oosupport translator/tool Message-ID: <20100528085746.31633282BD6@codespeak.net> Author: antocuni Date: Fri May 28 10:57:44 2010 New Revision: 74837 Added: pypy/branch/sys-prefix/pypy/tool/identity_dict.py - copied unchanged from r74817, pypy/branch/sys-prefix/pypy/lib/identity_dict.py pypy/branch/sys-prefix/pypy/tool/test/test_identitydict.py - copied, changed from r74817, pypy/branch/sys-prefix/pypy/lib/test2/test_identitydict.py Removed: pypy/branch/sys-prefix/pypy/lib/identity_dict.py pypy/branch/sys-prefix/pypy/lib/test2/test_identitydict.py Modified: pypy/branch/sys-prefix/pypy/annotation/bookkeeper.py pypy/branch/sys-prefix/pypy/lib/_sre.py pypy/branch/sys-prefix/pypy/module/cpyext/state.py pypy/branch/sys-prefix/pypy/module/cpyext/test/test_cpyext.py pypy/branch/sys-prefix/pypy/objspace/flow/model.py pypy/branch/sys-prefix/pypy/rpython/lltypesystem/lltype.py pypy/branch/sys-prefix/pypy/rpython/lltypesystem/rclass.py pypy/branch/sys-prefix/pypy/rpython/lltypesystem/test/test_lltype.py pypy/branch/sys-prefix/pypy/rpython/memory/gctypelayout.py pypy/branch/sys-prefix/pypy/rpython/ootypesystem/rclass.py pypy/branch/sys-prefix/pypy/tool/algo/graphlib.py pypy/branch/sys-prefix/pypy/translator/c/database.py pypy/branch/sys-prefix/pypy/translator/c/funcgen.py pypy/branch/sys-prefix/pypy/translator/geninterplevel.py pypy/branch/sys-prefix/pypy/translator/oosupport/function.py pypy/branch/sys-prefix/pypy/translator/tool/lltracker.py Log: move identitydict.py from pypy/lib to pypy/tool Modified: pypy/branch/sys-prefix/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/branch/sys-prefix/pypy/annotation/bookkeeper.py (original) +++ pypy/branch/sys-prefix/pypy/annotation/bookkeeper.py Fri May 28 10:57:44 2010 @@ -22,7 +22,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.ootypesystem import ootype from pypy.rpython import extregistry -from pypy.lib.identity_dict import identity_dict +from pypy.tool.identity_dict import identity_dict class Stats: Modified: pypy/branch/sys-prefix/pypy/lib/_sre.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/_sre.py (original) +++ pypy/branch/sys-prefix/pypy/lib/_sre.py Fri May 28 10:57:44 2010 @@ -19,7 +19,7 @@ import array, operator -from identity_dict import identity_dict +from __pypy__ import identity_dict from sre_constants import ATCODES, OPCODES, CHCODES, MAXREPEAT from sre_constants import SRE_INFO_PREFIX, SRE_INFO_LITERAL Modified: pypy/branch/sys-prefix/pypy/module/cpyext/state.py ============================================================================== --- pypy/branch/sys-prefix/pypy/module/cpyext/state.py (original) +++ pypy/branch/sys-prefix/pypy/module/cpyext/state.py Fri May 28 10:57:44 2010 @@ -1,6 +1,5 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.rpython.lltypesystem import rffi, lltype -from pypy.lib.identity_dict import identity_dict from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import lltype Modified: pypy/branch/sys-prefix/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/sys-prefix/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/sys-prefix/pypy/module/cpyext/test/test_cpyext.py Fri May 28 10:57:44 2010 @@ -15,7 +15,7 @@ from pypy.module.cpyext.pyobject import RefcountState from pypy.module.cpyext.pyobject import Py_DecRef, InvalidPointerException from pypy.translator.goal import autopath -from pypy.lib.identity_dict import identity_dict +from pypy.tool.identity_dict import identity_dict @api.cpython_api([], api.PyObject) def PyPy_Crash1(space): Modified: pypy/branch/sys-prefix/pypy/objspace/flow/model.py ============================================================================== --- pypy/branch/sys-prefix/pypy/objspace/flow/model.py (original) +++ pypy/branch/sys-prefix/pypy/objspace/flow/model.py Fri May 28 10:57:44 2010 @@ -7,7 +7,7 @@ from pypy.tool.uid import uid, Hashable from pypy.tool.descriptor import roproperty from pypy.tool.sourcetools import PY_IDENTIFIER, nice_repr_for_func -from pypy.lib.identity_dict import identity_dict +from pypy.tool.identity_dict import identity_dict """ memory size before and after introduction of __slots__ Modified: pypy/branch/sys-prefix/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/sys-prefix/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/sys-prefix/pypy/rpython/lltypesystem/lltype.py Fri May 28 10:57:44 2010 @@ -9,7 +9,7 @@ from pypy.rlib.objectmodel import Symbolic from pypy.tool.uid import Hashable from pypy.tool.tls import tlsobject -from pypy.lib.identity_dict import identity_dict +from pypy.tool.identity_dict import identity_dict from types import NoneType from sys import maxint import struct Modified: pypy/branch/sys-prefix/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/sys-prefix/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/sys-prefix/pypy/rpython/lltypesystem/rclass.py Fri May 28 10:57:44 2010 @@ -21,7 +21,7 @@ from pypy.annotation import model as annmodel from pypy.rlib.rarithmetic import intmask from pypy.rlib import objectmodel -from pypy.lib.identity_dict import identity_dict +from pypy.tool.identity_dict import identity_dict # # There is one "vtable" per user class, with the following structure: Modified: pypy/branch/sys-prefix/pypy/rpython/lltypesystem/test/test_lltype.py ============================================================================== --- pypy/branch/sys-prefix/pypy/rpython/lltypesystem/test/test_lltype.py (original) +++ pypy/branch/sys-prefix/pypy/rpython/lltypesystem/test/test_lltype.py Fri May 28 10:57:44 2010 @@ -1,7 +1,7 @@ import py from pypy.rpython.lltypesystem.lltype import * from pypy.rpython.lltypesystem import lltype, rffi -from pypy.lib.identity_dict import identity_dict +from pypy.tool.identity_dict import identity_dict def isweak(p, T): try: Modified: pypy/branch/sys-prefix/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/sys-prefix/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/sys-prefix/pypy/rpython/memory/gctypelayout.py Fri May 28 10:57:44 2010 @@ -3,7 +3,7 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.debug import ll_assert -from pypy.lib.identity_dict import identity_dict +from pypy.tool.identity_dict import identity_dict class GCData(object): Modified: pypy/branch/sys-prefix/pypy/rpython/ootypesystem/rclass.py ============================================================================== --- pypy/branch/sys-prefix/pypy/rpython/ootypesystem/rclass.py (original) +++ pypy/branch/sys-prefix/pypy/rpython/ootypesystem/rclass.py Fri May 28 10:57:44 2010 @@ -10,7 +10,7 @@ from pypy.rpython.exceptiondata import standardexceptions from pypy.tool.pairtype import pairtype from pypy.tool.sourcetools import func_with_new_name -from pypy.lib.identity_dict import identity_dict +from pypy.tool.identity_dict import identity_dict OBJECT = ootype.ROOT META = ootype.Instance("Meta", ootype.ROOT, Modified: pypy/branch/sys-prefix/pypy/tool/algo/graphlib.py ============================================================================== --- pypy/branch/sys-prefix/pypy/tool/algo/graphlib.py (original) +++ pypy/branch/sys-prefix/pypy/tool/algo/graphlib.py Fri May 28 10:57:44 2010 @@ -6,7 +6,7 @@ 'edges' is a dict mapping vertices to a list of edges with its source. Note that we can usually use 'edges' as the set of 'vertices' too. """ -from pypy.lib.identity_dict import identity_dict +from pypy.tool.identity_dict import identity_dict class Edge: Copied: pypy/branch/sys-prefix/pypy/tool/test/test_identitydict.py (from r74817, pypy/branch/sys-prefix/pypy/lib/test2/test_identitydict.py) ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/test2/test_identitydict.py (original) +++ pypy/branch/sys-prefix/pypy/tool/test/test_identitydict.py Fri May 28 10:57:44 2010 @@ -1,5 +1,5 @@ import py -from pypy.lib.identity_dict import identity_dict, IdentityDictPurePython +from pypy.tool.identity_dict import identity_dict, IdentityDictPurePython class TestIdentityDictNative: identity_dict = identity_dict Modified: pypy/branch/sys-prefix/pypy/translator/c/database.py ============================================================================== --- pypy/branch/sys-prefix/pypy/translator/c/database.py (original) +++ pypy/branch/sys-prefix/pypy/translator/c/database.py Fri May 28 10:57:44 2010 @@ -16,7 +16,7 @@ from pypy.translator.c.extfunc import do_the_getting from pypy import conftest from pypy.translator.c import gc -from pypy.lib.identity_dict import identity_dict +from pypy.tool.identity_dict import identity_dict class NoCorrespondingNode(Exception): Modified: pypy/branch/sys-prefix/pypy/translator/c/funcgen.py ============================================================================== --- pypy/branch/sys-prefix/pypy/translator/c/funcgen.py (original) +++ pypy/branch/sys-prefix/pypy/translator/c/funcgen.py Fri May 28 10:57:44 2010 @@ -13,7 +13,7 @@ from pypy.rpython.lltypesystem.llmemory import Address from pypy.translator.backendopt.ssa import SSI_to_SSA from pypy.translator.backendopt.innerloop import find_inner_loops -from pypy.lib.identity_dict import identity_dict +from pypy.tool.identity_dict import identity_dict PyObjPtr = Ptr(PyObject) Modified: pypy/branch/sys-prefix/pypy/translator/geninterplevel.py ============================================================================== --- pypy/branch/sys-prefix/pypy/translator/geninterplevel.py (original) +++ pypy/branch/sys-prefix/pypy/translator/geninterplevel.py Fri May 28 10:57:44 2010 @@ -62,7 +62,7 @@ from pypy.translator.gensupp import ordered_blocks, UniqueList, builtin_base, \ uniquemodulename, C_IDENTIFIER, NameManager -from pypy.lib.identity_dict import identity_dict +from pypy.tool.identity_dict import identity_dict import pypy # __path__ import py.path Modified: pypy/branch/sys-prefix/pypy/translator/oosupport/function.py ============================================================================== --- pypy/branch/sys-prefix/pypy/translator/oosupport/function.py (original) +++ pypy/branch/sys-prefix/pypy/translator/oosupport/function.py Fri May 28 10:57:44 2010 @@ -7,7 +7,7 @@ from pypy.rpython.ootypesystem import ootype from pypy.translator.oosupport.treebuilder import SubOperation from pypy.translator.oosupport.metavm import InstructionList, StoreResult -from pypy.lib.identity_dict import identity_dict +from pypy.tool.identity_dict import identity_dict def render_sub_op(sub_op, db, generator): Modified: pypy/branch/sys-prefix/pypy/translator/tool/lltracker.py ============================================================================== --- pypy/branch/sys-prefix/pypy/translator/tool/lltracker.py (original) +++ pypy/branch/sys-prefix/pypy/translator/tool/lltracker.py Fri May 28 10:57:44 2010 @@ -8,7 +8,7 @@ from pypy.rpython.memory.gcheader import header2obj from pypy.translator.tool.reftracker import BaseRefTrackerPage, MARKER from pypy.tool.uid import uid -from pypy.lib.identity_dict import identity_dict +from pypy.tool.identity_dict import identity_dict class LLRefTrackerPage(BaseRefTrackerPage): From arigo at codespeak.net Fri May 28 11:19:06 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 11:19:06 +0200 (CEST) Subject: [pypy-svn] r74838 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter metainterp metainterp/test Message-ID: <20100528091906.88E71282BD6@codespeak.net> Author: arigo Date: Fri May 28 11:19:04 2010 New Revision: 74838 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualizable.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/virtualizable.py Log: Lots of progress. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Fri May 28 11:19:04 2010 @@ -414,7 +414,7 @@ (v_base, arrayfielddescr, arraydescr) = vars kind = getkind(op.result.concretetype) return [SpaceOperation('-live-', [], None), - SpaceOperation('getarrayitem_vable_%s' % kind, + SpaceOperation('getarrayitem_vable_%s' % kind[0], [v_base, arrayfielddescr, arraydescr, op.args[1]], op.result)] # normal case follows @@ -432,9 +432,9 @@ if op.args[0] in self.vable_array_vars: # for virtualizables vars = self.vable_array_vars[op.args[0]] (v_base, arrayfielddescr, arraydescr) = vars - kind = getkind(op.result.concretetype) + kind = getkind(op.args[2].concretetype) return [SpaceOperation('-live-', [], None), - SpaceOperation('setarrayitem_vable_%s' % kind, + SpaceOperation('setarrayitem_vable_%s' % kind[0], [v_base, arrayfielddescr, arraydescr, op.args[1], op.args[2]], None)] arraydescr = self.cpu.arraydescrof(ARRAY) @@ -870,11 +870,24 @@ return SpaceOperation('new_array', [arraydescr, v_length], op.result) def do_fixed_list_len(self, op, args, arraydescr): + if args[0] in self.vable_array_vars: # virtualizable array + vars = self.vable_array_vars[args[0]] + (v_base, arrayfielddescr, arraydescr) = vars + return SpaceOperation('arraylen_vable', + [v_base, arrayfielddescr, arraydescr], + op.result) return SpaceOperation('arraylen_gc', [args[0], arraydescr], op.result) do_fixed_list_len_foldable = do_fixed_list_len def do_fixed_list_getitem(self, op, args, arraydescr, pure=False): + if args[0] in self.vable_array_vars: # virtualizable array + vars = self.vable_array_vars[args[0]] + (v_base, arrayfielddescr, arraydescr) = vars + kind = getkind(op.result.concretetype) + return SpaceOperation('getarrayitem_vable_%s' % kind[0], + [v_base, arrayfielddescr, arraydescr, + args[1]], op.result) v_index, extraop = self._prepare_list_getset(op, arraydescr, args, 'check_neg_index') extra = getkind(op.result.concretetype)[0] @@ -888,6 +901,13 @@ return self.do_fixed_list_getitem(op, args, arraydescr, pure=True) def do_fixed_list_setitem(self, op, args, arraydescr): + if args[0] in self.vable_array_vars: # virtualizable array + vars = self.vable_array_vars[args[0]] + (v_base, arrayfielddescr, arraydescr) = vars + kind = getkind(args[2].concretetype) + return SpaceOperation('setarrayitem_vable_%s' % kind[0], + [v_base, arrayfielddescr, arraydescr, + args[1], args[2]], None) v_index, extraop = self._prepare_list_getset(op, arraydescr, args, 'check_neg_index') kind = getkind(args[2].concretetype)[0] Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Fri May 28 11:19:04 2010 @@ -907,11 +907,6 @@ cpu.bh_setarrayitem_gc_f(arraydescr, items, index, nval) # ---------- - # virtualizables operations - - #xxx - - # ---------- # the following operations are directly implemented by the backend @arguments("cpu", "i", "d", "R", returns="i") @@ -1026,6 +1021,37 @@ def bhimpl_arraycopy(cpu, calldescr, func, x1, x2, x3, x4, x5, arraydescr): cpu.bh_call_v(func, calldescr, [x3, x4, x5], [x1, x2], None) + @arguments("cpu", "r", "d", "d", "i", returns="i") + def bhimpl_getarrayitem_vable_i(cpu, vable, fielddescr, arraydescr, index): + array = cpu.bh_getfield_gc_r(vable, fielddescr) + return cpu.bh_getarrayitem_gc_i(arraydescr, array, index) + @arguments("cpu", "r", "d", "d", "i", returns="r") + def bhimpl_getarrayitem_vable_r(cpu, vable, fielddescr, arraydescr, index): + array = cpu.bh_getfield_gc_r(vable, fielddescr) + return cpu.bh_getarrayitem_gc_r(arraydescr, array, index) + @arguments("cpu", "r", "d", "d", "i", returns="f") + def bhimpl_getarrayitem_vable_f(cpu, vable, fielddescr, arraydescr, index): + array = cpu.bh_getfield_gc_r(vable, fielddescr) + return cpu.bh_getarrayitem_gc_f(arraydescr, array, index) + + @arguments("cpu", "r", "d", "d", "i", "i") + def bhimpl_setarrayitem_vable_i(cpu, vable, fdescr, adescr, index, newval): + array = cpu.bh_getfield_gc_r(vable, fdescr) + cpu.bh_setarrayitem_gc_i(a, array, index, newval) + @arguments("cpu", "r", "d", "d", "i", "r") + def bhimpl_setarrayitem_vable_r(cpu, vable, fdescr, adescr, index, newval): + array = cpu.bh_getfield_gc_r(vable, fdescr) + cpu.bh_setarrayitem_gc_r(a, array, index, newval) + @arguments("cpu", "r", "d", "d", "i", "f") + def bhimpl_setarrayitem_vable_f(cpu, vable, fdescr, adescr, index, newval): + array = cpu.bh_getfield_gc_r(vable, fdescr) + cpu.bh_setarrayitem_gc_f(a, array, index, newval) + + @arguments("cpu", "r", "d", "d", returns="i") + def bhimpl_arraylen_vable(cpu, vable, fdescr, adescr): + array = cpu.bh_getfield_gc_r(vable, fdescr) + return cpu.bh_arraylen_gc(adescr, array) + @arguments("cpu", "r", "d", returns="i") def bhimpl_getfield_gc_i(cpu, struct, fielddescr): return cpu.bh_getfield_gc_i(struct, fielddescr) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Fri May 28 11:19:04 2010 @@ -29,7 +29,7 @@ return func return decorate -class FixME: # XXX temporary hack +class FixME: # XXX marks deprecated ootype-only operations def __init__(self, func): self.func = func def __getattr__(self, _): @@ -554,60 +554,63 @@ opimpl_setfield_vable_r = _opimpl_setfield_vable opimpl_setfield_vable_f = _opimpl_setfield_vable - def _get_arrayitem_vable_index(self, pc, arrayindex, indexbox): + def _get_arrayitem_vable_index(self, pc, arrayfielddescr, indexbox): indexbox = self.implement_guard_value(pc, indexbox) vinfo = self.metainterp.staticdata.virtualizable_info virtualizable_box = self.metainterp.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) + arrayindex = vinfo.array_field_by_descrs[arrayfielddescr] index = indexbox.getint() if index < 0: index += vinfo.get_array_length(virtualizable, arrayindex) assert 0 <= index < vinfo.get_array_length(virtualizable, arrayindex) return vinfo.get_index_in_array(virtualizable, arrayindex, index) - @FixME #arguments("orgpc", "box", "int", "box") - def opimpl_getarrayitem_vable(self, pc, basebox, arrayindex, indexbox): - if self._nonstandard_virtualizable(pc, basebox): - descr = self._get_virtualizable_array_field_descr(arrayindex) + @arguments("orgpc", "box", "descr", "descr", "box") + def _opimpl_getarrayitem_vable(self, pc, box, fdescr, adescr, indexbox): + if self._nonstandard_virtualizable(pc, box): arraybox = self.metainterp.execute_and_record(rop.GETFIELD_GC, - descr, basebox) - descr = self._get_virtualizable_array_descr(arrayindex) - self.execute_with_descr(rop.GETARRAYITEM_GC, descr, - arraybox, indexbox) - return + fdescr, box) + return self.execute_with_descr(rop.GETARRAYITEM_GC, adescr, + arraybox, indexbox) self.metainterp.check_synchronized_virtualizable() - index = self._get_arrayitem_vable_index(pc, arrayindex, indexbox) - resbox = self.metainterp.virtualizable_boxes[index] - self.make_result_box(resbox) - @FixME #arguments("orgpc", "box", "int", "box", "box") - def opimpl_setarrayitem_vable(self, pc, basebox, arrayindex, indexbox, + index = self._get_arrayitem_vable_index(pc, fdescr, indexbox) + return self.metainterp.virtualizable_boxes[index] + + opimpl_getarrayitem_vable_i = _opimpl_getarrayitem_vable + opimpl_getarrayitem_vable_r = _opimpl_getarrayitem_vable + opimpl_getarrayitem_vable_f = _opimpl_getarrayitem_vable + + @arguments("orgpc", "box", "descr", "descr", "box", "box") + def _opimpl_setarrayitem_vable(self, pc, box, fdescr, adescr, indexbox, valuebox): - if self._nonstandard_virtualizable(pc, basebox): - descr = self._get_virtualizable_array_field_descr(arrayindex) + if self._nonstandard_virtualizable(pc, box): arraybox = self.metainterp.execute_and_record(rop.GETFIELD_GC, - descr, basebox) - descr = self._get_virtualizable_array_descr(arrayindex) - self.execute_with_descr(rop.SETARRAYITEM_GC, descr, + fdescr, box) + self.execute_with_descr(rop.SETARRAYITEM_GC, adescr, arraybox, indexbox, valuebox) return - index = self._get_arrayitem_vable_index(pc, arrayindex, indexbox) + index = self._get_arrayitem_vable_index(pc, fdescr, indexbox) self.metainterp.virtualizable_boxes[index] = valuebox self.metainterp.synchronize_virtualizable() # XXX only the index'th field needs to be synchronized, really - @FixME #arguments("orgpc", "box", "int") - def opimpl_arraylen_vable(self, pc, basebox, arrayindex): - if self._nonstandard_virtualizable(pc, basebox): - descr = self._get_virtualizable_array_field_descr(arrayindex) + + opimpl_setarrayitem_vable_i = _opimpl_setarrayitem_vable + opimpl_setarrayitem_vable_r = _opimpl_setarrayitem_vable + opimpl_setarrayitem_vable_f = _opimpl_setarrayitem_vable + + @arguments("orgpc", "box", "descr", "descr") + def opimpl_arraylen_vable(self, pc, box, fdescr, adescr): + if self._nonstandard_virtualizable(pc, box): arraybox = self.metainterp.execute_and_record(rop.GETFIELD_GC, - descr, basebox) - descr = self._get_virtualizable_array_descr(arrayindex) - self.execute_with_descr(rop.ARRAYLEN_GC, descr, arraybox) - return + fdescr, box) + return self.execute_with_descr(rop.ARRAYLEN_GC, adescr, arraybox) vinfo = self.metainterp.staticdata.virtualizable_info virtualizable_box = self.metainterp.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) + arrayindex = vinfo.array_field_by_descrs[fdescr] result = vinfo.get_array_length(virtualizable, arrayindex) - self.make_result_box(ConstInt(result)) + return ConstInt(result) @arguments("jitcode", "boxes") def _opimpl_inline_call1(self, jitcode, argboxes): @@ -651,11 +654,6 @@ opimpl_residual_call_irf_f = _opimpl_residual_call3 opimpl_residual_call_irf_v = _opimpl_residual_call3 - @FixME #arguments("descr", "varargs") - def opimpl_residual_call_loopinvariant(self, calldescr, varargs): - return self.execute_varargs(rop.CALL_LOOPINVARIANT, varargs, calldescr, - exc=True) - @arguments("boxes3", "boxes3") def _opimpl_recursive_call(self, greenboxes, redboxes): allboxes = greenboxes + redboxes Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualizable.py Fri May 28 11:19:04 2010 @@ -462,7 +462,7 @@ def test_virtualizable_with_array(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 'frame', 'x'], + myjitdriver = JitDriver(greens = [], reds = ['n', 'x', 'frame'], virtualizables = ['frame']) class Frame(object): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/virtualizable.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/virtualizable.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/virtualizable.py Fri May 28 11:19:04 2010 @@ -74,6 +74,8 @@ for name in array_fields] self.static_field_by_descrs = dict( [(descr, i) for (i, descr) in enumerate(self.static_field_descrs)]) + self.array_field_by_descrs = dict( + [(descr, i) for (i, descr) in enumerate(self.array_field_descrs)]) # getlength = cpu.ts.getlength getarrayitem = cpu.ts.getarrayitem From arigo at codespeak.net Fri May 28 12:13:33 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 12:13:33 +0200 (CEST) Subject: [pypy-svn] r74839 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter metainterp metainterp/test Message-ID: <20100528101333.86076282BD6@codespeak.net> Author: arigo Date: Fri May 28 12:13:31 2010 New Revision: 74839 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualizable.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/virtualizable.py Log: test_virtualizable passes. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Fri May 28 12:13:31 2010 @@ -873,9 +873,10 @@ if args[0] in self.vable_array_vars: # virtualizable array vars = self.vable_array_vars[args[0]] (v_base, arrayfielddescr, arraydescr) = vars - return SpaceOperation('arraylen_vable', - [v_base, arrayfielddescr, arraydescr], - op.result) + return [SpaceOperation('-live-', [], None), + SpaceOperation('arraylen_vable', + [v_base, arrayfielddescr, arraydescr], + op.result)] return SpaceOperation('arraylen_gc', [args[0], arraydescr], op.result) do_fixed_list_len_foldable = do_fixed_list_len @@ -885,9 +886,10 @@ vars = self.vable_array_vars[args[0]] (v_base, arrayfielddescr, arraydescr) = vars kind = getkind(op.result.concretetype) - return SpaceOperation('getarrayitem_vable_%s' % kind[0], - [v_base, arrayfielddescr, arraydescr, - args[1]], op.result) + return [SpaceOperation('-live-', [], None), + SpaceOperation('getarrayitem_vable_%s' % kind[0], + [v_base, arrayfielddescr, arraydescr, + args[1]], op.result)] v_index, extraop = self._prepare_list_getset(op, arraydescr, args, 'check_neg_index') extra = getkind(op.result.concretetype)[0] @@ -905,9 +907,10 @@ vars = self.vable_array_vars[args[0]] (v_base, arrayfielddescr, arraydescr) = vars kind = getkind(args[2].concretetype) - return SpaceOperation('setarrayitem_vable_%s' % kind[0], - [v_base, arrayfielddescr, arraydescr, - args[1], args[2]], None) + return [SpaceOperation('-live-', [], None), + SpaceOperation('setarrayitem_vable_%s' % kind[0], + [v_base, arrayfielddescr, arraydescr, + args[1], args[2]], None)] v_index, extraop = self._prepare_list_getset(op, arraydescr, args, 'check_neg_index') kind = getkind(args[2].concretetype)[0] Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Fri May 28 12:13:31 2010 @@ -808,7 +808,8 @@ def get_portal_runner(self): metainterp_sd = self.builder.metainterp_sd - fnptr = llmemory.cast_ptr_to_adr(metainterp_sd.portal_runner_ptr) + fnptr = llmemory.cast_ptr_to_adr(metainterp_sd._portal_runner_ptr) + fnptr = llmemory.cast_adr_to_int(fnptr) calldescr = metainterp_sd.portal_code.calldescr return fnptr, calldescr Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py Fri May 28 12:13:31 2010 @@ -349,11 +349,8 @@ def handle_async_forcing(self, force_token): from pypy.jit.metainterp.resume import force_from_resumedata metainterp_sd = self.metainterp_sd - expect_virtualizable = metainterp_sd.virtualizable_info is not None all_virtuals = force_from_resumedata(metainterp_sd, self) - # Handle virtualizable_boxes: store them on the real virtualizable now - if expect_virtualizable: - metainterp_sd.virtualizable_info.forced_vable(virtualizable_boxes) + # The virtualizable data was stored on the real virtualizable above. # Handle all_virtuals: keep them for later blackholing from the # future failure of the GUARD_NOT_FORCED self.save_data(force_token, all_virtuals) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Fri May 28 12:13:31 2010 @@ -1019,9 +1019,9 @@ # xxx do something about code duplication resbox = self.metainterp.execute_and_record_varargs( rop.CALL_MAY_FORCE, allboxes, descr=descr) - self.metainterp.vable_and_vrefs_after_residual_call() if resbox is not None: self.make_result_of_lastop(resbox) + self.metainterp.vable_and_vrefs_after_residual_call() self.generate_guard(rop.GUARD_NOT_FORCED, None) self.metainterp.handle_possible_exception() return resbox @@ -1553,7 +1553,10 @@ # run it. from pypy.jit.metainterp.blackhole import convert_and_run_from_pyjitpl self.aborted_tracing(stb.reason) - convert_and_run_from_pyjitpl(self, stb.current_exc) + current_exc = lltype.nullptr(llmemory.GCREF.TO) + if self.last_exc_value_box is not None: + current_exc = self.last_exc_value_box.getref() + convert_and_run_from_pyjitpl(self, current_exc) assert False # ^^^ must raise def remove_consts_and_duplicates(self, boxes, endindex, duplicates): @@ -1826,7 +1829,8 @@ virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) if vinfo.tracing_after_residual_call(virtualizable): # the virtualizable escaped during CALL_MAY_FORCE. - raise XXX-SwitchToBlackhole(ABORT_ESCAPE, yyy) + self.load_fields_from_virtualizable() + raise SwitchToBlackhole(ABORT_ESCAPE) def stop_tracking_virtualref(self, i): virtualbox = self.virtualref_boxes[i] @@ -1916,8 +1920,8 @@ def load_fields_from_virtualizable(self): # Force a reload of the virtualizable fields into the local - # boxes (called only in escaping cases) - assert self.is_blackholing() + # boxes (called only in escaping cases). Only call this function + # just before SwitchToBlackhole. vinfo = self.staticdata.virtualizable_info if vinfo is not None: virtualizable_box = self.virtualizable_boxes[-1] @@ -2031,9 +2035,8 @@ it to reload the current top-of-stack frame that gets interpreted.""" class SwitchToBlackhole(Exception): - def __init__(self, reason, current_exc=lltype.nullptr(llmemory.GCREF.TO)): + def __init__(self, reason): self.reason = reason - self.current_exc = current_exc # ____________________________________________________________ Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py Fri May 28 12:13:31 2010 @@ -744,13 +744,17 @@ def force_from_resumedata(metainterp_sd, storage): resumereader = ResumeDataDirectReader(metainterp_sd.cpu, storage) + resumereader.handling_async_forcing() vinfo = metainterp_sd.virtualizable_info vrefinfo = metainterp_sd.virtualref_info resumereader.consume_vref_and_vable(vrefinfo, vinfo) return resumereader.virtuals class ResumeDataDirectReader(AbstractResumeDataReader): - resume_after_guard_not_forced = False + resume_after_guard_not_forced = 0 + # 0: not a GUARD_NOT_FORCED + # 1: in handle_async_forcing + # 2: resuming from the GUARD_NOT_FORCED def __init__(self, cpu, storage, all_virtuals=None): self._init(cpu, storage) @@ -759,9 +763,12 @@ else: # special case for resuming after a GUARD_NOT_FORCED: we already # have the virtuals - self.resume_after_guard_not_forced = True + self.resume_after_guard_not_forced = 2 self.virtuals = all_virtuals + def handling_async_forcing(self): + self.resume_after_guard_not_forced = 1 + def consume_one_section(self, blackholeinterp): self.blackholeinterp = blackholeinterp info = blackholeinterp.get_current_position_info() @@ -785,11 +792,16 @@ return len(nums) virtualizable = self.decode_ref(nums[-1]) virtualizable = vinfo.cast_gcref_to_vtype(virtualizable) - # just jumped away from assembler (case 4 in the comment in - # virtualizable.py) into tracing (case 2); check that vable_token - # is and stays 0. Note the call to reset_vable_token() in - # warmstate.py. - assert not virtualizable.vable_token + if self.resume_after_guard_not_forced == 1: + # in the middle of handle_async_forcing() + assert virtualizable.vable_token + virtualizable.vable_token = vinfo.TOKEN_NONE + else: + # just jumped away from assembler (case 4 in the comment in + # virtualizable.py) into tracing (case 2); check that vable_token + # is and stays 0. Note the call to reset_vable_token() in + # warmstate.py. + assert not virtualizable.vable_token return vinfo.write_from_resume_data_partial(virtualizable, self, nums) def load_value_of_type(self, TYPE, tagged): @@ -809,7 +821,7 @@ def consume_vref_and_vable(self, vrefinfo, vinfo): nums = self.cur_numb.nums self.cur_numb = self.cur_numb.prev - if not self.resume_after_guard_not_forced: + if self.resume_after_guard_not_forced != 2: end_vref = self.consume_vable_info(vinfo, nums) self.consume_virtualref_info(vrefinfo, nums, end_vref) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualizable.py Fri May 28 12:13:31 2010 @@ -522,7 +522,7 @@ def test_external_pass(self): - jitdriver = JitDriver(greens = [], reds = ['frame', 'n', 'z'], + jitdriver = JitDriver(greens = [], reds = ['n', 'z', 'frame'], virtualizables = ['frame']) class BaseFrame(object): @@ -822,7 +822,7 @@ assert res == f(123) def test_external_read_sometimes_recursive(self): - jitdriver = JitDriver(greens = [], reds = ['frame', 'rec'], + jitdriver = JitDriver(greens = [], reds = ['rec', 'frame'], virtualizables = ['frame']) class Frame(object): @@ -942,7 +942,7 @@ assert res == f(123) def test_promote_index_in_virtualizable_list(self): - jitdriver = JitDriver(greens = [], reds = ['frame', 'n'], + jitdriver = JitDriver(greens = [], reds = ['n', 'frame'], virtualizables = ['frame']) class Frame(object): _virtualizable2_ = ['stackpos', 'stack[*]'] @@ -1085,7 +1085,7 @@ res = self.meta_interp(f, [10]) assert res == 55 - self.check_loops(new_with_vtable=0, oois=1) + self.check_loops(new_with_vtable=0, ptr_eq=1) def test_virtual_child_frame_with_arrays(self): myjitdriver = JitDriver(greens = [], reds = ['frame'], @@ -1197,7 +1197,7 @@ if not self.basic: py.test.skip("purely frontend test") - myjitdriver = JitDriver(greens = [], reds = ['frame', 'fail'], + myjitdriver = JitDriver(greens = [], reds = ['fail', 'frame'], virtualizables = ['frame']) class Frame(object): @@ -1252,7 +1252,7 @@ self.x = x self.next = None - driver = JitDriver(greens=[], reds=['frame', 'result'], + driver = JitDriver(greens=[], reds=['result', 'frame'], virtualizables=['frame']) def interp(caller): @@ -1285,15 +1285,15 @@ self.n = n self.next = None - driver = JitDriver(greens=[], reds=['frame', 'result'], + driver = JitDriver(greens=[], reds=['result', 'frame'], virtualizables=['frame']) - def p(code, pc): + def p(pc, code): code = hlstr(code) return "%s %d %s" % (code, pc, code[pc]) - def c(code, pc): + def c(pc, code): return "l" not in hlstr(code) - myjitdriver = JitDriver(greens=['code', 'pc'], reds=['frame'], + myjitdriver = JitDriver(greens=['pc', 'code'], reds=['frame'], virtualizables=["frame"], get_printable_location=p, can_inline=c) def f(code, frame): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/virtualizable.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/virtualizable.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/virtualizable.py Fri May 28 12:13:31 2010 @@ -271,12 +271,6 @@ assert virtualizable.vable_token == self.TOKEN_NONE force_now._dont_inline_ = True - def forced_vable(self, virtualizable_boxes): - virtualizable_box = virtualizable_boxes[-1] - virtualizable = self.unwrap_virtualizable_box(virtualizable_box) - self.write_boxes(virtualizable, virtualizable_boxes) - virtualizable.vable_token = self.TOKEN_NONE - # ____________________________________________________________ # # The 'vable_token' field of a virtualizable is either 0, -1, or points From arigo at codespeak.net Fri May 28 12:38:42 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 12:38:42 +0200 (CEST) Subject: [pypy-svn] r74840 - in pypy/branch/blackhole-improvement/pypy/jit/metainterp: . test Message-ID: <20100528103842.DA2F2282BD6@codespeak.net> Author: arigo Date: Fri May 28 12:38:41 2010 New Revision: 74840 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_recursive.py Log: Fix CALL_ASSEMBLER generation. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py Fri May 28 12:38:41 2010 @@ -816,16 +816,6 @@ op = ResOperation(opnum, argboxes, resbox, descr) self.operations[position] = op - def slice_history_at(self, position): - """ a strange function that does this: - history : operation_at_position : rest - it'll kill operation_at_position, store everything before that - in history.operations and return rest - """ - rest = self.operations[position + 1:] - del self.operations[position:] - return rest - # ____________________________________________________________ Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Fri May 28 12:38:41 2010 @@ -40,8 +40,6 @@ class MIFrame(object): - env = property(lambda: xxx, lambda: xxx) # temporary: no read/write! - def __init__(self, metainterp): self.metainterp = metainterp self.registers_i = [None] * 256 @@ -659,36 +657,21 @@ allboxes = greenboxes + redboxes metainterp_sd = self.metainterp.staticdata portal_code = metainterp_sd.portal_code -## warmrunnerstate = metainterp_sd.state -## token = None -## if not self.is_blackholing() and warmrunnerstate.inlining: -## if warmrunnerstate.can_inline_callable(greenboxes): -## return self.metainterp.perform_call(portal_code, allboxes, -## greenkey=greenboxes) -## token = warmrunnerstate.get_assembler_token(greenboxes) -## call_position = 0 -## if token is not None: -## call_position = len(self.history.operations) -## # verify that we have all green args, needed to make sure -## # that assembler that we call is still correct -## self.verify_green_args(greenboxes) + warmrunnerstate = metainterp_sd.state + token = None + if warmrunnerstate.inlining: + if warmrunnerstate.can_inline_callable(greenboxes): + return self.metainterp.perform_call(portal_code, allboxes, + greenkey=greenboxes) + token = warmrunnerstate.get_assembler_token(greenboxes) + # verify that we have all green args, needed to make sure + # that assembler that we call is still correct + self.verify_green_args(greenboxes) # k = llmemory.cast_ptr_to_adr(metainterp_sd._portal_runner_ptr) funcbox = ConstInt(llmemory.cast_adr_to_int(k)) - resbox = self.do_residual_call(funcbox, portal_code.calldescr, - allboxes) - # -## if token is not None: -## # XXX fix the call position, -## while True: -## op = self.history.operations[call_position] -## if op.opnum == rop.CALL or op.opnum == rop.CALL_MAY_FORCE: -## break -## call_position += 1 -## # -## # this will substitute the residual call with assembler call -## self.direct_assembler_call(boxes, token, call_position) - return resbox + return self.do_residual_call(funcbox, portal_code.calldescr, + allboxes, assembler_call_token=token) opimpl_recursive_call_i = _opimpl_recursive_call opimpl_recursive_call_r = _opimpl_recursive_call @@ -1009,16 +992,20 @@ self.metainterp.assert_no_exception() return resbox - def do_residual_call(self, funcbox, descr, argboxes): + def do_residual_call(self, funcbox, descr, argboxes, + assembler_call_token=None): allboxes = [funcbox] + argboxes effectinfo = descr.get_extra_info() - if (effectinfo is None or effectinfo.extraeffect == - effectinfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE): + if (effectinfo is None or + effectinfo.extraeffect == + effectinfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE or + assembler_call_token is not None): # residual calls require attention to keep virtualizables in-sync self.metainterp.vable_and_vrefs_before_residual_call() - # xxx do something about code duplication resbox = self.metainterp.execute_and_record_varargs( rop.CALL_MAY_FORCE, allboxes, descr=descr) + if assembler_call_token is not None: + self.metainterp.direct_assembler_call(assembler_call_token) if resbox is not None: self.make_result_of_lastop(resbox) self.metainterp.vable_and_vrefs_after_residual_call() @@ -2007,20 +1994,22 @@ max_key = key return max_key - def direct_assembler_call(self, boxes, token, call_position): - """ Generate a direct call to assembler for portal entry point. + def direct_assembler_call(self, token): + """ Generate a direct call to assembler for portal entry point, + patching the CALL_MAY_FORCE that occurred just now. """ - assert not self.is_blackholing() # XXX + op = self.history.operations.pop() + assert op.opnum == rop.CALL_MAY_FORCE num_green_args = self.staticdata.num_green_args - args = boxes[num_green_args:] - resbox = self.history.operations[call_position].result - rest = self.history.slice_history_at(call_position) + args = op.args[num_green_args + 1:] if self.staticdata.virtualizable_info is not None: vindex = self.staticdata.virtualizable_info.index_of_virtualizable - vbox = boxes[vindex] + vbox = args[vindex - num_green_args] args += self.gen_load_from_other_virtualizable(vbox) - self.history.record(rop.CALL_ASSEMBLER, args[:], resbox, descr=token) - self.history.operations += rest + op.opnum = rop.CALL_ASSEMBLER + op.args = args + op.descr = token + self.history.operations.append(op) # ____________________________________________________________ Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_recursive.py Fri May 28 12:38:41 2010 @@ -137,7 +137,6 @@ return interpret - @py.test.mark.xfail() def test_inline(self): code = "021" subcode = "00" @@ -163,7 +162,6 @@ inline=True) == 42 self.check_loops(call_may_force = 1, call = 0) - @py.test.mark.xfail() def test_inline_faulty_can_inline(self): code = "021" subcode = "301" @@ -471,7 +469,6 @@ assert not res self.check_aborted_count(5) - @py.test.mark.xfail() def test_set_param_inlining(self): myjitdriver = JitDriver(greens=[], reds=['n', 'recurse']) def loop(n, recurse=False): @@ -538,7 +535,6 @@ self.check_tree_loop_count(3) self.check_history(int_add=1) - @py.test.mark.xfail() def test_dont_inline_huge_stuff(self): def p(pc, code): code = hlstr(code) @@ -583,7 +579,6 @@ self.check_history(call_assembler=1, call=0) self.check_tree_loop_count(3) - @py.test.mark.xfail() def test_directly_call_assembler(self): driver = JitDriver(greens = ['codeno'], reds = ['i'], get_printable_location = lambda codeno : str(codeno), @@ -621,7 +616,6 @@ self.check_history(call_assembler=0, call_may_force=1) self.check_enter_count_at_most(1) - @py.test.mark.xfail() def test_directly_call_assembler_return(self): driver = JitDriver(greens = ['codeno'], reds = ['i', 'k'], get_printable_location = lambda codeno : str(codeno), @@ -641,7 +635,6 @@ self.meta_interp(portal, [2], inline=True) self.check_history(call_assembler=1) - @py.test.mark.xfail() def test_directly_call_assembler_raise(self): class MyException(Exception): @@ -867,7 +860,6 @@ policy=StopAtXPolicy(change)) assert res == main(0) - @py.test.mark.xfail() def test_assembler_call_red_args(self): driver = JitDriver(greens = ['codeno'], reds = ['i', 'k'], get_printable_location = lambda codeno : str(codeno), From arigo at codespeak.net Fri May 28 12:50:59 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 12:50:59 +0200 (CEST) Subject: [pypy-svn] r74841 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100528105059.788D1282BD6@codespeak.net> Author: arigo Date: Fri May 28 12:50:58 2010 New Revision: 74841 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py Log: Finish fixing test_recursion.py. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Fri May 28 12:50:58 2010 @@ -794,17 +794,33 @@ pass @arguments("self", "I", "R", "F", "I", "R", "F") - def bhimpl_jit_merge_point(self, *results): + def bhimpl_jit_merge_point(self, *args): if self.nextblackholeinterp is None: # we are the last level CRN = self.builder.metainterp_sd.ContinueRunningNormally - raise CRN(*results) + raise CRN(*args) + # Note that the case above is an optimization: the case + # below would work too. But it keeps unnecessary stuff on + # the stack; the solution above first gets rid of the blackhole + # interpreter completely. else: # This occurs when we reach 'jit_merge_point' in the portal # function called by recursion. In this case, we can directly # call the interpreter main loop from here, and just return its # result. - XXX - raise LeaveFrame + sd = self.builder.metainterp_sd + if sd.result_type == 'void': + self.bhimpl_recursive_call_v(*args) + self.bhimpl_void_return() + elif sd.result_type == 'int': + x = self.bhimpl_recursive_call_i(*args) + self.bhimpl_int_return(x) + elif sd.result_type == 'ref': + x = self.bhimpl_recursive_call_r(*args) + self.bhimpl_ref_return(x) + elif sd.result_type == 'float': + x = self.bhimpl_recursive_call_f(*args) + self.bhimpl_float_return(x) + assert False def get_portal_runner(self): metainterp_sd = self.builder.metainterp_sd @@ -1295,7 +1311,6 @@ metainterp_sd.blackholeinterpbuilder, resumedescr, all_virtuals) - # XXX virtualizable current_exc = blackholeinterp._prepare_resume_from_failure( resumedescr.guard_opnum) try: Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py Fri May 28 12:50:58 2010 @@ -572,11 +572,11 @@ loop_token = fail_descr.handle_fail(self.metainterp_sd) fail_descr = self.cpu.execute_token(loop_token) except self.ContinueRunningNormally, e: - xxxxxxxxx args = () - for _, name, _ in portalfunc_ARGS: - v = getattr(e, name) - args = args + (v,) + for ARGTYPE, attrname, count in portalfunc_ARGS: + x = getattr(e, attrname)[count] + x = specialize_value(ARGTYPE, x) + args = args + (x,) return ll_portal_runner(*args) except self.DoneWithThisFrameVoid: assert result_kind == 'void' From arigo at codespeak.net Fri May 28 12:53:39 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 12:53:39 +0200 (CEST) Subject: [pypy-svn] r74842 - pypy/branch/blackhole-improvement/pypy/jit/metainterp/test Message-ID: <20100528105339.67A6C282BD6@codespeak.net> Author: arigo Date: Fri May 28 12:53:38 2010 New Revision: 74842 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_history.py Log: Remove the test for this removed hack. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_history.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_history.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_history.py Fri May 28 12:53:38 2010 @@ -9,10 +9,3 @@ s = lltype.cast_pointer(lltype.Ptr(S), t) const = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, s)) assert const._getrepr_() == "*T" - -def test_slicing(): - h = History() - h.operations = [1, 2, 3, 4, 5] - rest = h.slice_history_at(2) - assert rest == [4, 5] - assert h.operations == [1, 2] From arigo at codespeak.net Fri May 28 13:03:35 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 13:03:35 +0200 (CEST) Subject: [pypy-svn] r74843 - pypy/branch/blackhole-improvement/pypy/jit/metainterp/test Message-ID: <20100528110335.075DC282BD6@codespeak.net> Author: arigo Date: Fri May 28 13:03:34 2010 New Revision: 74843 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_resume.py Log: Fix tests. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_optimizeopt.py Fri May 28 13:03:34 2010 @@ -2561,7 +2561,7 @@ setfield_gc(p2, 5, descr=virtualrefindexdescr) escape(p2) setfield_gc(p2, p1, descr=virtualforceddescr) - setfield_gc(p2, -2, descr=virtualtokendescr) + setfield_gc(p2, -3, descr=virtualtokendescr) jump(p1) """ # XXX we should optimize a bit more the case of a nonvirtual. @@ -2603,7 +2603,7 @@ setfield_gc(p1b, 252, descr=valuedescr) setfield_gc(p1, p1b, descr=nextdescr) setfield_gc(p2, p1, descr=virtualforceddescr) - setfield_gc(p2, -2, descr=virtualtokendescr) + setfield_gc(p2, -3, descr=virtualtokendescr) jump(p0, i1) """ self.optimize_loop(ops, 'Not, Not', expected) @@ -2644,7 +2644,7 @@ setfield_gc(p1b, i1, descr=valuedescr) setfield_gc(p1, p1b, descr=nextdescr) setfield_gc(p2, p1, descr=virtualforceddescr) - setfield_gc(p2, -2, descr=virtualtokendescr) + setfield_gc(p2, -3, descr=virtualtokendescr) jump(p0, i1) """ # the point of this test is that 'i1' should show up in the fail_args @@ -2715,7 +2715,7 @@ escape(p2) p1 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p2, p1, descr=virtualforceddescr) - setfield_gc(p2, -2, descr=virtualtokendescr) + setfield_gc(p2, -3, descr=virtualtokendescr) call_may_force(i1, descr=mayforcevirtdescr) guard_not_forced() [] jump(i1) @@ -2741,7 +2741,7 @@ setfield_gc(p2, 23, descr=virtualrefindexdescr) escape(p2) setfield_gc(p2, p1, descr=virtualforceddescr) - setfield_gc(p2, -2, descr=virtualtokendescr) + setfield_gc(p2, -3, descr=virtualtokendescr) call_may_force(i1, descr=mayforcevirtdescr) guard_not_forced() [i1] jump(i1, p1) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_resume.py Fri May 28 13:03:34 2010 @@ -333,12 +333,7 @@ assert frame_info_list.pc == 15 snapshot = storage.rd_snapshot - assert snapshot.boxes == vbs - assert snapshot.boxes is not vbs - - snapshot = snapshot.prev - assert snapshot.boxes == vrs - assert snapshot.boxes is not vrs + assert snapshot.boxes == vrs + vbs # in the same list snapshot = snapshot.prev assert snapshot.prev is fs[2].parent_resumedata_snapshot @@ -829,9 +824,10 @@ class ResumeDataFakeReader(ResumeDataBoxReader): """Another subclass of AbstractResumeDataReader meant for tests.""" def __init__(self, storage, newboxes, metainterp): + self._init(metainterp.cpu, storage) self.liveboxes = newboxes self.metainterp = metainterp - self._prepare(metainterp.cpu, storage) + self._prepare(storage) def consume_boxes(self): self.lst = [] From arigo at codespeak.net Fri May 28 13:13:11 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 13:13:11 +0200 (CEST) Subject: [pypy-svn] r74844 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter codewriter/test metainterp/test Message-ID: <20100528111311.8A1E6282BD6@codespeak.net> Author: arigo Date: Fri May 28 13:13:09 2010 New Revision: 74844 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualref.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_warmspot.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_warmstate.py Log: Fix tests. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/call.py Fri May 28 13:13:09 2010 @@ -25,7 +25,7 @@ self.jitcodes = {} # map {graph: jitcode} self.unfinished_graphs = [] # list of graphs with pending jitcodes self.jitdriver = None - if cpu is not None: + if hasattr(cpu, 'rtyper'): # for tests self.rtyper = cpu.rtyper translator = self.rtyper.annotator.translator self.raise_analyzer = RaiseAnalyzer(translator) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_list.py Fri May 28 13:13:09 2010 @@ -45,6 +45,7 @@ v_result = varoftype(RESTYPE) tr = Transformer(FakeCPU(), FakeCallControl()) tr.immutable_arrays = {} + tr.vable_array_vars = {} if '/' in oopspec_name: oopspec_name, property = oopspec_name.split('/') def force_flags(op): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualref.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualref.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualref.py Fri May 28 13:13:09 2010 @@ -93,7 +93,7 @@ lst = [] vrefinfo.continue_tracing = lambda vref, virtual: \ lst.append((vref, virtual)) - resumereader.consume_virtualref_info(vrefinfo) + resumereader.consume_vref_and_vable(vrefinfo, None) del vrefinfo.continue_tracing assert len(lst) == 1 lltype.cast_opaque_ptr(lltype.Ptr(JIT_VIRTUAL_REF), Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_warmspot.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_warmspot.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_warmspot.py Fri May 28 13:13:09 2010 @@ -282,7 +282,7 @@ type_system = 'lltype' class TestOOWarmspot(WarmspotTests, OOJitMixin): - CPUClass = runner.OOtypeCPU + ##CPUClass = runner.OOtypeCPU type_system = 'ootype' class TestWarmspotDirect(object): @@ -304,7 +304,7 @@ raise metainterp_sd.warmrunnerdesc.DoneWithThisFrameInt(3) if self.no == 1: raise metainterp_sd.warmrunnerdesc.ContinueRunningNormally( - [BoxInt(0), BoxInt(1)]) + [0], [], [], [1], [], []) if self.no == 3: exc = lltype.malloc(OBJECT) exc.typeptr = exc_vtable @@ -317,6 +317,7 @@ supports_floats = False ts = llhelper translate_support_code = False + stats = "stats" def get_fail_descr_number(self, d): return -1 @@ -325,7 +326,7 @@ pass def nodescr(self, *args, **kwds): - pass + return "this is a descr" fielddescrof = nodescr calldescrof = nodescr sizeof = nodescr @@ -348,6 +349,7 @@ return red rtyper = annotate(f, [0]) + FakeCPU.rtyper = rtyper translator = rtyper.annotator.translator translator.config.translation.gc = 'hybrid' cls.desc = WarmRunnerDesc(translator, CPUClass=FakeCPU) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_warmstate.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_warmstate.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_warmstate.py Fri May 28 13:13:09 2010 @@ -128,8 +128,7 @@ class FakeWarmRunnerDesc: cpu = FakeCPU() red_args_types = ["int", "float"] - class metainterp_sd: - virtualizable_info = None + virtualizable_info = None # state = WarmEnterState(FakeWarmRunnerDesc()) set_future_values = state.make_set_future_values() From arigo at codespeak.net Fri May 28 13:52:19 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 13:52:19 +0200 (CEST) Subject: [pypy-svn] r74845 - in pypy/branch/blackhole-improvement/pypy/jit: backend/llgraph codewriter metainterp metainterp/test Message-ID: <20100528115219.28F04282BD6@codespeak.net> Author: arigo Date: Fri May 28 13:52:17 2010 New Revision: 74845 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_ztranslation.py Log: Translation fixes (step 1). Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py Fri May 28 13:52:17 2010 @@ -1496,8 +1496,10 @@ setannotation(frame_int_getvalue, annmodel.SomeInteger()) setannotation(frame_ptr_getvalue, annmodel.SomePtr(llmemory.GCREF)) setannotation(frame_float_getvalue, annmodel.SomeFloat()) +setannotation(frame_get_value_count, annmodel.SomeInteger()) +setannotation(frame_clear_latest_values, annmodel.s_None) -setannotation(grab_exc_value, annmodel.SomePtr(llmemory.GCREF)) +setannotation(grab_exc_value, annmodel.SomePtr(rclass.OBJECTPTR)) setannotation(force, annmodel.SomeInteger()) setannotation(get_forced_token_frame, s_Frame) setannotation(get_frame_forced_token, annmodel.SomeAddress()) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py Fri May 28 13:52:17 2010 @@ -46,9 +46,10 @@ assert sizedescr._corresponding_vtable == vtable else: assert lltype.typeOf(vtable) == VTABLETYPE - if not hasattr(cpu, '_all_size_descrs'): - cpu._all_size_descrs = [] - cpu._all_size_descrs.append(sizedescr) + if not hasattr(cpu, '_all_size_descrs_with_vtable'): + cpu._all_size_descrs_with_vtable = [] + cpu._vtable_to_descr_dict = None + cpu._all_size_descrs_with_vtable.append(sizedescr) sizedescr._corresponding_vtable = vtable def vtable2descr(cpu, vtable): @@ -59,9 +60,14 @@ # Build the dict {vtable: sizedescr} at runtime. # This is necessary because the 'vtables' are just pointers to # static data, so they can't be used as keys in prebuilt dicts. - XXX + d = cpu._vtable_to_descr_dict + if d is None: + d = cpu._vtable_to_descr_dict = {} + for descr in cpu._all_size_descrs_with_vtable: + d[descr._corresponding_vtable] = descr + return d[vtable] else: - for descr in cpu._all_size_descrs: + for descr in cpu._all_size_descrs_with_vtable: if descr._corresponding_vtable == vtable: return descr raise KeyError(vtable) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Fri May 28 13:52:17 2010 @@ -6,6 +6,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory, rclass from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.llinterp import LLException +from pypy.rpython.annlowlevel import cast_instance_to_base_ptr from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr from pypy.jit.codewriter import heaptracker @@ -40,7 +41,7 @@ def get_llexception(cpu, e): if we_are_translated(): - return XXX(e) + return cast_instance_to_base_ptr(e) if isinstance(e, LLException): return e.args[1] # ok if isinstance(e, OverflowError): @@ -376,7 +377,6 @@ etype = rclass.ll_type(e) raise LLException(etype, e) - # XXX must be specialized def copy_constants(self, registers, constants): """Copy jitcode.constants[0] to registers[255], jitcode.constants[1] to registers[254], @@ -389,6 +389,7 @@ assert j >= 0 registers[j] = constants[i] i -= 1 + copy_constants._annspecialcase_ = 'specialize:arglistitemtype(1)' # ---------- @@ -1172,6 +1173,7 @@ # helpers to resume running in blackhole mode when a guard failed def _resume_mainloop(self, current_exc): + assert lltype.typeOf(current_exc) == rclass.OBJECTPTR try: # if there is a current exception, raise it now # (it may be caught by a catch_operation in this frame) @@ -1201,7 +1203,7 @@ caller._setup_return_value_f(self.final_result_f()) else: assert kind == 'v' - return NULL + return lltype.nullptr(rclass.OBJECTPTR.TO) def _prepare_resume_from_failure(self, opnum): from pypy.jit.metainterp.resoperation import rop @@ -1248,7 +1250,7 @@ else: from pypy.jit.metainterp.resoperation import opname raise NotImplementedError(opname[opnum]) - return NULL + return lltype.nullptr(rclass.OBJECTPTR.TO) # connect the return of values from the called frame to the # 'xxx_call_yyy' instructions from the caller frame @@ -1298,7 +1300,6 @@ # ____________________________________________________________ def _run_forever(blackholeinterp, current_exc): - current_exc = lltype.cast_opaque_ptr(rclass.OBJECTPTR, current_exc) while True: current_exc = blackholeinterp._resume_mainloop(current_exc) blackholeinterp = blackholeinterp.nextblackholeinterp @@ -1319,7 +1320,7 @@ metainterp_sd.profiler.end_blackhole() debug_stop('jit-blackhole') -def convert_and_run_from_pyjitpl(metainterp, current_exc=NULL): +def convert_and_run_from_pyjitpl(metainterp): # Get a chain of blackhole interpreters and fill them by copying # 'metainterp.framestack'. Note that the order is important: the # first one we get must be the bottom one, in order to make @@ -1335,6 +1336,11 @@ nextbh = curbh firstbh = nextbh # + if metainterp.last_exc_value_box is not None: + current_exc = metainterp.last_exc_value_box.getref(rclass.OBJECTPTR) + else: + current_exc = lltype.nullptr(rclass.OBJECTPTR.TO) + # try: _run_forever(firstbh, current_exc) finally: Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py Fri May 28 13:52:17 2010 @@ -349,14 +349,14 @@ func = get_execute_function(opnum, len(argboxes)) return func(cpu, metainterp, *argboxes) # note that the 'argboxes' tuple # optionally ends with the descr -execute._annspecialcase_ = 'specialize:arg(1)' +execute._annspecialcase_ = 'specialize:arg(2)' def execute_varargs(cpu, metainterp, opnum, argboxes, descr): # only for opnums with a variable arity (calls, typically) check_descr(descr) func = get_execute_function(opnum, -1) return func(cpu, metainterp, argboxes, descr) -execute_varargs._annspecialcase_ = 'specialize:arg(1)' +execute_varargs._annspecialcase_ = 'specialize:arg(2)' def execute_nonspec(cpu, metainterp, opnum, argboxes, descr=None): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Fri May 28 13:52:17 2010 @@ -1,6 +1,5 @@ import py, os from pypy.rpython.lltypesystem import lltype, llmemory -from pypy.rpython.llinterp import LLException from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.debug import debug_start, debug_stop, debug_print @@ -29,12 +28,6 @@ return func return decorate -class FixME: # XXX marks deprecated ootype-only operations - def __init__(self, func): - self.func = func - def __getattr__(self, _): - raise Exception("@FixME: " + self.func.__name__) - # ____________________________________________________________ @@ -94,6 +87,7 @@ if argcode == 'I': reg = self.registers_i[index] elif argcode == 'R': reg = self.registers_r[index] elif argcode == 'F': reg = self.registers_f[index] + else: raise AssertionError(argcode) outvalue[startindex+i] = reg prepare_list_of_boxes._annspecialcase_ = 'specialize:arg(4)' @@ -336,20 +330,20 @@ cls = heaptracker.descr2vtable(cpu, sizedescr) return self.execute(rop.NEW_WITH_VTABLE, ConstInt(cls)) - @FixME #arguments("box") - def opimpl_runtimenew(self, classbox): - self.execute(rop.RUNTIMENEW, classbox) - - @FixME #arguments("orgpc", "box", "descr") - def opimpl_instanceof(self, pc, objbox, typedescr): - clsbox = self.cls_of_box(objbox) - if isinstance(objbox, Box): - self.generate_guard(pc, rop.GUARD_CLASS, objbox, [clsbox]) - self.execute_with_descr(rop.INSTANCEOF, typedescr, objbox) - - @FixME #arguments("box", "box") - def opimpl_subclassof(self, box1, box2): - self.execute(rop.SUBCLASSOF, box1, box2) +## @FixME #arguments("box") +## def opimpl_runtimenew(self, classbox): +## self.execute(rop.RUNTIMENEW, classbox) + +## @FixME #arguments("orgpc", "box", "descr") +## def opimpl_instanceof(self, pc, objbox, typedescr): +## clsbox = self.cls_of_box(objbox) +## if isinstance(objbox, Box): +## self.generate_guard(pc, rop.GUARD_CLASS, objbox, [clsbox]) +## self.execute_with_descr(rop.INSTANCEOF, typedescr, objbox) + +## @FixME #arguments("box", "box") +## def opimpl_subclassof(self, box1, box2): +## self.execute(rop.SUBCLASSOF, box1, box2) @arguments("descr", "box") def opimpl_new_array(self, itemsizedescr, countbox): @@ -678,21 +672,21 @@ opimpl_recursive_call_f = _opimpl_recursive_call opimpl_recursive_call_v = _opimpl_recursive_call - @FixME #arguments("orgpc", "methdescr", "varargs") - def opimpl_oosend(self, pc, methdescr, varargs): - objbox = varargs[0] - clsbox = self.cls_of_box(objbox) - if isinstance(objbox, Box): - self.generate_guard(pc, rop.GUARD_CLASS, objbox, [clsbox]) - oocls = clsbox.getref(ootype.Class) - jitcode = methdescr.get_jitcode_for_class(oocls) - if jitcode is not None: - # we should follow calls to this graph - return self.perform_call(jitcode, varargs) - else: - # but we should not follow calls to that graph - return self.execute_varargs(rop.OOSEND, varargs, - descr=methdescr, exc=True) +## @FixME #arguments("orgpc", "methdescr", "varargs") +## def opimpl_oosend(self, pc, methdescr, varargs): +## objbox = varargs[0] +## clsbox = self.cls_of_box(objbox) +## if isinstance(objbox, Box): +## self.generate_guard(pc, rop.GUARD_CLASS, objbox, [clsbox]) +## oocls = clsbox.getref(ootype.Class) +## jitcode = methdescr.get_jitcode_for_class(oocls) +## if jitcode is not None: +## # we should follow calls to this graph +## return self.perform_call(jitcode, varargs) +## else: +## # but we should not follow calls to that graph +## return self.execute_varargs(rop.OOSEND, varargs, +## descr=methdescr, exc=True) @arguments("box") def opimpl_strlen(self, strbox): @@ -726,20 +720,20 @@ def opimpl_newunicode(self, lengthbox): return self.execute(rop.NEWUNICODE, lengthbox) - @FixME #arguments("descr", "varargs") - def opimpl_residual_oosend_canraise(self, methdescr, varargs): - return self.execute_varargs(rop.OOSEND, varargs, descr=methdescr, - exc=True) - - @FixME #arguments("descr", "varargs") - def opimpl_residual_oosend_noraise(self, methdescr, varargs): - return self.execute_varargs(rop.OOSEND, varargs, descr=methdescr, - exc=False) - - @FixME #arguments("descr", "varargs") - def opimpl_residual_oosend_pure(self, methdescr, boxes): - return self.execute_varargs(rop.OOSEND_PURE, boxes, descr=methdescr, - exc=False) +## @FixME #arguments("descr", "varargs") +## def opimpl_residual_oosend_canraise(self, methdescr, varargs): +## return self.execute_varargs(rop.OOSEND, varargs, descr=methdescr, +## exc=True) + +## @FixME #arguments("descr", "varargs") +## def opimpl_residual_oosend_noraise(self, methdescr, varargs): +## return self.execute_varargs(rop.OOSEND, varargs, descr=methdescr, +## exc=False) + +## @FixME #arguments("descr", "varargs") +## def opimpl_residual_oosend_pure(self, methdescr, boxes): +## return self.execute_varargs(rop.OOSEND_PURE, boxes, descr=methdescr, +## exc=False) @arguments("orgpc", "box",) def _opimpl_guard_value(self, orgpc, box): @@ -1420,12 +1414,15 @@ op.name = self.framestack[-1].jitcode.name def execute_raised(self, exception, constant=False): + llexception = get_llexception(self.cpu, exception) + self.execute_ll_raised(llexception, constant) + + def execute_ll_raised(self, llexception, constant=False): # Exception handling: when execute.do_call() gets an exception it # calls metainterp.execute_raised(), which puts it into # 'self.last_exc_value_box'. This is used shortly afterwards # to generate either GUARD_EXCEPTION or GUARD_NO_EXCEPTION, and also # to handle the following opcodes 'goto_if_exception_mismatch'. - llexception = get_llexception(self.cpu, exception) llexception = self.cpu.ts.cast_to_ref(llexception) exc_value_box = self.cpu.ts.get_exc_value_box(llexception) if constant: @@ -1540,10 +1537,7 @@ # run it. from pypy.jit.metainterp.blackhole import convert_and_run_from_pyjitpl self.aborted_tracing(stb.reason) - current_exc = lltype.nullptr(llmemory.GCREF.TO) - if self.last_exc_value_box is not None: - current_exc = self.last_exc_value_box.getref() - convert_and_run_from_pyjitpl(self, current_exc) + convert_and_run_from_pyjitpl(self) assert False # ^^^ must raise def remove_consts_and_duplicates(self, boxes, endindex, duplicates): @@ -1632,11 +1626,7 @@ elif opnum == rop.GUARD_NO_EXCEPTION or opnum == rop.GUARD_EXCEPTION: exception = self.cpu.grab_exc_value() if exception: - if not we_are_translated(): - from pypy.rpython.lltypesystem import rclass - etype = rclass.ll_type(exception) - exception = LLException(etype, exception) - self.execute_raised(exception) + self.execute_ll_raised(exception) else: self.execute_did_not_raise() try: Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py Fri May 28 13:52:17 2010 @@ -4,7 +4,7 @@ from pypy.jit.metainterp.history import INT, REF, FLOAT, HOLE from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp import jitprof -from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rlib import rarithmetic from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.rlib.debug import have_debug_prints @@ -497,6 +497,7 @@ """ _mixin_ = True virtuals = None + virtual_default = None def _init(self, cpu, storage): self.cpu = cpu @@ -509,7 +510,7 @@ def _prepare_virtuals(self, virtuals): if virtuals: - self.virtuals = [None] * len(virtuals) + self.virtuals = [self.virtual_default] * len(virtuals) for i in range(len(virtuals)): vinfo = virtuals[i] if vinfo is not None: @@ -751,6 +752,7 @@ return resumereader.virtuals class ResumeDataDirectReader(AbstractResumeDataReader): + virtual_default = lltype.nullptr(llmemory.GCREF.TO) resume_after_guard_not_forced = 0 # 0: not a GUARD_NOT_FORCED # 1: in handle_async_forcing Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_ztranslation.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_ztranslation.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_ztranslation.py Fri May 28 13:52:17 2010 @@ -5,7 +5,6 @@ from pypy.rlib.jit import PARAMETERS, dont_look_inside from pypy.jit.metainterp.jitprof import Profiler from pypy.rpython.lltypesystem import lltype, llmemory -from pypy.rpython.ootypesystem import ootype class TranslationTest: @@ -40,7 +39,7 @@ def can_inline(): return False - jitdriver = JitDriver(greens = [], reds = ['frame', 'total'], + jitdriver = JitDriver(greens = [], reds = ['total', 'frame'], virtualizables = ['frame'], get_jitcell_at=get_jitcell_at, set_jitcell_at=set_jitcell_at, From arigo at codespeak.net Fri May 28 14:17:31 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 14:17:31 +0200 (CEST) Subject: [pypy-svn] r74846 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter metainterp Message-ID: <20100528121731.258CF282BD6@codespeak.net> Author: arigo Date: Fri May 28 14:17:29 2010 New Revision: 74846 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py Log: Translation fixes (step 2). Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py Fri May 28 14:17:29 2010 @@ -1,4 +1,4 @@ -from pypy.jit.codewriter import support +from pypy.jit.codewriter import support, heaptracker from pypy.jit.codewriter.regalloc import perform_register_allocation from pypy.jit.codewriter.flatten import flatten_graph, KINDS from pypy.jit.codewriter.assembler import Assembler, JitCode @@ -72,6 +72,7 @@ count += 1 if not count % 500: log.info("Produced %d jitcodes" % count) + heaptracker.finish_registering(self.cpu) log.info("there are %d JitCode instances." % count) def setup_vrefinfo(self, vrefinfo): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py Fri May 28 14:17:29 2010 @@ -52,10 +52,15 @@ cpu._all_size_descrs_with_vtable.append(sizedescr) sizedescr._corresponding_vtable = vtable +def finish_registering(cpu): + # annotation hack for small examples which have no vtable at all + if not hasattr(cpu, '_all_size_descrs_with_vtable'): + STRUCT = lltype.GcStruct('empty') + vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) + register_known_gctype(cpu, vtable, STRUCT) + def vtable2descr(cpu, vtable): assert lltype.typeOf(vtable) is lltype.Signed - vtable = llmemory.cast_int_to_adr(vtable) - vtable = llmemory.cast_adr_to_ptr(vtable, VTABLETYPE) if we_are_translated(): # Build the dict {vtable: sizedescr} at runtime. # This is necessary because the 'vtables' are just pointers to @@ -64,9 +69,14 @@ if d is None: d = cpu._vtable_to_descr_dict = {} for descr in cpu._all_size_descrs_with_vtable: - d[descr._corresponding_vtable] = descr + key = descr._corresponding_vtable + key = llmemory.cast_ptr_to_adr(key) + key = llmemory.cast_adr_to_int(key) + d[key] = descr return d[vtable] else: + vtable = llmemory.cast_int_to_adr(vtable) + vtable = llmemory.cast_adr_to_ptr(vtable, VTABLETYPE) for descr in cpu._all_size_descrs_with_vtable: if descr._corresponding_vtable == vtable: return descr Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py Fri May 28 14:17:29 2010 @@ -123,7 +123,7 @@ def get_register_index_f(self, index): return ord(self.live_f[index]) - def enumerate_vars(self, callback_i, callback_r, callback_f): + def enumerate_vars(self, callback_i, callback_r, callback_f, spec): index = 0 for i in range(self.get_register_count_i()): callback_i(index, self.get_register_index_i(i)) @@ -134,6 +134,7 @@ for i in range(self.get_register_count_f()): callback_f(index, self.get_register_index_f(i)) index += 1 + enumerate_vars._annspecialcase_ = 'specialize:arg(4)' _liveness_cache = {} Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py Fri May 28 14:17:29 2010 @@ -16,6 +16,7 @@ # ____________________________________________________________ def do_call(cpu, metainterp, argboxes, descr): + assert metainterp is not None # count the number of arguments of the different types count_i = count_r = count_f = 0 for i in range(1, len(argboxes)): @@ -161,6 +162,7 @@ [x1box.getref_base(), x2box.getref_base()], None) def do_int_add_ovf(cpu, metainterp, box1, box2): + assert metainterp is not None a = box1.getint() b = box2.getint() try: @@ -173,6 +175,7 @@ return BoxInt(z) def do_int_sub_ovf(cpu, metainterp, box1, box2): + assert metainterp is not None a = box1.getint() b = box2.getint() try: @@ -185,6 +188,7 @@ return BoxInt(z) def do_int_mul_ovf(cpu, metainterp, box1, box2): + assert metainterp is not None a = box1.getint() b = box2.getint() try: Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py Fri May 28 14:17:29 2010 @@ -232,6 +232,7 @@ class ConstInt(Const): type = INT + value = 0 _attrs_ = ('value',) def __init__(self, value): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Fri May 28 14:17:29 2010 @@ -4,6 +4,7 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.debug import debug_start, debug_stop, debug_print from pypy.rlib.debug import make_sure_not_resized +from pypy.rlib import nonconst from pypy.jit.metainterp import history, compile, resume from pypy.jit.metainterp.history import Const, ConstInt, ConstPtr, ConstFloat @@ -59,6 +60,8 @@ """Copy jitcode.constants[0] to registers[255], jitcode.constants[1] to registers[254], jitcode.constants[2] to registers[253], etc.""" + if nonconst.NonConstant(0): # force the right type + constants[0] = ConstClass.value # (useful for small tests) i = len(constants) - 1 while i >= 0: j = 255 - i @@ -138,7 +141,7 @@ count = self.jitcode.num_regs_f() registers = self.registers_f else: - assert 0, repr(oldbox) + assert 0, oldbox for i in range(count): if registers[i] is oldbox: registers[i] = newbox Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py Fri May 28 14:17:29 2010 @@ -531,7 +531,8 @@ # pypy.jit.codewriter.jitcode. Some tests give a different 'info'. info.enumerate_vars(self._callback_i, self._callback_r, - self._callback_f) + self._callback_f, + self.unique_id) # <-- annotation hack self.cur_numb = self.cur_numb.prev def _callback_i(self, index, register_index): @@ -569,6 +570,7 @@ return resumereader.liveboxes, virtualizable_boxes, virtualref_boxes class ResumeDataBoxReader(AbstractResumeDataReader): + unique_id = lambda: None def __init__(self, storage, metainterp): self._init(metainterp.cpu, storage) @@ -752,6 +754,7 @@ return resumereader.virtuals class ResumeDataDirectReader(AbstractResumeDataReader): + unique_id = lambda: None virtual_default = lltype.nullptr(llmemory.GCREF.TO) resume_after_guard_not_forced = 0 # 0: not a GUARD_NOT_FORCED From afa at codespeak.net Fri May 28 14:52:11 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 28 May 2010 14:52:11 +0200 (CEST) Subject: [pypy-svn] r74847 - in pypy/trunk/pypy: module/cpyext module/cpyext/test objspace/std Message-ID: <20100528125211.D14A5282BD6@codespeak.net> Author: afa Date: Fri May 28 14:52:10 2010 New Revision: 74847 Modified: pypy/trunk/pypy/module/cpyext/object.py pypy/trunk/pypy/module/cpyext/pyobject.py pypy/trunk/pypy/module/cpyext/test/test_typeobject.py pypy/trunk/pypy/module/cpyext/typeobject.py pypy/trunk/pypy/objspace/std/typeobject.py Log: Remove Typedef("C_type"), this type is no more visible at applevel, this fixes the metaclasses conflict. W_PyCTypeObject still exists, but it is now simply an alternate representation, it shares the same TypeDef as W_TypeObject. There is no visible difference between types defined in pypy and defined in C extensions, (except maybe the value of cls.__flags__) just like in CPython, type(cls)==type for both heap and builtin types At interp_level, use w_type.is_cpytype() to make the distinction. Modified: pypy/trunk/pypy/module/cpyext/object.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/object.py (original) +++ pypy/trunk/pypy/module/cpyext/object.py Fri May 28 14:52:10 2010 @@ -6,7 +6,7 @@ from pypy.module.cpyext.pyobject import ( PyObject, PyObjectP, create_ref, from_ref, Py_IncRef, Py_DecRef, track_reference, get_typedescr) -from pypy.module.cpyext.typeobject import PyTypeObjectPtr, W_PyCTypeObject +from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall from pypy.objspace.std.objectobject import W_ObjectObject from pypy.objspace.std.typeobject import W_TypeObject @@ -195,7 +195,8 @@ py_obj.c_ob_type = type py_obj.c_ob_refcnt = 1 w_type = from_ref(space, rffi.cast(PyObject, type)) - if isinstance(w_type, W_PyCTypeObject): + assert isinstance(w_type, W_TypeObject) + if w_type.is_cpytype(): w_obj = space.allocate_instance(W_ObjectObject, w_type) track_reference(space, py_obj, w_obj) from pypy.module.cpyext.typeobject import lifeline_dict, PyOLifeline Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Fri May 28 14:52:10 2010 @@ -334,7 +334,6 @@ return assert lltype.typeOf(obj) == PyObject - from pypy.module.cpyext.typeobject import W_PyCTypeObject obj.c_ob_refcnt -= 1 if DEBUG_REFCOUNT: debug_refcount("DECREF", obj, obj.c_ob_refcnt, frame_stackdepth=3) @@ -349,8 +348,7 @@ w_obj = state.py_objects_r2w[ptr] del state.py_objects_r2w[ptr] w_type = space.type(w_obj) - w_typetype = space.type(w_type) - if not space.is_w(w_typetype, space.gettypeobject(W_PyCTypeObject.typedef)): + if not w_type.is_cpytype(): _Py_Dealloc(space, obj) del state.py_objects_w2r[w_obj] # if the object was a container for borrowed references Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Fri May 28 14:52:10 2010 @@ -143,11 +143,11 @@ assert isinstance(x, module.MetaType) x() - @py.test.mark.xfail def test_metaclass_compatible(self): # metaclasses should not conflict here - print module.MetaType.__mro__ # (foo.Meta, type, object) - print type(module.fooType).__mro__ # (C_Type, type, object) + module = self.import_module(name='foo') + assert module.MetaType.__mro__ == (module.MetaType, type, object) + assert type(module.fooType).__mro__ == (type, object) y = module.MetaType('other', (module.fooType,), {}) assert isinstance(y, module.MetaType) y() Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Fri May 28 14:52:10 2010 @@ -4,12 +4,10 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.annlowlevel import llhelper from pypy.rlib.rweakref import RWeakKeyDictionary -from pypy.interpreter.gateway import ObjSpace, W_Root, Arguments -from pypy.interpreter.gateway import interp2app, unwrap_spec +from pypy.interpreter.gateway import W_Root from pypy.interpreter.baseobjspace import DescrMismatch -from pypy.objspace.std.typeobject import W_TypeObject, _CPYTYPE, call__Type -from pypy.objspace.std.typetype import _precheck_for_new -from pypy.interpreter.typedef import TypeDef, GetSetProperty +from pypy.objspace.std.typeobject import W_TypeObject, _CPYTYPE +from pypy.interpreter.typedef import GetSetProperty from pypy.module.cpyext.api import ( cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING, @@ -19,7 +17,6 @@ PyObject, make_ref, create_ref, from_ref, get_typedescr, make_typedescr, track_reference, RefcountState, borrow_from) from pypy.interpreter.module import Module -from pypy.interpreter.function import FunctionWithFixedCode, StaticMethod from pypy.module.cpyext import structmemberdefs from pypy.module.cpyext.modsupport import convert_method_defs, PyCFunction from pypy.module.cpyext.state import State @@ -31,9 +28,9 @@ PyTypeObjectPtr, PyTypeObject, PyGetSetDef, PyMemberDef, newfunc, PyNumberMethods) from pypy.module.cpyext.slotdefs import slotdefs -from pypy.interpreter.error import OperationError, operationerrfmt +from pypy.interpreter.error import OperationError from pypy.rlib.rstring import rsplit -from pypy.rlib.objectmodel import we_are_translated, specialize +from pypy.rlib.objectmodel import specialize from pypy.module.__builtin__.abstractinst import abstract_issubclass_w from pypy.module.__builtin__.interp_classobj import W_ClassObject @@ -266,67 +263,6 @@ check_descr(space, w_self, self.w_type) PyMember_SetOne(space, w_self, self.member, w_value) -def c_type_descr__call__(space, w_type, __args__): - if not isinstance(w_type, W_PyCTypeObject): - # XXX is this possible? - w_type = _precheck_for_new(space, w_type) - return call__Type(space, w_type, __args__) - - pyo = make_ref(space, w_type) - pto = rffi.cast(PyTypeObjectPtr, pyo) - tp_new = pto.c_tp_new - try: - if not tp_new: - raise operationerrfmt(space.w_TypeError, - "cannot create '%s' instances", w_type.getname(space, '?')) - - args_w, kw_w = __args__.unpack() - w_args = space.newtuple(args_w) - w_kw = space.newdict() - for key, w_obj in kw_w.items(): - space.setitem(w_kw, space.wrap(key), w_obj) - w_obj = generic_cpy_call(space, tp_new, pto, w_args, w_kw) - finally: - Py_DecRef(space, pyo) - - # If the returned object is not an instance of type, - # it won't be initialized. - w_obj_type = space.type(w_obj) - if not (space.is_w(w_obj_type, w_type) or - space.is_true(space.issubtype(w_obj_type, w_type))): - return w_obj - - # call tp_init - pyo = make_ref(space, w_obj_type) - pto = rffi.cast(PyTypeObjectPtr, pyo) - try: - if pto.c_tp_init: - res = generic_cpy_call(space, pto.c_tp_init, w_obj, w_args, w_kw) - if rffi.cast(lltype.Signed, res) < 0: - state = space.fromcache(State) - state.check_and_raise_exception() - finally: - Py_DecRef(space, pyo) - - return w_obj - -def c_type_descr__new__(space, w_typetype, w_name, w_bases, w_dict): - # copied from typetype.descr__new__, XXX missing logic: metaclass resolving - w_typetype = _precheck_for_new(space, w_typetype) - - bases_w = space.fixedview(w_bases) - name = space.str_w(w_name) - dict_w = {} - dictkeys_w = space.listview(w_dict) - for w_key in dictkeys_w: - key = space.str_w(w_key) - dict_w[key] = space.getitem(w_dict, w_key) - w_type = space.allocate_instance(W_PyCTypeObject, w_typetype) - W_TypeObject.__init__(w_type, space, name, bases_w or [space.w_object], - dict_w) - w_type.ready() - return w_type - class W_PyCTypeObject(W_TypeObject): def __init__(self, space, pto): bases_w = space.fixedview(from_ref(space, pto.c_tp_bases)) @@ -346,25 +282,14 @@ W_TypeObject.__init__(self, space, extension_name, bases_w or [space.w_object], dict_w) - self.__flags__ = _CPYTYPE # mainly disables lookup optimizations - -W_PyCTypeObject.typedef = TypeDef( - 'C_type', W_TypeObject.typedef, - __call__ = interp2app(c_type_descr__call__, unwrap_spec=[ObjSpace, W_Root, Arguments]), - __new__ = interp2app(c_type_descr__new__), - ) + self.__flags__ = _CPYTYPE @bootstrap_function def init_typeobject(space): make_typedescr(space.w_type.instancetypedef, basestruct=PyTypeObject, attach=type_attach, - realize=type_realize, - dealloc=type_dealloc) - make_typedescr(W_PyCTypeObject.typedef, - basestruct=PyTypeObject, - make_ref=pyctype_make_ref, - attach=type_attach, + make_ref=type_make_ref, realize=type_realize, dealloc=type_dealloc) @@ -425,17 +350,20 @@ # hopefully this does not clash with the memory model assumed in # extension modules -def pyctype_make_ref(space, w_type, w_obj, itemcount=0): - lifeline = lifeline_dict.get(w_obj) - if lifeline is not None: # make old PyObject ready for use in C code - py_obj = lifeline.pyo - assert py_obj.c_ob_refcnt == 0 - Py_IncRef(space, py_obj) - else: - typedescr = get_typedescr(w_obj.typedef) - py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) +def type_make_ref(space, w_type, w_obj, itemcount=0): + if w_type.is_cpytype(): + lifeline = lifeline_dict.get(w_obj) + if lifeline is not None: # make old PyObject ready for use in C code + py_obj = lifeline.pyo + assert py_obj.c_ob_refcnt == 0 + Py_IncRef(space, py_obj) + return py_obj + + typedescr = get_typedescr(w_obj.typedef) + py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) + if w_type.is_cpytype(): lifeline_dict.set(w_obj, PyOLifeline(space, py_obj)) - typedescr.attach(space, py_obj, w_obj) + typedescr.attach(space, py_obj, w_obj) return py_obj @cpython_api([PyObject, rffi.INTP], lltype.Signed, external=False, @@ -621,9 +549,7 @@ finish_type_1(space, py_type) - w_obj = space.allocate_instance( - W_PyCTypeObject, - space.gettypeobject(W_PyCTypeObject.typedef)) + w_obj = space.allocate_instance(W_PyCTypeObject, space.w_type) track_reference(space, py_obj, w_obj) w_obj.__init__(space, py_type) w_obj.ready() Modified: pypy/trunk/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/typeobject.py (original) +++ pypy/trunk/pypy/objspace/std/typeobject.py Fri May 28 14:52:10 2010 @@ -352,7 +352,7 @@ raise UnwrapError(w_self) def is_heaptype(w_self): - return w_self.__flags__&_HEAPTYPE + return w_self.__flags__ & _HEAPTYPE def is_cpytype(w_self): return w_self.__flags__ & _CPYTYPE @@ -581,6 +581,10 @@ w_bestbase = check_and_find_best_base(w_self.space, w_self.bases_w) w_self.instancetypedef = w_bestbase.instancetypedef w_self.__flags__ = _HEAPTYPE + for w_base in w_self.bases_w: + if not isinstance(w_base, W_TypeObject): + continue + w_self.__flags__ |= w_base.__flags__ hasoldstylebase = copy_flags_from_bases(w_self, w_bestbase) create_all_slots(w_self, hasoldstylebase) From afa at codespeak.net Fri May 28 15:33:22 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 28 May 2010 15:33:22 +0200 (CEST) Subject: [pypy-svn] r74848 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100528133322.6030E282BD6@codespeak.net> Author: afa Date: Fri May 28 15:33:20 2010 New Revision: 74848 Modified: pypy/trunk/pypy/module/cpyext/object.py pypy/trunk/pypy/module/cpyext/pyobject.py pypy/trunk/pypy/module/cpyext/test/test_cpyext.py pypy/trunk/pypy/module/cpyext/typeobject.py Log: Now there is one and only one "make_ref" function. Move lifeline_dict to pyobject, Modified: pypy/trunk/pypy/module/cpyext/object.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/object.py (original) +++ pypy/trunk/pypy/module/cpyext/object.py Fri May 28 15:33:20 2010 @@ -5,7 +5,7 @@ Py_GE, CONST_STRING, FILEP, fwrite) from pypy.module.cpyext.pyobject import ( PyObject, PyObjectP, create_ref, from_ref, Py_IncRef, Py_DecRef, - track_reference, get_typedescr) + track_reference, get_typedescr, lifeline_dict, PyOLifeline) from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall from pypy.objspace.std.objectobject import W_ObjectObject @@ -199,7 +199,6 @@ if w_type.is_cpytype(): w_obj = space.allocate_instance(W_ObjectObject, w_type) track_reference(space, py_obj, w_obj) - from pypy.module.cpyext.typeobject import lifeline_dict, PyOLifeline lifeline_dict.set(w_obj, PyOLifeline(space, py_obj)) else: assert False, "Please add more cases in PyObject_Init" Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Fri May 28 15:33:20 2010 @@ -8,6 +8,7 @@ from pypy.module.cpyext.state import State from pypy.objspace.std.typeobject import W_TypeObject from pypy.rlib.objectmodel import specialize, we_are_translated +from pypy.rlib.rweakref import RWeakKeyDictionary from pypy.rpython.annlowlevel import llhelper #________________________________________________________ @@ -20,8 +21,6 @@ raise NotImplementedError def allocate(self, space, w_type, itemcount=0): raise NotImplementedError - def make_ref(self, space, w_type, w_obj, itemcount=0): - raise NotImplementedError def attach(self, space, pyobj, w_obj): raise NotImplementedError def realize(self, space, ref): @@ -40,7 +39,6 @@ """ tp_basestruct = kw.pop('basestruct', PyObject.TO) - tp_make_ref = kw.pop('make_ref', None) tp_attach = kw.pop('attach', None) tp_realize = kw.pop('realize', None) tp_dealloc = kw.pop('dealloc', None) @@ -87,18 +85,6 @@ pyobj.c_ob_type = pytype return pyobj - # Specialized by meta-type - if tp_make_ref: - def make_ref(self, space, w_type, w_obj, itemcount=0): - return tp_make_ref(space, w_type, w_obj, itemcount=itemcount) - else: - def make_ref(self, space, w_type, w_obj, itemcount=0): - typedescr = get_typedescr(w_obj.typedef) - w_type = space.type(w_obj) - py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) - typedescr.attach(space, py_obj, w_obj) - return py_obj - if tp_attach: def attach(self, space, pyobj, w_obj): tp_attach(space, pyobj, w_obj) @@ -257,14 +243,26 @@ print >>sys.stderr, arg, print >>sys.stderr -def create_ref(space, w_obj, items=0): +def create_ref(space, w_obj, itemcount=0): """ Allocates a PyObject, and fills its fields with info from the given intepreter object. """ w_type = space.type(w_obj) - metatypedescr = get_typedescr(w_type.typedef) - return metatypedescr.make_ref(space, w_type, w_obj, itemcount=items) + if w_type.is_cpytype(): + lifeline = lifeline_dict.get(w_obj) + if lifeline is not None: # make old PyObject ready for use in C code + py_obj = lifeline.pyo + assert py_obj.c_ob_refcnt == 0 + Py_IncRef(space, py_obj) + return py_obj + + typedescr = get_typedescr(w_obj.typedef) + py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) + if w_type.is_cpytype(): + lifeline_dict.set(w_obj, PyOLifeline(space, py_obj)) + typedescr.attach(space, py_obj, w_obj) + return py_obj def track_reference(space, py_obj, w_obj, replace=False): """ @@ -381,6 +379,26 @@ generic_cpy_call_dont_decref(space, pto.c_tp_dealloc, obj) #___________________________________________________________ +# Support for "lifelines" +# +# Object structure must stay alive even when not referenced +# by any C code. + +class PyOLifeline(object): + def __init__(self, space, pyo): + self.pyo = pyo + self.space = space + + def __del__(self): + if self.pyo: + assert self.pyo.c_ob_refcnt == 0 + _Py_Dealloc(self.space, self.pyo) + self.pyo = lltype.nullptr(PyObject.TO) + # XXX handle borrowed objects here + +lifeline_dict = RWeakKeyDictionary(W_Root, PyOLifeline) + +#___________________________________________________________ # Support for borrowed references def make_borrowed_ref(space, w_container, w_borrowed): Modified: pypy/trunk/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Fri May 28 15:33:20 2010 @@ -10,9 +10,9 @@ from pypy.translator import platform from pypy.translator.gensupp import uniquemodulename from pypy.tool.udir import udir -from pypy.module.cpyext import api, typeobject +from pypy.module.cpyext import api from pypy.module.cpyext.state import State -from pypy.module.cpyext.pyobject import RefcountState +from pypy.module.cpyext.pyobject import RefcountState, lifeline_dict from pypy.module.cpyext.pyobject import Py_DecRef, InvalidPointerException from pypy.translator.goal import autopath from pypy.lib.identity_dict import identity_dict @@ -92,9 +92,9 @@ lost_objects_w.update((key, None) for key in self.frozen_refcounts.keys()) # Clear all lifelines, objects won't resurrect - for w_obj, obj in typeobject.lifeline_dict._dict.items(): + for w_obj, obj in lifeline_dict._dict.items(): if w_obj not in state.py_objects_w2r: - typeobject.lifeline_dict.set(w_obj, None) + lifeline_dict.set(w_obj, None) del obj gc.collect() @@ -107,7 +107,7 @@ if delta != 0: leaking = True print >>sys.stderr, "Leaking %r: %i references" % (w_obj, delta) - lifeline = typeobject.lifeline_dict.get(w_obj) + lifeline = lifeline_dict.get(w_obj) if lifeline is not None: refcnt = lifeline.pyo.c_ob_refcnt if refcnt > 0: Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Fri May 28 15:33:20 2010 @@ -3,8 +3,6 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.annlowlevel import llhelper -from pypy.rlib.rweakref import RWeakKeyDictionary -from pypy.interpreter.gateway import W_Root from pypy.interpreter.baseobjspace import DescrMismatch from pypy.objspace.std.typeobject import W_TypeObject, _CPYTYPE from pypy.interpreter.typedef import GetSetProperty @@ -212,20 +210,6 @@ pto.c_tp_new = base_pto.c_tp_new Py_DecRef(space, base_object_pyo) -class PyOLifeline(object): - def __init__(self, space, pyo): - self.pyo = pyo - self.space = space - - def __del__(self): - if self.pyo: - assert self.pyo.c_ob_refcnt == 0 - _Py_Dealloc(self.space, self.pyo) - self.pyo = lltype.nullptr(PyObject.TO) - # XXX handle borrowed objects here - -lifeline_dict = RWeakKeyDictionary(W_Root, PyOLifeline) - def check_descr(space, w_self, w_type): if not space.is_true(space.isinstance(w_self, w_type)): raise DescrMismatch() @@ -289,7 +273,6 @@ make_typedescr(space.w_type.instancetypedef, basestruct=PyTypeObject, attach=type_attach, - make_ref=type_make_ref, realize=type_realize, dealloc=type_dealloc) @@ -350,22 +333,6 @@ # hopefully this does not clash with the memory model assumed in # extension modules -def type_make_ref(space, w_type, w_obj, itemcount=0): - if w_type.is_cpytype(): - lifeline = lifeline_dict.get(w_obj) - if lifeline is not None: # make old PyObject ready for use in C code - py_obj = lifeline.pyo - assert py_obj.c_ob_refcnt == 0 - Py_IncRef(space, py_obj) - return py_obj - - typedescr = get_typedescr(w_obj.typedef) - py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) - if w_type.is_cpytype(): - lifeline_dict.set(w_obj, PyOLifeline(space, py_obj)) - typedescr.attach(space, py_obj, w_obj) - return py_obj - @cpython_api([PyObject, rffi.INTP], lltype.Signed, external=False, error=CANNOT_FAIL) def str_segcount(space, w_obj, ref): From afa at codespeak.net Fri May 28 15:44:28 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 28 May 2010 15:44:28 +0200 (CEST) Subject: [pypy-svn] r74849 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100528134428.75A42282BD6@codespeak.net> Author: afa Date: Fri May 28 15:44:26 2010 New Revision: 74849 Modified: pypy/trunk/pypy/module/cpyext/object.py pypy/trunk/pypy/module/cpyext/pyobject.py pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Log: Move the lifeline_dict instance into the RefcountState class. Modified: pypy/trunk/pypy/module/cpyext/object.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/object.py (original) +++ pypy/trunk/pypy/module/cpyext/object.py Fri May 28 15:44:26 2010 @@ -5,7 +5,7 @@ Py_GE, CONST_STRING, FILEP, fwrite) from pypy.module.cpyext.pyobject import ( PyObject, PyObjectP, create_ref, from_ref, Py_IncRef, Py_DecRef, - track_reference, get_typedescr, lifeline_dict, PyOLifeline) + track_reference, get_typedescr, RefcountState) from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall from pypy.objspace.std.objectobject import W_ObjectObject @@ -199,7 +199,8 @@ if w_type.is_cpytype(): w_obj = space.allocate_instance(W_ObjectObject, w_type) track_reference(space, py_obj, w_obj) - lifeline_dict.set(w_obj, PyOLifeline(space, py_obj)) + state = space.fromcache(RefcountState) + state.set_lifeline(w_obj, py_obj) else: assert False, "Please add more cases in PyObject_Init" return py_obj Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Fri May 28 15:44:26 2010 @@ -138,6 +138,8 @@ self.py_objects_w2r = {} # { w_obj -> raw PyObject } self.py_objects_r2w = {} # { addr of raw PyObject -> w_obj } + self.lifeline_dict = RWeakKeyDictionary(W_Root, PyOLifeline) + self.borrow_mapping = {None: {}} # { w_container -> { w_containee -> None } } # the None entry manages references borrowed during a call to @@ -165,6 +167,19 @@ for w_obj, obj in self.py_objects_w2r.items(): print "%r: %i" % (w_obj, obj.c_ob_refcnt) + def get_from_lifeline(self, w_obj): + lifeline = self.lifeline_dict.get(w_obj) + if lifeline is not None: # make old PyObject ready for use in C code + py_obj = lifeline.pyo + assert py_obj.c_ob_refcnt == 0 + return py_obj + else: + lltype.nullptr(PyObject.TO) + + def set_lifeline(self, w_obj, py_obj): + self.lifeline_dict.set(w_obj, + PyOLifeline(self.space, py_obj)) + def make_borrowed(self, w_container, w_borrowed): """ Create a borrowed reference, which will live as long as the container @@ -250,17 +265,16 @@ """ w_type = space.type(w_obj) if w_type.is_cpytype(): - lifeline = lifeline_dict.get(w_obj) - if lifeline is not None: # make old PyObject ready for use in C code - py_obj = lifeline.pyo - assert py_obj.c_ob_refcnt == 0 + state = space.fromcache(RefcountState) + py_obj = state.get_from_lifeline(w_obj) + if py_obj: Py_IncRef(space, py_obj) return py_obj typedescr = get_typedescr(w_obj.typedef) py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) if w_type.is_cpytype(): - lifeline_dict.set(w_obj, PyOLifeline(space, py_obj)) + state.set_lifeline(w_obj, py_obj) typedescr.attach(space, py_obj, w_obj) return py_obj @@ -396,8 +410,6 @@ self.pyo = lltype.nullptr(PyObject.TO) # XXX handle borrowed objects here -lifeline_dict = RWeakKeyDictionary(W_Root, PyOLifeline) - #___________________________________________________________ # Support for borrowed references Modified: pypy/trunk/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Fri May 28 15:44:26 2010 @@ -12,7 +12,7 @@ from pypy.tool.udir import udir from pypy.module.cpyext import api from pypy.module.cpyext.state import State -from pypy.module.cpyext.pyobject import RefcountState, lifeline_dict +from pypy.module.cpyext.pyobject import RefcountState from pypy.module.cpyext.pyobject import Py_DecRef, InvalidPointerException from pypy.translator.goal import autopath from pypy.lib.identity_dict import identity_dict @@ -92,9 +92,9 @@ lost_objects_w.update((key, None) for key in self.frozen_refcounts.keys()) # Clear all lifelines, objects won't resurrect - for w_obj, obj in lifeline_dict._dict.items(): + for w_obj, obj in state.lifeline_dict._dict.items(): if w_obj not in state.py_objects_w2r: - lifeline_dict.set(w_obj, None) + state.lifeline_dict.set(w_obj, None) del obj gc.collect() @@ -107,7 +107,7 @@ if delta != 0: leaking = True print >>sys.stderr, "Leaking %r: %i references" % (w_obj, delta) - lifeline = lifeline_dict.get(w_obj) + lifeline = state.lifeline_dict.get(w_obj) if lifeline is not None: refcnt = lifeline.pyo.c_ob_refcnt if refcnt > 0: From antocuni at codespeak.net Fri May 28 15:53:56 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 28 May 2010 15:53:56 +0200 (CEST) Subject: [pypy-svn] r74850 - in pypy/branch/sys-prefix/lib: . pypy1.2 pypy1.2/lib_pypy pypy1.2/lib_pypy/app_test pypy1.2/lib_pypy/ctypes_config_cache pypy1.2/lib_pypy/test2 Message-ID: <20100528135356.6E7D6282BD6@codespeak.net> Author: antocuni Date: Fri May 28 15:53:54 2010 New Revision: 74850 Added: pypy/branch/sys-prefix/lib/ pypy/branch/sys-prefix/lib/README pypy/branch/sys-prefix/lib/pypy1.2/ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/ - copied from r74817, pypy/branch/sys-prefix/pypy/lib/ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/_sre.py - copied unchanged from r74837, pypy/branch/sys-prefix/pypy/lib/_sre.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/py (contents, props changed) Removed: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/ctypes_config_cache/autopath.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/identity_dict.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/test2/test_identitydict.py Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/__init__.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/inprogress_test_binascii_extra.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_binascii.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_coroutine.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_ctypes_support.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_datetime.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_dbm_extra.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_defaultdict.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_deque_extra.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_exception_extra.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_functools.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_hashlib.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_locale.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_marshal_extra.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_md5_extra.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_pyexpat.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_resource.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_runpy.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_sha_extra.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_stackless.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_stackless_pickling.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_struct_extra.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_structseq.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_syslog.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/ctypes_config_cache/hashlib.ctc.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/ctypes_config_cache/locale.ctc.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/ctypes_config_cache/pyexpat.ctc.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/ctypes_config_cache/rebuild.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/ctypes_config_cache/resource.ctc.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/ctypes_config_cache/syslog.ctc.py Log: move pypy/lib/ to lib/pypy1.2/lib_pypy (part 1 of many). The final goal is to be able to use pypy-trunk as sys.prefix, so pypy-trunk/lib plays the role of /usr/lib in a normal system-wide installation. Since pypy.lib is no longer directly importable, all the tests in app_test now rely on relative imports to import the modules they are testing. There are still issues left; e.g., test_runpy fails, and the app-level tests in test2 should be moved somewhere else, because they need pypy/conftest.py to work Added: pypy/branch/sys-prefix/lib/README ============================================================================== --- (empty file) +++ pypy/branch/sys-prefix/lib/README Fri May 28 15:53:54 2010 @@ -0,0 +1,7 @@ +This directory plays the role of /usr/lib or /usr/local/lib in a normal +installation hierarchy. + +It is here so that we can freely use tools like virtualenv without any need to +install pypy system-wide, but just by using the root directory of the svn +checkout as sys.prefix: this way, the stdlib can always be found in +sys.prefix + '/lib/pypyX.Y' Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/__init__.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/__init__.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/__init__.py Fri May 28 15:53:54 2010 @@ -1,4 +1,7 @@ # This __init__.py shows up in PyPy's app-level standard library. -# Let's try to prevent that confusion... -if __name__ != 'pypy.lib': +# Let's try to prevent that confusion. +# Without this check, you would be able to do 'import __init__' from a pypy +# prompt + +if __name__ != 'lib_pypy': raise ImportError, '__init__' Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/inprogress_test_binascii_extra.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/app_test/inprogress_test_binascii_extra.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/inprogress_test_binascii_extra.py Fri May 28 15:53:54 2010 @@ -1,5 +1,5 @@ - -from pypy.lib import binascii +from __future__ import absolute_import +from .. import binascii def test_uu(): assert binascii.b2a_uu('1234567') == "',3(S-#4V-P \n" Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_binascii.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/app_test/test_binascii.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_binascii.py Fri May 28 15:53:54 2010 @@ -1,4 +1,6 @@ -from pypy.lib import binascii +from __future__ import absolute_import +import py +from .. import binascii # Create binary test data data = "The quick brown fox jumps over the lazy dog.\r\n" @@ -21,10 +23,10 @@ for prefix in prefixes: name = prefix + suffix assert callable(getattr(binascii, name)) - raises(TypeError, getattr(binascii, name)) + py.test.raises(TypeError, getattr(binascii, name)) for name in ("hexlify", "unhexlify"): assert callable(getattr(binascii, name)) - raises(TypeError, getattr(binascii, name)) + py.test.raises(TypeError, getattr(binascii, name)) def test_base64valid(): # Test base64 with valid data @@ -93,10 +95,10 @@ assert binascii.a2b_uu("\x7f") == "\x00"*31 assert binascii.a2b_uu("\x80") == "\x00"*32 assert binascii.a2b_uu("\xff") == "\x00"*31 - raises(binascii.Error, binascii.a2b_uu, "\xff\x00") - raises(binascii.Error, binascii.a2b_uu, "!!!!") + py.test.raises(binascii.Error, binascii.a2b_uu, "\xff\x00") + py.test.raises(binascii.Error, binascii.a2b_uu, "!!!!") - raises(binascii.Error, binascii.b2a_uu, 46*"!") + py.test.raises(binascii.Error, binascii.b2a_uu, 46*"!") def test_crc32(): crc = binascii.crc32("Test the CRC-32 of") @@ -106,7 +108,7 @@ crc = binascii.crc32('frotz\n', 0) assert crc == -372923920 - raises(TypeError, binascii.crc32) + py.test.raises(TypeError, binascii.crc32) def test_hex(): # test hexlification @@ -114,8 +116,8 @@ t = binascii.b2a_hex(s) u = binascii.a2b_hex(t) assert s == u - raises(TypeError, binascii.a2b_hex, t[:-1]) - raises(TypeError, binascii.a2b_hex, t[:-1] + 'q') + py.test.raises(TypeError, binascii.a2b_hex, t[:-1]) + py.test.raises(TypeError, binascii.a2b_hex, t[:-1] + 'q') # Verify the treatment of Unicode strings assert binascii.hexlify(unicode('a', 'ascii')) == '61' @@ -131,7 +133,7 @@ assert binascii.a2b_qp("= ") == "= " assert binascii.a2b_qp("==") == "=" assert binascii.a2b_qp("=AX") == "=AX" - raises(TypeError, binascii.b2a_qp, foo="bar") + py.test.raises(TypeError, binascii.b2a_qp, foo="bar") assert binascii.a2b_qp("=00\r\n=00") == "\x00\r\n\x00" assert binascii.b2a_qp("\xff\r\n\xff\n\xff") == "=FF\r\n=FF\r\n=FF" target = "0"*75+"=\r\n=FF\r\n=FF\r\n=FF" @@ -155,7 +157,7 @@ def test_wrong_padding(): s = 'CSixpLDtKSC/7Liuvsax4iC6uLmwMcijIKHaILzSwd/H0SC8+LCjwLsgv7W/+Mj3IQ' - raises(binascii.Error, binascii.a2b_base64, s) + py.test.raises(binascii.Error, binascii.a2b_base64, s) def test_crap_after_padding(): s = 'xxx=axxxx' @@ -163,4 +165,4 @@ def test_wrong_args(): # this should grow as a way longer list - raises(TypeError, binascii.a2b_base64, 42) + py.test.raises(TypeError, binascii.a2b_base64, 42) Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_coroutine.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/app_test/test_coroutine.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_coroutine.py Fri May 28 15:53:54 2010 @@ -1,7 +1,8 @@ +from __future__ import absolute_import from py.test import skip, raises try: - from pypy.lib.stackless import coroutine + from ..stackless import coroutine except ImportError, e: skip('cannot import stackless: %s' % (e,)) Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_ctypes_support.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/app_test/test_ctypes_support.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_ctypes_support.py Fri May 28 15:53:54 2010 @@ -1,10 +1,11 @@ +from __future__ import absolute_import import py from ctypes import * try: from ctypes_support import standard_c_lib, get_errno, set_errno except ImportError: # on top of cpython - from pypy.lib.ctypes_support import standard_c_lib, get_errno, set_errno + from ..ctypes_support import standard_c_lib, get_errno, set_errno def test_stdlib_and_errno(): Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_datetime.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/app_test/test_datetime.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_datetime.py Fri May 28 15:53:54 2010 @@ -1,5 +1,6 @@ +from __future__ import absolute_import -from pypy.lib import datetime +from .. import datetime def test_repr(): print datetime Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_dbm_extra.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/app_test/test_dbm_extra.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_dbm_extra.py Fri May 28 15:53:54 2010 @@ -1,7 +1,8 @@ +from __future__ import absolute_import import py from pypy.tool.udir import udir try: - from pypy.lib import dbm + from .. import dbm except ImportError, e: py.test.skip(e) Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_defaultdict.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/app_test/test_defaultdict.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_defaultdict.py Fri May 28 15:53:54 2010 @@ -1,16 +1,16 @@ # defaultdict Tests # from CPython2.5 - +from __future__ import absolute_import +import py import sys if sys.version_info < (2, 5): - import py # the app-level defaultdict relies on the interp-level dict # calling __missing__() py.test.skip("these tests only run on top of CPython 2.5") import copy -from pypy.lib.collections import defaultdict +from ..collections import defaultdict def foobar(): return list @@ -45,12 +45,12 @@ assert 12 not in d2.keys() d2.default_factory = None assert d2.default_factory == None - raises(KeyError, d2.__getitem__, 15) - raises(TypeError, defaultdict, 1) + py.test.raises(KeyError, d2.__getitem__, 15) + py.test.raises(TypeError, defaultdict, 1) def test_missing(self): d1 = defaultdict() - raises(KeyError, d1.__missing__, 42) + py.test.raises(KeyError, d1.__missing__, 42) d1.default_factory = list assert d1.__missing__(42) == [] Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_deque_extra.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/app_test/test_deque_extra.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_deque_extra.py Fri May 28 15:53:54 2010 @@ -1,11 +1,13 @@ # Deque Tests +from __future__ import absolute_import +import py n = 10 class Test_deque: def setup_method(self,method): - from pypy.lib.collections import deque + from ..collections import deque self.deque = deque self.d = deque(range(n)) @@ -20,22 +22,22 @@ def test_deque_iter(self): it = iter(self.d) - raises(TypeError, len, it) + py.test.raises(TypeError, len, it) assert it.next() == 0 self.d.pop() - raises(RuntimeError, it.next) + py.test.raises(RuntimeError, it.next) def test_deque_reversed(self): it = reversed(self.d) - raises(TypeError, len, it) + py.test.raises(TypeError, len, it) assert it.next() == n-1 assert it.next() == n-2 self.d.pop() - raises(RuntimeError, it.next) + py.test.raises(RuntimeError, it.next) def test_deque_remove(self): d = self.d - raises(ValueError, d.remove, "foobar") + py.test.raises(ValueError, d.remove, "foobar") def test_mutate_during_remove(self): # Handle evil mutator @@ -50,5 +52,5 @@ for match in (True, False): d = self.deque(['ab']) d.extend([MutateCmp(d, match), 'c']) - raises(IndexError, d.remove, 'c') + py.test.raises(IndexError, d.remove, 'c') assert len(d) == 0 Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_exception_extra.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/app_test/test_exception_extra.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_exception_extra.py Fri May 28 15:53:54 2010 @@ -1,4 +1,3 @@ - def app_test_environmenterror_repr(): import exceptions as ex e = ex.EnvironmentError("hello") Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_functools.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/app_test/test_functools.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_functools.py Fri May 28 15:53:54 2010 @@ -1,4 +1,5 @@ -from pypy.lib import functools +from __future__ import absolute_import +from .. import functools import unittest from test import test_support from weakref import proxy Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_hashlib.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/app_test/test_hashlib.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_hashlib.py Fri May 28 15:53:54 2010 @@ -1,7 +1,8 @@ -from pypy.lib.ctypes_config_cache import rebuild +from __future__ import absolute_import +from ..ctypes_config_cache import rebuild rebuild.rebuild_one('hashlib.ctc.py') -from pypy.lib import hashlib, _hashlib +from .. import hashlib, _hashlib def test_unicode(): assert isinstance(hashlib.new('sha1', u'xxx'), _hashlib.hash) Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_locale.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/app_test/test_locale.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_locale.py Fri May 28 15:53:54 2010 @@ -1,10 +1,11 @@ +from __future__ import absolute_import import py import sys -from pypy.lib.ctypes_config_cache import rebuild +from ..ctypes_config_cache import rebuild rebuild.rebuild_one('locale.ctc.py') -from pypy.lib import _locale +from .. import _locale def setup_module(mod): Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_marshal_extra.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/app_test/test_marshal_extra.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_marshal_extra.py Fri May 28 15:53:54 2010 @@ -1,7 +1,8 @@ +from __future__ import absolute_import import py import sys import marshal as cpy_marshal -from pypy.lib import _marshal as marshal +from .. import _marshal as marshal from pypy.tool.udir import udir Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_md5_extra.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/app_test/test_md5_extra.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_md5_extra.py Fri May 28 15:53:54 2010 @@ -4,8 +4,9 @@ 160 sec. per MB of data on a 233 MHz Intel Pentium CPU. """ +from __future__ import absolute_import import md5 # CPython's implementation in C. -from pypy.lib import md5 as pymd5 +from .. import md5 as pymd5 # Helpers... Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_pyexpat.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/app_test/test_pyexpat.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_pyexpat.py Fri May 28 15:53:54 2010 @@ -1,13 +1,14 @@ # XXX TypeErrors on calling handlers, or on bad return values from a # handler, are obscure and unhelpful. +from __future__ import absolute_import import StringIO, sys import unittest, py -from pypy.lib.ctypes_config_cache import rebuild +from ..ctypes_config_cache import rebuild rebuild.rebuild_one('pyexpat.ctc.py') -from pypy.lib import pyexpat +from .. import pyexpat #from xml.parsers import expat expat = pyexpat @@ -520,7 +521,7 @@ parser = expat.ParserCreate() parser.CharacterDataHandler = handler - raises(Exception, parser.Parse, xml) + py.test.raises(Exception, parser.Parse, xml) class TestChardataBuffer: """ @@ -539,9 +540,9 @@ def f(size): parser.buffer_size = size - raises(TypeError, f, sys.maxint+1) - raises(ValueError, f, -1) - raises(ValueError, f, 0) + py.test.raises(TypeError, f, sys.maxint+1) + py.test.raises(ValueError, f, -1) + py.test.raises(ValueError, f, 0) def test_unchanged_size(self): xml1 = ("%s" % ('a' * 512)) Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_resource.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/app_test/test_resource.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_resource.py Fri May 28 15:53:54 2010 @@ -1,7 +1,8 @@ -from pypy.lib.ctypes_config_cache import rebuild +from __future__ import absolute_import +from ..ctypes_config_cache import rebuild rebuild.rebuild_one('resource.ctc.py') -from pypy.lib import resource +from .. import resource def test_resource(): x = resource.getrusage(resource.RUSAGE_SELF) Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_runpy.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/app_test/test_runpy.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_runpy.py Fri May 28 15:53:54 2010 @@ -1,10 +1,11 @@ # Test the runpy module +from __future__ import absolute_import import unittest import os import os.path import sys import tempfile -from pypy.lib.runpy import _run_module_code, run_module +from ..runpy import _run_module_code, run_module verbose = 0 Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_sha_extra.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/app_test/test_sha_extra.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_sha_extra.py Fri May 28 15:53:54 2010 @@ -3,8 +3,8 @@ # use the three examples from Federal Information Processing Standards # Publication 180-1, Secure Hash Standard, 1995 April 17 # http://www.itl.nist.gov/div897/pubs/fip180-1.htm - -from pypy.lib import sha +from __future__ import absolute_import +from .. import sha class TestSHA: def check(self, data, digest): Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_stackless.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/app_test/test_stackless.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_stackless.py Fri May 28 15:53:54 2010 @@ -4,12 +4,13 @@ 2. CPython (with the stackless_new module in the path 3. pypy-c """ +from __future__ import absolute_import from py.test import skip try: import stackless except ImportError: try: - from pypy.lib import stackless + from .. import stackless except ImportError, e: skip('cannot import stackless: %s' % (e,)) Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_stackless_pickling.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/app_test/test_stackless_pickling.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_stackless_pickling.py Fri May 28 15:53:54 2010 @@ -2,12 +2,13 @@ this test should probably not run from CPython or py.py. I'm not entirely sure, how to do that. """ +from __future__ import absolute_import from py.test import skip try: import stackless except ImportError: try: - from pypy.lib import stackless as stackless + from .. import stackless as stackless except ImportError, e: skip('cannot import stackless: %s' % (e,)) Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_struct_extra.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/app_test/test_struct_extra.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_struct_extra.py Fri May 28 15:53:54 2010 @@ -1,4 +1,5 @@ -from pypy.lib import struct +from __future__ import absolute_import +from .. import struct def test_simple(): morezeros = '\x00' * (struct.calcsize('l')-4) Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_structseq.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/app_test/test_structseq.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_structseq.py Fri May 28 15:53:54 2010 @@ -1,5 +1,6 @@ +from __future__ import absolute_import import py -from pypy.lib._structseq import * +from .._structseq import * class mydata: Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_syslog.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/app_test/test_syslog.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_syslog.py Fri May 28 15:53:54 2010 @@ -1,10 +1,10 @@ - +from __future__ import absolute_import # XXX very minimal test -from pypy.lib.ctypes_config_cache import rebuild +from ..ctypes_config_cache import rebuild rebuild.rebuild_one('syslog.ctc.py') -from pypy.lib import syslog +from .. import syslog def test_syslog(): Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/ctypes_config_cache/hashlib.ctc.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/ctypes_config_cache/hashlib.ctc.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/ctypes_config_cache/hashlib.ctc.py Fri May 28 15:53:54 2010 @@ -3,7 +3,6 @@ Run this to rebuild _hashlib_cache.py. """ -import autopath from ctypes import * from ctypes_configure import configure, dumpcache Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/ctypes_config_cache/locale.ctc.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/ctypes_config_cache/locale.ctc.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/ctypes_config_cache/locale.ctc.py Fri May 28 15:53:54 2010 @@ -3,7 +3,6 @@ Run this to rebuild _locale_cache.py. """ -import autopath from ctypes_configure.configure import (configure, ExternalCompilationInfo, ConstantInteger, DefinedConstantInteger, SimpleType, check_eci) from ctypes_configure.dumpcache import dumpcache Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/ctypes_config_cache/pyexpat.ctc.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/ctypes_config_cache/pyexpat.ctc.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/ctypes_config_cache/pyexpat.ctc.py Fri May 28 15:53:54 2010 @@ -3,7 +3,6 @@ Run this to rebuild _pyexpat_cache.py. """ -import autopath import ctypes from ctypes import c_char_p, c_int, c_void_p, c_char from ctypes_configure import configure, dumpcache Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/ctypes_config_cache/rebuild.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/ctypes_config_cache/rebuild.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/ctypes_config_cache/rebuild.py Fri May 28 15:53:54 2010 @@ -1,7 +1,12 @@ #! /usr/bin/env python # Run this script to rebuild all caches from the *.ctc.py files. -import autopath +# hack: we cannot directly import autopath, as we are outside the pypy +# package. However, we pretend to be inside pypy/tool and manually run it, to +# get the correct path +autopath_py = '../../../../pypy/tool/autopath.py' +execfile(autopath_py, dict(__name__='autopath', __file__=autopath_py)) + import os, sys import py Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/ctypes_config_cache/resource.ctc.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/ctypes_config_cache/resource.ctc.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/ctypes_config_cache/resource.ctc.py Fri May 28 15:53:54 2010 @@ -3,8 +3,6 @@ Run this to rebuild _resource_cache.py. """ -import autopath - from ctypes import sizeof from ctypes_configure.dumpcache import dumpcache from ctypes_configure.configure import (configure, Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/ctypes_config_cache/syslog.ctc.py ============================================================================== --- pypy/branch/sys-prefix/pypy/lib/ctypes_config_cache/syslog.ctc.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/ctypes_config_cache/syslog.ctc.py Fri May 28 15:53:54 2010 @@ -3,7 +3,6 @@ Run this to rebuild _syslog_cache.py. """ -import autopath from ctypes_configure.configure import (configure, ExternalCompilationInfo, ConstantInteger, DefinedConstantInteger) from ctypes_configure.dumpcache import dumpcache Added: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/py ============================================================================== --- (empty file) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/py Fri May 28 15:53:54 2010 @@ -0,0 +1 @@ +link ../../../py \ No newline at end of file From antocuni at codespeak.net Fri May 28 15:55:55 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 28 May 2010 15:55:55 +0200 (CEST) Subject: [pypy-svn] r74851 - pypy/branch/sys-prefix/pypy/lib Message-ID: <20100528135555.3E94F282BD6@codespeak.net> Author: antocuni Date: Fri May 28 15:55:54 2010 New Revision: 74851 Removed: pypy/branch/sys-prefix/pypy/lib/ Log: (cfbolz, antocuni) after a fight with svn, finally remove pypy/lib, which has been moved to ../lib/pypy1.2/etc/etc/ From afa at codespeak.net Fri May 28 16:03:06 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 28 May 2010 16:03:06 +0200 (CEST) Subject: [pypy-svn] r74852 - pypy/trunk/pypy/module/cpyext Message-ID: <20100528140306.35BC3282BD6@codespeak.net> Author: afa Date: Fri May 28 16:03:04 2010 New Revision: 74852 Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py Log: Rename stubs for missing "wrap_xxx" functions, because the translated interpreter does not retain exception messages. At least the RPython traceback will show the missing function instead of "wrapper_10". Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/slotdefs.py (original) +++ pypy/trunk/pypy/module/cpyext/slotdefs.py Fri May 28 16:03:04 2010 @@ -13,8 +13,7 @@ from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.argument import Arguments from pypy.rlib.unroll import unrolling_iterable - -space = None +from pypy.tool.sourcetools import func_with_new_name def check_num_args(space, ob, n): @@ -148,12 +147,14 @@ if wrapper is Ellipsis: def wrapper(space, w_self, w_args, func, w_kwds): raise NotImplementedError("Wrapper for slot " + NAME) + wrapper = func_with_new_name(wrapper, WRAPPER) wrapper1 = None wrapper2 = wrapper else: if wrapper is Ellipsis: def wrapper(space, w_self, w_args, func): raise NotImplementedError("Wrapper for slot " + NAME) + wrapper = func_with_new_name(wrapper, WRAPPER) wrapper1 = wrapper wrapper2 = None return (NAME, slotname, function, wrapper1, wrapper2, DOC) From afa at codespeak.net Fri May 28 16:09:32 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 28 May 2010 16:09:32 +0200 (CEST) Subject: [pypy-svn] r74853 - pypy/trunk/pypy/module/cpyext Message-ID: <20100528140932.3956C282BD6@codespeak.net> Author: afa Date: Fri May 28 16:09:30 2010 New Revision: 74853 Modified: pypy/trunk/pypy/module/cpyext/pyobject.py Log: Fix translation Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Fri May 28 16:09:30 2010 @@ -274,6 +274,7 @@ typedescr = get_typedescr(w_obj.typedef) py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) if w_type.is_cpytype(): + state = space.fromcache(RefcountState) state.set_lifeline(w_obj, py_obj) typedescr.attach(space, py_obj, w_obj) return py_obj From afa at codespeak.net Fri May 28 16:17:22 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 28 May 2010 16:17:22 +0200 (CEST) Subject: [pypy-svn] r74854 - pypy/trunk/pypy/module/cpyext Message-ID: <20100528141722.5543F282BD6@codespeak.net> Author: afa Date: Fri May 28 16:17:20 2010 New Revision: 74854 Modified: pypy/trunk/pypy/module/cpyext/pyobject.py Log: Another translation fix Modified: pypy/trunk/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyobject.py (original) +++ pypy/trunk/pypy/module/cpyext/pyobject.py Fri May 28 16:17:20 2010 @@ -174,7 +174,7 @@ assert py_obj.c_ob_refcnt == 0 return py_obj else: - lltype.nullptr(PyObject.TO) + return lltype.nullptr(PyObject.TO) def set_lifeline(self, w_obj, py_obj): self.lifeline_dict.set(w_obj, From jcreigh at codespeak.net Fri May 28 16:20:34 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Fri, 28 May 2010 16:20:34 +0200 (CEST) Subject: [pypy-svn] r74855 - in pypy/branch/x86-64-jit-backend: . dotviewer lib-python py py/_cmdline py/_code py/_io py/_plugin py/_test pypy pypy/config pypy/doc pypy/interpreter pypy/interpreter/test pypy/jit/metainterp pypy/jit/metainterp/test pypy/lib pypy/lib/app_test pypy/lib/test2 pypy/module/__builtin__ pypy/module/cpyext pypy/module/cpyext/include pypy/module/cpyext/test pypy/module/imp pypy/module/posix pypy/module/posix/test pypy/module/pyexpat pypy/module/pyexpat/test pypy/module/sys pypy/module/sys/test pypy/objspace/flow pypy/objspace/flow/test pypy/objspace/std pypy/objspace/std/test pypy/rlib pypy/rlib/test pypy/rpython pypy/rpython/lltypesystem pypy/rpython/lltypesystem/test pypy/rpython/module pypy/rpython/module/test pypy/rpython/test pypy/tool/pytest pypy/tool/pytest/test pypy/translator/goal pypy/translator/platform pypy/translator/tool Message-ID: <20100528142034.33423282BD6@codespeak.net> Author: jcreigh Date: Fri May 28 16:20:28 2010 New Revision: 74855 Added: pypy/branch/x86-64-jit-backend/pypy/lib/app_test/test_pickle_extra.py - copied unchanged from r74853, pypy/trunk/pypy/lib/app_test/test_pickle_extra.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_iterator.py - copied unchanged from r74853, pypy/trunk/pypy/module/cpyext/test/test_iterator.py pypy/branch/x86-64-jit-backend/pypy/module/pyexpat/test/test_parser.py - copied unchanged from r74853, pypy/trunk/pypy/module/pyexpat/test/test_parser.py pypy/branch/x86-64-jit-backend/pypy/translator/goal/targetosreadbench.py - copied unchanged from r74853, pypy/trunk/pypy/translator/goal/targetosreadbench.py Removed: pypy/branch/x86-64-jit-backend/py/__init__.py.orig pypy/branch/x86-64-jit-backend/py/_plugin/pytest_pytester.py.orig pypy/branch/x86-64-jit-backend/py/_plugin/pytest_terminal.py.orig Modified: pypy/branch/x86-64-jit-backend/ (props changed) pypy/branch/x86-64-jit-backend/dotviewer/graphparse.py pypy/branch/x86-64-jit-backend/lib-python/conftest.py pypy/branch/x86-64-jit-backend/py/__init__.py pypy/branch/x86-64-jit-backend/py/_builtin.py pypy/branch/x86-64-jit-backend/py/_cmdline/pytest.py pypy/branch/x86-64-jit-backend/py/_code/code.py pypy/branch/x86-64-jit-backend/py/_code/source.py pypy/branch/x86-64-jit-backend/py/_io/capture.py pypy/branch/x86-64-jit-backend/py/_plugin/pytest__pytest.py pypy/branch/x86-64-jit-backend/py/_plugin/pytest_capture.py pypy/branch/x86-64-jit-backend/py/_plugin/pytest_default.py pypy/branch/x86-64-jit-backend/py/_plugin/pytest_genscript.py pypy/branch/x86-64-jit-backend/py/_plugin/pytest_helpconfig.py pypy/branch/x86-64-jit-backend/py/_plugin/pytest_junitxml.py pypy/branch/x86-64-jit-backend/py/_plugin/pytest_mark.py pypy/branch/x86-64-jit-backend/py/_plugin/pytest_pytester.py pypy/branch/x86-64-jit-backend/py/_plugin/pytest_resultlog.py pypy/branch/x86-64-jit-backend/py/_plugin/pytest_runner.py pypy/branch/x86-64-jit-backend/py/_plugin/pytest_skipping.py pypy/branch/x86-64-jit-backend/py/_plugin/pytest_terminal.py pypy/branch/x86-64-jit-backend/py/_test/cmdline.py pypy/branch/x86-64-jit-backend/py/_test/collect.py pypy/branch/x86-64-jit-backend/py/_test/pycollect.py pypy/branch/x86-64-jit-backend/py/_test/session.py pypy/branch/x86-64-jit-backend/pypy/config/pypyoption.py pypy/branch/x86-64-jit-backend/pypy/config/translationoption.py pypy/branch/x86-64-jit-backend/pypy/conftest.py pypy/branch/x86-64-jit-backend/pypy/doc/coding-guide.txt pypy/branch/x86-64-jit-backend/pypy/doc/faq.txt pypy/branch/x86-64-jit-backend/pypy/interpreter/gateway.py pypy/branch/x86-64-jit-backend/pypy/interpreter/module.py pypy/branch/x86-64-jit-backend/pypy/interpreter/test/test_code.py pypy/branch/x86-64-jit-backend/pypy/interpreter/test/test_zzpickle_and_slow.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/codewriter.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/compile.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/optimizeopt.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/pyjitpl.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_basic.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_codewriter.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_pyjitpl.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_recursive.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_warmspot.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/warmspot.py pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/warmstate.py pypy/branch/x86-64-jit-backend/pypy/lib/_locale.py pypy/branch/x86-64-jit-backend/pypy/lib/stackless.py pypy/branch/x86-64-jit-backend/pypy/lib/test2/test_stackless.py pypy/branch/x86-64-jit-backend/pypy/module/__builtin__/functional.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/api.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/dictobject.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/eval.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/funcobject.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/include/Python.h pypy/branch/x86-64-jit-backend/pypy/module/cpyext/include/funcobject.h pypy/branch/x86-64-jit-backend/pypy/module/cpyext/include/methodobject.h pypy/branch/x86-64-jit-backend/pypy/module/cpyext/include/pymem.h pypy/branch/x86-64-jit-backend/pypy/module/cpyext/include/pyport.h pypy/branch/x86-64-jit-backend/pypy/module/cpyext/include/sliceobject.h pypy/branch/x86-64-jit-backend/pypy/module/cpyext/include/unicodeobject.h pypy/branch/x86-64-jit-backend/pypy/module/cpyext/iterator.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/longobject.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/methodobject.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/modsupport.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/object.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/pyerrors.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/pyobject.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/sequence.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/sliceobject.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/slotdefs.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/state.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/stringobject.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/stubs.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/stubsactive.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/sysmodule.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/conftest.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/foo.c pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_api.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_bufferobject.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_cpyext.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_dictobject.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_eval.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_funcobject.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_longobject.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_object.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_pyerrors.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_sequence.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_sliceobject.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_stringobject.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_typeobject.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/typeobject.py pypy/branch/x86-64-jit-backend/pypy/module/cpyext/typeobjectdefs.py pypy/branch/x86-64-jit-backend/pypy/module/imp/importing.py pypy/branch/x86-64-jit-backend/pypy/module/posix/__init__.py pypy/branch/x86-64-jit-backend/pypy/module/posix/app_posix.py pypy/branch/x86-64-jit-backend/pypy/module/posix/test/test_posix2.py pypy/branch/x86-64-jit-backend/pypy/module/pyexpat/interp_pyexpat.py pypy/branch/x86-64-jit-backend/pypy/module/sys/__init__.py pypy/branch/x86-64-jit-backend/pypy/module/sys/test/test_sysmodule.py pypy/branch/x86-64-jit-backend/pypy/module/sys/version.py pypy/branch/x86-64-jit-backend/pypy/module/sys/vm.py pypy/branch/x86-64-jit-backend/pypy/objspace/flow/flowcontext.py pypy/branch/x86-64-jit-backend/pypy/objspace/flow/test/test_unroll.py pypy/branch/x86-64-jit-backend/pypy/objspace/std/floattype.py pypy/branch/x86-64-jit-backend/pypy/objspace/std/strutil.py pypy/branch/x86-64-jit-backend/pypy/objspace/std/test/test_floatobject.py pypy/branch/x86-64-jit-backend/pypy/objspace/std/test/test_listobject.py pypy/branch/x86-64-jit-backend/pypy/objspace/std/test/test_strutil.py pypy/branch/x86-64-jit-backend/pypy/objspace/std/typeobject.py pypy/branch/x86-64-jit-backend/pypy/rlib/jit.py pypy/branch/x86-64-jit-backend/pypy/rlib/rlocale.py pypy/branch/x86-64-jit-backend/pypy/rlib/rweakref.py pypy/branch/x86-64-jit-backend/pypy/rlib/rwin32.py pypy/branch/x86-64-jit-backend/pypy/rlib/test/test_jit.py pypy/branch/x86-64-jit-backend/pypy/rpython/extfunc.py pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/ll2ctypes.py pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/rffi.py pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/test/test_ll2ctypes.py pypy/branch/x86-64-jit-backend/pypy/rpython/module/ll_os.py pypy/branch/x86-64-jit-backend/pypy/rpython/module/ll_os_stat.py pypy/branch/x86-64-jit-backend/pypy/rpython/module/test/test_ll_os_stat.py pypy/branch/x86-64-jit-backend/pypy/rpython/test/test_extfuncregister.py pypy/branch/x86-64-jit-backend/pypy/tool/pytest/appsupport.py pypy/branch/x86-64-jit-backend/pypy/tool/pytest/test/test_appsupport.py pypy/branch/x86-64-jit-backend/pypy/translator/platform/__init__.py pypy/branch/x86-64-jit-backend/pypy/translator/platform/darwin.py pypy/branch/x86-64-jit-backend/pypy/translator/platform/freebsd7.py pypy/branch/x86-64-jit-backend/pypy/translator/tool/cbuild.py Log: merge changes from trunk through r74853 to x86-64-jit-backend Modified: pypy/branch/x86-64-jit-backend/dotviewer/graphparse.py ============================================================================== --- pypy/branch/x86-64-jit-backend/dotviewer/graphparse.py (original) +++ pypy/branch/x86-64-jit-backend/dotviewer/graphparse.py Fri May 28 16:20:28 2010 @@ -2,7 +2,9 @@ Graph file parsing. """ -import os, sys, re +import sys, re +import subprocess + import msgstruct re_nonword = re.compile(r'([^0-9a-zA-Z_.]+)') @@ -45,7 +47,9 @@ else: cmdline = 'neato -Tplain' #print >> sys.stderr, '* running:', cmdline - child_in, child_out = os.popen2(cmdline, 'b') + p = subprocess.Popen(cmdline, shell=True, close_fds=True, + stdin=subprocess.PIPE, stdout=subprocess.PIPE) + (child_in, child_out) = (p.stdin, p.stdout) try: import thread except ImportError: Modified: pypy/branch/x86-64-jit-backend/lib-python/conftest.py ============================================================================== --- pypy/branch/x86-64-jit-backend/lib-python/conftest.py (original) +++ pypy/branch/x86-64-jit-backend/lib-python/conftest.py Fri May 28 16:20:28 2010 @@ -14,7 +14,6 @@ # the following adds command line options as a side effect! from pypy.conftest import gettestobjspace, option as pypy_option -from test import pystone from pypy.tool.pytest import appsupport from pypy.tool.pytest.confpath import pypydir, libpythondir, \ @@ -40,6 +39,7 @@ option = py.test.config.option def gettimeout(): + from test import pystone timeout = option.timeout.lower() if timeout.endswith('mp'): megapystone = float(timeout[:-2]) Modified: pypy/branch/x86-64-jit-backend/py/__init__.py ============================================================================== --- pypy/branch/x86-64-jit-backend/py/__init__.py (original) +++ pypy/branch/x86-64-jit-backend/py/__init__.py Fri May 28 16:20:28 2010 @@ -8,7 +8,7 @@ (c) Holger Krekel and others, 2004-2010 """ -__version__ = version = "1.3.0" +__version__ = version = "1.3.1" import py.apipkg @@ -25,7 +25,6 @@ 'pytest': '._cmdline.pytest:main', 'pylookup': '._cmdline.pylookup:main', 'pycountloc': '._cmdline.pycountlog:main', - 'pytest': '._test.cmdline:main', 'pylookup': '._cmdline.pylookup:main', 'pycountloc': '._cmdline.pycountloc:main', 'pycleanup': '._cmdline.pycleanup:main', Modified: pypy/branch/x86-64-jit-backend/py/_builtin.py ============================================================================== --- pypy/branch/x86-64-jit-backend/py/_builtin.py (original) +++ pypy/branch/x86-64-jit-backend/py/_builtin.py Fri May 28 16:20:28 2010 @@ -151,7 +151,10 @@ return getattr(function, "__dict__", None) def _getcode(function): - return getattr(function, "func_code", None) + try: + return getattr(function, "__code__") + except AttributeError: + return getattr(function, "func_code", None) def print_(*args, **kwargs): """ minimal backport of py3k print statement. """ @@ -175,6 +178,7 @@ def exec_(obj, globals=None, locals=None): """ minimal backport of py3k exec statement. """ + __tracebackhide__ = True if globals is None: frame = sys._getframe(1) globals = frame.f_globals @@ -187,14 +191,17 @@ if sys.version_info >= (3,0): exec (""" def _reraise(cls, val, tb): + __tracebackhide__ = True assert hasattr(val, '__traceback__') raise val """) else: exec (""" def _reraise(cls, val, tb): + __tracebackhide__ = True raise cls, val, tb def exec2(obj, globals, locals): + __tracebackhide__ = True exec obj in globals, locals """) Modified: pypy/branch/x86-64-jit-backend/py/_cmdline/pytest.py ============================================================================== --- pypy/branch/x86-64-jit-backend/py/_cmdline/pytest.py (original) +++ pypy/branch/x86-64-jit-backend/py/_cmdline/pytest.py Fri May 28 16:20:28 2010 @@ -1,5 +1,5 @@ #!/usr/bin/env python import py -def main(args): - py.test.cmdline.main(args) +def main(args=None): + raise SystemExit(py.test.cmdline.main(args)) Modified: pypy/branch/x86-64-jit-backend/py/_code/code.py ============================================================================== --- pypy/branch/x86-64-jit-backend/py/_code/code.py (original) +++ pypy/branch/x86-64-jit-backend/py/_code/code.py Fri May 28 16:20:28 2010 @@ -23,64 +23,14 @@ def __ne__(self, other): return not self == other - def new(self, rec=False, **kwargs): - """ return new code object with modified attributes. - if rec-cursive is true then dive into code - objects contained in co_consts. - """ - if sys.platform.startswith("java"): - # XXX jython does not support the below co_filename hack - return self.raw - names = [x for x in dir(self.raw) if x[:3] == 'co_'] - for name in kwargs: - if name not in names: - raise TypeError("unknown code attribute: %r" %(name, )) - if rec and hasattr(self.raw, 'co_consts'): # jython - newconstlist = [] - co = self.raw - cotype = type(co) - for c in co.co_consts: - if isinstance(c, cotype): - c = self.__class__(c).new(rec=True, **kwargs) - newconstlist.append(c) - return self.new(rec=False, co_consts=tuple(newconstlist), **kwargs) - for name in names: - if name not in kwargs: - kwargs[name] = getattr(self.raw, name) - arglist = [ - kwargs['co_argcount'], - kwargs['co_nlocals'], - kwargs.get('co_stacksize', 0), # jython - kwargs.get('co_flags', 0), # jython - kwargs.get('co_code', ''), # jython - kwargs.get('co_consts', ()), # jython - kwargs.get('co_names', []), # - kwargs['co_varnames'], - kwargs['co_filename'], - kwargs['co_name'], - kwargs['co_firstlineno'], - kwargs.get('co_lnotab', ''), #jython - kwargs.get('co_freevars', None), #jython - kwargs.get('co_cellvars', None), # jython - ] - if sys.version_info >= (3,0): - arglist.insert(1, kwargs['co_kwonlyargcount']) - return self.raw.__class__(*arglist) - else: - return py.std.new.code(*arglist) - def path(self): """ return a path object pointing to source code""" - fn = self.raw.co_filename - try: - return fn.__path__ - except AttributeError: - p = py.path.local(self.raw.co_filename) - if not p.check(): - # XXX maybe try harder like the weird logic - # in the standard lib [linecache.updatecache] does? - p = self.raw.co_filename - return p + p = py.path.local(self.raw.co_filename) + if not p.check(): + # XXX maybe try harder like the weird logic + # in the standard lib [linecache.updatecache] does? + p = self.raw.co_filename + return p path = property(path, None, None, "path of this code object") @@ -466,7 +416,7 @@ args.append((argname, self._saferepr(argvalue))) return ReprFuncArgs(args) - def get_source(self, source, line_index=-1, excinfo=None): + def get_source(self, source, line_index=-1, excinfo=None, short=False): """ return formatted and marked up source lines. """ lines = [] if source is None: @@ -478,6 +428,8 @@ if i == line_index: prefix = self.flow_marker + " " else: + if short: + continue prefix = " " line = prefix + source[i] lines.append(line) @@ -532,24 +484,26 @@ line_index = entry.lineno - max(entry.getfirstlinesource(), 0) lines = [] - if self.style == "long": - reprargs = self.repr_args(entry) - lines.extend(self.get_source(source, line_index, excinfo)) - message = excinfo and excinfo.typename or "" + if self.style in ("short", "long"): + short = self.style == "short" + reprargs = None + if not short: + reprargs = self.repr_args(entry) + s = self.get_source(source, line_index, excinfo, short=short) + lines.extend(s) + if short: + message = "in %s" %(entry.name) + else: + message = excinfo and excinfo.typename or "" path = self._makepath(entry.path) filelocrepr = ReprFileLocation(path, entry.lineno+1, message) - localsrepr = self.repr_locals(entry.locals) - return ReprEntry(lines, reprargs, localsrepr, filelocrepr) - else: - if self.style == "short": - line = source[line_index].lstrip() - basename = os.path.basename(entry.frame.code.filename) - lines.append(' File "%s", line %d, in %s' % ( - basename, entry.lineno+1, entry.name)) - lines.append(" " + line) - if excinfo: - lines.extend(self.get_exconly(excinfo, indent=4)) - return ReprEntry(lines, None, None, None) + localsrepr = None + if not short: + localsrepr = self.repr_locals(entry.locals) + return ReprEntry(lines, reprargs, localsrepr, filelocrepr, short) + if excinfo: + lines.extend(self.get_exconly(excinfo, indent=4)) + return ReprEntry(lines, None, None, None, False) def _makepath(self, path): if not self.abspath: @@ -645,13 +599,21 @@ class ReprEntry(TerminalRepr): localssep = "_ " - def __init__(self, lines, reprfuncargs, reprlocals, filelocrepr): + def __init__(self, lines, reprfuncargs, reprlocals, filelocrepr, short): self.lines = lines self.reprfuncargs = reprfuncargs self.reprlocals = reprlocals self.reprfileloc = filelocrepr + self.short = short def toterminal(self, tw): + if self.short: + self.reprfileloc.toterminal(tw) + for line in self.lines: + red = line.startswith("E ") + tw.line(line, bold=True, red=red) + #tw.line("") + return if self.reprfuncargs: self.reprfuncargs.toterminal(tw) for line in self.lines: Modified: pypy/branch/x86-64-jit-backend/py/_code/source.py ============================================================================== --- pypy/branch/x86-64-jit-backend/py/_code/source.py (original) +++ pypy/branch/x86-64-jit-backend/py/_code/source.py Fri May 28 16:20:28 2010 @@ -2,6 +2,7 @@ import sys import inspect, tokenize import py +from types import ModuleType cpy_compile = compile try: @@ -25,6 +26,8 @@ partlines = [] if isinstance(part, Source): partlines = part.lines + elif isinstance(part, (tuple, list)): + partlines = [x.rstrip("\n") for x in part] elif isinstance(part, py.builtin._basestring): partlines = part.split('\n') if rstrip: @@ -171,7 +174,9 @@ try: #compile(source+'\n', "x", "exec") syntax_checker(source+'\n') - except SyntaxError: + except KeyboardInterrupt: + raise + except Exception: return False else: return True @@ -212,10 +217,17 @@ else: if flag & _AST_FLAG: return co - co_filename = MyStr(filename) - co_filename.__source__ = self - return py.code.Code(co).new(rec=1, co_filename=co_filename) - #return newcode_withfilename(co, co_filename) + lines = [(x + "\n") for x in self.lines] + if sys.version_info[0] >= 3: + # XXX py3's inspect.getsourcefile() checks for a module + # and a pep302 __loader__ ... we don't have a module + # at code compile-time so we need to fake it here + m = ModuleType("_pycodecompile_pseudo_module") + py.std.inspect.modulesbyfile[filename] = None + py.std.sys.modules[None] = m + m.__loader__ = 1 + py.std.linecache.cache[filename] = (1, None, lines, filename) + return co # # public API shortcut functions @@ -224,11 +236,9 @@ def compile_(source, filename=None, mode='exec', flags= generators.compiler_flag, dont_inherit=0): """ compile the given source to a raw code object, - which points back to the source code through - "co_filename.__source__". All code objects - contained in the code object will recursively - also have this special subclass-of-string - filename. + and maintain an internal cache which allows later + retrieval of the source code for the code object + and any recursively created code objects. """ if _ast is not None and isinstance(source, _ast.AST): # XXX should Source support having AST? @@ -262,44 +272,26 @@ # # helper functions # -class MyStr(str): - """ custom string which allows to add attributes. """ def findsource(obj): - obj = py.code.getrawcode(obj) try: - fullsource = obj.co_filename.__source__ - except AttributeError: - try: - sourcelines, lineno = py.std.inspect.findsource(obj) - except (KeyboardInterrupt, SystemExit): - raise - except: - return None, None - source = Source() - source.lines = [line.rstrip() for line in sourcelines] - return source, lineno - else: - lineno = obj.co_firstlineno - 1 - return fullsource, lineno - + sourcelines, lineno = py.std.inspect.findsource(obj) + except (KeyboardInterrupt, SystemExit): + raise + except: + return None, None + source = Source() + source.lines = [line.rstrip() for line in sourcelines] + return source, lineno def getsource(obj, **kwargs): obj = py.code.getrawcode(obj) try: - fullsource = obj.co_filename.__source__ - except AttributeError: - try: - strsrc = inspect.getsource(obj) - except IndentationError: - strsrc = "\"Buggy python version consider upgrading, cannot get source\"" - assert isinstance(strsrc, str) - return Source(strsrc, **kwargs) - else: - lineno = obj.co_firstlineno - 1 - end = fullsource.getblockend(lineno) - return Source(fullsource[lineno:end+1], deident=True) - + strsrc = inspect.getsource(obj) + except IndentationError: + strsrc = "\"Buggy python version consider upgrading, cannot get source\"" + assert isinstance(strsrc, str) + return Source(strsrc, **kwargs) def deindent(lines, offset=None): if offset is None: Modified: pypy/branch/x86-64-jit-backend/py/_io/capture.py ============================================================================== --- pypy/branch/x86-64-jit-backend/py/_io/capture.py (original) +++ pypy/branch/x86-64-jit-backend/py/_io/capture.py Fri May 28 16:20:28 2010 @@ -26,46 +26,56 @@ raise TypeError("not a byte value: %r" %(data,)) StringIO.write(self, data) +patchsysdict = {0: 'stdin', 1: 'stdout', 2: 'stderr'} + class FDCapture: """ Capture IO to/from a given os-level filedescriptor. """ - def __init__(self, targetfd, tmpfile=None): + def __init__(self, targetfd, tmpfile=None, now=True, patchsys=False): """ save targetfd descriptor, and open a new temporary file there. If no tmpfile is specified a tempfile.Tempfile() will be opened in text mode. """ self.targetfd = targetfd - if tmpfile is None: + if tmpfile is None and targetfd != 0: f = tempfile.TemporaryFile('wb+') tmpfile = dupfile(f, encoding="UTF-8") f.close() self.tmpfile = tmpfile - self._savefd = os.dup(targetfd) - os.dup2(self.tmpfile.fileno(), targetfd) - self._patched = [] - - def setasfile(self, name, module=sys): - """ patch . to self.tmpfile - """ - key = (module, name) - self._patched.append((key, getattr(module, name))) - setattr(module, name, self.tmpfile) + self._savefd = os.dup(self.targetfd) + if patchsys: + self._oldsys = getattr(sys, patchsysdict[targetfd]) + if now: + self.start() - def unsetfiles(self): - """ unpatch all patched items - """ - while self._patched: - (module, name), value = self._patched.pop() - setattr(module, name, value) + def start(self): + try: + os.fstat(self._savefd) + except OSError: + raise ValueError("saved filedescriptor not valid, " + "did you call start() twice?") + if self.targetfd == 0 and not self.tmpfile: + fd = os.open(devnullpath, os.O_RDONLY) + os.dup2(fd, 0) + os.close(fd) + if hasattr(self, '_oldsys'): + setattr(sys, patchsysdict[self.targetfd], DontReadFromInput()) + else: + fd = self.tmpfile.fileno() + os.dup2(self.tmpfile.fileno(), self.targetfd) + if hasattr(self, '_oldsys'): + setattr(sys, patchsysdict[self.targetfd], self.tmpfile) def done(self): """ unpatch and clean up, returns the self.tmpfile (file object) """ os.dup2(self._savefd, self.targetfd) - self.unsetfiles() os.close(self._savefd) - self.tmpfile.seek(0) + if self.targetfd != 0: + self.tmpfile.seek(0) + if hasattr(self, '_oldsys'): + setattr(sys, patchsysdict[self.targetfd], self._oldsys) return self.tmpfile def writeorg(self, data): @@ -100,7 +110,7 @@ if encoding is not None: mode = mode.replace("b", "") buffering = True - return os.fdopen(newfd, mode, buffering, encoding, closefd=False) + return os.fdopen(newfd, mode, buffering, encoding, closefd=True) else: f = os.fdopen(newfd, mode, buffering) if encoding is not None: @@ -146,90 +156,94 @@ def reset(self): """ reset sys.stdout/stderr and return captured output as strings. """ - if hasattr(self, '_suspended'): - outfile = self._kwargs['out'] - errfile = self._kwargs['err'] - del self._kwargs - else: - outfile, errfile = self.done() + outfile, errfile = self.done() out, err = "", "" - if outfile: + if outfile and not outfile.closed: out = outfile.read() outfile.close() - if errfile and errfile != outfile: + if errfile and errfile != outfile and not errfile.closed: err = errfile.read() errfile.close() return out, err def suspend(self): """ return current snapshot captures, memorize tempfiles. """ - assert not hasattr(self, '_suspended') - self._suspended = True outerr = self.readouterr() outfile, errfile = self.done() - self._kwargs['out'] = outfile - self._kwargs['err'] = errfile return outerr - def resume(self): - """ resume capturing with original temp files. """ - assert self._suspended - self._initialize(**self._kwargs) - del self._suspended - class StdCaptureFD(Capture): """ This class allows to capture writes to FD1 and FD2 and may connect a NULL file to FD0 (and prevent - reads from sys.stdin) + reads from sys.stdin). If any of the 0,1,2 file descriptors + is invalid it will not be captured. """ - def __init__(self, out=True, err=True, - mixed=False, in_=True, patchsys=True): - self._kwargs = locals().copy() - del self._kwargs['self'] - self._initialize(**self._kwargs) - - def _initialize(self, out=True, err=True, - mixed=False, in_=True, patchsys=True): + def __init__(self, out=True, err=True, mixed=False, + in_=True, patchsys=True, now=True): + self._options = locals() + self._save() + if now: + self.startall() + + def _save(self): + in_ = self._options['in_'] + out = self._options['out'] + err = self._options['err'] + mixed = self._options['mixed'] + patchsys = self._options['patchsys'] if in_: - self._oldin = (sys.stdin, os.dup(0)) - sys.stdin = DontReadFromInput() - fd = os.open(devnullpath, os.O_RDONLY) - os.dup2(fd, 0) - os.close(fd) - if out: + try: + self.in_ = FDCapture(0, tmpfile=None, now=False, + patchsys=patchsys) + except OSError: + pass + if out: tmpfile = None if hasattr(out, 'write'): tmpfile = out - self.out = py.io.FDCapture(1, tmpfile=tmpfile) - if patchsys: - self.out.setasfile('stdout') - if err: - if mixed and out: + try: + self.out = FDCapture(1, tmpfile=tmpfile, + now=False, patchsys=patchsys) + self._options['out'] = self.out.tmpfile + except OSError: + pass + if err: + if out and mixed: tmpfile = self.out.tmpfile elif hasattr(err, 'write'): tmpfile = err else: tmpfile = None - self.err = py.io.FDCapture(2, tmpfile=tmpfile) - if patchsys: - self.err.setasfile('stderr') + try: + self.err = FDCapture(2, tmpfile=tmpfile, + now=False, patchsys=patchsys) + self._options['err'] = self.err.tmpfile + except OSError: + pass + + def startall(self): + if hasattr(self, 'in_'): + self.in_.start() + if hasattr(self, 'out'): + self.out.start() + if hasattr(self, 'err'): + self.err.start() + + def resume(self): + """ resume capturing with original temp files. """ + self.startall() def done(self): """ return (outfile, errfile) and stop capturing. """ - if hasattr(self, 'out'): + outfile = errfile = None + if hasattr(self, 'out') and not self.out.tmpfile.closed: outfile = self.out.done() - else: - outfile = None - if hasattr(self, 'err'): + if hasattr(self, 'err') and not self.err.tmpfile.closed: errfile = self.err.done() - else: - errfile = None - if hasattr(self, '_oldin'): - oldsys, oldfd = self._oldin - os.dup2(oldfd, 0) - os.close(oldfd) - sys.stdin = oldsys + if hasattr(self, 'in_'): + tmpfile = self.in_.done() + self._save() return outfile, errfile def readouterr(self): @@ -252,69 +266,61 @@ modifies sys.stdout|stderr|stdin attributes and does not touch underlying File Descriptors (use StdCaptureFD for that). """ - def __init__(self, out=True, err=True, in_=True, mixed=False): - self._kwargs = locals().copy() - del self._kwargs['self'] - self._initialize(**self._kwargs) - - def _initialize(self, out, err, in_, mixed): - self._out = out - self._err = err - self._in = in_ - if out: - self._oldout = sys.stdout - if not hasattr(out, 'write'): - out = TextIO() - sys.stdout = self.out = out - if err: - self._olderr = sys.stderr - if out and mixed: - err = self.out + def __init__(self, out=True, err=True, in_=True, mixed=False, now=True): + self._oldout = sys.stdout + self._olderr = sys.stderr + self._oldin = sys.stdin + if out and not hasattr(out, 'file'): + out = TextIO() + self.out = out + if err: + if mixed: + err = out elif not hasattr(err, 'write'): err = TextIO() - sys.stderr = self.err = err - if in_: - self._oldin = sys.stdin - sys.stdin = self.newin = DontReadFromInput() + self.err = err + self.in_ = in_ + if now: + self.startall() + + def startall(self): + if self.out: + sys.stdout = self.out + if self.err: + sys.stderr = self.err + if self.in_: + sys.stdin = self.in_ = DontReadFromInput() def done(self): """ return (outfile, errfile) and stop capturing. """ - o,e = sys.stdout, sys.stderr - if self._out: - try: - sys.stdout = self._oldout - except AttributeError: - raise IOError("stdout capturing already reset") - del self._oldout + outfile = errfile = None + if self.out and not self.out.closed: + sys.stdout = self._oldout outfile = self.out outfile.seek(0) - else: - outfile = None - if self._err: - try: - sys.stderr = self._olderr - except AttributeError: - raise IOError("stderr capturing already reset") - del self._olderr + if self.err and not self.err.closed: + sys.stderr = self._olderr errfile = self.err errfile.seek(0) - else: - errfile = None - if self._in: + if self.in_: sys.stdin = self._oldin return outfile, errfile + def resume(self): + """ resume capturing with original temp files. """ + self.startall() + def readouterr(self): """ return snapshot value of stdout/stderr capturings. """ out = err = "" - if self._out: - out = sys.stdout.getvalue() - sys.stdout.truncate(0) - sys.stdout.seek(0) - if self._err: - err = sys.stderr.getvalue() - sys.stderr.truncate(0) - sys.stderr.seek(0) + if self.out: + out = self.out.getvalue() + self.out.truncate(0) + self.out.seek(0) + if self.err: + err = self.err.getvalue() + self.err.truncate(0) + self.err.seek(0) return out, err class DontReadFromInput: @@ -344,5 +350,3 @@ devnullpath = 'NUL' else: devnullpath = '/dev/null' - - Modified: pypy/branch/x86-64-jit-backend/py/_plugin/pytest__pytest.py ============================================================================== --- pypy/branch/x86-64-jit-backend/py/_plugin/pytest__pytest.py (original) +++ pypy/branch/x86-64-jit-backend/py/_plugin/pytest__pytest.py Fri May 28 16:20:28 2010 @@ -46,7 +46,8 @@ recorder = RecordCalls() self._recorders[hookspec] = recorder self._registry.register(recorder) - self.hook = HookRelay(hookspecs, registry=self._registry) + self.hook = HookRelay(hookspecs, registry=self._registry, + prefix="pytest_") def finish_recording(self): for recorder in self._recorders.values(): Modified: pypy/branch/x86-64-jit-backend/py/_plugin/pytest_capture.py ============================================================================== --- pypy/branch/x86-64-jit-backend/py/_plugin/pytest_capture.py (original) +++ pypy/branch/x86-64-jit-backend/py/_plugin/pytest_capture.py Fri May 28 16:20:28 2010 @@ -106,6 +106,14 @@ def pytest_configure(config): config.pluginmanager.register(CaptureManager(), 'capturemanager') +class NoCapture: + def startall(self): + pass + def resume(self): + pass + def suspend(self): + return "", "" + class CaptureManager: def __init__(self): self._method2capture = {} @@ -118,15 +126,17 @@ def _makestringio(self): return py.io.TextIO() - def _startcapture(self, method): + def _getcapture(self, method): if method == "fd": - return py.io.StdCaptureFD( + return py.io.StdCaptureFD(now=False, out=self._maketempfile(), err=self._maketempfile() ) elif method == "sys": - return py.io.StdCapture( + return py.io.StdCapture(now=False, out=self._makestringio(), err=self._makestringio() ) + elif method == "no": + return NoCapture() else: raise ValueError("unknown capturing method: %r" % method) @@ -152,27 +162,25 @@ if hasattr(self, '_capturing'): raise ValueError("cannot resume, already capturing with %r" % (self._capturing,)) - if method != "no": - cap = self._method2capture.get(method) - if cap is None: - cap = self._startcapture(method) - self._method2capture[method] = cap - else: - cap.resume() + cap = self._method2capture.get(method) self._capturing = method + if cap is None: + self._method2capture[method] = cap = self._getcapture(method) + cap.startall() + else: + cap.resume() def suspendcapture(self, item=None): self.deactivate_funcargs() if hasattr(self, '_capturing'): method = self._capturing - if method != "no": - cap = self._method2capture[method] + cap = self._method2capture.get(method) + if cap is not None: outerr = cap.suspend() - else: - outerr = "", "" del self._capturing if item: - outerr = (item.outerr[0] + outerr[0], item.outerr[1] + outerr[1]) + outerr = (item.outerr[0] + outerr[0], + item.outerr[1] + outerr[1]) return outerr return "", "" @@ -180,19 +188,17 @@ if not hasattr(pyfuncitem, 'funcargs'): return assert not hasattr(self, '_capturing_funcargs') - l = [] - for name, obj in pyfuncitem.funcargs.items(): - if name == 'capfd' and not hasattr(os, 'dup'): - py.test.skip("capfd funcarg needs os.dup") + self._capturing_funcargs = capturing_funcargs = [] + for name, capfuncarg in pyfuncitem.funcargs.items(): if name in ('capsys', 'capfd'): - obj._start() - l.append(obj) - if l: - self._capturing_funcargs = l + capturing_funcargs.append(capfuncarg) + capfuncarg._start() def deactivate_funcargs(self): - if hasattr(self, '_capturing_funcargs'): - for capfuncarg in self._capturing_funcargs: + capturing_funcargs = getattr(self, '_capturing_funcargs', None) + if capturing_funcargs is not None: + while capturing_funcargs: + capfuncarg = capturing_funcargs.pop() capfuncarg._finalize() del self._capturing_funcargs @@ -256,16 +262,19 @@ platform does not have ``os.dup`` (e.g. Jython) tests using this funcarg will automatically skip. """ + if not hasattr(os, 'dup'): + py.test.skip("capfd funcarg needs os.dup") return CaptureFuncarg(request, py.io.StdCaptureFD) class CaptureFuncarg: def __init__(self, request, captureclass): self._cclass = captureclass + self.capture = self._cclass(now=False) #request.addfinalizer(self._finalize) def _start(self): - self.capture = self._cclass() + self.capture.startall() def _finalize(self): if hasattr(self, 'capture'): @@ -276,6 +285,4 @@ return self.capture.readouterr() def close(self): - self.capture.reset() - del self.capture - + self._finalize() Modified: pypy/branch/x86-64-jit-backend/py/_plugin/pytest_default.py ============================================================================== --- pypy/branch/x86-64-jit-backend/py/_plugin/pytest_default.py (original) +++ pypy/branch/x86-64-jit-backend/py/_plugin/pytest_default.py Fri May 28 16:20:28 2010 @@ -62,9 +62,12 @@ def pytest_addoption(parser): group = parser.getgroup("general", "running and selection options") - group._addoption('-x', '--exitfirst', - action="store_true", dest="exitfirst", default=False, + group._addoption('-x', '--exitfirst', action="store_true", default=False, + dest="exitfirst", help="exit instantly on first error or failed test."), + group._addoption('--maxfail', metavar="num", + action="store", type="int", dest="maxfail", default=0, + help="exit after first num failures or errors.") group._addoption('-k', action="store", dest="keyword", default='', help="only run test items matching the given " @@ -89,6 +92,9 @@ def pytest_configure(config): setsession(config) + # compat + if config.getvalue("exitfirst"): + config.option.maxfail = 1 def setsession(config): val = config.getvalue Modified: pypy/branch/x86-64-jit-backend/py/_plugin/pytest_genscript.py ============================================================================== --- pypy/branch/x86-64-jit-backend/py/_plugin/pytest_genscript.py (original) +++ pypy/branch/x86-64-jit-backend/py/_plugin/pytest_genscript.py Fri May 28 16:20:28 2010 @@ -4,14 +4,7 @@ """ import os -import zlib -import base64 import sys -try: - import pickle -except Importerror: - import cPickle as pickle - def pytest_addoption(parser): group = parser.getgroup("debugconfig") group.addoption("--genscript", action="store", default=None, @@ -30,6 +23,13 @@ raise SystemExit(0) def main(pybasedir, outfile, infile): + import base64 + import zlib + try: + import pickle + except Importerror: + import cPickle as pickle + outfile = str(outfile) infile = str(infile) assert os.path.isabs(outfile) Modified: pypy/branch/x86-64-jit-backend/py/_plugin/pytest_helpconfig.py ============================================================================== --- pypy/branch/x86-64-jit-backend/py/_plugin/pytest_helpconfig.py (original) +++ pypy/branch/x86-64-jit-backend/py/_plugin/pytest_helpconfig.py Fri May 28 16:20:28 2010 @@ -45,7 +45,7 @@ options = [opt for opt in options if opt._long_opts] options.sort(key=lambda x: x._long_opts) for opt in options: - if not opt._long_opts: + if not opt._long_opts or not opt.dest: continue optstrings = list(opt._long_opts) # + list(opt._short_opts) optstrings = filter(None, optstrings) Modified: pypy/branch/x86-64-jit-backend/py/_plugin/pytest_junitxml.py ============================================================================== --- pypy/branch/x86-64-jit-backend/py/_plugin/pytest_junitxml.py (original) +++ pypy/branch/x86-64-jit-backend/py/_plugin/pytest_junitxml.py Fri May 28 16:20:28 2010 @@ -56,10 +56,15 @@ def append_failure(self, report): self._opentestcase(report) #msg = str(report.longrepr.reprtraceback.extraline) - self.appendlog('%s', - report.longrepr) + if "xfail" in report.keywords: + self.appendlog( + '') + self.skipped += 1 + else: + self.appendlog('%s', + report.longrepr) + self.failed += 1 self._closetestcase() - self.failed += 1 def _opentestcase_collectfailure(self, report): node = report.collector @@ -95,7 +100,12 @@ def append_skipped(self, report): self._opentestcase(report) - self.appendlog("") + if "xfail" in report.keywords: + self.appendlog( + '%s', + report.keywords['xfail']) + else: + self.appendlog("") self._closetestcase() self.skipped += 1 Modified: pypy/branch/x86-64-jit-backend/py/_plugin/pytest_mark.py ============================================================================== --- pypy/branch/x86-64-jit-backend/py/_plugin/pytest_mark.py (original) +++ pypy/branch/x86-64-jit-backend/py/_plugin/pytest_mark.py Fri May 28 16:20:28 2010 @@ -34,38 +34,45 @@ .. _`scoped-marking`: -Marking classes or modules +Marking whole classes or modules ---------------------------------------------------- -To mark all methods of a class set a ``pytestmark`` attribute like this:: +If you are programming with Python2.6 you may use ``py.test.mark`` decorators +with classes to apply markers to all its test methods:: + + @py.test.mark.webtest + class TestClass: + def test_startup(self): + ... + def test_startup_and_more(self): + ... + +This is equivalent to directly applying the decorator to the +two test functions. + +To remain compatible with Python2.5 you can also set a +``pytestmark`` attribute on a TestClass like this:: import py class TestClass: pytestmark = py.test.mark.webtest -You can re-use the same markers that you would use for decorating -a function - in fact this marker decorator will be applied -to all test methods of the class. +or if you need to use multiple markers you can use a list:: + + import py + + class TestClass: + pytestmark = [py.test.mark.webtest, pytest.mark.slowtest] You can also set a module level marker:: import py pytestmark = py.test.mark.webtest -in which case then the marker decorator will be applied to all functions and +in which case it will be applied to all functions and methods defined in the module. -The order in which marker functions are called is this:: - - per-function (upon import of module already) - per-class - per-module - -Later called markers may overwrite previous key-value settings. -Positional arguments are all appended to the same 'args' list -of the Marker object. - Using "-k MARKNAME" to select tests ---------------------------------------------------- @@ -105,15 +112,26 @@ """ if passed a single callable argument: decorate it with mark info. otherwise add *args/**kwargs in-place to mark information. """ if args: - if len(args) == 1 and hasattr(args[0], '__call__'): - func = args[0] - holder = getattr(func, self.markname, None) - if holder is None: - holder = MarkInfo(self.markname, self.args, self.kwargs) - setattr(func, self.markname, holder) + func = args[0] + if len(args) == 1 and hasattr(func, '__call__') or \ + hasattr(func, '__bases__'): + if hasattr(func, '__bases__'): + if hasattr(func, 'pytestmark'): + l = func.pytestmark + if not isinstance(l, list): + func.pytestmark = [l, self] + else: + l.append(self) + else: + func.pytestmark = [self] else: - holder.kwargs.update(self.kwargs) - holder.args.extend(self.args) + holder = getattr(func, self.markname, None) + if holder is None: + holder = MarkInfo(self.markname, self.args, self.kwargs) + setattr(func, self.markname, holder) + else: + holder.kwargs.update(self.kwargs) + holder.args.extend(self.args) return func else: self.args.extend(args) @@ -147,6 +165,10 @@ func = getattr(func, 'im_func', func) # py2 for parent in [x for x in (mod, cls) if x]: marker = getattr(parent.obj, 'pytestmark', None) - if isinstance(marker, MarkDecorator): - marker(func) + if marker is not None: + if not isinstance(marker, list): + marker = [marker] + for mark in marker: + if isinstance(mark, MarkDecorator): + mark(func) return item Modified: pypy/branch/x86-64-jit-backend/py/_plugin/pytest_pytester.py ============================================================================== --- pypy/branch/x86-64-jit-backend/py/_plugin/pytest_pytester.py (original) +++ pypy/branch/x86-64-jit-backend/py/_plugin/pytest_pytester.py Fri May 28 16:20:28 2010 @@ -110,7 +110,7 @@ def _makefile(self, ext, args, kwargs): items = list(kwargs.items()) if args: - source = "\n".join(map(str, args)) + source = "\n".join(map(str, args)) + "\n" basename = self.request.function.__name__ items.insert(0, (basename, source)) ret = None @@ -185,6 +185,7 @@ return reports[0] def inline_run(self, *args): + args = ("-s", ) + args # otherwise FD leakage config = self.parseconfig(*args) config.pluginmanager.do_configure(config) session = config.initsession() @@ -294,8 +295,10 @@ ret = popen.wait() f1.close() f2.close() - out = p1.read("rb").decode("utf-8").splitlines() - err = p2.read("rb").decode("utf-8").splitlines() + out = p1.read("rb") + out = getdecoded(out).splitlines() + err = p2.read("rb") + err = getdecoded(err).splitlines() def dump_lines(lines, fp): try: for line in lines: @@ -360,6 +363,13 @@ child.timeout = expect_timeout return child +def getdecoded(out): + try: + return out.decode("utf-8") + except UnicodeDecodeError: + return "INTERNAL not-utf8-decodeable, truncated string:\n%s" % ( + py.io.saferepr(out),) + class PseudoPlugin: def __init__(self, vars): self.__dict__.update(vars) Modified: pypy/branch/x86-64-jit-backend/py/_plugin/pytest_resultlog.py ============================================================================== --- pypy/branch/x86-64-jit-backend/py/_plugin/pytest_resultlog.py (original) +++ pypy/branch/x86-64-jit-backend/py/_plugin/pytest_resultlog.py Fri May 28 16:20:28 2010 @@ -73,7 +73,7 @@ code = report.shortrepr if code == 'x': longrepr = str(report.longrepr) - elif code == 'P': + elif code == 'X': longrepr = '' elif report.passed: longrepr = "" Modified: pypy/branch/x86-64-jit-backend/py/_plugin/pytest_runner.py ============================================================================== --- pypy/branch/x86-64-jit-backend/py/_plugin/pytest_runner.py (original) +++ pypy/branch/x86-64-jit-backend/py/_plugin/pytest_runner.py Fri May 28 16:20:28 2010 @@ -10,6 +10,7 @@ 'skip' : skip, 'importorskip' : importorskip, 'fail' : fail, + 'xfail' : xfail, 'exit' : exit, } @@ -187,7 +188,11 @@ self.passed = True self.result = result else: - self.longrepr = self.collector._repr_failure_py(excinfo) + style = "short" + if collector.config.getvalue("fulltrace"): + style = "long" + self.longrepr = self.collector._repr_failure_py(excinfo, + style=style) if excinfo.errisinstance(py.test.skip.Exception): self.skipped = True self.reason = str(excinfo.value) @@ -295,6 +300,10 @@ """ raised from an explicit call to py.test.fail() """ __module__ = 'builtins' +class XFailed(OutcomeException): + """ raised from an explicit call to py.test.xfail() """ + __module__ = 'builtins' + class ExceptionFailure(Failed): """ raised by py.test.raises on an exception-assertion mismatch. """ def __init__(self, expr, expected, msg=None, excinfo=None): @@ -335,6 +344,14 @@ fail.Exception = Failed +def xfail(reason=""): + """ xfail an executing test or setup functions, taking an optional + reason string. + """ + __tracebackhide__ = True + raise XFailed(reason) +xfail.Exception = XFailed + def raises(ExpectedException, *args, **kwargs): """ if args[0] is callable: raise AssertionError if calling it with the remaining arguments does not raise the expected exception. Modified: pypy/branch/x86-64-jit-backend/py/_plugin/pytest_skipping.py ============================================================================== --- pypy/branch/x86-64-jit-backend/py/_plugin/pytest_skipping.py (original) +++ pypy/branch/x86-64-jit-backend/py/_plugin/pytest_skipping.py Fri May 28 16:20:28 2010 @@ -60,6 +60,19 @@ # The ``pytestmark`` decorator will be applied to each test function. +If your code targets python2.6 or above you can equivalently use +the skipif decorator on classes:: + + @py.test.mark.skipif("sys.platform == 'win32'") + class TestPosixCalls: + + def test_function(self): + # will not be setup or run under 'win32' platform + # + +It is fine in general to apply multiple "skipif" decorators +on a single function - this means that if any of the conditions +apply the function will be skipped. .. _`whole class- or module level`: mark.html#scoped-marking @@ -82,10 +95,28 @@ depending on platform:: @py.test.mark.xfail("sys.version_info >= (3,0)") - def test_function(): ... +To not run a test and still regard it as "xfailed":: + + @py.test.mark.xfail(..., run=False) + +To specify an explicit reason to be shown with xfailure detail:: + + @py.test.mark.xfail(..., reason="my reason") + +imperative xfail from within a test or setup function +------------------------------------------------------ + +If you cannot declare xfail-conditions at import time +you can also imperatively produce an XFail-outcome from +within test or setup code. Example:: + + def test_function(): + if not valid_config(): + py.test.xfail("unsuppored configuration") + skipping on a missing import dependency -------------------------------------------------- @@ -115,11 +146,14 @@ py.test.skip("unsuppored configuration") """ -# XXX py.test.skip, .importorskip and the Skipped class -# should also be defined in this plugin, requires thought/changes import py +def pytest_addoption(parser): + group = parser.getgroup("general") + group.addoption('--runxfail', + action="store_true", dest="runxfail", default=False, + help="run tests even if they are marked xfail") class MarkEvaluator: def __init__(self, item, name): @@ -134,17 +168,20 @@ def istrue(self): if self.holder: d = {'os': py.std.os, 'sys': py.std.sys, 'config': self.item.config} - self.result = True - for expr in self.holder.args: - self.expr = expr - if isinstance(expr, str): - result = cached_eval(self.item.config, expr, d) - else: - result = expr - if not result: - self.result = False + if self.holder.args: + self.result = False + for expr in self.holder.args: self.expr = expr - break + if isinstance(expr, str): + result = cached_eval(self.item.config, expr, d) + else: + result = expr + if result: + self.result = True + self.expr = expr + break + else: + self.result = True return getattr(self, 'result', False) def get(self, attr, default=None): @@ -161,27 +198,54 @@ def pytest_runtest_setup(item): - expr, result = evalexpression(item, 'skipif') - if result: - py.test.skip(expr) + if not isinstance(item, py.test.collect.Function): + return + evalskip = MarkEvaluator(item, 'skipif') + if evalskip.istrue(): + py.test.skip(evalskip.getexplanation()) + item._evalxfail = MarkEvaluator(item, 'xfail') + if not item.config.getvalue("runxfail"): + if item._evalxfail.istrue(): + if not item._evalxfail.get('run', True): + py.test.skip("xfail") def pytest_runtest_makereport(__multicall__, item, call): - if call.when != "call": + if not isinstance(item, py.test.collect.Function): return - expr, result = evalexpression(item, 'xfail') - rep = __multicall__.execute() - if result: - if call.excinfo: + if not (call.excinfo and + call.excinfo.errisinstance(py.test.xfail.Exception)): + evalxfail = getattr(item, '_evalxfail', None) + if not evalxfail: + return + if call.excinfo and call.excinfo.errisinstance(py.test.xfail.Exception): + if not item.config.getvalue("runxfail"): + rep = __multicall__.execute() + rep.keywords['xfail'] = "reason: " + call.excinfo.value.msg rep.skipped = True - rep.failed = rep.passed = False + rep.failed = False + return rep + if call.when == "setup": + rep = __multicall__.execute() + if rep.skipped and evalxfail.istrue(): + expl = evalxfail.getexplanation() + if not evalxfail.get("run", True): + expl = "[NOTRUN] " + expl + rep.keywords['xfail'] = expl + return rep + elif call.when == "call": + rep = __multicall__.execute() + if not item.config.getvalue("runxfail") and evalxfail.istrue(): + if call.excinfo: + rep.skipped = True + rep.failed = rep.passed = False + else: + rep.skipped = rep.passed = False + rep.failed = True + rep.keywords['xfail'] = evalxfail.getexplanation() else: - rep.skipped = rep.passed = False - rep.failed = True - rep.keywords['xfail'] = expr - else: - if 'xfail' in rep.keywords: - del rep.keywords['xfail'] - return rep + if 'xfail' in rep.keywords: + del rep.keywords['xfail'] + return rep # called by terminalreporter progress reporting def pytest_report_teststatus(report): @@ -189,7 +253,7 @@ if report.skipped: return "xfailed", "x", "xfail" elif report.failed: - return "xpassed", "P", "xpass" + return "xpassed", "X", "XPASS" # called by the terminalreporter instance/plugin def pytest_terminal_summary(terminalreporter): @@ -229,13 +293,8 @@ xfailed = terminalreporter.stats.get("xfailed") if xfailed: for rep in xfailed: - entry = rep.longrepr.reprcrash - modpath = rep.item.getmodpath(includemodule=True) - pos = "%s %s:%d: " %(modpath, entry.path, entry.lineno) - reason = rep.longrepr.reprcrash.message - i = reason.find("\n") - if i != -1: - reason = reason[:i] + pos = terminalreporter.gettestid(rep.item) + reason = rep.keywords['xfail'] lines.append("XFAIL %s %s" %(pos, reason)) def show_xpassed(terminalreporter, lines): @@ -246,24 +305,6 @@ reason = rep.keywords['xfail'] lines.append("XPASS %s %s" %(pos, reason)) - -def evalexpression(item, keyword): - if isinstance(item, py.test.collect.Function): - markholder = getattr(item.obj, keyword, None) - result = False - if markholder: - d = {'os': py.std.os, 'sys': py.std.sys, 'config': item.config} - expr, result = None, True - for expr in markholder.args: - if isinstance(expr, str): - result = cached_eval(item.config, expr, d) - else: - result = expr - if not result: - break - return expr, result - return None, False - def cached_eval(config, expr, d): if not hasattr(config, '_evalcache'): config._evalcache = {} Modified: pypy/branch/x86-64-jit-backend/py/_plugin/pytest_terminal.py ============================================================================== --- pypy/branch/x86-64-jit-backend/py/_plugin/pytest_terminal.py (original) +++ pypy/branch/x86-64-jit-backend/py/_plugin/pytest_terminal.py Fri May 28 16:20:28 2010 @@ -141,6 +141,20 @@ else: return "???", dict(red=True) + def gettestid(self, item, relative=True): + fspath = item.fspath + chain = [x for x in item.listchain() if x.fspath == fspath] + chain = chain[1:] + names = [x.name for x in chain if x.name != "()"] + path = item.fspath + if relative: + relpath = path.relto(self.curdir) + if relpath: + path = relpath + names.insert(0, str(path)) + return "::".join(names) + + def pytest_internalerror(self, excrepr): for line in str(excrepr).split("\n"): self.write_line("INTERNALERROR> " + line) @@ -270,7 +284,7 @@ self._sessionstarttime = py.std.time.time() verinfo = ".".join(map(str, sys.version_info[:3])) - msg = "python: platform %s -- Python %s" % (sys.platform, verinfo) + msg = "platform %s -- Python %s" % (sys.platform, verinfo) msg += " -- pytest-%s" % (py.__version__) if self.config.option.verbose or self.config.option.debug or getattr(self.config.option, 'pastebin', None): msg += " -- " + str(sys.executable) @@ -298,12 +312,14 @@ self._keyboardinterrupt_memo = excinfo.getrepr(funcargs=True) def _report_keyboardinterrupt(self): - self.write_sep("!", "KEYBOARD INTERRUPT") excrepr = self._keyboardinterrupt_memo - if self.config.option.verbose: - excrepr.toterminal(self._tw) - else: - excrepr.reprcrash.toterminal(self._tw) + msg = excrepr.reprcrash.message + self.write_sep("!", msg) + if "KeyboardInterrupt" in msg: + if self.config.getvalue("fulltrace"): + excrepr.toterminal(self._tw) + else: + excrepr.reprcrash.toterminal(self._tw) def _getcrashline(self, report): try: Modified: pypy/branch/x86-64-jit-backend/py/_test/cmdline.py ============================================================================== --- pypy/branch/x86-64-jit-backend/py/_test/cmdline.py (original) +++ pypy/branch/x86-64-jit-backend/py/_test/cmdline.py Fri May 28 16:20:28 2010 @@ -16,8 +16,9 @@ colitems = config.getinitialnodes() exitstatus = session.main(colitems) config.pluginmanager.do_unconfigure(config) - raise SystemExit(exitstatus) except config.Error: e = sys.exc_info()[1] sys.stderr.write("ERROR: %s\n" %(e.args[0],)) - raise SystemExit(3) + exitstatus = 3 + py.test.config = py.test.config.__class__() + return exitstatus Modified: pypy/branch/x86-64-jit-backend/py/_test/collect.py ============================================================================== --- pypy/branch/x86-64-jit-backend/py/_test/collect.py (original) +++ pypy/branch/x86-64-jit-backend/py/_test/collect.py Fri May 28 16:20:28 2010 @@ -172,14 +172,15 @@ def _prunetraceback(self, traceback): return traceback - def _repr_failure_py(self, excinfo): + def _repr_failure_py(self, excinfo, style=None): excinfo.traceback = self._prunetraceback(excinfo.traceback) # XXX should excinfo.getrepr record all data and toterminal() # process it? - if self.config.option.tbstyle == "short": - style = "short" - else: - style = "long" + if style is None: + if self.config.option.tbstyle == "short": + style = "short" + else: + style = "long" return excinfo.getrepr(funcargs=True, showlocals=self.config.option.showlocals, style=style) Modified: pypy/branch/x86-64-jit-backend/py/_test/pycollect.py ============================================================================== --- pypy/branch/x86-64-jit-backend/py/_test/pycollect.py (original) +++ pypy/branch/x86-64-jit-backend/py/_test/pycollect.py Fri May 28 16:20:28 2010 @@ -253,7 +253,7 @@ traceback = ntraceback.filter() return traceback - def _repr_failure_py(self, excinfo): + def _repr_failure_py(self, excinfo, style="long"): if excinfo.errisinstance(funcargs.FuncargRequest.LookupError): fspath, lineno, msg = self.reportinfo() lines, _ = inspect.getsourcelines(self.obj) @@ -261,11 +261,13 @@ if line.strip().startswith('def'): return FuncargLookupErrorRepr(fspath, lineno, lines[:i+1], str(excinfo.value)) - return super(FunctionMixin, self)._repr_failure_py(excinfo) + return super(FunctionMixin, self)._repr_failure_py(excinfo, + style=style) def repr_failure(self, excinfo, outerr=None): assert outerr is None, "XXX outerr usage is deprecated" - return self._repr_failure_py(excinfo) + return self._repr_failure_py(excinfo, + style=self.config.getvalue("tbstyle")) shortfailurerepr = "F" @@ -393,5 +395,5 @@ def hasinit(obj): init = getattr(obj, '__init__', None) if init: - if not isinstance(init, type(object.__init__)): + if init != object.__init__: return True Modified: pypy/branch/x86-64-jit-backend/py/_test/session.py ============================================================================== --- pypy/branch/x86-64-jit-backend/py/_test/session.py (original) +++ pypy/branch/x86-64-jit-backend/py/_test/session.py Fri May 28 16:20:28 2010 @@ -20,11 +20,15 @@ class Session(object): nodeid = "" + class Interrupted(KeyboardInterrupt): + """ signals an interrupted test run. """ + __module__ = 'builtins' # for py3 + def __init__(self, config): self.config = config self.pluginmanager = config.pluginmanager # shortcut self.pluginmanager.register(self) - self._testsfailed = False + self._testsfailed = 0 self._nomatch = False self.shouldstop = False @@ -52,7 +56,7 @@ yield x self.config.hook.pytest_collectreport(report=rep) if self.shouldstop: - break + raise self.Interrupted(self.shouldstop) def filteritems(self, colitems): """ return items to process (some may be deselected)""" @@ -86,9 +90,11 @@ def pytest_runtest_logreport(self, report): if report.failed: - self._testsfailed = True - if self.config.option.exitfirst: - self.shouldstop = True + self._testsfailed += 1 + maxfail = self.config.getvalue("maxfail") + if maxfail and self._testsfailed >= maxfail: + self.shouldstop = "stopping after %d failures" % ( + self._testsfailed) pytest_collectreport = pytest_runtest_logreport def sessionfinishes(self, exitstatus): @@ -122,7 +128,8 @@ def _mainloop(self, colitems): for item in self.collect(colitems): - if self.shouldstop: - break if not self.config.option.collectonly: item.config.hook.pytest_runtest_protocol(item=item) + if self.shouldstop: + raise self.Interrupted(self.shouldstop) + Modified: pypy/branch/x86-64-jit-backend/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/config/pypyoption.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/config/pypyoption.py Fri May 28 16:20:28 2010 @@ -29,8 +29,7 @@ "rctime" , "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios", "zlib", "struct", "md5", "sha", "bz2", "_minimal_curses", "cStringIO", - "thread", "itertools", "pyexpat", "_ssl"] # "cpyext"] commented out until - # it stops adding _pyolifeline on W_Root + "thread", "itertools", "pyexpat", "_ssl", "cpyext"] )) working_oo_modules = default_modules.copy() Modified: pypy/branch/x86-64-jit-backend/pypy/config/translationoption.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/config/translationoption.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/config/translationoption.py Fri May 28 16:20:28 2010 @@ -23,7 +23,8 @@ "translation", "Translation Options", [ BoolOption("stackless", "enable stackless features during compilation", default=False, cmdline="--stackless", - requires=[("translation.type_system", "lltype")]), + requires=[("translation.type_system", "lltype"), + ("translation.gcremovetypeptr", False)]), # XXX? ChoiceOption("type_system", "Type system to use when RTyping", ["lltype", "ootype"], cmdline=None, default="lltype", requires={ @@ -375,6 +376,8 @@ config.translation.suggest(withsmallfuncsets=5) elif word == 'jit': config.translation.suggest(jit=True) + if config.translation.stackless: + raise NotImplementedError("JIT conflicts with stackless for now") elif word == 'removetypeptr': config.translation.suggest(gcremovetypeptr=True) else: Modified: pypy/branch/x86-64-jit-backend/pypy/conftest.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/conftest.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/conftest.py Fri May 28 16:20:28 2010 @@ -383,10 +383,15 @@ if option.runappdirect: return target() space = gettestobjspace() - func = app2interp_temp(target) + filename = self._getdynfilename(target) + func = app2interp_temp(target, filename=filename) print "executing", func self.execute_appex(space, func, space) + def _getdynfilename(self, func): + code = getattr(func, 'im_func', func).func_code + return "[%s:%s]" % (code.co_filename, code.co_firstlineno) + class AppTestMethod(AppTestFunction): def setup(self): @@ -410,7 +415,8 @@ if option.runappdirect: return target() space = target.im_self.space - func = app2interp_temp(target.im_func) + filename = self._getdynfilename(target) + func = app2interp_temp(target.im_func, filename=filename) w_instance = self.parent.w_instance self.execute_appex(space, func, space, w_instance) Modified: pypy/branch/x86-64-jit-backend/pypy/doc/coding-guide.txt ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/doc/coding-guide.txt (original) +++ pypy/branch/x86-64-jit-backend/pypy/doc/coding-guide.txt Fri May 28 16:20:28 2010 @@ -121,8 +121,8 @@ In any case, it should be obvious that the application-level implementation is definitely more readable, more elegant and more maintainable than the -interpreter-level one (and indeed, dict.update is really implemented at appleve -in PyPy). +interpreter-level one (and indeed, dict.update is really implemented at +applevel in PyPy). In fact, in almost all parts of PyPy, you find application level code in the middle of interpreter-level code. Apart from some bootstrapping Modified: pypy/branch/x86-64-jit-backend/pypy/doc/faq.txt ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/doc/faq.txt (original) +++ pypy/branch/x86-64-jit-backend/pypy/doc/faq.txt Fri May 28 16:20:28 2010 @@ -45,6 +45,11 @@ We list the differences we know about in `cpython_differences`_. +There is also an experimental support for CPython extension modules, so +they'll run without change (from current observation, rather with little +change) on trunk. It has not been released yet, although it should be a major +point of the next pypy release. + .. _`extension modules`: cpython_differences.html#extension-modules .. _`cpython_differences`: cpython_differences.html @@ -61,7 +66,9 @@ PyPy also basically works in a 64-bit Linux environment, but *it requires a 32-bit Intel CPU* for the JIT right now. (It works -fine in a 32-bit chroot of an Intel 64.) +fine in a 32-bit chroot of an Intel 64.). There is also an ongoing +summer of code project to provide 64bit support for JIT. It's scheduled +to be finished by end of summer 2010. ------------------------------------------------ Which Python version (2.x?) does PyPy implement? @@ -82,20 +89,11 @@ Note that PyPy also fully supports `stackless-like microthreads`_ (although both cannot be mixed yet). -As for other modules: The general rule of thumb is that pure-Python modules -work, C extension modules don't. However, many of the C extension modules -of the standard -library have been re-implemented in pure Python or as a mixed module (for some -there were also older pure-Python versions available). The list of supported -modules can be found as follows: - - * pure Python implementations: all the modules found in the `pypy/lib/`_ - directory - - * mixed module implementations: all the modules listed in - ``essential_modules``, ``default_modules`` and ``working_modules`` - in `pypy/config/pypyoption.py`_. - +All pure-python modules should work, unless they rely on ugly +cpython implementation details, in which case it's their fault. +There is an increasing number of compatible CPython extensions working, +including things like wxPython or PIL. This is an ongoing development effort +to bring as many CPython extension modules working as possible. .. _`stackless-like microthreads`: stackless.html @@ -104,16 +102,13 @@ Can I use CPython extension modules? ------------------------------------ -No and there are no short-term plans to support this. CPython extension modules -rely heavily on CPython's C API which contains a lot of implementation details -like reference counting, exact C-level object implementation and layout etc. - -A module based on ctypes is a different matter -- we support these nowadays. - -The long-term answer might be different. In principle, it should be possible -for PyPy to support the CPython C API (or at least a large subset of its -official part). It means that "in the fullness of time" you might be able to -simply recompile existing CPython extension modules and use them with PyPy. +Yes, but the feature is in alpha state and is available only on trunk +(not in the 1.2 release). However, we'll only ever support well-behaving +CPython extensions. Please consult PyPy developers on IRC or mailing list +for explanations if your favorite module works and how you can help to make +it happen in case it does not. + +We fully support ctypes-based extensions, however. ------------------------------------------ How do I write extension modules for PyPy? Modified: pypy/branch/x86-64-jit-backend/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/interpreter/gateway.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/interpreter/gateway.py Fri May 28 16:20:28 2010 @@ -821,9 +821,6 @@ # and now for something completely different ... # -class MyStr(str): - pass - class ApplevelClass: """NOT_RPYTHON A container for app-level source code that should be executed @@ -837,8 +834,7 @@ # HAAACK (but a good one) if filename is None: f = sys._getframe(1) - filename = MyStr('<%s:%d>' % (f.f_code.co_filename, f.f_lineno)) - filename.__source__ = py.code.Source(source) + filename = '<%s:%d>' % (f.f_code.co_filename, f.f_lineno) self.filename = filename self.source = str(py.code.Source(source).deindent()) self.modname = modname @@ -848,6 +844,9 @@ self.can_use_geninterp = False else: self.can_use_geninterp = True + # make source code available for tracebacks + lines = [x + "\n" for x in source.split("\n")] + py.std.linecache.cache[filename] = (1, None, lines, filename) def __repr__(self): return "" % (self.filename, self.can_use_geninterp) @@ -1081,14 +1080,14 @@ # ____________________________________________________________ -def appdef(source, applevel=ApplevelClass): +def appdef(source, applevel=ApplevelClass, filename=None): """ NOT_RPYTHON: build an app-level helper function, like for example: myfunc = appdef('''myfunc(x, y): return x+y ''') """ if not isinstance(source, str): - source = str(py.code.Source(source).strip()) + source = py.std.inspect.getsource(source).lstrip() while source.startswith('@py.test.mark.'): # these decorators are known to return the same function # object, we may ignore them @@ -1100,7 +1099,11 @@ assert p >= 0 funcname = source[:p].strip() source = source[p:] - return applevel("def %s%s\n" % (funcname, source)).interphook(funcname) + assert source.strip() + funcsource = "def %s%s\n" % (funcname, source) + #for debugging of wrong source code: py.std.parser.suite(funcsource) + a = applevel(funcsource, filename=filename) + return a.interphook(funcname) applevel = ApplevelClass # backward compatibility app2interp = appdef # backward compatibility @@ -1118,6 +1121,6 @@ return PyPyCacheDir.build_applevelinterp_dict(self, space) # app2interp_temp is used for testing mainly -def app2interp_temp(func, applevel_temp=applevel_temp): +def app2interp_temp(func, applevel_temp=applevel_temp, filename=None): """ NOT_RPYTHON """ - return appdef(func, applevel_temp) + return appdef(func, applevel_temp, filename=filename) Modified: pypy/branch/x86-64-jit-backend/pypy/interpreter/module.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/interpreter/module.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/interpreter/module.py Fri May 28 16:20:28 2010 @@ -73,7 +73,17 @@ ]) #already imported case w_import = space.builtin.get('__import__') - return space.newtuple([w_import, space.newtuple([w_name])]) + tup_return = [ + w_import, + space.newtuple([ + w_name, + space.w_None, + space.w_None, + space.newtuple([space.wrap('')]) + ]) + ] + + return space.newtuple(tup_return) def descr_module__repr__(self, space): from pypy.interpreter.mixedmodule import MixedModule Modified: pypy/branch/x86-64-jit-backend/pypy/interpreter/test/test_code.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/interpreter/test/test_code.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/interpreter/test/test_code.py Fri May 28 16:20:28 2010 @@ -6,11 +6,7 @@ def setup_class(cls): space = gettestobjspace() cls.space = space - if py.test.config.option.runappdirect: - filename = __file__ - else: - filename = gateway.__file__ - + filename = __file__ if filename[-3:] != '.py': filename = filename[:-1] Modified: pypy/branch/x86-64-jit-backend/pypy/interpreter/test/test_zzpickle_and_slow.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/interpreter/test/test_zzpickle_and_slow.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/interpreter/test/test_zzpickle_and_slow.py Fri May 28 16:20:28 2010 @@ -468,3 +468,18 @@ l = [] unbound_meth2(l, 1) assert l == [1] + + def test_pickle_submodule(self): + import pickle + import sys, new + + mod = new.module('pack.mod') + sys.modules['pack.mod'] = mod + pack = new.module('pack') + pack.mod = mod + sys.modules['pack'] = pack + + import pack.mod + pckl = pickle.dumps(pack.mod) + result = pickle.loads(pckl) + assert pack.mod is result Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/codewriter.py Fri May 28 16:20:28 2010 @@ -76,16 +76,13 @@ self.raise_analyzer = None self.jitdriver = None - def find_all_graphs(self, portal_graph, leave_graph, - policy, supports_floats): + def find_all_graphs(self, portal_graph, policy, supports_floats): from pypy.translator.simplify import get_graph def is_candidate(graph): return policy.look_inside_graph(graph) policy.set_supports_floats(supports_floats) todo = [portal_graph] - if leave_graph is not None: - todo.append(leave_graph) self.candidate_graphs = seen = set(todo) def callers(): @@ -178,15 +175,12 @@ return graph in self.candidate_graphs - def generate_bytecode(self, metainterp_sd, portal_graph, leave_graph, + def generate_bytecode(self, metainterp_sd, portal_graph, portal_runner_ptr): self._start(metainterp_sd, portal_runner_ptr) - leave_code = None - if leave_graph: - leave_code = self.make_one_bytecode((leave_graph, None), False) portal_code = self.make_portal_bytecode(portal_graph) - self.metainterp_sd.info_from_codewriter(portal_code, leave_code, + self.metainterp_sd.info_from_codewriter(portal_code, self.class_sizes, self.list_of_addr2name, portal_runner_ptr) @@ -1285,8 +1279,6 @@ calldescr, non_void_args = self.codewriter.getcalldescr(op.args[0], args, op.result) - self.emit('recursion_leave_prep') - self.emit_varargs(non_void_args) self.emit('recursive_call') self.emit(self.get_position(calldescr)) self.emit_varargs([op.args[0]] + non_void_args) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/compile.py Fri May 28 16:20:28 2010 @@ -255,12 +255,10 @@ counter = self._counters.see(box) return counter >= trace_eagerness - def reset_counter_from_failure(self, metainterp): + def reset_counter_from_failure(self): if self._counter >= 0: self._counter = 0 self._counters = None - warmrunnerstate = metainterp.staticdata.state - warmrunnerstate.disable_noninlinable_function(metainterp) def compile_and_attach(self, metainterp, new_loop): # We managed to create a bridge. Attach the new operations @@ -423,6 +421,9 @@ # general loop token old_loop_tokens.append(new_loop_token) + def reset_counter_from_failure(self): + pass + def compile_new_bridge(metainterp, old_loop_tokens, resumekey): """Try to compile a new bridge leading from the beginning of the history Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/optimizeopt.py Fri May 28 16:20:28 2010 @@ -954,7 +954,25 @@ self.optimize_default(op) resvalue = self.getvalue(op.result) self.loop_invariant_results[op.args[0].getint()] = resvalue - + + def optimize_INT_AND(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + if v1.is_null() or v2.is_null(): + self.make_constant_int(op.result, 0) + else: + self.optimize_default(op) + + def optimize_INT_OR(self, op): + v1 = self.getvalue(op.args[0]) + v2 = self.getvalue(op.args[1]) + if v1.is_null(): + self.make_equal_to(op.result, v2) + elif v2.is_null(): + self.make_equal_to(op.result, v1) + else: + self.optimize_default(op) + optimize_ops = _findall(Optimizer, 'optimize_') Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/pyjitpl.py Fri May 28 16:20:28 2010 @@ -647,19 +647,6 @@ @arguments("descr", "varargs") def opimpl_residual_call_loopinvariant(self, calldescr, varargs): return self.execute_varargs(rop.CALL_LOOPINVARIANT, varargs, calldescr, exc=True) - - @arguments("varargs") - def opimpl_recursion_leave_prep(self, varargs): - warmrunnerstate = self.metainterp.staticdata.state - if warmrunnerstate.inlining: - num_green_args = self.metainterp.staticdata.num_green_args - greenkey = varargs[:num_green_args] - if warmrunnerstate.can_inline_callable(greenkey): - return False - leave_code = self.metainterp.staticdata.leave_code - if leave_code is None: - return False - return self.perform_call(leave_code, varargs) @arguments("orgpc", "descr", "varargs") def opimpl_recursive_call(self, pc, calldescr, varargs): @@ -860,6 +847,11 @@ self.debug_merge_point() if self.metainterp.seen_can_enter_jit: self.metainterp.seen_can_enter_jit = False + assert not self.metainterp.in_recursion + # ^^^ it's impossible to arrive here with in_recursion set + # to a non-zero value: seen_can_enter_jit can only be set + # to True by opimpl_can_enter_jit, which should be executed + # just before opimpl_jit_merge_point (no recursion inbetween). try: self.metainterp.reached_can_enter_jit(self.env) except GiveUp: @@ -1128,7 +1120,6 @@ self.jit_starting_line = 'JIT starting (%s)' % backendmodule self.portal_code = None - self.leave_code = None self._class_sizes = None self._addr2name_keys = [] self._addr2name_values = [] @@ -1141,10 +1132,9 @@ def _freeze_(self): return True - def info_from_codewriter(self, portal_code, leave_code, class_sizes, + def info_from_codewriter(self, portal_code, class_sizes, list_of_addr2name, portal_runner_ptr): self.portal_code = portal_code - self.leave_code = leave_code self._class_sizes = class_sizes self._addr2name_keys = [key for key, value in list_of_addr2name] self._addr2name_values = [value for key, value in list_of_addr2name] @@ -1299,7 +1289,6 @@ self.staticdata = staticdata self.cpu = staticdata.cpu self.portal_trace_positions = [] - self.greenkey_of_huge_function = None def is_blackholing(self): return self.history is None @@ -1498,15 +1487,20 @@ self.staticdata.stats.aborted() self.staticdata.profiler.end_tracing() self.staticdata.profiler.start_blackhole() + self.resumekey.reset_counter_from_failure() switch_to_blackhole._dont_inline_ = True def switch_to_blackhole_if_trace_too_long(self): if not self.is_blackholing(): warmrunnerstate = self.staticdata.state if len(self.history.operations) > warmrunnerstate.trace_limit: - self.greenkey_of_huge_function = self.find_biggest_function() + greenkey_of_huge_function = self.find_biggest_function() self.portal_trace_positions = None self.switch_to_blackhole(ABORT_TOO_LONG) + if greenkey_of_huge_function is not None: + warmrunnerstate = self.staticdata.state + warmrunnerstate.disable_noninlinable_function( + greenkey_of_huge_function) def _interpret(self): # Execute the frames forward until we raise a DoneWithThisFrame, @@ -1575,7 +1569,6 @@ debug_stop('jit-tracing') def _handle_guard_failure(self, key): - from pypy.jit.metainterp.warmspot import ContinueRunningNormallyBase original_greenkey = key.original_greenkey # notice that here we just put the greenkey # use -1 to mark that we will have to give up @@ -1583,17 +1576,12 @@ self.current_merge_points = [(original_greenkey, -1)] self.resumekey = key self.seen_can_enter_jit = False - started_as_blackhole = self.is_blackholing() try: self.prepare_resume_from_failure(key.guard_opnum) self.interpret() assert False, "should always raise" except GenerateMergePoint, gmp: return self.designate_target_loop(gmp) - except ContinueRunningNormallyBase: - if not started_as_blackhole: - key.reset_counter_from_failure(self) - raise def remove_consts_and_duplicates(self, boxes, startindex, endindex, duplicates): Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_basic.py Fri May 28 16:20:28 2010 @@ -93,7 +93,7 @@ portal_graph = rtyper.annotator.translator.graphs[0] cw = codewriter.CodeWriter(rtyper) - graphs = cw.find_all_graphs(portal_graph, None, JitPolicy(), + graphs = cw.find_all_graphs(portal_graph, JitPolicy(), self.CPUClass.supports_floats) cw._start(metainterp.staticdata, None) portal_graph.func._jit_unroll_safe_ = True Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_codewriter.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_codewriter.py Fri May 28 16:20:28 2010 @@ -25,7 +25,7 @@ rtyper = support.annotate(i, [7]) cw = CodeWriter(rtyper) jitpolicy = JitPolicy() - res = cw.find_all_graphs(rtyper.annotator.translator.graphs[0], None, + res = cw.find_all_graphs(rtyper.annotator.translator.graphs[0], jitpolicy, True) translator = rtyper.annotator.translator @@ -41,13 +41,13 @@ cw = CodeWriter(rtyper) jitpolicy = JitPolicy() translator = rtyper.annotator.translator - res = cw.find_all_graphs(translator.graphs[0], None, jitpolicy, + res = cw.find_all_graphs(translator.graphs[0], jitpolicy, supports_floats=True) funcs = set([graph.func for graph in res]) assert funcs == set([f, g]) cw = CodeWriter(rtyper) - res = cw.find_all_graphs(translator.graphs[0], None, jitpolicy, + res = cw.find_all_graphs(translator.graphs[0], jitpolicy, supports_floats=False) funcs = [graph.func for graph in res] assert funcs == [f] @@ -75,7 +75,7 @@ cw = CodeWriter(rtyper) jitpolicy = JitPolicy() translator = rtyper.annotator.translator - res = cw.find_all_graphs(translator.graphs[0], None, jitpolicy, + res = cw.find_all_graphs(translator.graphs[0], jitpolicy, supports_floats=True) funcs = set([graph.func for graph in res]) assert funcs == set([f, h]) @@ -96,7 +96,7 @@ cw = CodeWriter(rtyper) jitpolicy = JitPolicy() translator = rtyper.annotator.translator - res = cw.find_all_graphs(translator.graphs[0], None, jitpolicy, + res = cw.find_all_graphs(translator.graphs[0], jitpolicy, supports_floats=True) funcs = set([graph.func for graph in res]) assert funcs == set([g, h]) @@ -110,7 +110,7 @@ jitpolicy = JitPolicy() translator = rtyper.annotator.translator # does not explode - cw.find_all_graphs(translator.graphs[0], None, jitpolicy, True) + cw.find_all_graphs(translator.graphs[0], jitpolicy, True) class SomeLabel(object): def __eq__(self, other): Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_optimizeopt.py Fri May 28 16:20:28 2010 @@ -1999,6 +1999,22 @@ """ self.optimize_loop(ops, 'Not', expected) + def test_int_and_or_with_zero(self): + ops = """ + [i0, i1] + i2 = int_and(i0, 0) + i3 = int_and(0, i2) + i4 = int_or(i2, i1) + i5 = int_or(i0, i3) + jump(i4, i5) + """ + expected = """ + [i0, i1] + jump(i1, i0) + """ + self.optimize_loop(ops, 'Not, Not', expected) + + # ---------- def make_fail_descr(self): Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_pyjitpl.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_pyjitpl.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_pyjitpl.py Fri May 28 16:20:28 2010 @@ -121,7 +121,7 @@ def __init__(self): pass metainterp_sd = FakeMetaInterpSd() - metainterp_sd.info_from_codewriter(None, None, None, + metainterp_sd.info_from_codewriter(None, None, [(123, "a"), (456, "b")], None) assert metainterp_sd.get_name_from_address(123) == 'a' Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_recursive.py Fri May 28 16:20:28 2010 @@ -494,77 +494,6 @@ res = self.meta_interp(main, [1], optimizer=OPTIMIZER_SIMPLE, trace_limit=TRACE_LIMIT) self.check_loops(call_may_force=0, call=0) - def test_leave_jit_hook(self): - from pypy.rpython.annlowlevel import hlstr - def p(code, pc): - code = hlstr(code) - return "%s %d %s" % (code, pc, code[pc]) - def c(code, pc): - return "L" not in hlstr(code) - - def really_leave(frame): - frame.hookcalled = True - - def leave(code, pc, frame): - really_leave(frame) - - class ExpectedHook(Exception): - pass - class UnexpectedHook(Exception): - pass - - myjitdriver = JitDriver(greens=['code', 'pc'], reds=['self'], - get_printable_location=p, can_inline=c, - leave=leave) - class Frame(object): - hookcalled = True - - def __init__(self, n): - self.n = n - self.hookcalled = False - def f(self, code): - pc = 0 - while pc < len(code): - - myjitdriver.jit_merge_point(self=self, code=code, pc=pc) - op = code[pc] - if op == "-": - self.n -= 1 - elif op == "c": - frame = Frame(self.n) - self.n = frame.f("---i---") - if we_are_jitted(): - if frame.hookcalled: - raise UnexpectedHook - elif op == "C": - frame = Frame(self.n) - self.n = frame.f("cL") - if we_are_jitted(): - if not frame.hookcalled: - raise ExpectedHook - elif op == "i": - if self.n % 5 == 1: - return self.n - elif op == "l": - if self.n > 0: - myjitdriver.can_enter_jit(self=self, code=code, pc=0) - pc = 0 - continue - elif op == "L": - if self.n > 50: - myjitdriver.can_enter_jit(self=self, code=code, pc=0) - pc = 0 - continue - else: - assert 0 - pc += 1 - return self.n - def main(n): - frame = Frame(n) - return frame.f("C-l") - res = self.meta_interp(main, [100], optimizer=OPTIMIZER_SIMPLE, inline=True) - assert res == main(100) - def test_trace_from_start(self): def p(code, pc): code = hlstr(code) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_warmspot.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_warmspot.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/test/test_warmspot.py Fri May 28 16:20:28 2010 @@ -131,7 +131,7 @@ assert warmrunnerdescr.state.optimize_loop is optimize.optimize_loop assert warmrunnerdescr.state.optimize_bridge is optimize.optimize_bridge - def test_static_debug_level(self): + def test_static_debug_level(self, capfd): py.test.skip("debug_level is being deprecated") from pypy.rlib.jit import DEBUG_PROFILE, DEBUG_OFF, DEBUG_STEPS from pypy.jit.metainterp.jitprof import EmptyProfiler, Profiler @@ -144,36 +144,31 @@ n -= 1 return n - outerr = py.io.StdCaptureFD() + capfd.readouterr() self.meta_interp(f, [10], debug_level=DEBUG_OFF, ProfilerClass=Profiler) - out, errf = outerr.done() - err = errf.read() + out, err = capfd.readouterr() assert not 'ENTER' in err assert not 'LEAVE' in err assert not "Running asm" in err - outerr = py.io.StdCaptureFD() self.meta_interp(f, [10], debug_level=DEBUG_PROFILE, ProfilerClass=Profiler) - out, errf = outerr.done() - err = errf.read() + out, err = capfd.readouterr() assert not 'ENTER' in err assert not 'LEAVE' in err assert not 'compiled new' in err assert "Running asm" in err - outerr = py.io.StdCaptureFD() + self.meta_interp(f, [10], debug_level=DEBUG_STEPS, ProfilerClass=Profiler) - out, errf = outerr.done() - err = errf.read() + out, err = capfd.readouterr() assert 'ENTER' in err assert 'LEAVE' in err assert "Running asm" in err - outerr = py.io.StdCaptureFD() + self.meta_interp(f, [10], debug_level=DEBUG_STEPS, ProfilerClass=EmptyProfiler) - out, errf = outerr.done() - err = errf.read() + out, err = capfd.readouterr() assert 'ENTER' in err assert 'LEAVE' in err assert not "Running asm" in err Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/warmspot.py Fri May 28 16:20:28 2010 @@ -150,10 +150,8 @@ policy = JitPolicy() self.set_translator(translator) self.find_portal() - self.make_leave_jit_graph() self.codewriter = codewriter.CodeWriter(self.rtyper) graphs = self.codewriter.find_all_graphs(self.portal_graph, - self.leave_graph, policy, CPUClass.supports_floats) policy.dump_unsafe_loops() @@ -177,7 +175,6 @@ self.codewriter.generate_bytecode(self.metainterp_sd, self.portal_graph, - self.leave_graph, self.portal_runner_ptr ) self.rewrite_can_enter_jit() @@ -332,6 +329,7 @@ self.metainterp_sd.DoneWithThisFrameFloat = DoneWithThisFrameFloat self.metainterp_sd.ExitFrameWithExceptionRef = ExitFrameWithExceptionRef self.metainterp_sd.ContinueRunningNormally = ContinueRunningNormally + def make_enter_function(self): from pypy.jit.metainterp.warmstate import WarmEnterState state = WarmEnterState(self) @@ -369,18 +367,6 @@ maybe_compile_and_run(*args) maybe_enter_from_start._always_inline_ = True self.maybe_enter_from_start_fn = maybe_enter_from_start - - - def make_leave_jit_graph(self): - self.leave_graph = None - if self.jitdriver.leave: - args_s = self.portal_args_s - from pypy.annotation import model as annmodel - annhelper = MixLevelHelperAnnotator(self.translator.rtyper) - s_result = annmodel.s_None - self.leave_graph = annhelper.getgraph(self.jitdriver.leave, - args_s, s_result) - annhelper.finish() def make_driverhook_graphs(self): from pypy.rlib.jit import BaseJitCell Modified: pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/metainterp/warmstate.py Fri May 28 16:20:28 2010 @@ -157,16 +157,14 @@ if self.profiler is not None: self.profiler.set_printing(value >= DEBUG_PROFILE) - def disable_noninlinable_function(self, metainterp): - greenkey = metainterp.greenkey_of_huge_function - if greenkey is not None: - cell = self.jit_cell_at_key(greenkey) - cell.dont_trace_here = True - debug_start("jit-disableinlining") - sd = self.warmrunnerdesc.metainterp_sd - loc = sd.state.get_location_str(greenkey) - debug_print("disabled inlining", loc) - debug_stop("jit-disableinlining") + def disable_noninlinable_function(self, greenkey): + cell = self.jit_cell_at_key(greenkey) + cell.dont_trace_here = True + debug_start("jit-disableinlining") + sd = self.warmrunnerdesc.metainterp_sd + loc = sd.state.get_location_str(greenkey) + debug_print("disabled inlining", loc) + debug_stop("jit-disableinlining") def attach_unoptimized_bridge_from_interp(self, greenkey, entry_loop_token): @@ -183,7 +181,6 @@ metainterp_sd = self.warmrunnerdesc.metainterp_sd vinfo = metainterp_sd.virtualizable_info - ContinueRunningNormally = self.warmrunnerdesc.ContinueRunningNormally num_green_args = self.warmrunnerdesc.num_green_args get_jitcell = self.make_jitcell_getter() set_future_values = self.make_set_future_values() @@ -227,11 +224,6 @@ cell.counter = -2 try: loop_token = metainterp.compile_and_run_once(*args) - except ContinueRunningNormally: - # the trace got too long, reset the counter - cell.counter = 0 - self.disable_noninlinable_function(metainterp) - raise finally: if cell.counter == -2: cell.counter = 0 Modified: pypy/branch/x86-64-jit-backend/pypy/lib/_locale.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/lib/_locale.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/lib/_locale.py Fri May 28 16:20:28 2010 @@ -4,7 +4,7 @@ """ from ctypes import (Structure, POINTER, create_string_buffer, - c_ubyte, c_int, c_char_p, c_wchar_p) + c_ubyte, c_int, c_char_p, c_wchar_p, c_size_t) from ctypes_support import standard_c_lib as libc from ctypes_support import get_errno @@ -12,9 +12,6 @@ from ctypes_config_cache._locale_cache import * -size_t = c_int - - # Ubuntu Gusty i386 structure class lconv(Structure): _fields_ = ( @@ -94,8 +91,8 @@ _wcscoll.restype = c_int _strxfrm = libc.strxfrm -_strxfrm.argtypes = (c_char_p, c_char_p, size_t) -_strxfrm.restype = size_t +_strxfrm.argtypes = (c_char_p, c_char_p, c_size_t) +_strxfrm.restype = c_size_t HAS_LIBINTL = hasattr(libc, 'gettext') if HAS_LIBINTL: Modified: pypy/branch/x86-64-jit-backend/pypy/lib/stackless.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/lib/stackless.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/lib/stackless.py Fri May 28 16:20:28 2010 @@ -488,6 +488,19 @@ # we want to get rid of the parent thing. # for now, we just drop it a, b, c, d = coro_state + + # Removing all frames related to stackless.py. + # They point to stuff we don't want to be pickled. + frame_list = list(b) + new_frame_list = [] + for frame in frame_list: + if frame.f_code == schedule.func_code: + # Removing everything including and after the + # call to stackless.schedule() + break + new_frame_list.append(frame) + b = tuple(new_frame_list) + if d: assert isinstance(d, coroutine) coro_state = a, b, c, None Modified: pypy/branch/x86-64-jit-backend/pypy/lib/test2/test_stackless.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/lib/test2/test_stackless.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/lib/test2/test_stackless.py Fri May 28 16:20:28 2010 @@ -54,3 +54,36 @@ ''' in mod.__dict__ finally: del sys.modules['mod'] + + def test_pickle2(self): + # To test a bug where too much stuff gets pickled when + # a tasklet halted on stackless.schedule() is pickled. + import new, sys + + mod = new.module('mod') + sys.modules['mod'] = mod + mod.can_unpickle = self.can_unpickle + mod.skip = skip + try: + exec ''' +import pickle, sys +import stackless +import socket + +def task_should_be_picklable(): + stackless.schedule() + +def task_socket(): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + stackless.schedule() + +def task_pickle(ref_task): + p = pickle.dumps(ref_task) + +ref_task = stackless.tasklet(task_should_be_picklable)() +stackless.tasklet(task_socket)() +stackless.tasklet(task_pickle)(ref_task) +stackless.run() +''' in mod.__dict__ + finally: + del sys.modules['mod'] Modified: pypy/branch/x86-64-jit-backend/pypy/module/__builtin__/functional.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/__builtin__/functional.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/__builtin__/functional.py Fri May 28 16:20:28 2010 @@ -186,7 +186,7 @@ while True: cont = False args_w = [space.w_None] * num_iterators - for i in range(len(iterators_w)): + for i in range(num_iterators): try: args_w[i] = space.next(iterators_w[i]) except OperationError, e: @@ -194,8 +194,8 @@ raise else: cont = True - w_args = space.newtuple(args_w) if cont: + w_args = space.newtuple(args_w) if none_func: result_w.append(w_args) else: Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/api.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/api.py Fri May 28 16:20:28 2010 @@ -17,7 +17,11 @@ from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.gateway import ObjSpace, unwrap_spec -from pypy.interpreter.nestedscope import Cell +from pypy.interpreter.nestedscope import Cell +from pypy.interpreter.module import Module +from pypy.interpreter.function import StaticMethod +from pypy.objspace.std.sliceobject import W_SliceObject +from pypy.module.__builtin__.descriptor import W_Property from pypy.rlib.entrypoint import entrypoint from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.objectmodel import specialize @@ -65,6 +69,23 @@ assert CONST_STRING is not rffi.CCHARP assert CONST_WSTRING is not rffi.CWCHARP +# FILE* interface +FILEP = rffi.COpaquePtr('FILE') +fopen = rffi.llexternal('fopen', [CONST_STRING, CONST_STRING], FILEP) +fclose = rffi.llexternal('fclose', [FILEP], rffi.INT) +fwrite = rffi.llexternal('fwrite', + [rffi.VOIDP, rffi.SIZE_T, rffi.SIZE_T, FILEP], + rffi.SIZE_T) +fread = rffi.llexternal('fread', + [rffi.VOIDP, rffi.SIZE_T, rffi.SIZE_T, FILEP], + rffi.SIZE_T) +feof = rffi.llexternal('feof', [FILEP], rffi.INT) +if sys.platform == 'win32': + fileno = rffi.llexternal('_fileno', [FILEP], rffi.INT) +else: + fileno = rffi.llexternal('fileno', [FILEP], rffi.INT) + + constant_names = """ Py_TPFLAGS_READY Py_TPFLAGS_READYING METH_COEXIST METH_STATIC METH_CLASS @@ -262,7 +283,9 @@ '_PyArg_NoKeywords', 'PyString_FromFormat', 'PyString_FromFormatV', 'PyModule_AddObject', 'PyModule_AddIntConstant', 'PyModule_AddStringConstant', - 'Py_BuildValue', 'PyTuple_Pack', 'PyErr_Format', 'PyErr_NewException', + 'Py_BuildValue', 'Py_VaBuildValue', 'PyTuple_Pack', + + 'PyErr_Format', 'PyErr_NewException', 'PyEval_CallFunction', 'PyEval_CallMethod', 'PyObject_CallFunction', 'PyObject_CallMethod', 'PyObject_CallFunctionObjArgs', 'PyObject_CallMethodObjArgs', @@ -312,10 +335,15 @@ 'None': 'space.type(space.w_None)', 'NotImplemented': 'space.type(space.w_NotImplemented)', 'Cell': 'space.gettypeobject(Cell.typedef)', + 'Module': 'space.gettypeobject(Module.typedef)', + 'Property': 'space.gettypeobject(W_Property.typedef)', + 'Slice': 'space.gettypeobject(W_SliceObject.typedef)', + 'StaticMethod': 'space.gettypeobject(StaticMethod.typedef)', + 'CFunction': 'space.gettypeobject(cpyext.methodobject.W_PyCFunctionObject.typedef)', }.items(): GLOBALS['Py%s_Type#' % (cpyname, )] = ('PyTypeObject*', pypyexpr) - for cpyname in 'Method List Int Long Dict Tuple'.split(): + for cpyname in 'Method List Int Long Dict Tuple Class'.split(): FORWARD_DECLS.append('typedef struct { PyObject_HEAD } ' 'Py%sObject' % (cpyname, )) build_exported_objects() @@ -579,6 +607,7 @@ # populate static data for name, (type, expr) in GLOBALS.iteritems(): + from pypy.module import cpyext w_obj = eval(expr) name = name.replace("#", "") INTERPLEVEL_API[name] = w_obj @@ -751,6 +780,7 @@ # populate static data for name, (type, expr) in GLOBALS.iteritems(): name = name.replace("#", "") + from pypy.module import cpyext w_obj = eval(expr) struct_ptr = make_ref(space, w_obj) struct = rffi.cast(get_structtype_for_ctype(type), struct_ptr)._obj @@ -811,6 +841,7 @@ @specialize.memo() def make_generic_cpy_call(FT, decref_args, expect_null): from pypy.module.cpyext.pyobject import make_ref, from_ref, Py_DecRef + from pypy.module.cpyext.pyobject import RefcountState from pypy.module.cpyext.pyerrors import PyErr_Occurred unrolling_arg_types = unrolling_iterable(enumerate(FT.ARGS)) RESULT_TYPE = FT.RESULT @@ -856,8 +887,17 @@ boxed_args += (arg,) else: boxed_args += (arg,) - result = call_external_function(func, *boxed_args) + try: + # create a new container for borrowed references + state = space.fromcache(RefcountState) + old_container = state.swap_borrow_container(None) + try: + # Call the function + result = call_external_function(func, *boxed_args) + finally: + state.swap_borrow_container(old_container) + if RESULT_TYPE is PyObject: if result is None: ret = result Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/dictobject.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/dictobject.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/dictobject.py Fri May 28 16:20:28 2010 @@ -1,7 +1,9 @@ from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, build_type_checkers,\ - Py_ssize_t, PyObjectP, CONST_STRING -from pypy.module.cpyext.pyobject import PyObject, borrow_from +from pypy.module.cpyext.api import ( + cpython_api, CANNOT_FAIL, build_type_checkers, Py_ssize_t, + Py_ssize_tP, CONST_STRING) +from pypy.module.cpyext.pyobject import PyObject, PyObjectP, borrow_from +from pypy.module.cpyext.pyobject import RefcountState from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.interpreter.error import OperationError @@ -102,8 +104,8 @@ dictionary, as in the dictionary method dict.items().""" return space.call_method(w_obj, "items") - at cpython_api([PyObject, Py_ssize_t, PyObjectP, PyObjectP], rffi.INT_real, error=CANNOT_FAIL) -def PyDict_Next(space, w_obj, ppos, pkey, pvalue): + at cpython_api([PyObject, Py_ssize_tP, PyObjectP, PyObjectP], rffi.INT_real, error=CANNOT_FAIL) +def PyDict_Next(space, w_dict, ppos, pkey, pvalue): """Iterate over all key-value pairs in the dictionary p. The Py_ssize_t referred to by ppos must be initialized to 0 prior to the first call to this function to start the iteration; the @@ -144,8 +146,30 @@ } Py_DECREF(o); }""" - if w_obj is None: + if w_dict is None: return 0 - raise NotImplementedError + + # Note: this is not efficient. Storing an iterator would probably + # work, but we can't work out how to not leak it if iteration does + # not complete. + + try: + w_iter = space.call_method(w_dict, "iteritems") + pos = ppos[0] + while pos: + space.call_method(w_iter, "next") + pos -= 1 + + w_item = space.call_method(w_iter, "next") + w_key, w_value = space.fixedview(w_item, 2) + state = space.fromcache(RefcountState) + pkey[0] = state.make_borrowed(w_dict, w_key) + pvalue[0] = state.make_borrowed(w_dict, w_value) + ppos[0] += 1 + except OperationError, e: + if not e.match(space, space.w_StopIteration): + raise + return 0 + return 1 Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/eval.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/eval.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/eval.py Fri May 28 16:20:28 2010 @@ -1,12 +1,48 @@ from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import ( - cpython_api, PyObject, CANNOT_FAIL, CONST_STRING) + cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, fread, feof, Py_ssize_tP) +from pypy.module.cpyext.pyobject import PyObject, borrow_from +from pypy.module.cpyext.pyerrors import PyErr_SetFromErrno +from pypy.module.__builtin__ import compiling @cpython_api([PyObject, PyObject, PyObject], PyObject) def PyEval_CallObjectWithKeywords(space, w_obj, w_arg, w_kwds): return space.call(w_obj, w_arg, w_kwds) + at cpython_api([], PyObject) +def PyEval_GetBuiltins(space): + """Return a dictionary of the builtins in the current execution + frame, or the interpreter of the thread state if no frame is + currently executing.""" + caller = space.getexecutioncontext().gettopframe_nohidden() + if caller is not None: + w_globals = caller.w_globals + w_builtins = space.getitem(w_globals, space.wrap('__builtins__')) + if not space.isinstance_w(w_builtins, space.w_dict): + w_builtins = w_builtins.getdict() + else: + w_builtins = space.builtin.getdict() + return borrow_from(None, w_builtins) + + at cpython_api([], PyObject, error=CANNOT_FAIL) +def PyEval_GetLocals(space): + """Return a dictionary of the local variables in the current execution + frame, or NULL if no frame is currently executing.""" + caller = space.getexecutioncontext().gettopframe_nohidden() + if caller is None: + return None + return borrow_from(None, caller.getdictscope()) + + at cpython_api([], PyObject, error=CANNOT_FAIL) +def PyEval_GetGlobals(space): + """Return a dictionary of the global variables in the current execution + frame, or NULL if no frame is currently executing.""" + caller = space.getexecutioncontext().gettopframe_nohidden() + if caller is None: + return None + return borrow_from(None, caller.w_globals) + @cpython_api([PyObject, PyObject], PyObject) def PyObject_CallObject(space, w_obj, w_arg): """ @@ -33,13 +69,8 @@ Py_file_input = 257 Py_eval_input = 258 - at cpython_api([CONST_STRING, rffi.INT_real, PyObject, PyObject], PyObject) -def PyRun_String(space, str, start, w_globals, w_locals): - """This is a simplified interface to PyRun_StringFlags() below, leaving - flags set to NULL.""" - from pypy.module.__builtin__ import compiling - w_source = space.wrap(rffi.charp2str(str)) - filename = "" +def run_string(space, source, filename, start, w_globals, w_locals): + w_source = space.wrap(source) start = rffi.cast(lltype.Signed, start) if start == Py_file_input: mode = 'exec' @@ -53,3 +84,49 @@ w_code = compiling.compile(space, w_source, filename, mode) return compiling.eval(space, w_code, w_globals, w_locals) + at cpython_api([CONST_STRING, rffi.INT_real,PyObject, PyObject], PyObject) +def PyRun_String(space, source, start, w_globals, w_locals): + """This is a simplified interface to PyRun_StringFlags() below, leaving + flags set to NULL.""" + source = rffi.charp2str(source) + filename = "" + return run_string(space, source, filename, start, w_globals, w_locals) + + at cpython_api([FILEP, CONST_STRING, rffi.INT_real, PyObject, PyObject], PyObject) +def PyRun_File(space, fp, filename, start, w_globals, w_locals): + """This is a simplified interface to PyRun_FileExFlags() below, leaving + closeit set to 0 and flags set to NULL.""" + BUF_SIZE = 8192 + source = "" + filename = rffi.charp2str(filename) + buf = lltype.malloc(rffi.CCHARP.TO, BUF_SIZE, flavor='raw') + try: + while True: + count = fread(buf, 1, BUF_SIZE, fp) + source += rffi.charpsize2str(buf, count) + if count < BUF_SIZE: + if feof(fp): + break + PyErr_SetFromErrno(space, space.w_IOError) + finally: + lltype.free(buf, flavor='raw') + return run_string(space, source, filename, start, w_globals, w_locals) + +# Undocumented function! + at cpython_api([PyObject, Py_ssize_tP], rffi.INT_real, error=0) +def _PyEval_SliceIndex(space, w_obj, pi): + """Extract a slice index from a PyInt or PyLong or an object with the + nb_index slot defined, and store in *pi. + Silently reduce values larger than PY_SSIZE_T_MAX to PY_SSIZE_T_MAX, + and silently boost values less than -PY_SSIZE_T_MAX-1 to -PY_SSIZE_T_MAX-1. + + Return 0 on error, 1 on success. + + Note: If v is NULL, return success without storing into *pi. This + is because_PyEval_SliceIndex() is called by apply_slice(), which can be + called by the SLICE opcode with v and/or w equal to NULL. + """ + if w_obj is not None: + pi[0] = space.getindex_w(w_obj, None) + return 1 + Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/funcobject.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/funcobject.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/funcobject.py Fri May 28 16:20:28 2010 @@ -50,6 +50,13 @@ return borrow_from(w_method, w_method.w_function) @cpython_api([PyObject], PyObject) +def PyMethod_Self(space, w_method): + """Return the instance associated with the method meth if it is bound, + otherwise return NULL.""" + assert isinstance(w_method, Method) + return borrow_from(w_method, w_method.w_instance) + + at cpython_api([PyObject], PyObject) def PyMethod_Class(space, w_method): """Return the class object from which the method meth was created; if this was created from an instance, it will be the class of the instance.""" Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/include/Python.h Fri May 28 16:20:28 2010 @@ -35,38 +35,7 @@ #define Py_USING_UNICODE -// from pyport.h -#ifdef SIZE_MAX -#define PY_SIZE_MAX SIZE_MAX -#else -#define PY_SIZE_MAX ((size_t)-1) -#endif -/* uintptr_t is the C9X name for an unsigned integral type such that a - * legitimate void* can be cast to uintptr_t and then back to void* again - * without loss of information. Similarly for intptr_t, wrt a signed - * integral type. - */ -#ifdef HAVE_UINTPTR_T -typedef uintptr_t Py_uintptr_t; -typedef intptr_t Py_intptr_t; - -#elif SIZEOF_VOID_P <= SIZEOF_INT -typedef unsigned int Py_uintptr_t; -typedef int Py_intptr_t; - -#elif SIZEOF_VOID_P <= SIZEOF_LONG -typedef unsigned long Py_uintptr_t; -typedef long Py_intptr_t; - -#elif defined(HAVE_LONG_LONG) && (SIZEOF_VOID_P <= SIZEOF_LONG_LONG) -typedef unsigned PY_LONG_LONG Py_uintptr_t; -typedef PY_LONG_LONG Py_intptr_t; - -#else -# error "Python needs a typedef for Py_uintptr_t in pyport.h." -#endif /* HAVE_UINTPTR_T */ - - +#include /* Convert a possibly signed character to a nonnegative int */ /* XXX This assumes characters are 8 bits wide */ @@ -98,7 +67,6 @@ #include #include #include -#include #include "pyconfig.h" Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/include/funcobject.h ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/include/funcobject.h (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/include/funcobject.h Fri May 28 16:20:28 2010 @@ -12,8 +12,9 @@ PyObject *func_name; /* The __name__ attribute, a string object */ } PyFunctionObject; -#define PyMethod_GET_CLASS(obj) PyMethod_Class(obj) -#define PyMethod_GET_FUNCTION(obj) PyMethod_Function(obj) +#define PyMethod_GET_FUNCTION(obj) PyMethod_Function((PyObject*)(obj)) +#define PyMethod_GET_SELF(obj) PyMethod_Self((PyObject*)(obj)) +#define PyMethod_GET_CLASS(obj) PyMethod_Class((PyObject*)(obj)) #ifdef __cplusplus } Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/include/methodobject.h ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/include/methodobject.h (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/include/methodobject.h Fri May 28 16:20:28 2010 @@ -49,6 +49,8 @@ #define METH_COEXIST 0x0040 +#define PyCFunction_New(ml, self) PyCFunction_NewEx((ml), (self), NULL) + /* Macros for direct access to these values. Type checks are *not* done, so use with care. */ #define PyCFunction_GET_FUNCTION(func) \ Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/include/pymem.h ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/include/pymem.h (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/include/pymem.h Fri May 28 16:20:28 2010 @@ -5,12 +5,6 @@ #define PyMem_REALLOC(p, n) realloc((p), (n) ? (n) : 1) #define PyMem_FREE free -/* XXX use obmalloc like cpython and pypy do, otherwise we might get segfaults */ -#define PyObject_MALLOC PyMem_MALLOC -// we won't support this -// #define PyObject_REALLOC PyMem_REALLOC -#define PyObject_FREE PyMem_FREE - #define PyMem_Malloc PyMem_MALLOC #define PyMem_Free PyMem_FREE #define PyMem_Realloc PyMem_REALLOC Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/include/pyport.h ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/include/pyport.h (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/include/pyport.h Fri May 28 16:20:28 2010 @@ -16,4 +16,29 @@ #define PY_SIZE_MAX ((size_t)-1) #endif +/* uintptr_t is the C9X name for an unsigned integral type such that a + * legitimate void* can be cast to uintptr_t and then back to void* again + * without loss of information. Similarly for intptr_t, wrt a signed + * integral type. + */ +#ifdef HAVE_UINTPTR_T +typedef uintptr_t Py_uintptr_t; +typedef intptr_t Py_intptr_t; + +#elif SIZEOF_VOID_P <= SIZEOF_INT +typedef unsigned int Py_uintptr_t; +typedef int Py_intptr_t; + +#elif SIZEOF_VOID_P <= SIZEOF_LONG +typedef unsigned long Py_uintptr_t; +typedef long Py_intptr_t; + +#elif defined(HAVE_LONG_LONG) && (SIZEOF_VOID_P <= SIZEOF_LONG_LONG) +typedef unsigned PY_LONG_LONG Py_uintptr_t; +typedef PY_LONG_LONG Py_intptr_t; + +#else +# error "Python needs a typedef for Py_uintptr_t in pyport.h." +#endif /* HAVE_UINTPTR_T */ + #endif /* Py_PYPORT_H */ Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/include/sliceobject.h ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/include/sliceobject.h (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/include/sliceobject.h Fri May 28 16:20:28 2010 @@ -12,6 +12,9 @@ typedef struct { PyObject_HEAD + PyObject *start; + PyObject *stop; + PyObject *step; } PySliceObject; #ifdef __cplusplus Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/include/unicodeobject.h ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/include/unicodeobject.h (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/include/unicodeobject.h Fri May 28 16:20:28 2010 @@ -16,6 +16,7 @@ #endif typedef PY_UNICODE_TYPE Py_UNICODE; +#define Py_UNICODE_REPLACEMENT_CHARACTER ((Py_UNICODE) 0xFFFD) typedef struct { PyObject_HEAD Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/iterator.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/iterator.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/iterator.py Fri May 28 16:20:28 2010 @@ -1,6 +1,8 @@ -from pypy.module.cpyext.api import generic_cpy_call, cpython_api, PyObject from pypy.interpreter.error import OperationError +from pypy.module.cpyext.api import (generic_cpy_call, cpython_api, PyObject, + CANNOT_FAIL) import pypy.module.__builtin__.operation as operation +from pypy.rpython.lltypesystem import rffi @cpython_api([PyObject, PyObject], PyObject) @@ -12,3 +14,34 @@ """ return operation.iter_sentinel(space, w_callable, w_sentinel) + at cpython_api([PyObject], PyObject) +def PyObject_GetIter(space, w_obj): + """This is equivalent to the Python expression iter(o). It returns a new + iterator for the object argument, or the object itself if the object is + already an iterator. Raises TypeError and returns NULL if the object + cannot be iterated.""" + return space.iter(w_obj) + + at cpython_api([PyObject], PyObject, error=CANNOT_FAIL) +def PyIter_Next(space, w_obj): + """Return the next value from the iteration o. If the object is an + iterator, this retrieves the next value from the iteration, and returns + NULL with no exception set if there are no remaining items. If the object + is not an iterator, TypeError is raised, or if there is an error in + retrieving the item, returns NULL and passes along the exception.""" + try: + return space.next(w_obj) + except OperationError, e: + if not e.match(space, space.w_StopIteration): + raise + return None + + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) +def PyIter_Check(space, w_obj): + """Return true if the object o supports the iterator protocol.""" + try: + w_attr = space.getattr(space.type(w_obj), space.wrap("next")) + except: + return False + else: + return space.is_true(space.callable(w_attr)) Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/longobject.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/longobject.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/longobject.py Fri May 28 16:20:28 2010 @@ -18,6 +18,12 @@ on failure.""" return space.wrap(val) + at cpython_api([rffi.ULONG], PyObject) +def PyLong_FromUnsignedLong(space, val): + """Return a new PyLongObject object from a C unsigned long, or + NULL on failure.""" + return space.wrap(val) + @cpython_api([rffi.ULONGLONG], PyObject) def PyLong_FromUnsignedLongLong(space, val): """Return a new PyLongObject object from a C unsigned long long, Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/methodobject.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/methodobject.py Fri May 28 16:20:28 2010 @@ -1,11 +1,11 @@ from pypy.interpreter.baseobjspace import Wrappable, W_Root -from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.gateway import ObjSpace, W_Root from pypy.interpreter.argument import Arguments from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.interpreter.function import BuiltinFunction, Method +from pypy.interpreter.function import BuiltinFunction, Method, StaticMethod from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.pyobject import (PyObject, from_ref, make_ref, make_typedescr, Py_DecRef) @@ -57,10 +57,10 @@ PyObject_dealloc(space, py_obj) class W_PyCFunctionObject(Wrappable): - def __init__(self, space, ml, w_self, doc=None): + def __init__(self, space, ml, w_self, w_module=None): self.ml = ml self.w_self = w_self - self.doc = doc + self.w_module = w_module def call(self, space, w_self, w_args, w_kw): # Call the C function @@ -100,6 +100,9 @@ w_arg = w_args return generic_cpy_call(space, self.ml.c_ml_meth, w_self, w_arg) + def get_doc(space, self): + return space.wrap(rffi.charp2str(self.ml.c_ml_doc)) + class W_PyCMethodObject(W_PyCFunctionObject): w_self = None @@ -192,7 +195,8 @@ W_PyCFunctionObject.typedef = TypeDef( 'builtin_function_or_method', __call__ = interp2app(cfunction_descr_call), - __doc__ = interp_attrproperty('doc', cls=W_PyCFunctionObject), + __doc__ = GetSetProperty(W_PyCFunctionObject.get_doc), + __module__ = interp_attrproperty_w('w_module', cls=W_PyCFunctionObject), ) W_PyCFunctionObject.typedef.acceptable_as_base_class = False @@ -220,9 +224,13 @@ W_PyCWrapperObject.typedef.acceptable_as_base_class = False -def PyCFunction_NewEx(space, ml, w_self): # not exactly the API sig - return space.wrap(W_PyCFunctionObject(space, ml, w_self)) - + at cpython_api([lltype.Ptr(PyMethodDef), PyObject, PyObject], PyObject) +def PyCFunction_NewEx(space, ml, w_self, w_name): + return space.wrap(W_PyCFunctionObject(space, ml, w_self, w_name)) + + at cpython_api([PyObject], PyObject) +def PyStaticMethod_New(space, w_func): + return space.wrap(StaticMethod(w_func)) def PyDescr_NewMethod(space, w_type, method): return space.wrap(W_PyCMethodObject(space, method, w_type)) @@ -233,7 +241,7 @@ wrapper_func, doc, flags, func)) @cpython_api([lltype.Ptr(PyMethodDef), PyObject, CONST_STRING], PyObject) -def Py_FindMethod(space, table, w_ob, name_ptr): +def Py_FindMethod(space, table, w_obj, name_ptr): """Return a bound method object for an extension type implemented in C. This can be useful in the implementation of a tp_getattro or tp_getattr handler that does not use the @@ -253,7 +261,7 @@ if name == "__methods__": method_list_w.append(space.wrap(rffi.charp2str(method.c_ml_name))) elif rffi.charp2str(method.c_ml_name) == name: # XXX expensive copying - return PyCFunction_NewEx(space, method, w_ob) + return space.wrap(W_PyCFunctionObject(space, method, w_obj)) if name == "__methods__": return space.newlist(method_list_w) raise OperationError(space.w_AttributeError, space.wrap(name)) Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/modsupport.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/modsupport.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/modsupport.py Fri May 28 16:20:28 2010 @@ -3,7 +3,9 @@ METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, CONST_STRING from pypy.module.cpyext.pyobject import PyObject, borrow_from from pypy.interpreter.module import Module -from pypy.module.cpyext.methodobject import W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod, PyMethodDef, PyCFunction +from pypy.module.cpyext.methodobject import ( + W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod, + PyMethodDef, PyCFunction, PyStaticMethod_New) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.module.cpyext.state import State from pypy.interpreter.error import OperationError @@ -52,17 +54,17 @@ w_mod = PyImport_AddModule(space, state.package_context) dict_w = {} - convert_method_defs(space, dict_w, methods, None, w_self) + convert_method_defs(space, dict_w, methods, None, w_self, modname) for key, w_value in dict_w.items(): space.setattr(w_mod, space.wrap(key), w_value) if doc: space.setattr(w_mod, space.wrap("__doc__"), space.wrap(rffi.charp2str(doc))) - # we cannot borrow here - return w_mod + return borrow_from(None, w_mod) -def convert_method_defs(space, dict_w, methods, w_type, w_self=None): +def convert_method_defs(space, dict_w, methods, w_type, w_self=None, name=None): + w_name = space.wrap(name) methods = rffi.cast(rffi.CArrayPtr(PyMethodDef), methods) if methods: i = -1 @@ -73,16 +75,12 @@ methodname = rffi.charp2str(method.c_ml_name) flags = rffi.cast(lltype.Signed, method.c_ml_flags) - if method.c_ml_doc: - doc = rffi.charp2str(method.c_ml_doc) - else: - doc = None if w_type is None: if flags & METH_CLASS or flags & METH_STATIC: raise OperationError(space.w_ValueError, space.wrap("module functions cannot set METH_CLASS or METH_STATIC")) - w_obj = space.wrap(W_PyCFunctionObject(space, method, w_self, doc)) + w_obj = space.wrap(W_PyCFunctionObject(space, method, w_self, w_name)) else: if methodname in dict_w and not (flags & METH_COEXIST): continue @@ -93,9 +91,8 @@ #w_obj = PyDescr_NewClassMethod(space, w_type, method) w_obj = space.w_Ellipsis # XXX elif flags & METH_STATIC: - w_func = PyCFunction_NewEx(space, method, None) - w_obj = space.w_Ellipsis # XXX - #w_obj = PyStaticMethod_New(space, w_func) + w_func = PyCFunction_NewEx(space, method, None, None) + w_obj = PyStaticMethod_New(space, w_func) else: w_obj = PyDescr_NewMethod(space, w_type, method) Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/object.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/object.py Fri May 28 16:20:28 2010 @@ -1,11 +1,12 @@ from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import cpython_api, generic_cpy_call, CANNOT_FAIL,\ - Py_ssize_t, PyVarObject, Py_TPFLAGS_HEAPTYPE,\ - Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT, Py_GE, CONST_STRING +from pypy.module.cpyext.api import ( + cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP, + PyVarObject, Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT, + Py_GE, CONST_STRING, FILEP, fwrite) from pypy.module.cpyext.pyobject import ( PyObject, PyObjectP, create_ref, from_ref, Py_IncRef, Py_DecRef, - track_reference) -from pypy.module.cpyext.typeobject import PyTypeObjectPtr, W_PyCTypeObject + track_reference, get_typedescr, RefcountState) +from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall from pypy.objspace.std.objectobject import W_ObjectObject from pypy.objspace.std.typeobject import W_TypeObject @@ -13,19 +14,32 @@ import pypy.module.__builtin__.operation as operation + at cpython_api([Py_ssize_t], rffi.VOIDP, error=lltype.nullptr(rffi.VOIDP.TO)) +def PyObject_MALLOC(space, size): + return lltype.malloc(rffi.VOIDP.TO, size, + flavor='raw', zero=True) + + at cpython_api([rffi.VOIDP_real], lltype.Void) +def PyObject_FREE(space, ptr): + lltype.free(ptr, flavor='raw') + @cpython_api([PyTypeObjectPtr], PyObject) def _PyObject_New(space, type): return _PyObject_NewVar(space, type, 0) @cpython_api([PyTypeObjectPtr, Py_ssize_t], PyObject) -def _PyObject_NewVar(space, type, size): +def _PyObject_NewVar(space, type, itemcount): w_type = from_ref(space, rffi.cast(PyObject, type)) - if isinstance(w_type, W_PyCTypeObject): - w_obj = space.allocate_instance(W_ObjectObject, w_type) - py_obj = create_ref(space, w_obj, items=size) - track_reference(space, py_obj, w_obj) - return py_obj - assert False, "Please add more cases in _PyObject_New" + assert isinstance(w_type, W_TypeObject) + typedescr = get_typedescr(w_type.instancetypedef) + py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) + py_obj.c_ob_refcnt = 0 + if type.c_tp_itemsize == 0: + w_obj = PyObject_Init(space, py_obj, type) + else: + py_objvar = rffi.cast(PyVarObject, py_obj) + w_obj = PyObject_InitVar(space, py_objvar, type, itemcount) + return py_obj @cpython_api([rffi.VOIDP_real], lltype.Void) def PyObject_Del(space, obj): @@ -127,6 +141,14 @@ space.delattr(w_obj, w_name) return 0 + at cpython_api([PyObject, CONST_STRING], rffi.INT_real, error=-1) +def PyObject_DelAttrString(space, w_obj, name_ptr): + """Delete attribute named attr_name, for object o. Returns -1 on failure. + This is the equivalent of the Python statement del o.attr_name.""" + w_name = space.wrap(rffi.charp2str(name_ptr)) + space.delattr(w_obj, w_name) + return 0 + @cpython_api([PyObject], lltype.Void) def PyObject_ClearWeakRefs(space, w_object): w_object.clear_all_weakrefs() @@ -147,29 +169,50 @@ This is the equivalent of the Python expression o[key].""" return space.getitem(w_obj, w_key) + at cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) +def PyObject_SetItem(space, w_obj, w_key, w_value): + """Map the object key to the value v. Returns -1 on failure. This is the + equivalent of the Python statement o[key] = v.""" + space.setitem(w_obj, w_key, w_value) + return 0 + + at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) +def PyObject_DelItem(space, w_obj, w_key): + """Delete the mapping for key from o. Returns -1 on failure. This is the + equivalent of the Python statement del o[key].""" + space.delitem(w_obj, w_key) + return 0 + @cpython_api([PyObject, PyTypeObjectPtr], PyObject) -def PyObject_Init(space, op, type): +def PyObject_Init(space, py_obj, type): """Initialize a newly-allocated object op with its type and initial reference. Returns the initialized object. If type indicates that the object participates in the cyclic garbage detector, it is added to the detector's set of observed objects. Other fields of the object are not affected.""" - if not op: + if not py_obj: PyErr_NoMemory(space) - op.c_ob_type = type - op.c_ob_refcnt = 1 - return from_ref(space, op) # XXX will give an exception + py_obj.c_ob_type = type + py_obj.c_ob_refcnt = 1 + w_type = from_ref(space, rffi.cast(PyObject, type)) + assert isinstance(w_type, W_TypeObject) + if w_type.is_cpytype(): + w_obj = space.allocate_instance(W_ObjectObject, w_type) + track_reference(space, py_obj, w_obj) + state = space.fromcache(RefcountState) + state.set_lifeline(w_obj, py_obj) + else: + assert False, "Please add more cases in PyObject_Init" + return py_obj @cpython_api([PyVarObject, PyTypeObjectPtr, Py_ssize_t], PyObject) -def PyObject_InitVar(space, op, type, size): +def PyObject_InitVar(space, py_obj, type, size): """This does everything PyObject_Init() does, and also initializes the length information for a variable-size object.""" - if not op: + if not py_obj: PyErr_NoMemory(space) - op.c_ob_size = size - op.c_ob_type = type - op.c_ob_refcnt = 1 - return from_ref(space, rffi.cast(PyObject, op)) # XXX likewise + py_obj.c_ob_size = size + return PyObject_Init(space, rffi.cast(PyObject, py_obj), type) @cpython_api([PyObject], PyObject) def PyObject_Type(space, w_obj): @@ -341,3 +384,62 @@ Compute and return the hash value of an object o. On failure, return -1. This is the equivalent of the Python expression hash(o).""" return space.int_w(space.hash(w_obj)) + + at cpython_api([PyObject, rffi.CCHARPP, Py_ssize_tP], rffi.INT_real, error=-1) +def PyObject_AsCharBuffer(space, obj, bufferp, sizep): + """Returns a pointer to a read-only memory location usable as + character-based input. The obj argument must support the single-segment + character buffer interface. On success, returns 0, sets buffer to the + memory location and size to the buffer length. Returns -1 and sets a + TypeError on error. + """ + pto = obj.c_ob_type + + pb = pto.c_tp_as_buffer + if not (pb and pb.c_bf_getreadbuffer and pb.c_bf_getsegcount): + raise OperationError(space.w_TypeError, space.wrap( + "expected a character buffer object")) + if generic_cpy_call(space, pb.c_bf_getsegcount, + obj, lltype.nullptr(rffi.INTP.TO)) != 1: + raise OperationError(space.w_TypeError, space.wrap( + "expected a single-segment buffer object")) + size = generic_cpy_call(space, pb.c_bf_getreadbuffer, + obj, 0, bufferp) + if size < 0: + return -1 + sizep[0] = size + return 0 + +# Also in include/object.h +Py_PRINT_RAW = 1 # No string quotes etc. + + at cpython_api([PyObject, FILEP, rffi.INT_real], rffi.INT_real, error=-1) +def PyObject_Print(space, w_obj, fp, flags): + """Print an object o, on file fp. Returns -1 on error. The flags argument + is used to enable certain printing options. The only option currently + supported is Py_PRINT_RAW; if given, the str() of the object is written + instead of the repr().""" + if rffi.cast(lltype.Signed, flags) & Py_PRINT_RAW: + w_str = space.str(w_obj) + else: + w_str = space.repr(w_obj) + + count = space.int_w(space.len(w_str)) + data = space.str_w(w_str) + buf = rffi.get_nonmovingbuffer(data) + try: + fwrite(buf, 1, count, fp) + finally: + rffi.free_nonmovingbuffer(data, buf) + return 0 + + at cpython_api([CONST_STRING, CONST_STRING], PyObject) +def PyFile_FromString(space, filename, mode): + """ + On success, return a new file object that is opened on the file given by + filename, with a file mode given by mode, where mode has the same + semantics as the standard C routine fopen(). On failure, return NULL.""" + w_filename = space.wrap(rffi.charp2str(filename)) + w_mode = space.wrap(rffi.charp2str(mode)) + return space.call_method(space.builtin, 'file', w_filename, w_mode) + Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/pyerrors.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/pyerrors.py Fri May 28 16:20:28 2010 @@ -1,7 +1,7 @@ import os from pypy.rpython.lltypesystem import rffi, lltype -from pypy.interpreter.error import OperationError, wrap_oserror +from pypy.interpreter.error import OperationError from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING from pypy.module.exceptions.interp_exceptions import W_RuntimeWarning from pypy.module.cpyext.pyobject import ( @@ -53,10 +53,11 @@ if operror: ptype[0] = make_ref(space, operror.w_type) pvalue[0] = make_ref(space, operror.get_w_value(space)) + ptraceback[0] = make_ref(space, space.wrap(operror.application_traceback)) else: ptype[0] = lltype.nullptr(PyObject.TO) pvalue[0] = lltype.nullptr(PyObject.TO) - ptraceback[0] = lltype.nullptr(PyObject.TO) + ptraceback[0] = lltype.nullptr(PyObject.TO) @cpython_api([PyObject, PyObject, PyObject], lltype.Void) def PyErr_Restore(space, w_type, w_value, w_traceback): @@ -115,7 +116,9 @@ Return value: always NULL.""" # XXX Doesn't actually do anything with PyErr_CheckSignals. errno = get_errno() - raise wrap_oserror(space, OSError(errno, "PyErr_SetFromErrno")) + msg = os.strerror(errno) + w_error = space.call_function(w_type, space.wrap(errno), space.wrap(msg)) + raise OperationError(w_type, w_error) @cpython_api([], rffi.INT_real, error=-1) def PyErr_CheckSignals(space): @@ -208,4 +211,33 @@ Deprecated; use PyErr_WarnEx() instead.""" return PyErr_WarnEx(space, w_category, message, 1) + at cpython_api([rffi.INT_real], lltype.Void) +def PyErr_PrintEx(space, set_sys_last_vars): + """Print a standard traceback to sys.stderr and clear the error indicator. + Call this function only when the error indicator is set. (Otherwise it will + cause a fatal error!) + + If set_sys_last_vars is nonzero, the variables sys.last_type, + sys.last_value and sys.last_traceback will be set to the + type, value and traceback of the printed exception, respectively.""" + if not PyErr_Occurred(space): + PyErr_BadInternalCall(space) + state = space.fromcache(State) + operror = state.clear_exception() + + w_type = operror.w_type + w_value = operror.get_w_value(space) + w_tb = space.wrap(operror.application_traceback) + + if rffi.cast(lltype.Signed, set_sys_last_vars): + space.sys.setdictvalue(space, "last_type", w_type) + space.sys.setdictvalue(space, "last_value", w_value) + space.sys.setdictvalue(space, "last_traceback", w_tb) + space.call_function(space.sys.get("excepthook"), + w_type, w_value, w_tb) + + at cpython_api([], lltype.Void) +def PyErr_Print(space): + """Alias for PyErr_PrintEx(1).""" + PyErr_PrintEx(space, 1) Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/pyobject.py Fri May 28 16:20:28 2010 @@ -8,6 +8,7 @@ from pypy.module.cpyext.state import State from pypy.objspace.std.typeobject import W_TypeObject from pypy.rlib.objectmodel import specialize, we_are_translated +from pypy.rlib.rweakref import RWeakKeyDictionary from pypy.rpython.annlowlevel import llhelper #________________________________________________________ @@ -20,8 +21,6 @@ raise NotImplementedError def allocate(self, space, w_type, itemcount=0): raise NotImplementedError - def make_ref(self, space, w_type, w_obj, itemcount=0): - raise NotImplementedError def attach(self, space, pyobj, w_obj): raise NotImplementedError def realize(self, space, ref): @@ -40,7 +39,6 @@ """ tp_basestruct = kw.pop('basestruct', PyObject.TO) - tp_make_ref = kw.pop('make_ref', None) tp_attach = kw.pop('attach', None) tp_realize = kw.pop('realize', None) tp_dealloc = kw.pop('dealloc', None) @@ -87,18 +85,6 @@ pyobj.c_ob_type = pytype return pyobj - # Specialized by meta-type - if tp_make_ref: - def make_ref(self, space, w_type, w_obj, itemcount=0): - return tp_make_ref(space, w_type, w_obj, itemcount=itemcount) - else: - def make_ref(self, space, w_type, w_obj, itemcount=0): - typedescr = get_typedescr(w_obj.typedef) - w_type = space.type(w_obj) - py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) - typedescr.attach(space, py_obj, w_obj) - return py_obj - if tp_attach: def attach(self, space, pyobj, w_obj): tp_attach(space, pyobj, w_obj) @@ -151,19 +137,27 @@ self.space = space self.py_objects_w2r = {} # { w_obj -> raw PyObject } self.py_objects_r2w = {} # { addr of raw PyObject -> w_obj } - self.borrow_mapping = {} # { w_container -> { w_containee -> None } } - self.borrowed_objects = {} # { addr of containee -> None } + + self.lifeline_dict = RWeakKeyDictionary(W_Root, PyOLifeline) + + self.borrow_mapping = {None: {}} + # { w_container -> { w_containee -> None } } + # the None entry manages references borrowed during a call to + # generic_cpy_call() + self.borrowed_objects = {} + # { addr of containee -> None } + # For tests self.non_heaptypes_w = [] def _freeze_(self): - assert not self.borrowed_objects and not self.borrow_mapping + assert not self.borrowed_objects + assert self.borrow_mapping == {None: {}} self.py_objects_r2w.clear() # is not valid anymore after translation return False def init_r2w_from_w2r(self): """Rebuilds the dict py_objects_r2w on startup""" - from pypy.module.cpyext.api import ADDR for w_obj, obj in self.py_objects_w2r.items(): ptr = rffi.cast(ADDR, obj) self.py_objects_r2w[ptr] = w_obj @@ -173,12 +167,82 @@ for w_obj, obj in self.py_objects_w2r.items(): print "%r: %i" % (w_obj, obj.c_ob_refcnt) + def get_from_lifeline(self, w_obj): + lifeline = self.lifeline_dict.get(w_obj) + if lifeline is not None: # make old PyObject ready for use in C code + py_obj = lifeline.pyo + assert py_obj.c_ob_refcnt == 0 + return py_obj + else: + lltype.nullptr(PyObject.TO) + + def set_lifeline(self, w_obj, py_obj): + self.lifeline_dict.set(w_obj, + PyOLifeline(self.space, py_obj)) + + def make_borrowed(self, w_container, w_borrowed): + """ + Create a borrowed reference, which will live as long as the container + has a living reference (as a PyObject!) + """ + ref = make_ref(self.space, w_borrowed) + obj_ptr = rffi.cast(ADDR, ref) + if obj_ptr not in self.borrowed_objects: + # borrowed_objects owns the reference + self.borrowed_objects[obj_ptr] = None + else: + Py_DecRef(self.space, ref) # already in borrowed list + + borrowees = self.borrow_mapping.setdefault(w_container, {}) + borrowees[w_borrowed] = None + return ref + def reset_borrowed_references(self): + "Used in tests" while self.borrowed_objects: addr, _ = self.borrowed_objects.popitem() w_obj = self.py_objects_r2w[addr] Py_DecRef(self.space, w_obj) - self.borrow_mapping = {} + self.borrow_mapping = {None: {}} + + def delete_borrower(self, w_obj): + """ + Called when a potential container for borrowed references has lost its + last reference. Removes the borrowed references it contains. + """ + if w_obj in self.borrow_mapping: # move to lifeline __del__ + for w_containee in self.borrow_mapping[w_obj]: + self.forget_borrowee(w_containee) + del self.borrow_mapping[w_obj] + + def swap_borrow_container(self, container): + """switch the current default contained with the given one.""" + if container is None: + old_container = self.borrow_mapping[None] + self.borrow_mapping[None] = {} + return old_container + else: + old_container = self.borrow_mapping[None] + self.borrow_mapping[None] = container + for w_containee in old_container: + self.forget_borrowee(w_containee) + + def forget_borrowee(self, w_obj): + "De-register an object from the list of borrowed references" + ref = self.py_objects_w2r.get(w_obj, lltype.nullptr(PyObject.TO)) + if not ref: + if DEBUG_REFCOUNT: + print >>sys.stderr, "Borrowed object is already gone:", \ + hex(containee) + return + + containee_ptr = rffi.cast(ADDR, ref) + try: + del self.borrowed_objects[containee_ptr] + except KeyError: + pass + else: + Py_DecRef(self.space, ref) class InvalidPointerException(Exception): pass @@ -194,14 +258,26 @@ print >>sys.stderr, arg, print >>sys.stderr -def create_ref(space, w_obj, items=0): +def create_ref(space, w_obj, itemcount=0): """ Allocates a PyObject, and fills its fields with info from the given intepreter object. """ w_type = space.type(w_obj) - metatypedescr = get_typedescr(w_type.typedef) - return metatypedescr.make_ref(space, w_type, w_obj, itemcount=items) + if w_type.is_cpytype(): + state = space.fromcache(RefcountState) + py_obj = state.get_from_lifeline(w_obj) + if py_obj: + Py_IncRef(space, py_obj) + return py_obj + + typedescr = get_typedescr(w_obj.typedef) + py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) + if w_type.is_cpytype(): + state = space.fromcache(RefcountState) + state.set_lifeline(w_obj, py_obj) + typedescr.attach(space, py_obj, w_obj) + return py_obj def track_reference(space, py_obj, w_obj, replace=False): """ @@ -271,7 +347,6 @@ return assert lltype.typeOf(obj) == PyObject - from pypy.module.cpyext.typeobject import W_PyCTypeObject obj.c_ob_refcnt -= 1 if DEBUG_REFCOUNT: debug_refcount("DECREF", obj, obj.c_ob_refcnt, frame_stackdepth=3) @@ -279,21 +354,18 @@ state = space.fromcache(RefcountState) ptr = rffi.cast(ADDR, obj) if ptr not in state.py_objects_r2w: - w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type)) - if space.is_w(w_type, space.w_str) or space.is_w(w_type, space.w_unicode): - # this is a half-allocated string, lets call the deallocator - # without modifying the r2w/w2r dicts - _Py_Dealloc(space, obj) + # this is a half-allocated object, lets call the deallocator + # without modifying the r2w/w2r dicts + _Py_Dealloc(space, obj) else: w_obj = state.py_objects_r2w[ptr] del state.py_objects_r2w[ptr] w_type = space.type(w_obj) - w_typetype = space.type(w_type) - if not space.is_w(w_typetype, space.gettypeobject(W_PyCTypeObject.typedef)): + if not w_type.is_cpytype(): _Py_Dealloc(space, obj) del state.py_objects_w2r[w_obj] # if the object was a container for borrowed references - delete_borrower(space, w_obj) + state.delete_borrower(w_obj) else: if not we_are_translated() and obj.c_ob_refcnt < 0: message = "Negative refcount for obj %s with type %s" % ( @@ -322,6 +394,24 @@ generic_cpy_call_dont_decref(space, pto.c_tp_dealloc, obj) #___________________________________________________________ +# Support for "lifelines" +# +# Object structure must stay alive even when not referenced +# by any C code. + +class PyOLifeline(object): + def __init__(self, space, pyo): + self.pyo = pyo + self.space = space + + def __del__(self): + if self.pyo: + assert self.pyo.c_ob_refcnt == 0 + _Py_Dealloc(self.space, self.pyo) + self.pyo = lltype.nullptr(PyObject.TO) + # XXX handle borrowed objects here + +#___________________________________________________________ # Support for borrowed references def make_borrowed_ref(space, w_container, w_borrowed): @@ -329,23 +419,11 @@ Create a borrowed reference, which will live as long as the container has a living reference (as a PyObject!) """ - ref = make_ref(space, w_borrowed) - if not ref: - return ref + if w_borrowed is None: + return lltype.nullptr(PyObject.TO) - # state.borrowed_objects owns the reference state = space.fromcache(RefcountState) - obj_ptr = rffi.cast(ADDR, ref) - if obj_ptr not in state.borrowed_objects: - state.borrowed_objects[obj_ptr] = None - else: - Py_DecRef(space, ref) # already in borrowed list - - if w_container is None: # self-managed - return ref - borrowees = state.borrow_mapping.setdefault(w_container, {}) - borrowees[w_borrowed] = None - return ref + return state.make_borrowed(w_container, w_borrowed) class BorrowPair: """ @@ -361,36 +439,6 @@ def borrow_from(container, borrowed): return BorrowPair(container, borrowed) -def forget_borrowee(space, w_obj): - "De-register an object from the list of borrowed references" - state = space.fromcache(RefcountState) - ref = state.py_objects_w2r.get(w_obj, lltype.nullptr(PyObject.TO)) - if not ref: - if DEBUG_REFCOUNT: - print >>sys.stderr, "Borrowed object is already gone:", \ - hex(containee) - return - - containee_ptr = rffi.cast(ADDR, ref) - try: - del state.borrowed_objects[containee_ptr] - except KeyError: - pass - else: - Py_DecRef(space, ref) - -def delete_borrower(space, w_obj): - """ - Called when a potential container for borrowed references has lost its - last reference. Removes the borrowed references it contains. - """ - state = space.fromcache(RefcountState) - if w_obj in state.borrow_mapping: # move to lifeline __del__ - for w_containee in state.borrow_mapping[w_obj]: - forget_borrowee(space, w_containee) - del state.borrow_mapping[w_obj] - - #___________________________________________________________ @cpython_api([rffi.VOIDP_real], lltype.Signed, error=CANNOT_FAIL) Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/sequence.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/sequence.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/sequence.py Fri May 28 16:20:28 2010 @@ -67,12 +67,23 @@ @cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject) def PySequence_GetSlice(space, w_obj, start, end): """Return the slice of sequence object o between i1 and i2, or NULL on - failure. This is the equivalent of the Python expression o[i1:i2]. - - This function used an int type for i1 and i2. This might - require changes in your code for properly supporting 64-bit systems.""" + failure. This is the equivalent of the Python expression o[i1:i2].""" return space.getslice(w_obj, space.wrap(start), space.wrap(end)) + at cpython_api([PyObject, Py_ssize_t, Py_ssize_t, PyObject], rffi.INT_real, error=-1) +def PySequence_SetSlice(space, w_obj, start, end, w_value): + """Assign the sequence object v to the slice in sequence object o from i1 to + i2. This is the equivalent of the Python statement o[i1:i2] = v.""" + space.setslice(w_obj, space.wrap(start), space.wrap(end), w_value) + return 0 + + at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], rffi.INT_real, error=-1) +def PySequence_DelSlice(space, w_obj, start, end): + """Delete the slice in sequence object o from i1 to i2. Returns -1 on + failure. This is the equivalent of the Python statement del o[i1:i2].""" + space.delslice(w_obj, space.wrap(start), space.wrap(end)) + return 0 + @cpython_api([PyObject, Py_ssize_t], PyObject) def PySequence_GetItem(space, w_obj, i): """Return the ith element of o, or NULL on failure. This is the equivalent of Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/sliceobject.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/sliceobject.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/sliceobject.py Fri May 28 16:20:28 2010 @@ -1,13 +1,63 @@ from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, Py_ssize_t,\ - Py_ssize_tP, build_type_checkers -from pypy.module.cpyext.pyobject import Py_DecRef, PyObject +from pypy.module.cpyext.api import ( + cpython_api, cpython_struct, bootstrap_function, build_type_checkers, + CANNOT_FAIL, Py_ssize_t, Py_ssize_tP, PyObjectFields) +from pypy.module.cpyext.pyobject import ( + Py_DecRef, PyObject, make_ref, make_typedescr) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.interpreter.error import OperationError from pypy.objspace.std.sliceobject import W_SliceObject +# Slice objects directly expose their members as PyObject. +# Don't change them! + +PySliceObjectStruct = lltype.ForwardReference() +PySliceObject = lltype.Ptr(PySliceObjectStruct) +PySliceObjectFields = PyObjectFields + \ + (("start", PyObject), ("step", PyObject), ("stop", PyObject), ) +cpython_struct("PySliceObject", PySliceObjectFields, PySliceObjectStruct) + + at bootstrap_function +def init_sliceobject(space): + "Type description of PySliceObject" + make_typedescr(W_SliceObject.typedef, + basestruct=PySliceObject.TO, + attach=slice_attach, + dealloc=slice_dealloc) + +def slice_attach(space, py_obj, w_obj): + """ + Fills a newly allocated PySliceObject with the given slice object. The + fields must not be modified. + """ + py_slice = rffi.cast(PySliceObject, py_obj) + assert isinstance(w_obj, W_SliceObject) + py_slice.c_start = make_ref(space, w_obj.w_start) + py_slice.c_stop = make_ref(space, w_obj.w_stop) + py_slice.c_step = make_ref(space, w_obj.w_step) + + at cpython_api([PyObject], lltype.Void, external=False) +def slice_dealloc(space, py_obj): + """Frees allocated PyStringObject resources. + """ + py_slice = rffi.cast(PySliceObject, py_obj) + Py_DecRef(space, py_slice.c_start) + Py_DecRef(space, py_slice.c_stop) + Py_DecRef(space, py_slice.c_step) + from pypy.module.cpyext.object import PyObject_dealloc + PyObject_dealloc(space, py_obj) + PySlice_Check, PySlice_CheckExact = build_type_checkers("Slice") + at cpython_api([PyObject, PyObject, PyObject], PyObject) +def PySlice_New(space, w_start, w_stop, w_step): + """Return a new slice object with the given values. The start, stop, and + step parameters are used as the values of the slice object attributes of + the same names. Any of the values may be NULL, in which case the + None will be used for the corresponding attribute. Return NULL if + the new object could not be allocated.""" + return W_SliceObject(w_start, w_stop, w_step) + @cpython_api([PyObject, Py_ssize_t, Py_ssize_tP, Py_ssize_tP, Py_ssize_tP, Py_ssize_tP], rffi.INT_real, error=-1) def PySlice_GetIndicesEx(space, w_slice, length, start_p, stop_p, Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/slotdefs.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/slotdefs.py Fri May 28 16:20:28 2010 @@ -13,8 +13,7 @@ from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.argument import Arguments from pypy.rlib.unroll import unrolling_iterable - -space = None +from pypy.tool.sourcetools import func_with_new_name def check_num_args(space, ob, n): @@ -106,7 +105,7 @@ raise OperationError(space.w_StopIteration, space.w_None) return w_res - at cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject, external=True) + at cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject, external=False) def slot_tp_new(space, type, w_args, w_kwds): from pypy.module.cpyext.tupleobject import PyTuple_Check pyo = rffi.cast(PyObject, type) @@ -117,13 +116,21 @@ w_args_new = space.newtuple(args_w) return space.call(w_func, w_args_new, w_kwds) - at cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) + at cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1, external=False) def slot_tp_init(space, w_self, w_args, w_kwds): w_descr = space.lookup(w_self, '__init__') args = Arguments.frompacked(space, w_args, w_kwds) space.get_and_call_args(w_descr, w_self, args) return 0 + at cpython_api([PyObject, PyObject, PyObject], PyObject, external=False) +def slot_tp_call(space, w_self, w_args, w_kwds): + return space.call(w_self, w_args, w_kwds) + + at cpython_api([PyObject], PyObject, external=False) +def slot_nb_int(space, w_self): + return space.int(w_self) + PyWrapperFlag_KEYWORDS = 1 # adopted from typeobject.c @@ -140,12 +147,14 @@ if wrapper is Ellipsis: def wrapper(space, w_self, w_args, func, w_kwds): raise NotImplementedError("Wrapper for slot " + NAME) + wrapper = func_with_new_name(wrapper, WRAPPER) wrapper1 = None wrapper2 = wrapper else: if wrapper is Ellipsis: def wrapper(space, w_self, w_args, func): raise NotImplementedError("Wrapper for slot " + NAME) + wrapper = func_with_new_name(wrapper, WRAPPER) wrapper1 = wrapper wrapper2 = None return (NAME, slotname, function, wrapper1, wrapper2, DOC) @@ -180,18 +189,6 @@ return ETSLOT(NAME, "tp_as_number.c_" + SLOT, FUNCTION, "wrap_binaryfunc_r", \ "x." + NAME + "(y) <==> " + DOC) -slotdef_replacements = ( - ("\s+", " "), - ("static [^{]*{", "("), - ("};", ")"), - (r"(?P +..SLOT\([^,]*, )(?P[^,]*), (?P[^,]*), (?P[^,]*)", r"\g'\g', '\g', '\g'"), - (r"(?P *R?[^ ]{3}SLOT(NOTINFIX)?\([^,]*, )(?P[^,]*), (?P[^,]*)", r"\g'\g', '\g'"), - ("'NULL'", "None"), - ("{NULL}", ""), - ("\(wrapperfunc\)", ""), - ("\),", "),\n"), -) - """ /* Heap types defining __add__/__mul__ have sq_concat/sq_repeat == NULL. The logic in abstract.c always falls back to nb_add/nb_multiply in @@ -203,7 +200,7 @@ # Copy new slotdefs from typeobject.c # Remove comments and tabs # Done. -slotdefs_str = """ +slotdefs_str = r""" static slotdef slotdefs[] = { SQSLOT("__len__", sq_length, slot_sq_length, wrap_lenfunc, "x.__len__() <==> len(x)"), @@ -395,6 +392,21 @@ {NULL} }; """ + +# Convert the above string into python code +slotdef_replacements = ( + ("\s+", " "), # all on one line + ("static [^{]*{", "("), # remove first line... + ("};", ")"), # ...last line... + ("{NULL}", ""), # ...and sentinel + # add quotes around function name, slot name, and wrapper name + (r"(?P +..SLOT\([^,]*, )(?P[^,]*), (?P[^,]*), (?P[^,]*)", r"\g'\g', '\g', '\g'"), + (r"(?P *R?[^ ]{3}SLOT(NOTINFIX)?\([^,]*, )(?P[^,]*), (?P[^,]*)", r"\g'\g', '\g'"), + ("'NULL'", "None"), # but NULL becomes None + ("\(wrapperfunc\)", ""), # casts are not needed in python tuples + ("\),", "),\n"), # add newlines again +) + for regex, repl in slotdef_replacements: slotdefs_str = re.sub(regex, repl, slotdefs_str) Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/state.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/state.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/state.py Fri May 28 16:20:28 2010 @@ -29,17 +29,8 @@ self.operror = operror def clear_exception(self): - """Clear the current exception state, and return the operror. - Also frees the borrowed reference returned by PyErr_Occurred() - """ - from pypy.module.cpyext.api import ADDR - # handling of borrowed objects, remove when we have - # a weakkeydict + """Clear the current exception state, and return the operror.""" operror = self.operror - if operror is not None: - # we have no explicit container - from pypy.module.cpyext.pyobject import forget_borrowee - forget_borrowee(self.space, operror.w_type) self.operror = None return operror Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/stringobject.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/stringobject.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/stringobject.py Fri May 28 16:20:28 2010 @@ -238,3 +238,12 @@ args. The args argument must be a tuple.""" return space.mod(w_format, w_args) + at cpython_api([CONST_STRING], PyObject) +def PyString_InternFromString(space, string): + """A combination of PyString_FromString() and + PyString_InternInPlace(), returning either a new string object that has + been interned, or a new ("owned") reference to an earlier interned string + object with the same value.""" + s = rffi.charp2str(string) + return space.new_interned_str(s) + Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/stubs.py Fri May 28 16:20:28 2010 @@ -826,17 +826,6 @@ """ raise NotImplementedError - at cpython_api([rffi.INT_real], lltype.Void) -def PyErr_PrintEx(space, set_sys_last_vars): - """Print a standard traceback to sys.stderr and clear the error indicator. - Call this function only when the error indicator is set. (Otherwise it will - cause a fatal error!) - - If set_sys_last_vars is nonzero, the variables sys.last_type, - sys.last_value and sys.last_traceback will be set to the - type, value and traceback of the printed exception, respectively.""" - raise NotImplementedError - @cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void) def PyErr_NormalizeException(space, exc, val, tb): """Under certain circumstances, the values returned by PyErr_Fetch() below @@ -993,17 +982,6 @@ """ raise NotImplementedError - at cpython_api([rffi.CCHARP, rffi.CCHARP], PyObject) -def PyFile_FromString(space, filename, mode): - """ - - - - On success, return a new file object that is opened on the file given by - filename, with a file mode given by mode, where mode has the same - semantics as the standard C routine fopen(). On failure, return NULL.""" - raise NotImplementedError - @cpython_api([FILE, rffi.CCHARP, rffi.CCHARP, rffi.INT_real], PyObject) def PyFile_FromFile(space, fp, name, mode, close): """Create a new PyFileObject from the already-open standard C file @@ -2084,11 +2062,6 @@ raise NotImplementedError @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyIter_Check(space, o): - """Return true if the object o supports the iterator protocol.""" - raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PySeqIter_Check(space, seq): """Return true if the type of op is PySeqIter_Type. """ @@ -2291,19 +2264,6 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject], PyObject) -def PyMethod_Self(space, meth): - """Return the instance associated with the method meth if it is bound, otherwise - return NULL.""" - borrow_from() - raise NotImplementedError - - at cpython_api([PyObject], PyObject) -def PyMethod_GET_SELF(space, meth): - """Macro version of PyMethod_Self() which avoids error checking.""" - borrow_from() - raise NotImplementedError - @cpython_api([], rffi.INT_real, error=CANNOT_FAIL) def PyMethod_ClearFreeList(space, ): """Clear the free list. Return the total number of freed items. @@ -2373,26 +2333,6 @@ """ raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARPP, Py_ssize_t], rffi.INT_real, error=-1) -def PyObject_AsCharBuffer(space, obj, buffer, buffer_len): - """Returns a pointer to a read-only memory location usable as character-based - input. The obj argument must support the single-segment character buffer - interface. On success, returns 0, sets buffer to the memory location - and buffer_len to the buffer length. Returns -1 and sets a - TypeError on error. - - - - This function used an int * type for buffer_len. This might - require changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - - at cpython_api([PyObject, rffi.CCHARP], rffi.INT_real, error=-1) -def PyObject_DelAttrString(space, o, attr_name): - """Delete attribute named attr_name, for object o. Returns -1 on failure. - This is the equivalent of the Python statement del o.attr_name.""" - raise NotImplementedError - @cpython_api([PyObject, PyObject, rffi.INTP], rffi.INT_real, error=-1) def PyObject_Cmp(space, o1, o2, result): """ @@ -2421,18 +2361,6 @@ """ raise NotImplementedError - at cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) -def PyObject_SetItem(space, o, key, v): - """Map the object key to the value v. Returns -1 on failure. This is the - equivalent of the Python statement o[key] = v.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) -def PyObject_DelItem(space, o, key): - """Delete the mapping for key from o. Returns -1 on failure. This is the - equivalent of the Python statement del o[key].""" - raise NotImplementedError - @cpython_api([PyObject], PyObject) def PyObject_Dir(space, o): """This is equivalent to the Python expression dir(o), returning a (possibly @@ -2442,27 +2370,6 @@ is active then NULL is returned but PyErr_Occurred() will return false.""" raise NotImplementedError - at cpython_api([], PyObject) -def PyEval_GetBuiltins(space, ): - """Return a dictionary of the builtins in the current execution frame, - or the interpreter of the thread state if no frame is currently executing.""" - borrow_from() - raise NotImplementedError - - at cpython_api([], PyObject) -def PyEval_GetLocals(space, ): - """Return a dictionary of the local variables in the current execution frame, - or NULL if no frame is currently executing.""" - borrow_from() - raise NotImplementedError - - at cpython_api([], PyObject) -def PyEval_GetGlobals(space, ): - """Return a dictionary of the global variables in the current execution frame, - or NULL if no frame is currently executing.""" - borrow_from() - raise NotImplementedError - @cpython_api([], PyFrameObject) def PyEval_GetFrame(space, ): """Return the current thread state's frame, which is NULL if no frame is @@ -2541,24 +2448,6 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, Py_ssize_t, Py_ssize_t, PyObject], rffi.INT_real, error=-1) -def PySequence_SetSlice(space, o, i1, i2, v): - """Assign the sequence object v to the slice in sequence object o from i1 to - i2. This is the equivalent of the Python statement o[i1:i2] = v. - - This function used an int type for i1 and i2. This might - require changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - - at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], rffi.INT_real, error=-1) -def PySequence_DelSlice(space, o, i1, i2): - """Delete the slice in sequence object o from i1 to i2. Returns -1 on - failure. This is the equivalent of the Python statement del o[i1:i2]. - - This function used an int type for i1 and i2. This might - require changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([PyObject, PyObject], Py_ssize_t, error=-1) def PySequence_Count(space, o, value): """Return the number of occurrences of value in o, that is, return the number @@ -2731,15 +2620,6 @@ """Empty an existing set of all elements.""" raise NotImplementedError - at cpython_api([PyObject, PyObject, PyObject], PyObject) -def PySlice_New(space, start, stop, step): - """Return a new slice object with the given values. The start, stop, and - step parameters are used as the values of the slice object attributes of - the same names. Any of the values may be NULL, in which case the - None will be used for the corresponding attribute. Return NULL if - the new object could not be allocated.""" - raise NotImplementedError - @cpython_api([PyObject, Py_ssize_t, Py_ssize_t, Py_ssize_t, Py_ssize_t], rffi.INT_real, error=-1) def PySlice_GetIndices(space, slice, length, start, stop, step): """Retrieve the start, stop and step indices from the slice object slice, @@ -2789,16 +2669,6 @@ This function is not available in 3.x and does not have a PyBytes alias.""" raise NotImplementedError - at cpython_api([rffi.CCHARP], PyObject) -def PyString_InternFromString(space, v): - """A combination of PyString_FromString() and - PyString_InternInPlace(), returning either a new string object that has - been interned, or a new ("owned") reference to an earlier interned string object - with the same value. - - This function is not available in 3.x and does not have a PyBytes alias.""" - raise NotImplementedError - @cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.CCHARP], PyObject) def PyString_Decode(space, s, size, encoding, errors): """Create an object by decoding size bytes of the encoded buffer s using the @@ -3711,12 +3581,6 @@ exception was raised.""" raise NotImplementedError - at cpython_api([FILE, rffi.CCHARP, rffi.INT_real, PyObject, PyObject], PyObject) -def PyRun_File(space, fp, filename, start, globals, locals): - """This is a simplified interface to PyRun_FileExFlags() below, leaving - closeit set to 0 and flags set to NULL.""" - raise NotImplementedError - @cpython_api([FILE, rffi.CCHARP, rffi.INT_real, PyObject, PyObject, rffi.INT_real], PyObject) def PyRun_FileEx(space, fp, filename, start, globals, locals, closeit): """This is a simplified interface to PyRun_FileExFlags() below, leaving Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/stubsactive.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/stubsactive.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/stubsactive.py Fri May 28 16:20:28 2010 @@ -1,10 +1,11 @@ from pypy.module.cpyext.pyobject import PyObject from pypy.module.cpyext.api import cpython_api, Py_ssize_t, CANNOT_FAIL, CConfig +from pypy.module.cpyext.object import FILEP from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.pystate import PyThreadState, PyInterpreterState - at cpython_api([PyObject], rffi.VOIDP, error=CANNOT_FAIL) #XXX + at cpython_api([PyObject], FILEP, error=CANNOT_FAIL) def PyFile_AsFile(space, p): """Return the file object associated with p as a FILE*. @@ -13,44 +14,6 @@ PyFile_DecUseCount() functions described below as appropriate.""" raise NotImplementedError - at cpython_api([PyObject], PyObject) -def PyObject_GetIter(space, o): - """This is equivalent to the Python expression iter(o). It returns a new - iterator for the object argument, or the object itself if the object is already - an iterator. Raises TypeError and returns NULL if the object cannot be - iterated.""" - raise NotImplementedError - - at cpython_api([PyObject], PyObject) -def PyIter_Next(space, o): - """Return the next value from the iteration o. If the object is an iterator, - this retrieves the next value from the iteration, and returns NULL with no - exception set if there are no remaining items. If the object is not an - iterator, TypeError is raised, or if there is an error in retrieving the - item, returns NULL and passes along the exception.""" - raise NotImplementedError - - at cpython_api([rffi.ULONG], PyObject) -def PyLong_FromUnsignedLong(space, v): - """Return a new PyLongObject object from a C unsigned long, or - NULL on failure.""" - raise NotImplementedError - -FILE = rffi.VOIDP_real.TO -FILEP = lltype.Ptr(FILE) - at cpython_api([PyObject, FILEP, rffi.INT_real], rffi.INT_real, error=-1) -def PyObject_Print(space, o, fp, flags): - """Print an object o, on file fp. Returns -1 on error. The flags argument - is used to enable certain printing options. The only option currently supported - is Py_PRINT_RAW; if given, the str() of the object is written - instead of the repr().""" - raise NotImplementedError - - at cpython_api([], lltype.Void) -def PyErr_Print(space): - """Alias for PyErr_PrintEx(1).""" - raise NotImplementedError - @cpython_api([PyInterpreterState], PyThreadState, error=CANNOT_FAIL) def PyThreadState_New(space, interp): """Create a new thread state object belonging to the given interpreter object. Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/sysmodule.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/sysmodule.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/sysmodule.py Fri May 28 16:20:28 2010 @@ -10,7 +10,7 @@ name = rffi.charp2str(name) w_dict = space.sys.getdict() w_obj = space.finditem_str(w_dict, name) - return borrow_from(w_dict, w_obj) + return borrow_from(None, w_obj) @cpython_api([CONST_STRING, PyObject], rffi.INT_real, error=-1) def PySys_SetObject(space, name, w_obj): Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/conftest.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/conftest.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/conftest.py Fri May 28 16:20:28 2010 @@ -1,15 +1,12 @@ import py from pypy.conftest import option, gettestobjspace -class Directory(py.test.collect.Directory): - def collect(self): - if option.runappdirect: - py.test.skip("cannot be run by py.test -A") +def pytest_collect_directory(parent): + if parent.config.option.runappdirect: + py.test.skip("cannot be run by py.test -A") - # ensure additional functions are registered - import pypy.module.cpyext.test.test_cpyext - - return super(Directory, self).collect() + # ensure additional functions are registered + import pypy.module.cpyext.test.test_cpyext def pytest_funcarg__space(request): return gettestobjspace(usemodules=['cpyext', 'thread']) Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/foo.c ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/foo.c (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/foo.c Fri May 28 16:20:28 2010 @@ -53,6 +53,12 @@ } static PyObject * +foo_create(fooobject *self) +{ + return newfooobject(); +} + +static PyObject * foo_unset(fooobject *self) { self->foo_string = NULL; @@ -62,6 +68,7 @@ static PyMethodDef foo_methods[] = { {"copy", (PyCFunction)foo_copy, METH_NOARGS, NULL}, + {"create", (PyCFunction)foo_create, METH_NOARGS|METH_STATIC, NULL}, {"unset_string_member", (PyCFunction)foo_unset, METH_NOARGS, NULL}, {NULL, NULL} /* sentinel */ }; @@ -159,6 +166,8 @@ foo_getseters, /*tp_getset*/ }; +/* A type that inherits from 'unicode */ + typedef struct { PyUnicodeObject HEAD; int val; @@ -305,6 +314,68 @@ }; +/* A Metatype */ + +PyTypeObject MetaType = { + PyObject_HEAD_INIT(NULL) + 0, + "foo.Meta", + sizeof(PyTypeObject), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + 0, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_CHECKTYPES, /*tp_flags*/ + 0, /*tp_doc*/ + + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + + /* Attribute descriptor and subclassing stuff */ + + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + + 0, /*tp_init*/ + 0, /*tp_alloc*/ + 0, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0 /*tp_weaklist*/ +}; + + /* foo functions */ static PyObject * @@ -337,6 +408,7 @@ FuuType.tp_base = &PyUnicode_Type; Fuu2Type.tp_base = &FuuType; + MetaType.tp_base = &PyType_Type; if (PyType_Ready(&footype) < 0) return; @@ -344,15 +416,20 @@ return; if (PyType_Ready(&Fuu2Type) < 0) return; + if (PyType_Ready(&MetaType) < 0) + return; m = Py_InitModule("foo", foo_functions); if (m == NULL) return; d = PyModule_GetDict(m); - if (d) { - if (PyDict_SetItemString(d, "fooType", (PyObject *)&footype) < 0) - return; - PyDict_SetItemString(d, "FuuType", (PyObject *) &FuuType); - PyDict_SetItemString(d, "Fuu2Type", (PyObject *) &Fuu2Type); - } - /* No need to check the error here, the caller will do that */ + if (d == NULL) + return; + if (PyDict_SetItemString(d, "fooType", (PyObject *)&footype) < 0) + return; + if (PyDict_SetItemString(d, "FuuType", (PyObject *) &FuuType) < 0) + return; + if(PyDict_SetItemString(d, "Fuu2Type", (PyObject *) &Fuu2Type) < 0) + return; + if (PyDict_SetItemString(d, "MetaType", (PyObject *) &MetaType) < 0) + return; } Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_api.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_api.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_api.py Fri May 28 16:20:28 2010 @@ -22,12 +22,15 @@ # warm up reference counts: # - the posix module allocates a HCRYPTPROV on Windows - # - writing to stderr allocates a file lock + # - writing to stdout and stderr allocates a file lock space.getbuiltinmodule("cpyext") space.getbuiltinmodule(os.name) space.call_function(space.getattr(space.sys.get("stderr"), space.wrap("write")), space.wrap("")) + space.call_function(space.getattr(space.sys.get("stdout"), + space.wrap("write")), + space.wrap("")) class CAPI: def __getattr__(self, name): Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_bufferobject.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_bufferobject.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_bufferobject.py Fri May 28 16:20:28 2010 @@ -20,10 +20,30 @@ """ free(cbuf); Py_RETURN_NONE; + """), + ("check_ascharbuffer", "METH_O", + """ + char *ptr; + Py_ssize_t size; + if (PyObject_AsCharBuffer(args, &ptr, &size) < 0) + return NULL; + return PyString_FromStringAndSize(ptr, size); """) ], prologue = """ static char* cbuf = NULL; """) - buffer = module.get_FromMemory() - assert str(buffer) == 'abc\0' + buf = module.get_FromMemory() + assert str(buf) == 'abc\0' + + assert module.check_ascharbuffer(buf) == 'abc\0' + module.free_buffer() + + def test_Buffer_New(self): + module = self.import_extension('foo', [ + ("buffer_new", "METH_NOARGS", + """ + return PyBuffer_New(150); + """), + ]) + module.buffer_new() Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_cpyext.py Fri May 28 16:20:28 2010 @@ -10,7 +10,7 @@ from pypy.translator import platform from pypy.translator.gensupp import uniquemodulename from pypy.tool.udir import udir -from pypy.module.cpyext import api, typeobject +from pypy.module.cpyext import api from pypy.module.cpyext.state import State from pypy.module.cpyext.pyobject import RefcountState from pypy.module.cpyext.pyobject import Py_DecRef, InvalidPointerException @@ -90,6 +90,14 @@ gc.collect() lost_objects_w = identity_dict() lost_objects_w.update((key, None) for key in self.frozen_refcounts.keys()) + + # Clear all lifelines, objects won't resurrect + for w_obj, obj in state.lifeline_dict._dict.items(): + if w_obj not in state.py_objects_w2r: + state.lifeline_dict.set(w_obj, None) + del obj + gc.collect() + for w_obj, obj in state.py_objects_w2r.iteritems(): base_refcnt = self.frozen_refcounts.get(w_obj) delta = obj.c_ob_refcnt @@ -99,7 +107,7 @@ if delta != 0: leaking = True print >>sys.stderr, "Leaking %r: %i references" % (w_obj, delta) - lifeline = typeobject.lifeline_dict.get(w_obj) + lifeline = state.lifeline_dict.get(w_obj) if lifeline is not None: refcnt = lifeline.pyo.c_ob_refcnt if refcnt > 0: @@ -249,14 +257,11 @@ def unimport_module(self, name): """ - Remove the named module from the space's sys.modules and discard the - reference (owned by "the test") to it. + Remove the named module from the space's sys.modules. """ w_modules = self.space.sys.get('modules') w_name = self.space.wrap(name) - w_mod = self.space.getitem(w_modules, w_name) self.space.delitem(w_modules, w_name) - Py_DecRef(self.space, w_mod) def teardown_method(self, func): for name in self.imported_module_names: @@ -301,6 +306,7 @@ assert 'return_pi' in dir(module) assert module.return_pi is not None assert module.return_pi() == 3.14 + assert module.return_pi.__module__ == 'foo' def test_export_docstring(self): @@ -649,3 +655,25 @@ h = mod.get_hash() assert h != 0 assert h % 4 == 0 # it's the pointer value + + def test_types(self): + """test the presence of random types""" + + mod = self.import_extension('foo', [ + ('get_names', 'METH_NOARGS', + ''' + /* XXX in tests, the C type is not correct */ + #define NAME(type) ((PyTypeObject*)&type)->tp_name + return Py_BuildValue("sssss", + NAME(PyCell_Type), + NAME(PyModule_Type), + NAME(PyProperty_Type), + NAME(PyStaticMethod_Type), + NAME(PyCFunction_Type) + ); + ''' + ), + ]) + assert mod.get_names() == ('cell', 'module', 'property', + 'staticmethod', + 'builtin_function_or_method') Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_dictobject.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_dictobject.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_dictobject.py Fri May 28 16:20:28 2010 @@ -1,5 +1,7 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.api import Py_ssize_tP, PyObjectP +from pypy.module.cpyext.pyobject import make_ref, from_ref class TestDictObject(BaseApiTest): def test_dict(self, space, api): @@ -71,3 +73,28 @@ api.PyDict_Update(w_d, w_d2) assert space.unwrap(w_d) == dict(a='b', c='d', e='f') + + def test_iter(self, space, api): + w_dict = space.sys.getdict() + py_dict = make_ref(space, w_dict) + + ppos = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw') + ppos[0] = 0 + pkey = lltype.malloc(PyObjectP.TO, 1, flavor='raw') + pvalue = lltype.malloc(PyObjectP.TO, 1, flavor='raw') + + try: + w_copy = space.newdict() + while api.PyDict_Next(w_dict, ppos, pkey, pvalue): + w_key = from_ref(space, pkey[0]) + w_value = from_ref(space, pvalue[0]) + space.setitem(w_copy, w_key, w_value) + finally: + lltype.free(ppos, flavor='raw') + lltype.free(pkey, flavor='raw') + lltype.free(pvalue, flavor='raw') + + api.Py_DecRef(py_dict) # release borrowed references + + assert space.eq_w(space.len(w_copy), space.len(w_dict)) + assert space.eq_w(w_copy, w_dict) Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_eval.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_eval.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_eval.py Fri May 28 16:20:28 2010 @@ -1,7 +1,12 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.test.test_api import BaseApiTest -from pypy.module.cpyext.eval import Py_single_input, Py_file_input, Py_eval_input +from pypy.module.cpyext.eval import ( + Py_single_input, Py_file_input, Py_eval_input) +from pypy.module.cpyext.api import fopen, fclose, fileno, Py_ssize_tP +from pypy.interpreter.gateway import interp2app +from pypy.tool.udir import udir +import sys, os class TestEval(BaseApiTest): def test_eval(self, space, api): @@ -76,6 +81,75 @@ assert 42 * 43 == space.unwrap( api.PyObject_GetItem(w_globals, space.wrap("a"))) + def test_run_file(self, space, api): + filepath = udir / "cpyext_test_runfile.py" + filepath.write("raise ZeroDivisionError") + fp = fopen(str(filepath), "rb") + filename = rffi.str2charp(str(filepath)) + w_globals = w_locals = space.newdict() + api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals) + fclose(fp) + assert api.PyErr_Occurred() is space.w_ZeroDivisionError + api.PyErr_Clear() + + # try again, but with a closed file + fp = fopen(str(filepath), "rb") + os.close(fileno(fp)) + api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals) + fclose(fp) + assert api.PyErr_Occurred() is space.w_IOError + api.PyErr_Clear() + + rffi.free_charp(filename) + + def test_getbuiltins(self, space, api): + assert api.PyEval_GetBuiltins() is space.builtin.w_dict + + def cpybuiltins(space): + return api.PyEval_GetBuiltins() + w_cpybuiltins = space.wrap(interp2app(cpybuiltins)) + + w_result = space.appexec([w_cpybuiltins], """(cpybuiltins): + return cpybuiltins() is __builtins__.__dict__ + """) + assert space.is_true(w_result) + + w_result = space.appexec([w_cpybuiltins], """(cpybuiltins): + d = dict(__builtins__={'len':len}, cpybuiltins=cpybuiltins) + return eval("cpybuiltins()", d, d) + """) + assert space.int_w(space.len(w_result)) == 1 + + def test_getglobals(self, space, api): + assert api.PyEval_GetLocals() is None + assert api.PyEval_GetGlobals() is None + + def cpyvars(space): + return space.newtuple([api.PyEval_GetGlobals(), + api.PyEval_GetLocals()]) + w_cpyvars = space.wrap(interp2app(cpyvars)) + + w_result = space.appexec([w_cpyvars], """(cpyvars): + x = 1 + return cpyvars() + \ny = 2 + """) + globals, locals = space.unwrap(w_result) + assert sorted(locals) == ['cpyvars', 'x'] + assert sorted(globals) == ['__builtins__', 'anonymous', 'y'] + + def test_sliceindex(self, space, api): + pi = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw') + assert api._PyEval_SliceIndex(space.w_None, pi) == 0 + api.PyErr_Clear() + + assert api._PyEval_SliceIndex(space.wrap(123), pi) == 1 + assert pi[0] == 123 + + assert api._PyEval_SliceIndex(space.wrap(1 << 66), pi) == 1 + assert pi[0] == sys.maxint + + lltype.free(pi, flavor='raw') class AppTestCall(AppTestCpythonExtensionBase): def test_CallFunction(self): Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_funcobject.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_funcobject.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_funcobject.py Fri May 28 16:20:28 2010 @@ -26,9 +26,12 @@ """) w_function = space.getattr(w_method, space.wrap("im_func")) + w_self = space.getattr(w_method, space.wrap("im_self")) + w_class = space.getattr(w_method, space.wrap("im_class")) + assert space.is_w(api.PyMethod_Function(w_method), w_function) + assert space.is_w(api.PyMethod_Self(w_method), w_self) + assert space.is_w(api.PyMethod_Class(w_method), w_class) - w_class = space.getattr(w_method, space.wrap("im_class")) - w_self = space.getattr(w_method, space.wrap("im_self")) w_method2 = api.PyMethod_New(w_function, w_self, w_class) assert space.eq_w(w_method, w_method2) Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_longobject.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_longobject.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_longobject.py Fri May 28 16:20:28 2010 @@ -70,6 +70,14 @@ assert api.PyLong_AsVoidPtr(w_l) == lltype.nullptr(rffi.VOIDP_real.TO) class AppTestLongObject(AppTestCpythonExtensionBase): + def test_fromunsignedlong(self): + module = self.import_extension('foo', [ + ("from_unsignedlong", "METH_NOARGS", + """ + return PyLong_FromUnsignedLong((unsigned long)-1); + """)]) + assert module.from_unsignedlong() == (1<<32) - 1 + def test_fromlonglong(self): module = self.import_extension('foo', [ ("from_longlong", "METH_NOARGS", Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_object.py Fri May 28 16:20:28 2010 @@ -1,9 +1,11 @@ import py from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import Py_LT, Py_LE, Py_NE, Py_EQ,\ Py_GE, Py_GT +from pypy.tool.udir import udir class TestObject(BaseApiTest): def test_IsTrue(self, space, api): @@ -61,6 +63,9 @@ assert not api.PyObject_GetAttrString(space.wrap(""), charp2) assert api.PyErr_Occurred() is space.w_AttributeError api.PyErr_Clear() + assert api.PyObject_DelAttrString(space.wrap(""), charp1) == -1 + assert api.PyErr_Occurred() is space.w_AttributeError + api.PyErr_Clear() rffi.free_charp(charp1) rffi.free_charp(charp2) @@ -76,6 +81,14 @@ space.setitem(w_d, space.wrap("a key!"), space.wrap(72)) assert space.unwrap(api.PyObject_GetItem(w_d, space.wrap("a key!"))) == 72 + assert api.PyObject_SetItem(w_d, space.wrap("key"), space.w_None) == 0 + assert space.getitem(w_d, space.wrap("key")) is space.w_None + + assert api.PyObject_DelItem(w_d, space.wrap("key")) == 0 + assert api.PyObject_GetItem(w_d, space.wrap("key")) is None + assert api.PyErr_Occurred() is space.w_KeyError + api.PyErr_Clear() + def test_size(self, space, api): assert api.PyObject_Size(space.newlist([space.w_None])) == 1 @@ -178,3 +191,39 @@ assert space.unwrap(api.PyObject_Unicode(space.wrap("e"))) == u"e" assert api.PyObject_Unicode(space.wrap("\xe9")) is None api.PyErr_Clear() + + def test_file_fromstring(self, space, api): + filename = rffi.str2charp(str(udir / "_test_file")) + mode = rffi.str2charp("wb") + w_file = api.PyFile_FromString(filename, mode) + rffi.free_charp(filename) + rffi.free_charp(mode) + + space.call_method(w_file, "write", space.wrap("text")) + space.call_method(w_file, "close") + assert (udir / "_test_file").read() == "text" + +class AppTestObjectPrint(AppTestCpythonExtensionBase): + def setup_class(cls): + AppTestCpythonExtensionBase.setup_class.im_func(cls) + cls.w_tmpname = cls.space.wrap(str(py.test.ensuretemp("out", dir=0))) + + def test_print(self): + module = self.import_extension('foo', [ + ("dump", "METH_VARARGS", + """ + PyObject *fname = PyTuple_GetItem(args, 0); + PyObject *obj = PyTuple_GetItem(args, 1); + + FILE *fp = fopen(PyString_AsString(fname), "wb"); + int ret; + if (fp == NULL) + Py_RETURN_NONE; + ret = PyObject_Print(obj, fp, Py_PRINT_RAW); + fclose(fp); + if (ret < 0) + return NULL; + Py_RETURN_TRUE; + """)]) + assert module.dump(self.tmpname, None) + assert open(self.tmpname).read() == 'None' Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_pyerrors.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_pyerrors.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_pyerrors.py Fri May 28 16:20:28 2010 @@ -1,3 +1,6 @@ +import sys +import StringIO + from pypy.module.cpyext.state import State from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase @@ -79,6 +82,13 @@ assert ": UserWarning: this is a warning" in err rffi.free_charp(message) + def test_print_err(self, space, api, capfd): + api.PyErr_SetObject(space.w_Exception, space.wrap("cpyext is cool")) + api.PyErr_Print() + out, err = capfd.readouterr() + assert "cpyext is cool" in err + assert not api.PyErr_Occurred() + class AppTestFetch(AppTestCpythonExtensionBase): def setup_class(cls): AppTestCpythonExtensionBase.setup_class.im_func(cls) Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_sequence.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_sequence.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_sequence.py Fri May 28 16:20:28 2010 @@ -41,9 +41,14 @@ rffi.free_charp(message) def test_get_slice(self, space, api): - w_t = space.wrap((1, 2, 3, 4, 5)) - assert space.unwrap(api.PySequence_GetSlice(w_t, 2, 4)) == (3, 4) - assert space.unwrap(api.PySequence_GetSlice(w_t, 1, -1)) == (2, 3, 4) + w_t = space.wrap([1, 2, 3, 4, 5]) + assert space.unwrap(api.PySequence_GetSlice(w_t, 2, 4)) == [3, 4] + assert space.unwrap(api.PySequence_GetSlice(w_t, 1, -1)) == [2, 3, 4] + + assert api.PySequence_DelSlice(w_t, 1, 4) == 0 + assert space.eq_w(w_t, space.wrap([1, 5])) + assert api.PySequence_SetSlice(w_t, 1, 1, space.wrap((3,))) == 0 + assert space.eq_w(w_t, space.wrap([1, 3, 5])) def test_iter(self, space, api): w_t = space.wrap((1, 2)) Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_sliceobject.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_sliceobject.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_sliceobject.py Fri May 28 16:20:28 2010 @@ -25,3 +25,21 @@ lltype.free(values, flavor='raw') return rv assert get_indices(w(10), w(20), w(1), 200) == (10, 20, 1, 10) + +class AppTestSliceMembers(AppTestCpythonExtensionBase): + def test_members(self): + module = self.import_extension('foo', [ + ("clone", "METH_O", + """ + PySliceObject *slice = (PySliceObject *)args; + if (slice->ob_type != &PySlice_Type) { + PyErr_SetNone(PyExc_ValueError); + return NULL; + } + return PySlice_New(slice->start, + slice->stop, + slice->step); + """), + ]) + s = slice(10, 20, 30) + assert module.clone(s) == s Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_stringobject.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_stringobject.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_stringobject.py Fri May 28 16:20:28 2010 @@ -2,7 +2,7 @@ from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.stringobject import new_empty_str, PyStringObject -from pypy.module.cpyext.api import PyObjectP, PyObject +from pypy.module.cpyext.api import PyObjectP, PyObject, Py_ssize_tP from pypy.module.cpyext.pyobject import Py_DecRef, from_ref, make_ref import py @@ -234,3 +234,22 @@ def test_format(self, space, api): assert "1 2" == space.unwrap( api.PyString_Format(space.wrap('%s %d'), space.wrap((1, 2)))) + + def test_asbuffer(self, space, api): + bufp = lltype.malloc(rffi.VOIDPP.TO, 1, flavor='raw') + lenp = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw') + + w_text = space.wrap("text") + assert api.PyObject_AsCharBuffer(w_text, bufp, lenp) == 0 + assert lenp[0] == 4 + assert rffi.charp2str(bufp[0]) == 'text' + + lltype.free(bufp, flavor='raw') + lltype.free(lenp, flavor='raw') + + def test_intern(self, space, api): + buf = rffi.str2charp("test") + w_s1 = api.PyString_InternFromString(buf) + w_s2 = api.PyString_InternFromString(buf) + rffi.free_charp(buf) + assert w_s1 is w_s2 Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/test/test_typeobject.py Fri May 28 16:20:28 2010 @@ -89,6 +89,13 @@ raises(TypeError, "obj.char_member = 'spam'") raises(TypeError, "obj.char_member = 42") + def test_staticmethod(self): + module = self.import_module(name="foo") + obj = module.fooType.create() + assert obj.foo == 42 + obj2 = obj.create() + assert obj2.foo == 42 + def test_new(self): module = self.import_module(name='foo') obj = module.new() @@ -128,6 +135,23 @@ assert newobj.get_val() == 42 assert newobj.foobar == 32 + def test_metatype(self): + module = self.import_module(name='foo') + assert module.MetaType.__mro__ == (module.MetaType, type, object) + x = module.MetaType('name', (), {}) + assert isinstance(x, type) + assert isinstance(x, module.MetaType) + x() + + def test_metaclass_compatible(self): + # metaclasses should not conflict here + module = self.import_module(name='foo') + assert module.MetaType.__mro__ == (module.MetaType, type, object) + assert type(module.fooType).__mro__ == (type, object) + y = module.MetaType('other', (module.fooType,), {}) + assert isinstance(y, module.MetaType) + y() + def test_sre(self): module = self.import_module(name='_sre') import sre_compile @@ -173,3 +197,51 @@ """) ref = make_ref(space, w_class) api.Py_DecRef(ref) + + def test_lookup(self, space, api): + w_type = space.w_str + w_obj = api._PyType_Lookup(w_type, space.wrap("upper")) + assert space.is_w(w_obj, space.w_str.getdictvalue(space, "upper")) + + w_obj = api._PyType_Lookup(w_type, space.wrap("__invalid")) + assert w_obj is None + assert api.PyErr_Occurred() is None + +class AppTestSlots(AppTestCpythonExtensionBase): + def test_nb_int(self): + module = self.import_extension('foo', [ + ("nb_int", "METH_O", + ''' + if (!args->ob_type->tp_as_number || + !args->ob_type->tp_as_number->nb_int) + { + PyErr_SetNone(PyExc_ValueError); + return NULL; + } + return args->ob_type->tp_as_number->nb_int(args); + ''' + ) + ]) + assert module.nb_int(10) == 10 + assert module.nb_int(-12.3) == -12 + raises(ValueError, module.nb_int, "123") + + def test_tp_call(self): + module = self.import_extension('foo', [ + ("tp_call", "METH_VARARGS", + ''' + PyObject *obj = PyTuple_GET_ITEM(args, 0); + PyObject *c_args = PyTuple_GET_ITEM(args, 1); + if (!obj->ob_type->tp_call) + { + PyErr_SetNone(PyExc_ValueError); + return NULL; + } + return obj->ob_type->tp_call(obj, c_args, NULL); + ''' + ) + ]) + class C: + def __call__(self, *args): + return args + assert module.tp_call(C(), ('x', 2)) == ('x', 2) Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/typeobject.py Fri May 28 16:20:28 2010 @@ -3,13 +3,9 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.annlowlevel import llhelper -from pypy.rlib.rweakref import RWeakKeyDictionary -from pypy.interpreter.gateway import ObjSpace, W_Root, Arguments -from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.baseobjspace import DescrMismatch -from pypy.objspace.std.typeobject import W_TypeObject, _CPYTYPE, call__Type -from pypy.objspace.std.typetype import _precheck_for_new -from pypy.interpreter.typedef import TypeDef, GetSetProperty +from pypy.objspace.std.typeobject import W_TypeObject, _CPYTYPE +from pypy.interpreter.typedef import GetSetProperty from pypy.module.cpyext.api import ( cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING, @@ -17,22 +13,22 @@ PyBufferProcs, build_type_checkers) from pypy.module.cpyext.pyobject import ( PyObject, make_ref, create_ref, from_ref, get_typedescr, make_typedescr, - track_reference, RefcountState) + track_reference, RefcountState, borrow_from) from pypy.interpreter.module import Module -from pypy.interpreter.function import FunctionWithFixedCode, StaticMethod from pypy.module.cpyext import structmemberdefs from pypy.module.cpyext.modsupport import convert_method_defs, PyCFunction from pypy.module.cpyext.state import State -from pypy.module.cpyext.methodobject import PyDescr_NewWrapper, \ - PyCFunction_NewEx +from pypy.module.cpyext.methodobject import ( + PyDescr_NewWrapper, PyCFunction_NewEx) from pypy.module.cpyext.pyobject import Py_IncRef, Py_DecRef, _Py_Dealloc from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne -from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr, PyTypeObject, \ - PyGetSetDef, PyMemberDef, newfunc +from pypy.module.cpyext.typeobjectdefs import ( + PyTypeObjectPtr, PyTypeObject, PyGetSetDef, PyMemberDef, newfunc, + PyNumberMethods) from pypy.module.cpyext.slotdefs import slotdefs -from pypy.interpreter.error import OperationError, operationerrfmt +from pypy.interpreter.error import OperationError from pypy.rlib.rstring import rsplit -from pypy.rlib.objectmodel import we_are_translated, specialize +from pypy.rlib.objectmodel import specialize from pypy.module.__builtin__.abstractinst import abstract_issubclass_w from pypy.module.__builtin__.interp_classobj import W_ClassObject @@ -104,10 +100,10 @@ dict_w[name] = w_descr i += 1 -def update_all_slots(space, w_obj, pto): +def update_all_slots(space, w_type, pto): # XXX fill slots in pto for method_name, slot_name, slot_func, _, _, _ in slotdefs: - w_descr = space.lookup(w_obj, method_name) + w_descr = w_type.lookup(method_name) if w_descr is None: # XXX special case iternext continue @@ -119,17 +115,20 @@ slot_func.api_func.get_wrapper(space)) # XXX special case wrapper-functions and use a "specific" slot func - # the special case of __new__ in CPython works a bit differently, hopefully - # this matches the semantics - if method_name == "__new__" and not pto.c_tp_new: - continue if len(slot_name) == 1: setattr(pto, slot_name[0], slot_func_helper) else: assert len(slot_name) == 2 struct = getattr(pto, slot_name[0]) if not struct: - continue + if slot_name[0] == 'c_tp_as_number': + STRUCT_TYPE = PyNumberMethods + else: + raise AssertionError( + "Structure not allocated: %s" % (slot_name[0],)) + struct = lltype.malloc(STRUCT_TYPE, flavor='raw', zero=True) + setattr(pto, slot_name[0], struct) + setattr(struct, slot_name[1], slot_func_helper) def add_operators(space, dict_w, pto): @@ -199,7 +198,7 @@ return pyo = rffi.cast(PyObject, pto) dict_w["__new__"] = PyCFunction_NewEx(space, get_new_method_def(space), - from_ref(space, pyo)) + from_ref(space, pyo), None) def inherit_special(space, pto, base_pto): # XXX missing: copy basicsize and flags in a magical way @@ -211,20 +210,6 @@ pto.c_tp_new = base_pto.c_tp_new Py_DecRef(space, base_object_pyo) -class PyOLifeline(object): - def __init__(self, space, pyo): - self.pyo = pyo - self.space = space - - def __del__(self): - if self.pyo: - assert self.pyo.c_ob_refcnt == 0 - _Py_Dealloc(self.space, self.pyo) - self.pyo = lltype.nullptr(PyObject.TO) - # XXX handle borrowed objects here - -lifeline_dict = RWeakKeyDictionary(W_Root, PyOLifeline) - def check_descr(space, w_self, w_type): if not space.is_true(space.isinstance(w_self, w_type)): raise DescrMismatch() @@ -262,67 +247,6 @@ check_descr(space, w_self, self.w_type) PyMember_SetOne(space, w_self, self.member, w_value) -def c_type_descr__call__(space, w_type, __args__): - if not isinstance(w_type, W_PyCTypeObject): - # XXX is this possible? - w_type = _precheck_for_new(space, w_type) - return call__Type(space, w_type, __args__) - - pyo = make_ref(space, w_type) - pto = rffi.cast(PyTypeObjectPtr, pyo) - tp_new = pto.c_tp_new - try: - if not tp_new: - raise operationerrfmt(space.w_TypeError, - "cannot create '%s' instances", w_type.getname(space, '?')) - - args_w, kw_w = __args__.unpack() - w_args = space.newtuple(args_w) - w_kw = space.newdict() - for key, w_obj in kw_w.items(): - space.setitem(w_kw, space.wrap(key), w_obj) - w_obj = generic_cpy_call(space, tp_new, pto, w_args, w_kw) - finally: - Py_DecRef(space, pyo) - - # If the returned object is not an instance of type, - # it won't be initialized. - w_obj_type = space.type(w_obj) - if not (space.is_w(w_obj_type, w_type) or - space.is_true(space.issubtype(w_obj_type, w_type))): - return w_obj - - # call tp_init - pyo = make_ref(space, w_obj_type) - pto = rffi.cast(PyTypeObjectPtr, pyo) - try: - if pto.c_tp_init: - res = generic_cpy_call(space, pto.c_tp_init, w_obj, w_args, w_kw) - if rffi.cast(lltype.Signed, res) < 0: - state = space.fromcache(State) - state.check_and_raise_exception() - finally: - Py_DecRef(space, pyo) - - return w_obj - -def c_type_descr__new__(space, w_typetype, w_name, w_bases, w_dict): - # copied from typetype.descr__new__, XXX missing logic: metaclass resolving - w_typetype = _precheck_for_new(space, w_typetype) - - bases_w = space.fixedview(w_bases) - name = space.str_w(w_name) - dict_w = {} - dictkeys_w = space.listview(w_dict) - for w_key in dictkeys_w: - key = space.str_w(w_key) - dict_w[key] = space.getitem(w_dict, w_key) - w_type = space.allocate_instance(W_PyCTypeObject, w_typetype) - W_TypeObject.__init__(w_type, space, name, bases_w or [space.w_object], - dict_w) - w_type.ready() - return w_type - class W_PyCTypeObject(W_TypeObject): def __init__(self, space, pto): bases_w = space.fixedview(from_ref(space, pto.c_tp_bases)) @@ -342,13 +266,7 @@ W_TypeObject.__init__(self, space, extension_name, bases_w or [space.w_object], dict_w) - self.__flags__ = _CPYTYPE # mainly disables lookup optimizations - -W_PyCTypeObject.typedef = TypeDef( - 'C_type', W_TypeObject.typedef, - __call__ = interp2app(c_type_descr__call__, unwrap_spec=[ObjSpace, W_Root, Arguments]), - __new__ = interp2app(c_type_descr__new__), - ) + self.__flags__ = _CPYTYPE @bootstrap_function def init_typeobject(space): @@ -357,12 +275,6 @@ attach=type_attach, realize=type_realize, dealloc=type_dealloc) - make_typedescr(W_PyCTypeObject.typedef, - basestruct=PyTypeObject, - make_ref=pyctype_make_ref, - attach=type_attach, - realize=type_realize, - dealloc=type_dealloc) # some types are difficult to create because of cycles. # - object.ob_type = type @@ -421,18 +333,6 @@ # hopefully this does not clash with the memory model assumed in # extension modules -def pyctype_make_ref(space, w_type, w_obj, itemcount=0): - lifeline = lifeline_dict.get(w_obj) - if lifeline is not None: # make old PyObject ready for use in C code - py_obj = lifeline.pyo - assert py_obj.c_ob_refcnt == 0 - Py_IncRef(space, py_obj) - else: - typedescr = get_typedescr(w_obj.typedef) - py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) - lifeline_dict.set(w_obj, PyOLifeline(space, py_obj)) - return py_obj - @cpython_api([PyObject, rffi.INTP], lltype.Signed, external=False, error=CANNOT_FAIL) def str_segcount(space, w_obj, ref): @@ -472,12 +372,15 @@ if obj_pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE: if obj_pto.c_tp_as_buffer: lltype.free(obj_pto.c_tp_as_buffer, flavor='raw') + if obj_pto.c_tp_as_number: + lltype.free(obj_pto.c_tp_as_number, flavor='raw') Py_DecRef(space, base_pyo) rffi.free_charp(obj_pto.c_tp_name) obj_pto_voidp = rffi.cast(rffi.VOIDP_real, obj_pto) generic_cpy_call(space, type_pto.c_tp_free, obj_pto_voidp) - pto = rffi.cast(PyObject, type_pto) - Py_DecRef(space, pto) + if type_pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE: + pto = rffi.cast(PyObject, type_pto) + Py_DecRef(space, pto) def type_attach(space, py_obj, w_type): @@ -613,9 +516,7 @@ finish_type_1(space, py_type) - w_obj = space.allocate_instance( - W_PyCTypeObject, - space.gettypeobject(W_PyCTypeObject.typedef)) + w_obj = space.allocate_instance(W_PyCTypeObject, space.w_type) track_reference(space, py_obj, w_obj) w_obj.__init__(space, py_type) w_obj.ready() @@ -675,4 +576,18 @@ return generic_cpy_call( space, type.c_tp_alloc, type, 0) + at cpython_api([PyTypeObjectPtr, PyObject], PyObject, error=CANNOT_FAIL) +def _PyType_Lookup(space, type, w_name): + """Internal API to look for a name through the MRO. + This returns a borrowed reference, and doesn't set an exception!""" + w_type = from_ref(space, rffi.cast(PyObject, type)) + assert isinstance(w_type, W_TypeObject) + + if not space.isinstance_w(w_name, space.w_str): + return None + name = space.str_w(w_name) + w_obj = w_type.lookup(name) + return borrow_from(w_type, w_obj) + + Modified: pypy/branch/x86-64-jit-backend/pypy/module/cpyext/typeobjectdefs.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/cpyext/typeobjectdefs.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/cpyext/typeobjectdefs.py Fri May 28 16:20:28 2010 @@ -2,7 +2,7 @@ from pypy.rpython.lltypesystem.lltype import Ptr, FuncType, Void from pypy.module.cpyext.api import cpython_struct, \ PyVarObjectFields, Py_ssize_t, Py_TPFLAGS_READYING, \ - Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE, ADDR, \ + Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE, \ PyTypeObject, PyTypeObjectPtr, PyBufferProcs from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref from pypy.module.cpyext.modsupport import PyMethodDef Modified: pypy/branch/x86-64-jit-backend/pypy/module/imp/importing.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/imp/importing.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/imp/importing.py Fri May 28 16:20:28 2010 @@ -356,6 +356,12 @@ if pkgdir is not None: space.setattr(w_mod, w('__path__'), space.newlist([w(pkgdir)])) +def load_c_extension(space, filename, modulename): + # the next line is mandatory to init cpyext + space.getbuiltinmodule("cpyext") + from pypy.module.cpyext.api import load_extension_module + load_extension_module(space, filename, modulename) + @jit.dont_look_inside def load_module(space, w_modulename, find_info, reuse=False): if find_info is None: @@ -408,10 +414,7 @@ w_mod = check_sys_modules(space, w_modulename) return w_mod elif find_info.modtype == C_EXTENSION and space.config.objspace.usemodules.cpyext: - # the next line is mandantory to init cpyext - space.getbuiltinmodule("cpyext") - from pypy.module.cpyext.api import load_extension_module - load_extension_module(space, find_info.filename, space.str_w(w_modulename)) + load_c_extension(space, find_info.filename, space.str_w(w_modulename)) return check_sys_modules(space, w_modulename) except OperationError: w_mods = space.sys.get('modules') Modified: pypy/branch/x86-64-jit-backend/pypy/module/posix/__init__.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/posix/__init__.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/posix/__init__.py Fri May 28 16:20:28 2010 @@ -2,7 +2,7 @@ from pypy.interpreter.mixedmodule import MixedModule from pypy.rpython.module.ll_os import RegisterOs -import os +import os, sys exec 'import %s as posix' % os.name class Module(MixedModule): @@ -81,7 +81,7 @@ interpleveldefs['putenv'] = 'interp_posix.putenv' if hasattr(posix, 'unsetenv'): # note: emulated in os interpleveldefs['unsetenv'] = 'interp_posix.unsetenv' - if hasattr(os, 'kill'): + if hasattr(os, 'kill') and sys.platform != 'win32': interpleveldefs['kill'] = 'interp_posix.kill' interpleveldefs['abort'] = 'interp_posix.abort' if hasattr(os, 'getpid'): Modified: pypy/branch/x86-64-jit-backend/pypy/module/posix/app_posix.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/posix/app_posix.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/posix/app_posix.py Fri May 28 16:20:28 2010 @@ -62,13 +62,16 @@ if self.st_ctime is None: self.__dict__['st_ctime'] = self[9] +# Capture file.fdopen at import time, as some code replaces +# __builtins__.file with a custom function. +_fdopen = file.fdopen def fdopen(fd, mode='r', buffering=-1): """fdopen(fd [, mode='r' [, buffering]]) -> file_object Return an open file object connected to a file descriptor.""" - return file.fdopen(fd, mode, buffering) + return _fdopen(fd, mode, buffering) def tmpfile(): Modified: pypy/branch/x86-64-jit-backend/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/posix/test/test_posix2.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/posix/test/test_posix2.py Fri May 28 16:20:28 2010 @@ -249,6 +249,19 @@ f = posix.fdopen(fd, "r") f.close() + def test_fdopen_hackedbuiltins(self): + "Same test, with __builtins__.file removed" + _file = __builtins__.file + __builtins__.file = None + try: + path = self.path + posix = self.posix + fd = posix.open(path, posix.O_RDONLY, 0777) + f = posix.fdopen(fd, "r") + f.close() + finally: + __builtins__.file = _file + def test_getcwd(self): assert isinstance(self.posix.getcwd(), str) assert isinstance(self.posix.getcwdu(), unicode) @@ -477,6 +490,12 @@ if not hasattr(os, "fork"): skip("Need fork() to test wait()") + if hasattr(os, "waitpid") and hasattr(os, "WNOHANG"): + try: + while os.waitpid(-1, os.WNOHANG)[0]: + pass + except OSError: # until we get "No child processes", hopefully + pass child = os.fork() if child == 0: # in child os._exit(exit_status) Modified: pypy/branch/x86-64-jit-backend/pypy/module/pyexpat/interp_pyexpat.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/pyexpat/interp_pyexpat.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/pyexpat/interp_pyexpat.py Fri May 28 16:20:28 2010 @@ -93,6 +93,9 @@ class Storage: "Store objects under a non moving ID" def __init__(self): + self.clear() + + def clear(self): self.next_id = 0 self.storage = {} Modified: pypy/branch/x86-64-jit-backend/pypy/module/sys/__init__.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/sys/__init__.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/sys/__init__.py Fri May 28 16:20:28 2010 @@ -1,5 +1,6 @@ from pypy.interpreter.mixedmodule import MixedModule from pypy.interpreter.error import OperationError +import sys class Module(MixedModule): """Sys Builtin Module. """ @@ -71,6 +72,10 @@ 'setdefaultencoding' : 'interp_encoding.setdefaultencoding', 'getfilesystemencoding' : 'interp_encoding.getfilesystemencoding', } + + if sys.platform == 'win32': + interpleveldefs['winver'] = 'version.get_winver(space)' + interpleveldefs['getwindowsversion'] = 'vm.getwindowsversion' appleveldefs = { 'excepthook' : 'app.excepthook', Modified: pypy/branch/x86-64-jit-backend/pypy/module/sys/test/test_sysmodule.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/sys/test/test_sysmodule.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/sys/test/test_sysmodule.py Fri May 28 16:20:28 2010 @@ -331,6 +331,10 @@ assert isinstance(v[3], int) assert isinstance(v[4], str) + def test_winver(self): + if hasattr(sys, "winver"): + assert sys.winver == sys.version[:3] + def test_dlopenflags(self): if hasattr(sys, "setdlopenflags"): assert hasattr(sys, "getdlopenflags") Modified: pypy/branch/x86-64-jit-backend/pypy/module/sys/version.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/sys/version.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/sys/version.py Fri May 28 16:20:28 2010 @@ -58,6 +58,11 @@ PYPY_VERSION[1], PYPY_VERSION[2])) +def get_winver(space): + return space.wrap("%d.%d" % ( + CPYTHON_VERSION[0], + CPYTHON_VERSION[1])) + def get_hexversion(space): return space.wrap(tuple2hex(CPYTHON_VERSION)) Modified: pypy/branch/x86-64-jit-backend/pypy/module/sys/vm.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/module/sys/vm.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/module/sys/vm.py Fri May 28 16:20:28 2010 @@ -117,3 +117,12 @@ saved, and restored afterwards. This is intended to be called from a debugger from a checkpoint, to recursively debug some other code.""" return space.getexecutioncontext().call_tracing(w_func, w_args) + +def getwindowsversion(space): + from pypy.rlib import rwin32 + info = rwin32.GetVersionEx() + return space.newtuple([space.wrap(info[0]), + space.wrap(info[1]), + space.wrap(info[2]), + space.wrap(info[3]), + space.wrap(info[4])]) Modified: pypy/branch/x86-64-jit-backend/pypy/objspace/flow/flowcontext.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/objspace/flow/flowcontext.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/objspace/flow/flowcontext.py Fri May 28 16:20:28 2010 @@ -388,11 +388,14 @@ def replace_in_stack(self, oldvalue, newvalue): w_new = Constant(newvalue) stack_items_w = self.crnt_frame.valuestack_w - for i in range(self.crnt_frame.valuestackdepth): + for i in range(self.crnt_frame.valuestackdepth-1, -1, -1): w_v = stack_items_w[i] if isinstance(w_v, Constant): if w_v.value is oldvalue: + # replace the topmost item of the stack that is equal + # to 'oldvalue' with 'newvalue'. stack_items_w[i] = w_new + break class FlowSpaceFrame(pyframe.PyFrame): """ Modified: pypy/branch/x86-64-jit-backend/pypy/objspace/flow/test/test_unroll.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/objspace/flow/test/test_unroll.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/objspace/flow/test/test_unroll.py Fri May 28 16:20:28 2010 @@ -64,3 +64,15 @@ 'ne': 1, 'gt': 1, 'ge': 1} + + def test_unroll_twice(self): + operations = unrolling_iterable([1, 2, 3]) + def f(x): + for num1 in operations: + for num2 in operations: + x = x + (num1 + num2) + return x + + graph = self.codetest(f) + ops = self.all_operations(graph) + assert ops['add'] == 9 Modified: pypy/branch/x86-64-jit-backend/pypy/objspace/std/floattype.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/objspace/std/floattype.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/objspace/std/floattype.py Fri May 28 16:20:28 2010 @@ -1,21 +1,16 @@ from pypy.interpreter import gateway from pypy.interpreter.error import OperationError from pypy.objspace.std.stdtypedef import StdTypeDef -from pypy.objspace.std.strutil import string_to_float, ParseStringError +from pypy.objspace.std.strutil import ParseStringError from pypy.objspace.std.strutil import interp_string_to_float -USE_NEW_S2F = True - def descr__new__(space, w_floattype, w_x=0.0): from pypy.objspace.std.floatobject import W_FloatObject w_value = w_x # 'x' is the keyword argument name in CPython if space.is_true(space.isinstance(w_value, space.w_str)): strvalue = space.str_w(w_value) try: - if USE_NEW_S2F: - value = interp_string_to_float(space, strvalue) - else: - value = string_to_float(strvalue) + value = interp_string_to_float(space, strvalue) except ParseStringError, e: raise OperationError(space.w_ValueError, space.wrap(e.msg)) @@ -26,10 +21,7 @@ from unicodeobject import unicode_to_decimal_w strvalue = unicode_to_decimal_w(space, w_value) try: - if USE_NEW_S2F: - value = interp_string_to_float(space, strvalue) - else: - value = string_to_float(strvalue) + value = interp_string_to_float(space, strvalue) except ParseStringError, e: raise OperationError(space.w_ValueError, space.wrap(e.msg)) Modified: pypy/branch/x86-64-jit-backend/pypy/objspace/std/strutil.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/objspace/std/strutil.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/objspace/std/strutil.py Fri May 28 16:20:28 2010 @@ -2,7 +2,8 @@ Pure Python implementation of string utilities. """ -from pypy.rlib.rarithmetic import ovfcheck, break_up_float, parts_to_float +from pypy.rlib.rarithmetic import ovfcheck, break_up_float, parts_to_float,\ + INFINITY, NAN from pypy.interpreter.error import OperationError import math @@ -138,33 +139,6 @@ return w_result w_result = space.add(space.mul(w_result,w_base), space.newlong(digit)) -def string_to_float(s): - """ - Conversion of string to float. - This version tries to only raise on invalid literals. - Overflows should be converted to infinity whenever possible. - """ - - s = strip_spaces(s) - - if not s: - raise ParseStringError("empty string for float()") - - # 1) parse the string into pieces. - try: - sign, before_point, after_point, exponent = break_up_float(s) - except ValueError: - raise ParseStringError("invalid literal for float()") - - if not before_point and not after_point: - raise ParseStringError("invalid literal for float()") - - try: - return parts_to_float(sign, before_point, after_point, exponent) - except ValueError: - raise ParseStringError("invalid literal for float()") - - # Tim's comment: # 57 bits are more than needed in any case. # to allow for some rounding, we take one @@ -187,140 +161,6 @@ del calc_mantissa_bits MANTISSA_DIGITS = len(str( (1L << MANTISSA_BITS)-1 )) + 1 -# we keep this version for reference. -def applevel_string_to_float(s): - """ - Conversion of string to float. - This version tries to only raise on invalid literals. - Overflows should be converted to infinity whenever possible. - """ - # this version was triggered by Python 2.4 which adds - # a test that breaks on overflow. - # XXX The test still breaks for a different reason: - # float must implement rich comparisons, where comparison - # between infinity and a too large long does not overflow! - - # The problem: - # there can be extreme notations of floats which are not - # infinity. - # For instance, this works in CPython: - # float('1' + '0'*1000 + 'e-1000') - # should evaluate to 1.0. - # note: float('1' + '0'*10000 + 'e-10000') - # does not work in CPython, but PyPy can do it, now. - - # The idea: - # in order to compensate between very long digit strings - # and extreme exponent numbers, we try to avoid overflows - # by adjusting the exponent by the number of mantissa - # digits. For simplicity, all computations are done in - # long math. - - # The plan: - # 1) parse the string into pieces. - # 2) pre-calculate digit exponent dexp. - # 3) truncate and adjust dexp. - # 4) compute the exponent and truncate to +-400. - # 5) compute the value using long math and proper rounding. - - # Positive results: - # The algorithm appears appears to produce correct round-trip - # values for the perfect input of _float_formatting. - # Note: - # XXX: the builtin rounding of long->float does not work, correctly. - # Ask Tim Peters for the reasons why no correct rounding is done. - # XXX: limitations: - # - It is possibly not too efficient. - # - Really optimum results need a more sophisticated algorithm - # like Bellerophon from William D. Clinger, cf. - # http://citeseer.csail.mit.edu/clinger90how.html - - s = strip_spaces(s) - - if not s: - raise ParseStringError("empty string for float()") - - # 1) parse the string into pieces. - try: - sign, before_point, after_point, exponent = break_up_float(s) - except ValueError: - raise ParseStringError("invalid literal for float()") - - digits = before_point + after_point - if digits: - raise ParseStringError("invalid literal for float()") - - # 2) pre-calculate digit exponent dexp. - dexp = len(before_point) - - # 3) truncate and adjust dexp. - p = 0 - plim = dexp + len(after_point) - while p < plim and digits[p] == '0': - p += 1 - dexp -= 1 - digits = digits[p : p + MANTISSA_DIGITS] - p = len(digits) - 1 - while p >= 0 and digits[p] == '0': - p -= 1 - dexp -= p + 1 - digits = digits[:p+1] - if len(digits) == 0: - digits = '0' - - # 4) compute the exponent and truncate to +-400 - if not exponent: - exponent = '0' - e = long(exponent) + dexp - if e >= 400: - e = 400 - elif e <= -400: - e = -400 - - # 5) compute the value using long math and proper rounding. - lr = long(digits) - if e >= 0: - bits = 0 - m = lr * 10L ** e - else: - # compute a sufficiently large scale - prec = MANTISSA_DIGITS * 2 + 22 # 128, maybe - bits = - (int(math.ceil(-e / math.log10(2.0) - 1e-10)) + prec) - scale = 2L ** -bits - pten = 10L ** -e - m = (lr * scale) // pten - - # we now have a fairly large mantissa. - # Shift it and round the last bit. - - # first estimate the bits and do a big shift - if m: - mbits = int(math.ceil(math.log(m, 2) - 1e-10)) - needed = MANTISSA_BITS - if mbits > needed: - if mbits > needed+1: - shifted = mbits - (needed+1) - m >>= shifted - bits += shifted - # do the rounding - bits += 1 - m = (m >> 1) + (m & 1) - - try: - r = math.ldexp(m, bits) - except OverflowError: - r = 1e200 * 1e200 # produce inf, hopefully - - if sign == '-': - r = -r - - return r - - -# the "real" implementation. -# for comments, see above. -# XXX probably this very specific thing should go into longobject? - def interp_string_to_float(space, s): """ Conversion of string to float. @@ -336,6 +176,15 @@ raise OperationError(space.w_ValueError, space.wrap( "empty string for float()")) + + low = s.lower() + if low == "-inf": + return -INFINITY + elif low == "inf": + return INFINITY + elif low == "nan" or low == "-nan": + return NAN + # 1) parse the string into pieces. try: sign, before_point, after_point, exponent = break_up_float(s) @@ -432,7 +281,7 @@ if r == 2*r and r != 0.0: raise OverflowError except OverflowError: - r = 1e200 * 1e200 # produce inf, hopefully + r = INFINITY if sign == '-': r = -r Modified: pypy/branch/x86-64-jit-backend/pypy/objspace/std/test/test_floatobject.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/objspace/std/test/test_floatobject.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/objspace/std/test/test_floatobject.py Fri May 28 16:20:28 2010 @@ -92,6 +92,9 @@ def test_float_string(self): assert 42 == float("42") assert 42.25 == float("42.25") + assert str(float("inf")).startswith("inf") + assert str(float("-INf")).startswith("-inf") + assert str(float("-nAn")).startswith("nan") def test_float_unicode(self): # u00A0 and u2000 are some kind of spaces Modified: pypy/branch/x86-64-jit-backend/pypy/objspace/std/test/test_listobject.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/objspace/std/test/test_listobject.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/objspace/std/test/test_listobject.py Fri May 28 16:20:28 2010 @@ -5,7 +5,7 @@ from pypy.conftest import gettestobjspace -class TestW_ListObject: +class TestW_ListObject(object): def test_is_true(self): w = self.space.wrap @@ -342,7 +342,7 @@ self.space.w_True) -class AppTestW_ListObject: +class AppTestW_ListObject(object): def test_call_list(self): assert list('') == [] assert list('abc') == ['a', 'b', 'c'] @@ -576,6 +576,12 @@ l *= 2 assert l == [0, 1, 0, 1] + def test_mul_errors(self): + try: + [1, 2, 3] * (3,) + except TypeError: + pass + def test_index(self): c = range(10) assert c.index(0) == 0 Modified: pypy/branch/x86-64-jit-backend/pypy/objspace/std/test/test_strutil.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/objspace/std/test/test_strutil.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/objspace/std/test/test_strutil.py Fri May 28 16:20:28 2010 @@ -1,5 +1,6 @@ import py, random from pypy.objspace.std.strutil import * +from pypy.interpreter.error import OperationError import py @@ -132,50 +133,53 @@ assert string_to_w_long(space, '123L', 21).longval() == 441 + 42 + 3 assert string_to_w_long(space, '1891234174197319').longval() == 1891234174197319 -def test_string_to_float(): - assert string_to_float('0') == 0.0 - assert string_to_float('1') == 1.0 - assert string_to_float('-1.5') == -1.5 - assert string_to_float('1.5E2') == 150.0 - assert string_to_float('2.5E-1') == 0.25 - assert string_to_float('1e1111111111111') == float('1e1111111111111') - assert string_to_float('1e-1111111111111') == float('1e-1111111111111') - assert string_to_float('-1e1111111111111') == float('-1e1111111111111') - assert string_to_float('-1e-1111111111111') == float('-1e-1111111111111') - assert string_to_float('1e111111111111111111111') == float('1e111111111111111111111') - assert string_to_float('1e-111111111111111111111') == float('1e-111111111111111111111') - assert string_to_float('-1e111111111111111111111') == float('-1e111111111111111111111') - assert string_to_float('-1e-111111111111111111111') == float('-1e-111111111111111111111') - - valid_parts = [['', ' ', ' \f\n\r\t\v'], - ['', '+', '-'], - ['00', '90', '.5', '2.4', '3.', '0.07', - '12.3489749871982471987198371293717398256187563298638726' - '2187362820947193247129871083561249818451804287437824015' - '013816418758104762348932657836583048761487632840726386'], - ['', 'e0', 'E+1', 'E-01', 'E42'], - ['', ' ', ' \f\n\r\t\v'], - ] - invalid_parts = [['#'], - ['++', '+-', '-+', '--'], - ['', '1.2.3', '.', '5..6'], - ['E+', 'E-', 'e', 'e++', 'E++2'], - ['#'], - ] - for part0 in valid_parts[0]: - for part1 in valid_parts[1]: - for part2 in valid_parts[2]: - for part3 in valid_parts[3]: - for part4 in valid_parts[4]: - s = part0+part1+part2+part3+part4 - assert (abs(string_to_float(s) - float(s)) <= - 1E-13 * abs(float(s))) - - for j in range(len(invalid_parts)): - for invalid in invalid_parts[j]: - for i in range(20): - parts = [random.choice(lst) for lst in valid_parts] - parts[j] = invalid - s = ''.join(parts) - print repr(s) - py.test.raises(ParseStringError, string_to_float, s) + def test_string_to_float(self): + def string_to_float(x): + return interp_string_to_float(self.space, x) + assert string_to_float('0') == 0.0 + assert string_to_float('1') == 1.0 + assert string_to_float('-1.5') == -1.5 + assert string_to_float('1.5E2') == 150.0 + assert string_to_float('2.5E-1') == 0.25 + assert string_to_float('1e1111111111111') == float('1e1111111111111') + assert string_to_float('1e-1111111111111') == float('1e-1111111111111') + assert string_to_float('-1e1111111111111') == float('-1e1111111111111') + assert string_to_float('-1e-1111111111111') == float('-1e-1111111111111') + assert string_to_float('1e111111111111111111111') == float('1e111111111111111111111') + assert string_to_float('1e-111111111111111111111') == float('1e-111111111111111111111') + assert string_to_float('-1e111111111111111111111') == float('-1e111111111111111111111') + assert string_to_float('-1e-111111111111111111111') == float('-1e-111111111111111111111') + + valid_parts = [['', ' ', ' \f\n\r\t\v'], + ['', '+', '-'], + ['00', '90', '.5', '2.4', '3.', '0.07', + '12.3489749871982471987198371293717398256187563298638726' + '2187362820947193247129871083561249818451804287437824015' + '013816418758104762348932657836583048761487632840726386'], + ['', 'e0', 'E+1', 'E-01', 'E42'], + ['', ' ', ' \f\n\r\t\v'], + ] + invalid_parts = [['#'], + ['++', '+-', '-+', '--'], + ['', '1.2.3', '.', '5..6'], + ['E+', 'E-', 'e', 'e++', 'E++2'], + ['#'], + ] + for part0 in valid_parts[0]: + for part1 in valid_parts[1]: + for part2 in valid_parts[2]: + for part3 in valid_parts[3]: + for part4 in valid_parts[4]: + s = part0+part1+part2+part3+part4 + assert (abs(string_to_float(s) - float(s)) <= + 1E-13 * abs(float(s))) + + for j in range(len(invalid_parts)): + for invalid in invalid_parts[j]: + for i in range(20): + parts = [random.choice(lst) for lst in valid_parts] + parts[j] = invalid + s = ''.join(parts) + print repr(s) + if s.strip(): # empty s raises OperationError directly + py.test.raises(ParseStringError, string_to_float, s) Modified: pypy/branch/x86-64-jit-backend/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/objspace/std/typeobject.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/objspace/std/typeobject.py Fri May 28 16:20:28 2010 @@ -352,7 +352,7 @@ raise UnwrapError(w_self) def is_heaptype(w_self): - return w_self.__flags__&_HEAPTYPE + return w_self.__flags__ & _HEAPTYPE def is_cpytype(w_self): return w_self.__flags__ & _CPYTYPE @@ -581,6 +581,10 @@ w_bestbase = check_and_find_best_base(w_self.space, w_self.bases_w) w_self.instancetypedef = w_bestbase.instancetypedef w_self.__flags__ = _HEAPTYPE + for w_base in w_self.bases_w: + if not isinstance(w_base, W_TypeObject): + continue + w_self.__flags__ |= w_base.__flags__ hasoldstylebase = copy_flags_from_bases(w_self, w_bestbase) create_all_slots(w_self, hasoldstylebase) Modified: pypy/branch/x86-64-jit-backend/pypy/rlib/jit.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rlib/jit.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rlib/jit.py Fri May 28 16:20:28 2010 @@ -224,8 +224,7 @@ def __init__(self, greens=None, reds=None, virtualizables=None, get_jitcell_at=None, set_jitcell_at=None, can_inline=None, get_printable_location=None, - confirm_enter_jit=None, - leave=None): # XXX 'leave' is deprecated + confirm_enter_jit=None): if greens is not None: self.greens = greens if reds is not None: @@ -243,7 +242,6 @@ self.get_printable_location = get_printable_location self.can_inline = can_inline self.confirm_enter_jit = confirm_enter_jit - self.leave = leave def _freeze_(self): return True @@ -358,7 +356,6 @@ **kwds_s) self.annotate_hook(driver.can_inline, driver.greens, **kwds_s) self.annotate_hook(driver.get_printable_location, driver.greens, **kwds_s) - self.annotate_hook(driver.leave, driver.greens + driver.reds, **kwds_s) def annotate_hook(self, func, variables, args_s=[], **kwds_s): if func is None: Modified: pypy/branch/x86-64-jit-backend/pypy/rlib/rlocale.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rlib/rlocale.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rlib/rlocale.py Fri May 28 16:20:28 2010 @@ -17,6 +17,7 @@ class CConfig: includes = ['locale.h', 'limits.h'] + if HAVE_LANGINFO: includes += ['langinfo.h'] if HAVE_LIBINTL: @@ -24,7 +25,7 @@ if sys.platform == 'win32': includes += ['windows.h'] _compilation_info_ = ExternalCompilationInfo( - includes=includes, + includes=includes ) HAVE_BIND_TEXTDOMAIN_CODESET = platform.Has('bind_textdomain_codeset') lconv = platform.Struct("struct lconv", [ Modified: pypy/branch/x86-64-jit-backend/pypy/rlib/rweakref.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rlib/rweakref.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rlib/rweakref.py Fri May 28 16:20:28 2010 @@ -5,7 +5,6 @@ """ import weakref -from weakref import ref class RWeakValueDictionary(object): Modified: pypy/branch/x86-64-jit-backend/pypy/rlib/rwin32.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rlib/rwin32.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rlib/rwin32.py Fri May 28 16:20:28 2010 @@ -50,6 +50,15 @@ SYSTEMTIME = rffi_platform.Struct('SYSTEMTIME', []) + OSVERSIONINFO = rffi_platform.Struct( + 'OSVERSIONINFO', + [('dwOSVersionInfoSize', rffi.UINT), + ('dwMajorVersion', rffi.UINT), + ('dwMinorVersion', rffi.UINT), + ('dwBuildNumber', rffi.UINT), + ('dwPlatformId', rffi.UINT), + ('szCSDVersion', rffi.CFixedArray(lltype.Char, 1))]) + LPSECURITY_ATTRIBUTES = rffi_platform.SimpleType( "LPSECURITY_ATTRIBUTES", rffi.CCHARP) @@ -184,3 +193,23 @@ return ''.join([buf[i] for i in range(res)]) finally: lltype.free(buf, flavor='raw') + + _GetVersionEx = winexternal('GetVersionExA', + [lltype.Ptr(OSVERSIONINFO)], + DWORD) + + def GetVersionEx(): + info = lltype.malloc(OSVERSIONINFO, flavor='raw') + rffi.setintfield(info, 'c_dwOSVersionInfoSize', + rffi.sizeof(OSVERSIONINFO)) + try: + if not _GetVersionEx(info): + raise lastWindowsError() + return (rffi.cast(lltype.Signed, info.c_dwMajorVersion), + rffi.cast(lltype.Signed, info.c_dwMinorVersion), + rffi.cast(lltype.Signed, info.c_dwBuildNumber), + rffi.cast(lltype.Signed, info.c_dwPlatformId), + rffi.charp2str(rffi.cast(rffi.CCHARP, + info.c_szCSDVersion))) + finally: + lltype.free(info, flavor='raw') Modified: pypy/branch/x86-64-jit-backend/pypy/rlib/test/test_jit.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rlib/test/test_jit.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rlib/test/test_jit.py Fri May 28 16:20:28 2010 @@ -61,12 +61,10 @@ def can_inline(m): pass def get_printable_location(m): pass - def leave(m, n): pass myjitdriver = JitDriver(greens=['m'], reds=['n'], can_inline=can_inline, - get_printable_location=get_printable_location, - leave=leave) + get_printable_location=get_printable_location) def fn(n): m = 42.5 while n > 0: @@ -83,9 +81,6 @@ return [v.concretetype for v in graph.getargs()] raise Exception, 'function %r has not been annotated' % func - leave_args = getargs(leave) - assert leave_args == [lltype.Float, lltype.Signed] - can_inline_args = getargs(can_inline) get_printable_location_args = getargs(get_printable_location) assert can_inline_args == get_printable_location_args == [lltype.Float] Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/extfunc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/extfunc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/extfunc.py Fri May 28 16:20:28 2010 @@ -58,13 +58,16 @@ return method return decorator -def registering_if(ns, name): +def registering_if(ns, name, condition=True): try: func = getattr(ns, name) except AttributeError: - return lambda method: None - else: + condition = False + + if condition: return registering(func) + else: + return lambda method: None class LazyRegisteringMeta(type): def __new__(self, _name, _type, _vars): Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/ll2ctypes.py Fri May 28 16:20:28 2010 @@ -208,6 +208,8 @@ else: restype = get_ctypes_type(T.TO.RESULT) return ctypes.CFUNCTYPE(restype, *argtypes) + elif isinstance(T.TO, lltype.OpaqueType): + return ctypes.c_void_p else: return ctypes.POINTER(get_ctypes_type(T.TO, delayed_builders)) elif T is lltype.Void: @@ -635,10 +637,15 @@ cbuf = ctypes.create_string_buffer(T.TO.hints['getsize']()) else: cbuf = ctypes.create_string_buffer("\x00") + cbuf = ctypes.cast(cbuf, ctypes.c_void_p) add_storage(container, _parentable_mixin, cbuf) else: raise NotImplementedError(T) container._ctypes_storage_was_allocated() + + if isinstance(T.TO, lltype.OpaqueType): + return container._storage + storage = container._storage p = ctypes.pointer(storage) if index: @@ -729,6 +736,7 @@ container = _llgcopaque(cobj) else: container = lltype._opaque(T.TO) + container._storage = ctypes.cast(cobj, ctypes.c_void_p) else: raise NotImplementedError(T) llobj = lltype._ptr(T, container, solid=True) Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/rffi.py Fri May 28 16:20:28 2010 @@ -645,7 +645,7 @@ """ str_chars_offset = (offsetof(STRTYPE, 'chars') + \ itemoffsetof(STRTYPE.chars, 0)) - gc_buf = rgc.malloc_nonmovable(STRTYPE, count) + gc_buf = lltype.nullptr(STRTYPE) # rgc.malloc_nonmovable(STRTYPE, count) if gc_buf: realbuf = cast_ptr_to_adr(gc_buf) + str_chars_offset raw_buf = cast(TYPEP, realbuf) Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Fri May 28 16:20:28 2010 @@ -343,6 +343,17 @@ lltype.free(ll_timevalp, flavor='raw') assert not ALLOCATED # detects memory leaks in the test + def test_opaque_obj_2(self): + FILEP = rffi.COpaquePtr('FILE') + fopen = rffi.llexternal('fopen', [rffi.CCHARP, rffi.CCHARP], FILEP) + fclose = rffi.llexternal('fclose', [FILEP], rffi.INT) + tmppath = udir.join('test_ll2ctypes.test_opaque_obj_2') + ll_file = fopen(str(tmppath), "w") + assert ll_file + fclose(ll_file) + assert tmppath.check(file=1) + assert not ALLOCATED # detects memory leaks in the test + def test_simple_cast(self): assert rffi.cast(rffi.SIGNEDCHAR, 0x123456) == 0x56 assert rffi.cast(rffi.SIGNEDCHAR, 0x123481) == -127 Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/module/ll_os.py Fri May 28 16:20:28 2010 @@ -1329,7 +1329,7 @@ return extdef([int], int, llimpl=umask_llimpl, export_name="ll_os.ll_os_umask") - @registering_if(os, 'kill') + @registering_if(os, 'kill', sys.platform != 'win32') def register_os_kill(self): os_kill = self.llexternal('kill', [rffi.PID_T, rffi.INT], rffi.INT) Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/module/ll_os_stat.py Fri May 28 16:20:28 2010 @@ -317,6 +317,11 @@ 'ERROR_SHARING_VIOLATION') _S_IFDIR = platform.ConstantInteger('_S_IFDIR') _S_IFREG = platform.ConstantInteger('_S_IFREG') + _S_IFCHR = platform.ConstantInteger('_S_IFCHR') + _S_IFIFO = platform.ConstantInteger('_S_IFIFO') + FILE_TYPE_UNKNOWN = platform.ConstantInteger('FILE_TYPE_UNKNOWN') + FILE_TYPE_CHAR = platform.ConstantInteger('FILE_TYPE_CHAR') + FILE_TYPE_PIPE = platform.ConstantInteger('FILE_TYPE_PIPE') WIN32_FILE_ATTRIBUTE_DATA = platform.Struct( 'WIN32_FILE_ATTRIBUTE_DATA', @@ -365,6 +370,12 @@ rwin32.BOOL, calling_conv='win') + GetFileType = rffi.llexternal( + 'GetFileType', + [rwin32.HANDLE], + rwin32.DWORD, + calling_conv='win') + FindFirstFile = rffi.llexternal( 'FindFirstFileA', [rffi.CCHARP, lltype.Ptr(WIN32_FIND_DATA)], @@ -477,9 +488,29 @@ win32_lstat_llimpl = win32_stat_llimpl def win32_fstat_llimpl(fd): - info = lltype.malloc(BY_HANDLE_FILE_INFORMATION, flavor='raw') + handle = rwin32._get_osfhandle(fd) + + filetype = GetFileType(handle) + if filetype == FILE_TYPE_CHAR: + # console or LPT device + return make_stat_result((_S_IFCHR, + 0, 0, 0, 0, 0, + 0, 0, 0, 0)) + elif filetype == FILE_TYPE_PIPE: + # socket or named pipe + return make_stat_result((_S_IFIFO, + 0, 0, 0, 0, 0, + 0, 0, 0, 0)) + elif filetype == FILE_TYPE_UNKNOWN: + error = rwin32.GetLastError() + if error != 0: + raise WindowsError(error, "os_fstat failed") + # else: unknown but valid file + + # normal disk file (FILE_TYPE_DISK) + info = lltype.malloc(BY_HANDLE_FILE_INFORMATION, flavor='raw', + zero=True) try: - handle = rwin32._get_osfhandle(fd) res = GetFileInformationByHandle(handle, info) if res == 0: raise WindowsError(rwin32.GetLastError(), "os_fstat failed") Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/module/test/test_ll_os_stat.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/module/test/test_ll_os_stat.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/module/test/test_ll_os_stat.py Fri May 28 16:20:28 2010 @@ -15,3 +15,7 @@ check('c:/') check('c:/temp') check('c:/pagefile.sys') + + def test_fstat(self): + stat = ll_os_stat.win32_fstat_llimpl(0) # stdout + assert stat.st_mode != 0 Modified: pypy/branch/x86-64-jit-backend/pypy/rpython/test/test_extfuncregister.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/rpython/test/test_extfuncregister.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/rpython/test/test_extfuncregister.py Fri May 28 16:20:28 2010 @@ -105,3 +105,9 @@ assert bar is None + @registering_if(A, 'f', False) + def baz(): + pass + + assert baz is None + Modified: pypy/branch/x86-64-jit-backend/pypy/tool/pytest/appsupport.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/tool/pytest/appsupport.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/tool/pytest/appsupport.py Fri May 28 16:20:28 2010 @@ -11,7 +11,8 @@ self.raw = pycode self.w_file = space.getattr(pycode, space.wrap('co_filename')) self.name = space.getattr(pycode, space.wrap('co_name')) - self.firstlineno = space.unwrap(space.getattr(pycode, space.wrap('co_firstlineno'))) + self.firstlineno = space.unwrap( + space.getattr(pycode, space.wrap('co_firstlineno'))) - 1 #try: # self.path = space.unwrap(space.getattr(self.w_file, space.wrap('__path__'))) #except OperationError: @@ -20,13 +21,14 @@ self.space = space def fullsource(self): + filename = self.space.str_w(self.w_file) + source = py.code.Source(py.std.linecache.getlines(filename)) + if source.lines: + return source try: - return self.space.str_w(self.w_file).__source__ - except AttributeError: - try: - return py.code.Source(self.path.read(mode="rU")) - except py.error.Error: - return None + return py.code.Source(self.path.read(mode="rU")) + except py.error.Error: + return None fullsource = property(fullsource, None, None, "Full source of AppCode") def getargs(self): @@ -211,12 +213,19 @@ source = py.code.Source(expr) frame = space.getexecutioncontext().gettopframe() w_locals = frame.getdictscope() + pycode = frame.pycode + filename = "<%s:%s>" %(pycode.co_filename, frame.f_lineno) + lines = [x + "\n" for x in expr.split("\n")] + py.std.linecache.cache[filename] = (1, None, lines, filename) w_locals = space.call_method(w_locals, 'copy') for key, w_value in kwds_w.items(): space.setitem(w_locals, space.wrap(key), w_value) + #filename = __file__ + #if filename.endswith("pyc"): + # filename = filename[:-1] try: space.exec_(str(source), frame.w_globals, w_locals, - filename=__file__) + filename=filename) except OperationError, e: if e.match(space, w_ExpectedException): return _exc_info(space, e) Modified: pypy/branch/x86-64-jit-backend/pypy/tool/pytest/test/test_appsupport.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/tool/pytest/test/test_appsupport.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/tool/pytest/test/test_appsupport.py Fri May 28 16:20:28 2010 @@ -70,6 +70,49 @@ assert result.ret == 0 result.stdout.fnmatch_lines(["*2 passed*"]) +def test_applevel_raises_simple_display(testdir): + setpypyconftest(testdir) + p = testdir.makepyfile(""" + def app_test_raises(): + raises(ValueError, x) + class AppTestRaises: + def test_func(self): + raises (ValueError, x) + # + """) + result = testdir.runpytest(p, "-s") + assert result.ret == 1 + result.stdout.fnmatch_lines([ + "*E*application-level*NameError*x*not defined", + "*test_func(self)*", + ">*raises*ValueError*", + "*E*application-level*NameError*x*not defined", + "*test_applevel_raises_simple_display*", + ]) + result = testdir.runpytest(p) # this time we may run the pyc file + assert result.ret == 1 + result.stdout.fnmatch_lines([ + "*E*application-level*NameError*x*not defined", + ]) + +def test_applevel_raises_display(testdir): + setpypyconftest(testdir) + p = testdir.makepyfile(""" + def app_test_raises(): + raises(ValueError, "x") + pass + """) + result = testdir.runpytest(p, "-s") + assert result.ret == 1 + result.stdout.fnmatch_lines([ + "*E*application-level*NameError*x*not defined", + ]) + result = testdir.runpytest(p) # this time we may run the pyc file + assert result.ret == 1 + result.stdout.fnmatch_lines([ + "*E*application-level*NameError*x*not defined", + ]) + def app_test_raises(): info = raises(TypeError, id) assert info.type is TypeError Modified: pypy/branch/x86-64-jit-backend/pypy/translator/platform/__init__.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/platform/__init__.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/platform/__init__.py Fri May 28 16:20:28 2010 @@ -113,7 +113,6 @@ log.WARNING(line) def _preprocess_include_dirs(self, include_dirs): - # hook for maemo return include_dirs def _compile_args_from_eci(self, eci, standalone): @@ -125,9 +124,13 @@ extra = self.shared_only cflags = self.cflags + extra return (cflags + list(eci.compile_extra) + args) + + def _preprocess_library_dirs(self, library_dirs): + return library_dirs def _link_args_from_eci(self, eci, standalone): - library_dirs = self._libdirs(eci.library_dirs) + library_dirs = self._preprocess_library_dirs(eci.library_dirs) + library_dirs = self._libdirs(library_dirs) libraries = self._libs(eci.libraries) link_files = self._linkfiles(eci.link_files) return (library_dirs + self.link_flags + Modified: pypy/branch/x86-64-jit-backend/pypy/translator/platform/darwin.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/platform/darwin.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/platform/darwin.py Fri May 28 16:20:28 2010 @@ -20,6 +20,18 @@ def _args_for_shared(self, args): return (self.shared_only + ['-dynamiclib', '-undefined', 'dynamic_lookup'] + args) + + def _preprocess_include_dirs(self, include_dirs): + res_incl_dirs = list(include_dirs) + res_incl_dirs.append('/usr/local/include') # Homebrew + res_incl_dirs.append('/opt/local/include') # MacPorts + return res_incl_dirs + + def _preprocess_library_dirs(self, library_dirs): + res_lib_dirs = list(library_dirs) + res_lib_dirs.append('/usr/local/lib') # Homebrew + res_lib_dirs.append('/opt/local/lib') # MacPorts + return res_lib_dirs def include_dirs_for_libffi(self): return ['/usr/include/ffi'] Modified: pypy/branch/x86-64-jit-backend/pypy/translator/platform/freebsd7.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/platform/freebsd7.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/platform/freebsd7.py Fri May 28 16:20:28 2010 @@ -15,6 +15,16 @@ def _args_for_shared(self, args): return ['-shared'] + args + def _preprocess_include_dirs(self, include_dirs): + res_incl_dirs = list(include_dirs) + res_incl_dirs.append('/usr/local/include') + return res_incl_dirs + + def _preprocess_library_dirs(self, library_dirs): + res_lib_dirs = list(library_dirs) + res_lib_dirs.append('/usr/local/lib') + return res_lib_dirs + def include_dirs_for_libffi(self): return ['/usr/local/include'] Modified: pypy/branch/x86-64-jit-backend/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/translator/tool/cbuild.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/translator/tool/cbuild.py Fri May 28 16:20:28 2010 @@ -282,7 +282,7 @@ for sym in self.export_symbols: f.write("%s;\n" % (sym,)) f.write("};") - d['link_extra'] += ("-Wl,--dynamic-list=" + str(file_name), ) + d['link_extra'] += ("-Wl,--export-dynamic,--version-script=" + str(file_name), ) f.close() d['export_symbols'] = () return ExternalCompilationInfo(**d) From arigo at codespeak.net Fri May 28 16:23:48 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 16:23:48 +0200 (CEST) Subject: [pypy-svn] r74856 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100528142348.0BE80282BD6@codespeak.net> Author: arigo Date: Fri May 28 16:23:47 2010 New Revision: 74856 Added: pypy/branch/blackhole-improvement/pypy/jit/metainterp/jitexc.py Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/jitprof.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizeutil.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py Log: More translation fixes. The first test in test_ztranslation passes. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Fri May 28 16:23:47 2010 @@ -6,9 +6,9 @@ from pypy.rpython.lltypesystem import lltype, llmemory, rclass from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.llinterp import LLException -from pypy.rpython.annlowlevel import cast_instance_to_base_ptr from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr from pypy.jit.codewriter import heaptracker +from pypy.jit.metainterp.jitexc import JitException, get_llexception def arguments(*argtypes, **kwds): @@ -20,7 +20,7 @@ return function return decorate -class LeaveFrame(Exception): +class LeaveFrame(JitException): pass class MissingValue(object): @@ -33,21 +33,6 @@ NULL = lltype.nullptr(llmemory.GCREF.TO) -def _get_standard_error(rtyper, Class): - exdata = rtyper.getexceptiondata() - clsdef = rtyper.annotator.bookkeeper.getuniqueclassdef(Class) - evalue = exdata.get_standard_ll_exc_instance(rtyper, clsdef) - return evalue - -def get_llexception(cpu, e): - if we_are_translated(): - return cast_instance_to_base_ptr(e) - if isinstance(e, LLException): - return e.args[1] # ok - if isinstance(e, OverflowError): - return _get_standard_error(cpu.rtyper, OverflowError) - raise # leave other exceptions to be propagated - # ____________________________________________________________ @@ -305,6 +290,8 @@ self.dispatch_loop(self, self.jitcode.code, self.position) except LeaveFrame: break + except JitException: + raise # go through except Exception, e: lle = get_llexception(self.cpu, e) self.handle_exception_in_frame(lle) @@ -1183,6 +1170,8 @@ # we now proceed to interpret the bytecode in this frame self.run() # + except JitException: + raise # go through except Exception, e: # if we get an exception, return it to the caller frame current_exc = get_llexception(self.cpu, e) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py Fri May 28 16:23:47 2010 @@ -252,7 +252,7 @@ execute = execute_by_num_args[num_args] # if execute[value] is not None: - raise Exception("duplicate entry for op number %d" % value) + raise AssertionError("duplicate entry for op number %d"% value) if key.endswith('_PURE'): key = key[:-5] # @@ -276,7 +276,14 @@ if func is not None: execute[value] = func continue - print "XXX warning: missing", key + if value in (rop.FORCE_TOKEN, + rop.CALL_ASSEMBLER, + rop.COND_CALL_GC_WB, + rop.DEBUG_MERGE_POINT, + rop.SETARRAYITEM_RAW, + ): # list of opcodes never executed by pyjitpl + continue + raise AssertionError("missing %r" % (key,)) return execute_by_num_args def make_execute_function_with_boxes(name, func): Added: pypy/branch/blackhole-improvement/pypy/jit/metainterp/jitexc.py ============================================================================== --- (empty file) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/jitexc.py Fri May 28 16:23:47 2010 @@ -0,0 +1,28 @@ +from pypy.rpython.annlowlevel import cast_instance_to_base_ptr +from pypy.rpython.llinterp import LLException +from pypy.rlib.objectmodel import we_are_translated + + +class JitException(Exception): + """The base class for exceptions raised and caught in the JIT. + The point is that the places that catch any user exception should avoid + catching exceptions that inherit from JitException. + """ + _go_through_llinterp_uncaught_ = True # ugh + + +def _get_standard_error(rtyper, Class): + exdata = rtyper.getexceptiondata() + clsdef = rtyper.annotator.bookkeeper.getuniqueclassdef(Class) + evalue = exdata.get_standard_ll_exc_instance(rtyper, clsdef) + return evalue + +def get_llexception(cpu, e): + if we_are_translated(): + return cast_instance_to_base_ptr(e) + assert not isinstance(e, JitException) + if isinstance(e, LLException): + return e.args[1] # ok + if isinstance(e, OverflowError): + return _get_standard_error(cpu.rtyper, OverflowError) + raise # leave other exceptions to be propagated Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/jitprof.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/jitprof.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/jitprof.py Fri May 28 16:23:47 2010 @@ -5,6 +5,7 @@ import os import time from pypy.rlib.debug import debug_print +from pypy.jit.metainterp.jitexc import JitException counters=""" TRACING @@ -192,5 +193,5 @@ -class BrokenProfilerData(Exception): +class BrokenProfilerData(JitException): pass Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizeutil.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizeutil.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizeutil.py Fri May 28 16:23:47 2010 @@ -2,8 +2,9 @@ from pypy.rlib.rarithmetic import intmask from pypy.rlib.unroll import unrolling_iterable from pypy.jit.metainterp import resoperation, history +from pypy.jit.metainterp.jitexc import JitException -class InvalidLoop(Exception): +class InvalidLoop(JitException): """Raised when the optimize*.py detect that the loop that we are trying to build cannot possibly make sense as a long-running loop (e.g. it cannot run 2 complete iterations).""" Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Fri May 28 16:23:47 2010 @@ -15,7 +15,7 @@ from pypy.jit.metainterp.jitprof import EmptyProfiler from pypy.jit.metainterp.jitprof import GUARDS, RECORDED_OPS, ABORT_ESCAPE from pypy.jit.metainterp.jitprof import ABORT_TOO_LONG, ABORT_BRIDGE -from pypy.jit.metainterp.blackhole import get_llexception +from pypy.jit.metainterp.jitexc import JitException, get_llexception from pypy.rlib.rarithmetic import intmask from pypy.rlib.objectmodel import specialize from pypy.rlib.jit import DEBUG_OFF, DEBUG_PROFILE, DEBUG_STEPS, DEBUG_DETAILED @@ -1287,16 +1287,7 @@ assert False def finishframe_exception(self): - # detect and propagate some exceptions early: - # - AssertionError - # - all subclasses of JitException excvaluebox = self.last_exc_value_box - if we_are_translated(): - from pypy.jit.metainterp.warmspot import JitException - e = self.cpu.ts.get_exception_obj(excvaluebox) - if isinstance(e, JitException) or isinstance(e, AssertionError): - raise Exception, e - # while self.framestack: frame = self.framestack[-1] code = frame.bytecode @@ -1332,7 +1323,7 @@ else: print " ", print jitcode.name - raise Exception + raise AssertionError def create_empty_history(self): warmrunnerstate = self.staticdata.state @@ -1417,6 +1408,8 @@ op.name = self.framestack[-1].jitcode.name def execute_raised(self, exception, constant=False): + if isinstance(exception, JitException): + raise JitException, exception # go through llexception = get_llexception(self.cpu, exception) self.execute_ll_raised(llexception, constant) @@ -2006,17 +1999,17 @@ # ____________________________________________________________ -class GenerateMergePoint(Exception): +class GenerateMergePoint(JitException): def __init__(self, args, target_loop_token): assert target_loop_token is not None self.argboxes = args self.target_loop_token = target_loop_token -class ChangeFrame(Exception): +class ChangeFrame(JitException): """Raised after we mutated metainterp.framestack, in order to force it to reload the current top-of-stack frame that gets interpreted.""" -class SwitchToBlackhole(Exception): +class SwitchToBlackhole(JitException): def __init__(self, reason): self.reason = reason Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py Fri May 28 16:23:47 2010 @@ -20,6 +20,7 @@ from pypy.jit.metainterp.pyjitpl import MetaInterpStaticData, MetaInterp from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper from pypy.jit.metainterp.jitprof import Profiler, EmptyProfiler +from pypy.jit.metainterp.jitexc import JitException from pypy.jit.codewriter import support, codewriter from pypy.jit.codewriter.policy import JitPolicy from pypy.rlib.jit import DEBUG_STEPS, DEBUG_DETAILED, DEBUG_OFF, DEBUG_PROFILE @@ -130,9 +131,6 @@ stats.maybe_view() stats.check_consistency() -class JitException(Exception): - _go_through_llinterp_uncaught_ = True # ugh - class ContinueRunningNormallyBase(JitException): pass From arigo at codespeak.net Fri May 28 16:25:39 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 16:25:39 +0200 (CEST) Subject: [pypy-svn] r74857 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100528142539.868DA282BD6@codespeak.net> Author: arigo Date: Fri May 28 16:25:37 2010 New Revision: 74857 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: Kill is_blackholing(). Yay. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Fri May 28 16:25:37 2010 @@ -844,21 +844,18 @@ # ConstPtr(NULL). # metainterp = self.metainterp - if metainterp.is_blackholing(): - resbox = box # good enough when blackholing - else: - vrefinfo = metainterp.staticdata.virtualref_info - obj = box.getref_base() - vref = vrefinfo.virtual_ref_during_tracing(obj) - resbox = history.BoxPtr(vref) - cindex = history.ConstInt(len(metainterp.virtualref_boxes) // 2) - metainterp.history.record(rop.VIRTUAL_REF, [box, cindex], resbox) - # Note: we allocate a JIT_VIRTUAL_REF here - # (in virtual_ref_during_tracing()), in order to detect when - # the virtual escapes during tracing already. We record it as a - # VIRTUAL_REF operation. Later, optimizeopt.py should either kill - # that operation or replace it with a NEW_WITH_VTABLE followed by - # SETFIELD_GCs. + vrefinfo = metainterp.staticdata.virtualref_info + obj = box.getref_base() + vref = vrefinfo.virtual_ref_during_tracing(obj) + resbox = history.BoxPtr(vref) + cindex = history.ConstInt(len(metainterp.virtualref_boxes) // 2) + metainterp.history.record(rop.VIRTUAL_REF, [box, cindex], resbox) + # Note: we allocate a JIT_VIRTUAL_REF here + # (in virtual_ref_during_tracing()), in order to detect when + # the virtual escapes during tracing already. We record it as a + # VIRTUAL_REF operation. Later, optimizeopt.py should either kill + # that operation or replace it with a NEW_WITH_VTABLE followed by + # SETFIELD_GCs. metainterp.virtualref_boxes.append(box) metainterp.virtualref_boxes.append(resbox) return resbox @@ -871,12 +868,11 @@ vrefbox = metainterp.virtualref_boxes.pop() lastbox = metainterp.virtualref_boxes.pop() assert box.getref_base() == lastbox.getref_base() - if not metainterp.is_blackholing(): - vrefinfo = metainterp.staticdata.virtualref_info - vref = vrefbox.getref_base() - if vrefinfo.is_virtual_ref(vref): - metainterp.history.record(rop.VIRTUAL_REF_FINISH, - [vrefbox, lastbox], None) + vrefinfo = metainterp.staticdata.virtualref_info + vref = vrefbox.getref_base() + if vrefinfo.is_virtual_ref(vref): + metainterp.history.record(rop.VIRTUAL_REF_FINISH, + [vrefbox, lastbox], None) # ------------------------------ @@ -922,8 +918,6 @@ if isinstance(box, Const): # no need for a guard return metainterp = self.metainterp - if metainterp.is_blackholing(): - return if box is not None: moreargs = [box] + extraargs else: @@ -1226,9 +1220,6 @@ self.free_frames_list = [] self.last_exc_value_box = None - def is_blackholing(self): - return False # XXX get rid of this method - def perform_call(self, jitcode, boxes, greenkey=None): # causes the metainterp to enter the given subfunction # with a special case for recursive portal calls @@ -1239,7 +1230,7 @@ def newframe(self, jitcode, greenkey=None): if jitcode is self.staticdata.portal_code: self.in_recursion += 1 - if greenkey is not None and not self.is_blackholing(): + if greenkey is not None: self.portal_trace_positions.append( (greenkey, len(self.history.operations))) if len(self.free_frames_list) > 0: @@ -1254,7 +1245,7 @@ frame = self.framestack.pop() if frame.jitcode is self.staticdata.portal_code: self.in_recursion -= 1 - if frame.greenkey is not None and not self.is_blackholing(): + if frame.greenkey is not None: self.portal_trace_positions.append( (None, len(self.history.operations))) # we save the freed MIFrames to avoid needing to re-create new @@ -1363,15 +1354,12 @@ profiler.count_ops(opnum) resbox = executor.execute_varargs(self.cpu, self, opnum, argboxes, descr) - if self.is_blackholing(): - profiler.count_ops(opnum, BLACKHOLED_OPS) + # check if the operation can be constant-folded away + argboxes = list(argboxes) + if rop._ALWAYS_PURE_FIRST <= opnum <= rop._ALWAYS_PURE_LAST: + resbox = self._record_helper_pure_varargs(opnum, resbox, descr, argboxes) else: - # check if the operation can be constant-folded away - argboxes = list(argboxes) - if rop._ALWAYS_PURE_FIRST <= opnum <= rop._ALWAYS_PURE_LAST: - resbox = self._record_helper_pure_varargs(opnum, resbox, descr, argboxes) - else: - resbox = self._record_helper_nonpure_varargs(opnum, resbox, descr, argboxes) + resbox = self._record_helper_nonpure_varargs(opnum, resbox, descr, argboxes) return resbox def _record_helper_pure(self, opnum, resbox, descr, *argboxes): @@ -1760,9 +1748,6 @@ vinfo.clear_vable_token(virtualizable) def vable_and_vrefs_before_residual_call(self): - if self.is_blackholing(): - return - # vrefinfo = self.staticdata.virtualref_info for i in range(1, len(self.virtualref_boxes), 2): vrefbox = self.virtualref_boxes[i] @@ -1957,8 +1942,6 @@ boxes[i] = newbox def find_biggest_function(self): - assert not self.is_blackholing() - start_stack = [] max_size = 0 max_key = None From jcreigh at codespeak.net Fri May 28 16:27:29 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Fri, 28 May 2010 16:27:29 +0200 (CEST) Subject: [pypy-svn] r74858 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test Message-ID: <20100528142729.5BFA4282BD6@codespeak.net> Author: jcreigh Date: Fri May 28 16:27:28 2010 New Revision: 74858 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/conftest.py Log: enable tests on x86-64 (of course they're all broken, but you have to start somewhere) Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/conftest.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/conftest.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/test/conftest.py Fri May 28 16:27:28 2010 @@ -3,9 +3,5 @@ cpu = detect_cpu.autodetect() def pytest_runtest_setup(item): - # XXX - if "test_rx86" in str(item.fspath): - # Don't skip - pass - elif cpu != 'x86': - py.test.skip("x86 tests skipped: cpu is %r" % (cpu,)) + if cpu not in ('x86', 'x86_64'): + py.test.skip("x86/x86_64 tests skipped: cpu is %r" % (cpu,)) From jcreigh at codespeak.net Fri May 28 16:34:32 2010 From: jcreigh at codespeak.net (jcreigh at codespeak.net) Date: Fri, 28 May 2010 16:34:32 +0200 (CEST) Subject: [pypy-svn] r74859 - pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86 Message-ID: <20100528143432.AED88282BD6@codespeak.net> Author: jcreigh Date: Fri May 28 16:34:31 2010 New Revision: 74859 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Log: remove workaround for unrolling_iterable bug that was fixed in r74829 Modified: pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/x86-64-jit-backend/pypy/jit/backend/x86/regloc.py Fri May 28 16:34:31 2010 @@ -139,12 +139,7 @@ eax, ecx, edx, ebx, esp, ebp, esi, edi = REGLOCS xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 = XMMREGLOCS -possible_location_codes = list("rbsmajix") -# Separate objects are required because you can't use the same -# unrolling_iterable instance in more than once place -_binop_pc_outer = unrolling_iterable(possible_location_codes) -_binop_pc_inner = unrolling_iterable(possible_location_codes) -_unaryop_pc = unrolling_iterable(possible_location_codes) +unrolling_location_codes = unrolling_iterable(list("rbsmajix")) class LocationCodeBuilder(object): _mixin_ = True @@ -153,9 +148,9 @@ def INSN(self, loc1, loc2): code1 = loc1.location_code() code2 = loc2.location_code() - for possible_code1 in _binop_pc_outer: + for possible_code1 in unrolling_location_codes: if code1 == possible_code1: - for possible_code2 in _binop_pc_inner: + for possible_code2 in unrolling_location_codes: if code2 == possible_code2: methname = name + "_" + possible_code1 + possible_code2 if hasattr(rx86.AbstractX86CodeBuilder, methname): @@ -171,7 +166,7 @@ def _unaryop(name): def INSN(self, loc): code = loc.location_code() - for possible_code in _unaryop_pc: + for possible_code in unrolling_location_codes: if code == possible_code: methname = name + "_" + possible_code if hasattr(rx86.AbstractX86CodeBuilder, methname): From arigo at codespeak.net Fri May 28 16:53:06 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 16:53:06 +0200 (CEST) Subject: [pypy-svn] r74860 - in pypy/branch/blackhole-improvement/pypy: annotation jit/codewriter rpython rpython/lltypesystem rpython/test translator/c/src translator/c/test Message-ID: <20100528145306.D890F282BD6@codespeak.net> Author: arigo Date: Fri May 28 16:53:05 2010 New Revision: 74860 Modified: pypy/branch/blackhole-improvement/pypy/annotation/builtin.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/lloperation.py pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/opimpl.py pypy/branch/blackhole-improvement/pypy/rpython/raddress.py pypy/branch/blackhole-improvement/pypy/rpython/rbuiltin.py pypy/branch/blackhole-improvement/pypy/rpython/test/test_rdict.py pypy/branch/blackhole-improvement/pypy/translator/c/src/int.h pypy/branch/blackhole-improvement/pypy/translator/c/test/test_lladdresses.py Log: Use and support dictionaries with addresses as keys. Add a test in rpython/test and translator/c/test for this case, which used to work fine before introduction of the AddressAsKey. Modified: pypy/branch/blackhole-improvement/pypy/annotation/builtin.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/annotation/builtin.py (original) +++ pypy/branch/blackhole-improvement/pypy/annotation/builtin.py Fri May 28 16:53:05 2010 @@ -346,6 +346,9 @@ def llmemory_cast_int_to_adr(s): return SomeAddress() +def llmemory_get_inthash_from_int(s): + return SomeInteger() # xxx + ##def rarith_ovfcheck(s_obj): ## if isinstance(s_obj, SomeInteger) and s_obj.unsigned: @@ -387,6 +390,7 @@ BUILTIN_ANALYZERS[pypy.rpython.lltypesystem.llmemory.cast_adr_to_ptr] = llmemory_cast_adr_to_ptr BUILTIN_ANALYZERS[pypy.rpython.lltypesystem.llmemory.cast_adr_to_int] = llmemory_cast_adr_to_int BUILTIN_ANALYZERS[pypy.rpython.lltypesystem.llmemory.cast_int_to_adr] = llmemory_cast_int_to_adr +BUILTIN_ANALYZERS[pypy.rpython.lltypesystem.llmemory.get_inthash_from_int] = llmemory_get_inthash_from_int BUILTIN_ANALYZERS[getattr(OSError.__init__, 'im_func', OSError.__init__)] = ( OSError_init) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py Fri May 28 16:53:05 2010 @@ -61,6 +61,7 @@ def vtable2descr(cpu, vtable): assert lltype.typeOf(vtable) is lltype.Signed + vtable = llmemory.cast_int_to_adr(vtable) if we_are_translated(): # Build the dict {vtable: sizedescr} at runtime. # This is necessary because the 'vtables' are just pointers to @@ -71,11 +72,9 @@ for descr in cpu._all_size_descrs_with_vtable: key = descr._corresponding_vtable key = llmemory.cast_ptr_to_adr(key) - key = llmemory.cast_adr_to_int(key) d[key] = descr return d[vtable] else: - vtable = llmemory.cast_int_to_adr(vtable) vtable = llmemory.cast_adr_to_ptr(vtable, VTABLETYPE) for descr in cpu._all_size_descrs_with_vtable: if descr._corresponding_vtable == vtable: Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py Fri May 28 16:53:05 2010 @@ -635,6 +635,11 @@ ptr = lltype.cast_int_to_ptr(_NONGCREF, int) return cast_ptr_to_adr(ptr) +def get_inthash_from_int(int): + if isinstance(int, AddressAsInt): + int = lltype.cast_ptr_to_int(int.adr.ptr) + return int + # ____________________________________________________________ # Weakrefs. # Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/lloperation.py Fri May 28 16:53:05 2010 @@ -410,7 +410,8 @@ 'cast_ptr_to_adr': LLOp(sideeffects=False), 'cast_adr_to_ptr': LLOp(canfold=True), 'cast_adr_to_int': LLOp(sideeffects=False), - 'cast_int_to_adr': LLOp(canfold=True), # not implemented in llinterp + 'cast_int_to_adr': LLOp(canfold=True), + 'get_inthash_from_int': LLOp(canfold=True), 'get_group_member': LLOp(canfold=True), 'get_next_group_member':LLOp(canfold=True), Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/opimpl.py Fri May 28 16:53:05 2010 @@ -340,6 +340,9 @@ def op_cast_int_to_adr(int): return llmemory.cast_int_to_adr(int) +def op_get_inthash_from_int(int): + return llmemory.get_inthash_from_int(int) + ##def op_cast_int_to_adr(x): ## assert type(x) is int ## return llmemory.cast_int_to_adr(x) Modified: pypy/branch/blackhole-improvement/pypy/rpython/raddress.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/raddress.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/raddress.py Fri May 28 16:53:05 2010 @@ -2,11 +2,13 @@ from pypy.tool.pairtype import pairtype from pypy.annotation import model as annmodel from pypy.rpython.lltypesystem.llmemory import NULL, Address, \ - cast_adr_to_int, fakeaddress + cast_adr_to_int, get_inthash_from_int, fakeaddress from pypy.rpython.rmodel import Repr, IntegerRepr from pypy.rpython.rptr import PtrRepr from pypy.rpython.lltypesystem import lltype from pypy.rlib.rarithmetic import r_uint +from pypy.rlib.objectmodel import we_are_translated + class __extend__(annmodel.SomeAddress): def rtyper_makerepr(self, rtyper): @@ -33,7 +35,7 @@ def ll_str(self, a): from pypy.rpython.lltypesystem.rstr import ll_str - id = cast_adr_to_int(a) + id = ll_addrhash(a) return ll_str.ll_int2hex(r_uint(id), True) def rtype_getattr(self, hop): @@ -55,7 +57,9 @@ get_ll_fasthash_function = get_ll_hash_function def ll_addrhash(addr1): - return cast_adr_to_int(addr1) + result = cast_adr_to_int(addr1) + # we don't want to have an AddressAsInt instance in this case + return get_inthash_from_int(result) address_repr = AddressRepr() Modified: pypy/branch/blackhole-improvement/pypy/rpython/rbuiltin.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/rbuiltin.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/rbuiltin.py Fri May 28 16:53:05 2010 @@ -615,9 +615,15 @@ return hop.genop('cast_int_to_adr', [v_input], resulttype = llmemory.Address) +def rtype_get_inthash_from_int(hop): + v_input, = hop.inputargs(lltype.Signed) + hop.exception_cannot_occur() + return hop.genop('get_inthash_from_int', [v_input], + resulttype = lltype.Signed) + BUILTIN_TYPER[llmemory.cast_ptr_to_adr] = rtype_cast_ptr_to_adr BUILTIN_TYPER[llmemory.cast_adr_to_ptr] = rtype_cast_adr_to_ptr BUILTIN_TYPER[llmemory.cast_adr_to_int] = rtype_cast_adr_to_int BUILTIN_TYPER[llmemory.cast_int_to_adr] = rtype_cast_int_to_adr - +BUILTIN_TYPER[llmemory.get_inthash_from_int] = rtype_get_inthash_from_int Modified: pypy/branch/blackhole-improvement/pypy/rpython/test/test_rdict.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/test/test_rdict.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/test/test_rdict.py Fri May 28 16:53:05 2010 @@ -781,6 +781,24 @@ assert lltype.typeOf(res.item1) == lltype.typeOf(res.item2) assert lltype.typeOf(res.item1) == lltype.typeOf(res.item3) + def test_dict_of_addresses(self): + from pypy.rpython.lltypesystem import llmemory + TP = lltype.Struct('x') + a = lltype.malloc(TP, flavor='raw', immortal=True) + b = lltype.malloc(TP, flavor='raw', immortal=True) + + def func(i): + d = {} + d[llmemory.cast_ptr_to_adr(a)] = 123 + d[llmemory.cast_ptr_to_adr(b)] = 456 + if i > 5: + key = llmemory.cast_ptr_to_adr(a) + else: + key = llmemory.cast_ptr_to_adr(b) + return d[key] + + assert self.interpret(func, [3]) == 456 + def test_prebuilt_list_of_addresses(self): from pypy.rpython.lltypesystem import llmemory Modified: pypy/branch/blackhole-improvement/pypy/translator/c/src/int.h ============================================================================== --- pypy/branch/blackhole-improvement/pypy/translator/c/src/int.h (original) +++ pypy/branch/blackhole-improvement/pypy/translator/c/src/int.h Fri May 28 16:53:05 2010 @@ -297,3 +297,5 @@ #define OP_ULLONG_AND OP_LLONG_AND #define OP_ULLONG_OR OP_LLONG_OR #define OP_ULLONG_XOR OP_LLONG_XOR + +#define OP_GET_INTHASH_FROM_INT(x, r) r = x; Modified: pypy/branch/blackhole-improvement/pypy/translator/c/test/test_lladdresses.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/translator/c/test/test_lladdresses.py (original) +++ pypy/branch/blackhole-improvement/pypy/translator/c/test/test_lladdresses.py Fri May 28 16:53:05 2010 @@ -211,3 +211,24 @@ fc = compile(f, [int]) res = fc(5) assert res == 42 + +def test_dict_of_addresses(): + TP = lltype.Struct('x') + a = lltype.malloc(TP, flavor='raw', immortal=True) + b = lltype.malloc(TP, flavor='raw', immortal=True) + + def f(i): + d = {} + d[cast_ptr_to_adr(a)] = 123 + d[cast_ptr_to_adr(b)] = 456 + if i > 5: + key = cast_ptr_to_adr(a) + else: + key = cast_ptr_to_adr(b) + return d[key] + + fc = compile(f, [int]) + res = fc(-77) + assert res == 456 + res = fc(77) + assert res == 123 From arigo at codespeak.net Fri May 28 16:57:32 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 16:57:32 +0200 (CEST) Subject: [pypy-svn] r74861 - in pypy/branch/blackhole-improvement/pypy: jit/metainterp rpython rpython/lltypesystem Message-ID: <20100528145732.44DE2282BD6@codespeak.net> Author: arigo Date: Fri May 28 16:57:30 2010 New Revision: 74861 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/logger.py pypy/branch/blackhole-improvement/pypy/rpython/llinterp.py pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py Log: Fixes. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/logger.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/logger.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/logger.py Fri May 28 16:57:30 2010 @@ -108,6 +108,7 @@ def int_could_be_an_address(x): if we_are_translated(): + x = llmemory.get_inthash_from_int(x) return not (-32768 <= x <= 32767) else: return isinstance(x, llmemory.AddressAsInt) Modified: pypy/branch/blackhole-improvement/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/llinterp.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/llinterp.py Fri May 28 16:57:30 2010 @@ -117,7 +117,7 @@ continue try: logline += " " + self.typer.annotator.annotated[frame.curr_block].func.__module__ - except (KeyError, AttributeError): + except (KeyError, AttributeError, TypeError): logline += " " lines.append(logline) for i, operation in enumerate(frame.curr_block.operations): Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py Fri May 28 16:57:30 2010 @@ -636,6 +636,9 @@ return cast_ptr_to_adr(ptr) def get_inthash_from_int(int): + """Just returns 'int' after translation. Before translation, it convert + an AddressAsInt into a real integer, to use e.g. as a hash. Do not store + the real integer (or the dict using this as a hash) across translation.""" if isinstance(int, AddressAsInt): int = lltype.cast_ptr_to_int(int.adr.ptr) return int From getxsick at codespeak.net Fri May 28 17:01:58 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Fri, 28 May 2010 17:01:58 +0200 (CEST) Subject: [pypy-svn] r74862 - pypy/branch/fast-ctypes Message-ID: <20100528150158.1E36136C538@codespeak.net> Author: getxsick Date: Fri May 28 17:01:56 2010 New Revision: 74862 Added: pypy/branch/fast-ctypes/ (props changed) - copied from r74861, pypy/trunk/ Log: fast ctypes branch From arigo at codespeak.net Fri May 28 17:19:53 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 17:19:53 +0200 (CEST) Subject: [pypy-svn] r74863 - pypy/branch/blackhole-improvement/pypy/jit/backend/test Message-ID: <20100528151953.242B1282BD6@codespeak.net> Author: arigo Date: Fri May 28 17:19:51 2010 New Revision: 74863 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py Log: Fix test_exceptions. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py Fri May 28 17:19:51 2010 @@ -239,7 +239,7 @@ res = self.cpu.get_latest_value_int(0) assert res == 20 - def test_make_boxes_from_latest_values(self): + def test_get_latest_value_count(self): i0 = BoxInt() i1 = BoxInt() i2 = BoxInt() @@ -258,12 +258,12 @@ self.cpu.set_future_value_int(0, 2) fail = self.cpu.execute_token(looptoken) assert fail is faildescr1 - boxes = self.cpu.make_boxes_from_latest_values(faildescr1) - assert len(boxes) == 3 - assert boxes[0] is None - assert isinstance(boxes[1], BoxInt) - assert boxes[1].value == 10 - assert boxes[2] is None + + count = self.cpu.get_latest_value_count() + assert count == 3 + assert self.cpu.get_latest_value_int(1) == 10 + assert self.cpu.get_latest_value_int(1) == 10 # multiple reads ok + self.cpu.clear_latest_values() def test_finish(self): i0 = BoxInt() @@ -1104,7 +1104,7 @@ @classmethod def get_funcbox(cls, cpu, func_ptr): addr = llmemory.cast_ptr_to_adr(func_ptr) - return BoxInt(cpu.cast_adr_to_int(addr)) + return ConstInt(llmemory.cast_adr_to_int(addr)) MY_VTABLE = rclass.OBJECT_VTABLE # for tests only @@ -1307,11 +1307,13 @@ self.cpu.execute_token(loop.token) assert self.cpu.get_latest_value_int(0) == 0 assert self.cpu.get_latest_value_ref(1) == xptr - self.cpu.clear_exception() + excvalue = self.cpu.grab_exc_value() + assert not excvalue self.cpu.set_future_value_int(0, 0) self.cpu.execute_token(loop.token) assert self.cpu.get_latest_value_int(0) == 1 - self.cpu.clear_exception() + excvalue = self.cpu.grab_exc_value() + assert not excvalue ytp = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) ytp.subclassrange_min = 2 @@ -1329,7 +1331,9 @@ self.cpu.set_future_value_int(0, 1) self.cpu.execute_token(loop.token) assert self.cpu.get_latest_value_int(0) == 1 - self.cpu.clear_exception() + excvalue = self.cpu.grab_exc_value() + assert excvalue == yptr + assert not self.cpu.grab_exc_value() # cleared exc_tp = xtp exc_ptr = xptr @@ -1345,11 +1349,13 @@ self.cpu.set_future_value_int(0, 1) self.cpu.execute_token(loop.token) assert self.cpu.get_latest_value_int(0) == 1 - self.cpu.clear_exception() + excvalue = self.cpu.grab_exc_value() + assert excvalue == xptr self.cpu.set_future_value_int(0, 0) self.cpu.execute_token(loop.token) assert self.cpu.get_latest_value_int(0) == 0 - self.cpu.clear_exception() + excvalue = self.cpu.grab_exc_value() + assert not excvalue def test_cond_call_gc_wb(self): def func_void(a, b): From arigo at codespeak.net Fri May 28 18:09:10 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 18:09:10 +0200 (CEST) Subject: [pypy-svn] r74864 - in pypy/branch/blackhole-improvement/pypy/jit: backend backend/test metainterp/test Message-ID: <20100528160910.31B4E282BD6@codespeak.net> Author: arigo Date: Fri May 28 18:09:08 2010 New Revision: 74864 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/model.py pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_string.py Log: Fix tests, cont. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/model.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/model.py Fri May 28 18:09:08 2010 @@ -174,6 +174,10 @@ raise NotImplementedError def bh_new_array(self, arraydescr, length): raise NotImplementedError + def bh_newstr(self, length): + raise NotImplementedError + def bh_newunicode(self, length): + raise NotImplementedError def bh_arraylen_gc(self, arraydescr, array): raise NotImplementedError Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py Fri May 28 18:09:08 2010 @@ -16,6 +16,8 @@ from pypy.rpython.annlowlevel import llhelper from pypy.rpython.llinterp import LLException from pypy.jit.metainterp.test.oparser import parse +from pypy.jit.codewriter import heaptracker + class Runner(object): @@ -378,8 +380,8 @@ looptoken = LoopToken() self.cpu.compile_loop([v1, v2], ops, looptoken) for x, y, z in testcases: - assert not self.cpu.get_exception() - assert not self.cpu.get_exc_value() + excvalue = self.cpu.grab_exc_value() + assert not excvalue self.cpu.set_future_value_int(0, x) self.cpu.set_future_value_int(1, y) fail = self.cpu.execute_token(looptoken) @@ -389,13 +391,13 @@ assert fail.identifier == 2 if z != boom: assert self.cpu.get_latest_value_int(0) == z - assert not self.cpu.get_exception() - assert not self.cpu.get_exc_value() + excvalue = self.cpu.grab_exc_value() + assert not excvalue def test_ovf_operations_reversed(self): self.test_ovf_operations(reversed=True) - def test_do_call(self): + def test_bh_call(self): cpu = self.cpu # def func(c): @@ -403,24 +405,21 @@ FPTR = self.Ptr(self.FuncType([lltype.Char], lltype.Char)) func_ptr = llhelper(FPTR, func) calldescr = cpu.calldescrof(deref(FPTR), (lltype.Char,), lltype.Char) - x = cpu.do_call( - [self.get_funcbox(cpu, func_ptr), - BoxInt(ord('A'))], - calldescr) - assert x.value == ord('B') + x = cpu.bh_call_i(self.get_funcbox(cpu, func_ptr).value, + calldescr, [ord('A')], None, None) + assert x == ord('B') if cpu.supports_floats: def func(f, i): - return float(i) + f + return f - float(i) FPTR = self.Ptr(self.FuncType([lltype.Float, lltype.Signed], lltype.Float)) func_ptr = llhelper(FPTR, func) FTP = deref(FPTR) calldescr = cpu.calldescrof(FTP, FTP.ARGS, FTP.RESULT) - x = cpu.do_call( - [self.get_funcbox(cpu, func_ptr), - BoxFloat(3.5), BoxInt(42)], - calldescr) - assert x.value == 42 + 3.5 + x = cpu.bh_call_f(self.get_funcbox(cpu, func_ptr).value, + calldescr, + [42], None, [3.5]) + assert x == 3.5 - 42 def test_call(self): @@ -614,31 +613,31 @@ def test_ooops(self): u1_box, U_box = self.alloc_instance(self.U) u2_box, U_box = self.alloc_instance(self.U) - r = self.execute_operation(rop.OOIS, [u1_box, - u1_box.clonebox()], 'int') + r = self.execute_operation(rop.PTR_EQ, [u1_box, + u1_box.clonebox()], 'int') assert r.value == 1 - r = self.execute_operation(rop.OOISNOT, [u2_box, - u2_box.clonebox()], 'int') + r = self.execute_operation(rop.PTR_NE, [u2_box, + u2_box.clonebox()], 'int') assert r.value == 0 - r = self.execute_operation(rop.OOIS, [u1_box, u2_box], 'int') + r = self.execute_operation(rop.PTR_EQ, [u1_box, u2_box], 'int') assert r.value == 0 - r = self.execute_operation(rop.OOISNOT, [u2_box, u1_box], 'int') + r = self.execute_operation(rop.PTR_NE, [u2_box, u1_box], 'int') assert r.value == 1 # null_box = self.null_instance() - r = self.execute_operation(rop.OOIS, [null_box, - null_box.clonebox()], 'int') + r = self.execute_operation(rop.PTR_EQ, [null_box, + null_box.clonebox()], 'int') assert r.value == 1 - r = self.execute_operation(rop.OOIS, [u1_box, null_box], 'int') + r = self.execute_operation(rop.PTR_EQ, [u1_box, null_box], 'int') assert r.value == 0 - r = self.execute_operation(rop.OOIS, [null_box, u2_box], 'int') + r = self.execute_operation(rop.PTR_EQ, [null_box, u2_box], 'int') assert r.value == 0 - r = self.execute_operation(rop.OOISNOT, [null_box, - null_box.clonebox()], 'int') + r = self.execute_operation(rop.PTR_NE, [null_box, + null_box.clonebox()], 'int') assert r.value == 0 - r = self.execute_operation(rop.OOISNOT, [u2_box, null_box], 'int') + r = self.execute_operation(rop.PTR_NE, [u2_box, null_box], 'int') assert r.value == 1 - r = self.execute_operation(rop.OOISNOT, [null_box, u1_box], 'int') + r = self.execute_operation(rop.PTR_NE, [null_box, u1_box], 'int') assert r.value == 1 def test_array_basic(self): @@ -771,10 +770,10 @@ assert r.value == 153 def test_do_unicode_basic(self): - u_box = self.cpu.do_newunicode(ConstInt(5)) - self.cpu.do_unicodesetitem(u_box, BoxInt(4), BoxInt(123)) - r = self.cpu.do_unicodegetitem(u_box, BoxInt(4)) - assert r.value == 123 + u = self.cpu.bh_newunicode(5) + self.cpu.bh_unicodesetitem(u, 4, 123) + r = self.cpu.bh_unicodegetitem(u, 4) + assert r == 123 def test_unicode_basic(self): u_box = self.alloc_unicode(u"hello\u1234") @@ -811,17 +810,10 @@ assert r.value == 5.5 def test_virtual_ref(self): - # if VIRTUAL_REF reaches the backend, it should just be a SAME_AS - u_box = self.alloc_unicode(u"hello\u1234") - r = self.execute_operation(rop.VIRTUAL_REF, [u_box, ConstInt(2)], - 'ref') - assert r.value == u_box.value + pass # VIRTUAL_REF must not reach the backend nowadays def test_virtual_ref_finish(self): - # if VIRTUAL_REF_FINISH reaches the backend, it is a no-op - self.execute_operation(rop.VIRTUAL_REF_FINISH, - [BoxInt(123), BoxInt(234)], - 'void') + pass # VIRTUAL_REF_FINISH must not reach the backend nowadays def test_jump(self): # this test generates small loops where the JUMP passes many @@ -1136,7 +1128,7 @@ elif T == self.U: t.parent.parent.parent.typeptr = vtable_for_T t_box = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, t)) - T_box = ConstInt(self.cpu.cast_adr_to_int(vtable_for_T_addr)) + T_box = ConstInt(llmemory.cast_adr_to_int(vtable_for_T_addr)) return t_box, T_box def null_instance(self): @@ -1171,17 +1163,17 @@ x = lltype.cast_opaque_ptr(llmemory.GCREF, x) res = self.execute_operation(rop.CAST_PTR_TO_INT, [BoxPtr(x)], 'int').value - assert res == self.cpu.cast_gcref_to_int(x) + assert res == llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(x)) res = self.execute_operation(rop.CAST_PTR_TO_INT, [ConstPtr(x)], 'int').value - assert res == self.cpu.cast_gcref_to_int(x) + assert res == llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(x)) def test_ooops_non_gc(self): x = lltype.malloc(lltype.Struct('x'), flavor='raw') - v = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x)) - r = self.execute_operation(rop.OOIS, [BoxInt(v), BoxInt(v)], 'int') + v = llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(x)) + r = self.execute_operation(rop.PTR_EQ, [BoxInt(v), BoxInt(v)], 'int') assert r.value == 1 - r = self.execute_operation(rop.OOISNOT, [BoxInt(v), BoxInt(v)], 'int') + r = self.execute_operation(rop.PTR_NE, [BoxInt(v), BoxInt(v)], 'int') assert r.value == 0 lltype.free(x, flavor='raw') @@ -1208,7 +1200,7 @@ S = lltype.Struct('S', ('x', lltype.Signed)) s = lltype.malloc(S, flavor='raw') sa = llmemory.cast_ptr_to_adr(s) - s_box = BoxInt(self.cpu.cast_adr_to_int(sa)) + s_box = BoxInt(llmemory.cast_adr_to_int(sa)) for get_op, set_op in ((rop.GETFIELD_RAW, rop.SETFIELD_RAW), (rop.GETFIELD_RAW_PURE, rop.SETFIELD_RAW)): fd = self.cpu.fielddescrof(S, 'x') @@ -1220,7 +1212,9 @@ def test_new_with_vtable(self): cpu = self.cpu t_box, T_box = self.alloc_instance(self.T) - cpu.set_class_sizes({T_box.value: cpu.sizeof(self.T)}) + vtable = llmemory.cast_adr_to_ptr( + llmemory.cast_int_to_adr(T_box.value), heaptracker.VTABLETYPE) + heaptracker.register_known_gctype(cpu, vtable, self.T) r1 = self.execute_operation(rop.NEW_WITH_VTABLE, [T_box], 'ref') r2 = self.execute_operation(rop.NEW_WITH_VTABLE, [T_box], 'ref') assert r1.value != r2.value @@ -1236,11 +1230,11 @@ s = lltype.cast_opaque_ptr(lltype.Ptr(self.T), r1.value) assert s.parent.chr1 == chr(190) assert s.parent.chr2 == chr(150) - r = self.cpu.do_getfield_gc(r1, descrshort) - assert r.value == 1313 - self.cpu.do_setfield_gc(r1, BoxInt(1333), descrshort) - r = self.cpu.do_getfield_gc(r1, descrshort) - assert r.value == 1333 + r = self.cpu.bh_getfield_gc_i(r1.value, descrshort) + assert r == 1313 + self.cpu.bh_setfield_gc_i(r1.value, descrshort, 1333) + r = self.cpu.bh_getfield_gc_i(r1.value, descrshort) + assert r == 1333 r = self.execute_operation(rop.GETFIELD_GC, [r1], 'int', descr=descrshort) assert r.value == 1333 @@ -1532,108 +1526,92 @@ A = lltype.GcArray(lltype.Char) descr_A = cpu.arraydescrof(A) a = lltype.malloc(A, 5) - x = cpu.do_arraylen_gc( - BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, a)), - descr_A) - assert x.value == 5 + x = cpu.bh_arraylen_gc(descr_A, + lltype.cast_opaque_ptr(llmemory.GCREF, a)) + assert x == 5 # a[2] = 'Y' - x = cpu.do_getarrayitem_gc( - BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, a)), BoxInt(2), - descr_A) - assert x.value == ord('Y') + x = cpu.bh_getarrayitem_gc_i( + descr_A, lltype.cast_opaque_ptr(llmemory.GCREF, a), 2) + assert x == ord('Y') # B = lltype.GcArray(lltype.Ptr(A)) descr_B = cpu.arraydescrof(B) b = lltype.malloc(B, 4) b[3] = a - x = cpu.do_getarrayitem_gc( - BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, b)), BoxInt(3), - descr_B) - assert isinstance(x, BoxPtr) - assert x.getref(lltype.Ptr(A)) == a + x = cpu.bh_getarrayitem_gc_r( + descr_B, lltype.cast_opaque_ptr(llmemory.GCREF, b), 3) + assert lltype.cast_opaque_ptr(lltype.Ptr(A), x) == a if self.cpu.supports_floats: C = lltype.GcArray(lltype.Float) c = lltype.malloc(C, 6) c[3] = 3.5 descr_C = cpu.arraydescrof(C) - x = cpu.do_getarrayitem_gc( - BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, c)), BoxInt(3), - descr_C) - assert isinstance(x, BoxFloat) - assert x.getfloat() == 3.5 - cpu.do_setarrayitem_gc( - BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, c)), BoxInt(4), - BoxFloat(4.5), descr_C) + x = cpu.bh_getarrayitem_gc_f( + descr_C, lltype.cast_opaque_ptr(llmemory.GCREF, c), 3) + assert x == 3.5 + cpu.bh_setarrayitem_gc_f( + descr_C, lltype.cast_opaque_ptr(llmemory.GCREF, c), 4, 4.5) assert c[4] == 4.5 s = rstr.mallocstr(6) - x = cpu.do_strlen( - BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, s))) - assert x.value == 6 + x = cpu.bh_strlen(lltype.cast_opaque_ptr(llmemory.GCREF, s)) + assert x == 6 # s.chars[3] = 'X' - x = cpu.do_strgetitem( - BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, s)), BoxInt(3)) - assert x.value == ord('X') + x = cpu.bh_strgetitem(lltype.cast_opaque_ptr(llmemory.GCREF, s), 3) + assert x == ord('X') # S = lltype.GcStruct('S', ('x', lltype.Char), ('y', lltype.Ptr(A)), ('z', lltype.Float)) descrfld_x = cpu.fielddescrof(S, 'x') s = lltype.malloc(S) s.x = 'Z' - x = cpu.do_getfield_gc( - BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, s)), - descrfld_x) - assert x.value == ord('Z') - # - cpu.do_setfield_gc( - BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, s)), - BoxInt(ord('4')), - descrfld_x) + x = cpu.bh_getfield_gc_i(lltype.cast_opaque_ptr(llmemory.GCREF, s), + descrfld_x) + assert x == ord('Z') + # + cpu.bh_setfield_gc_i(lltype.cast_opaque_ptr(llmemory.GCREF, s), + descrfld_x, + ord('4')) assert s.x == '4' # descrfld_y = cpu.fielddescrof(S, 'y') s.y = a - x = cpu.do_getfield_gc( - BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, s)), - descrfld_y) - assert isinstance(x, BoxPtr) - assert x.getref(lltype.Ptr(A)) == a + x = cpu.bh_getfield_gc_r(lltype.cast_opaque_ptr(llmemory.GCREF, s), + descrfld_y) + assert lltype.cast_opaque_ptr(lltype.Ptr(A), x) == a # s.y = lltype.nullptr(A) - cpu.do_setfield_gc( - BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, s)), x, - descrfld_y) + cpu.bh_setfield_gc_r(lltype.cast_opaque_ptr(llmemory.GCREF, s), + descrfld_y, x) assert s.y == a # RS = lltype.Struct('S', ('x', lltype.Char)) #, ('y', lltype.Ptr(A))) descrfld_rx = cpu.fielddescrof(RS, 'x') rs = lltype.malloc(RS, immortal=True) rs.x = '?' - x = cpu.do_getfield_raw( - BoxInt(cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(rs))), + x = cpu.bh_getfield_raw_i( + llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(rs)), descrfld_rx) - assert x.value == ord('?') + assert x == ord('?') # - cpu.do_setfield_raw( - BoxInt(cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(rs))), - BoxInt(ord('!')), - descrfld_rx) + cpu.bh_setfield_raw_i( + llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(rs)), + descrfld_rx, ord('!')) assert rs.x == '!' # if self.cpu.supports_floats: descrfld_z = cpu.fielddescrof(S, 'z') - cpu.do_setfield_gc( - BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, s)), - BoxFloat(3.5), - descrfld_z) + cpu.bh_setfield_gc_f( + lltype.cast_opaque_ptr(llmemory.GCREF, s), + descrfld_z, 3.5) assert s.z == 3.5 s.z = 3.2 - x = cpu.do_getfield_gc( - BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, s)), + x = cpu.bh_getfield_gc_f( + lltype.cast_opaque_ptr(llmemory.GCREF, s), descrfld_z) - assert x.getfloat() == 3.2 + assert x == 3.2 ### we don't support in the JIT for now GC pointers ### stored inside non-GC structs. #descrfld_ry = cpu.fielddescrof(RS, 'y') @@ -1651,45 +1629,40 @@ #assert rs.y == a # descrsize = cpu.sizeof(S) - x = cpu.do_new(descrsize) - assert isinstance(x, BoxPtr) - x.getref(lltype.Ptr(S)) + x = cpu.bh_new(descrsize) + lltype.cast_opaque_ptr(lltype.Ptr(S), x) # type check # descrsize2 = cpu.sizeof(rclass.OBJECT) vtable2 = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) - vtable2_int = cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(vtable2)) - cpu.set_class_sizes({vtable2_int: descrsize2}) - x = cpu.do_new_with_vtable(ConstInt(vtable2_int)) - assert isinstance(x, BoxPtr) + vtable2_int = llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(vtable2)) + heaptracker.register_known_gctype(cpu, vtable2, rclass.OBJECT) + x = cpu.bh_new_with_vtable(descrsize2, vtable2_int) + lltype.cast_opaque_ptr(lltype.Ptr(rclass.OBJECT), x) # type check # well... #assert x.getref(rclass.OBJECTPTR).typeptr == vtable2 # arraydescr = cpu.arraydescrof(A) - x = cpu.do_new_array(BoxInt(7), arraydescr) - assert isinstance(x, BoxPtr) - assert len(x.getref(lltype.Ptr(A))) == 7 - # - cpu.do_setarrayitem_gc( - x, BoxInt(5), BoxInt(ord('*')), descr_A) - assert x.getref(lltype.Ptr(A))[5] == '*' - # - cpu.do_setarrayitem_gc( - BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, b)), - BoxInt(1), x, - descr_B) - assert b[1] == x.getref(lltype.Ptr(A)) - # - x = cpu.do_newstr(BoxInt(5)) - assert isinstance(x, BoxPtr) - assert len(x.getref(lltype.Ptr(rstr.STR)).chars) == 5 - # - cpu.do_strsetitem(x, BoxInt(4), BoxInt(ord('/'))) - assert x.getref(lltype.Ptr(rstr.STR)).chars[4] == '/' - # - x = cpu.do_newstr(BoxInt(5)) - y = cpu.do_cast_ptr_to_int(x) - assert isinstance(y, BoxInt) - assert y.value == cpu.cast_gcref_to_int(x.value) + x = cpu.bh_new_array(arraydescr, 7) + array = lltype.cast_opaque_ptr(lltype.Ptr(A), x) + assert len(array) == 7 + # + cpu.bh_setarrayitem_gc_i(descr_A, x, 5, ord('*')) + assert array[5] == '*' + # + cpu.bh_setarrayitem_gc_r( + descr_B, lltype.cast_opaque_ptr(llmemory.GCREF, b), 1, x) + assert b[1] == array + # + x = cpu.bh_newstr(5) + str = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), x) + assert len(str.chars) == 5 + # + cpu.bh_strsetitem(x, 4, ord('/')) + assert str.chars[4] == '/' + # + x = cpu.bh_newstr(5) + y = cpu.bh_cast_ptr_to_int(x) + assert y == llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(x)) def test_sorting_of_fields(self): S = self.S @@ -1702,7 +1675,7 @@ def test_guards_nongc(self): x = lltype.malloc(lltype.Struct('x'), flavor='raw') - v = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x)) + v = llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(x)) vbox = BoxInt(v) ops = [ (rop.GUARD_NONNULL, vbox, False), Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_string.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_string.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_string.py Fri May 28 18:09:08 2010 @@ -44,6 +44,34 @@ assert res == 5 self.check_loops(**{self.CALL_PURE: 0}) + def test_newstr(self): + jitdriver = JitDriver(greens = [], reds = ['n', 'm']) + def f(n, m): + while True: + jitdriver.can_enter_jit(m=m, n=n) + jitdriver.jit_merge_point(m=m, n=n) + bytecode = 'adlfkj' + chr(n) + res = bytecode[n] + m -= 1 + if m < 0: + return ord(res) + res = self.meta_interp(f, [6, 10]) + assert res == 6 + + def test_newunicode(self): + jitdriver = JitDriver(greens = [], reds = ['n', 'm']) + def f(n, m): + while True: + jitdriver.can_enter_jit(m=m, n=n) + jitdriver.jit_merge_point(m=m, n=n) + bytecode = u'adlfkj' + unichr(n) + res = bytecode[n] + m -= 1 + if m < 0: + return ord(res) + res = self.meta_interp(f, [6, 10]) + assert res == 6 + class TestOOtype(StringTests, OOJitMixin): CALL_PURE = "oosend_pure" From arigo at codespeak.net Fri May 28 18:10:46 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 18:10:46 +0200 (CEST) Subject: [pypy-svn] r74865 - in pypy/branch/blackhole-improvement/pypy/jit/backend: llgraph/test test Message-ID: <20100528161046.2C9E2282BD6@codespeak.net> Author: arigo Date: Fri May 28 18:10:44 2010 New Revision: 74865 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/test/test_llgraph.py pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py Log: Finish test_llgraph. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/test/test_llgraph.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/test/test_llgraph.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/test/test_llgraph.py Fri May 28 18:10:44 2010 @@ -36,6 +36,7 @@ ## from pypy.jit.backend.llgraph.runner import OOtypeCPU as cpu_type def test_fielddescr_ootype(): + py.test.skip("ootype tests skipped") from pypy.rpython.ootypesystem import ootype from pypy.jit.backend.llgraph.runner import OOtypeCPU A = ootype.Instance("A", ootype.ROOT, {"foo": ootype.Signed}) Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py Fri May 28 18:10:44 2010 @@ -1778,6 +1778,9 @@ malloc = staticmethod(ootype.new) nullptr = staticmethod(ootype.null) + def setup_class(cls): + py.test.skip("ootype tests skipped") + @classmethod def get_funcbox(cls, cpu, func_ptr): return BoxObj(ootype.cast_to_object(func_ptr)) From afa at codespeak.net Fri May 28 18:12:46 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 28 May 2010 18:12:46 +0200 (CEST) Subject: [pypy-svn] r74866 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100528161246.ED05F282BD6@codespeak.net> Author: afa Date: Fri May 28 18:12:45 2010 New Revision: 74866 Modified: pypy/trunk/pypy/module/cpyext/object.py pypy/trunk/pypy/module/cpyext/slotdefs.py pypy/trunk/pypy/module/cpyext/test/foo.c pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Log: Implement forwarding calls from __setattr__ and __delattr__ to tp_setattro Also a test for PyObject_GenericSetAttr when used to delete attributes. Modified: pypy/trunk/pypy/module/cpyext/object.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/object.py (original) +++ pypy/trunk/pypy/module/cpyext/object.py Fri May 28 18:12:45 2010 @@ -320,9 +320,14 @@ over setting the attribute in the instance dictionary. Otherwise, the attribute is set in the object's __dict__ (if present). Otherwise, an AttributeError is raised and -1 is returned.""" - from pypy.objspace.descroperation import object_setattr - w_descr = object_setattr(space) - return space.get_and_call_function(w_descr, w_obj, w_name, w_value) + from pypy.objspace.descroperation import object_setattr, object_delattr + if w_value is not None: + w_descr = object_setattr(space) + space.get_and_call_function(w_descr, w_obj, w_name, w_value) + else: + w_descr = object_delattr(space) + space.get_and_call_function(w_descr, w_obj, w_name) + return 0 @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) def PyObject_IsInstance(space, w_inst, w_cls): Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/slotdefs.py (original) +++ pypy/trunk/pypy/module/cpyext/slotdefs.py Fri May 28 18:12:45 2010 @@ -1,12 +1,11 @@ import re from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import generic_cpy_call, cpython_api, \ - PyObject -from pypy.module.cpyext.typeobjectdefs import unaryfunc, wrapperfunc,\ - ternaryfunc, PyTypeObjectPtr, binaryfunc, getattrfunc, lenfunc,\ - ssizeargfunc, ssizessizeargfunc, ssizeobjargproc, iternextfunc,\ - initproc +from pypy.module.cpyext.api import generic_cpy_call, cpython_api, PyObject +from pypy.module.cpyext.typeobjectdefs import ( + unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc, + getattrfunc, setattrofunc, lenfunc, ssizeargfunc, ssizessizeargfunc, + ssizeobjargproc, iternextfunc, initproc) from pypy.module.cpyext.pyobject import from_ref from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.state import State @@ -53,6 +52,24 @@ finally: rffi.free_charp(name_ptr) +def wrap_setattr(space, w_self, w_args, func): + func_target = rffi.cast(setattrofunc, func) + check_num_args(space, w_args, 2) + w_name, w_value = space.fixedview(w_args) + # XXX "Carlo Verre hack"? + if generic_cpy_call(space, func_target, w_self, w_name, w_value) < 0: + space.fromcache(State).check_and_raise_exception(always=True) + return space.w_None + +def wrap_delattr(space, w_self, w_args, func): + func_target = rffi.cast(setattrofunc, func) + check_num_args(space, w_args, 1) + w_name, = space.fixedview(w_args) + # XXX "Carlo Verre hack"? + if generic_cpy_call(space, func_target, w_self, w_name, None) < 0: + space.fromcache(State).check_and_raise_exception(always=True) + return space.w_None + def wrap_call(space, w_self, w_args, func, w_kwds): func_target = rffi.cast(ternaryfunc, func) return generic_cpy_call(space, func_target, w_self, w_args, w_kwds) @@ -135,11 +152,15 @@ # adopted from typeobject.c def FLSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC, FLAGS): - wrapper = globals().get(WRAPPER, Ellipsis) if WRAPPER is None: wrapper = None + else: + wrapper = globals().get(WRAPPER, Ellipsis) + + # irregular interface, because of tp_getattr/tp_getattro confusion if NAME == "__getattr__": wrapper = wrap_getattr + function = globals().get(FUNCTION, None) slotname = ("c_" + SLOT).split(".") assert FLAGS == 0 or FLAGS == PyWrapperFlag_KEYWORDS Modified: pypy/trunk/pypy/module/cpyext/test/foo.c ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/foo.c (original) +++ pypy/trunk/pypy/module/cpyext/test/foo.c Fri May 28 18:12:45 2010 @@ -114,6 +114,25 @@ return kwds; } +static int +foo_setattro(fooobject *self, PyObject *name, PyObject *value) +{ + char *name_str; + if (!PyString_Check(name)) { + PyErr_SetObject(PyExc_AttributeError, name); + return -1; + } + name_str = PyString_AsString(name); + if (strcmp(name_str, "set_foo") == 0) + { + long v = PyInt_AsLong(value); + if (v == -1 && PyErr_Occurred()) + return -1; + self->foo = v; + } + return PyObject_GenericSetAttr(self, name, value); +} + static PyMemberDef foo_members[] = { {"int_member", T_INT, offsetof(fooobject, foo), 0, "A helpful docstring."}, @@ -151,7 +170,7 @@ foo_call, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ - 0, /*tp_setattro*/ + foo_setattro, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT, /*tp_flags*/ 0, /*tp_doc*/ Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Fri May 28 18:12:45 2010 @@ -72,6 +72,9 @@ del obj.object_member_ex raises(AttributeError, "del obj.object_member_ex") + obj.set_foo = 32 + assert obj.foo == 32 + def test_typeobject_string_member(self): module = self.import_module(name='foo') obj = module.new() From afa at codespeak.net Fri May 28 18:28:13 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 28 May 2010 18:28:13 +0200 (CEST) Subject: [pypy-svn] r74867 - pypy/trunk/pypy/module/cpyext Message-ID: <20100528162813.80692282BD6@codespeak.net> Author: afa Date: Fri May 28 18:28:11 2010 New Revision: 74867 Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py Log: translation fix Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/slotdefs.py (original) +++ pypy/trunk/pypy/module/cpyext/slotdefs.py Fri May 28 18:28:11 2010 @@ -57,18 +57,18 @@ check_num_args(space, w_args, 2) w_name, w_value = space.fixedview(w_args) # XXX "Carlo Verre hack"? - if generic_cpy_call(space, func_target, w_self, w_name, w_value) < 0: + res = generic_cpy_call(space, func_target, w_self, w_name, w_value) + if rffi.cast(lltype.Signed, res) == -1: space.fromcache(State).check_and_raise_exception(always=True) - return space.w_None def wrap_delattr(space, w_self, w_args, func): func_target = rffi.cast(setattrofunc, func) check_num_args(space, w_args, 1) w_name, = space.fixedview(w_args) # XXX "Carlo Verre hack"? - if generic_cpy_call(space, func_target, w_self, w_name, None) < 0: + res = generic_cpy_call(space, func_target, w_self, w_name, None) + if rffi.cast(lltype.Signed, res) == -1: space.fromcache(State).check_and_raise_exception(always=True) - return space.w_None def wrap_call(space, w_self, w_args, func, w_kwds): func_target = rffi.cast(ternaryfunc, func) From afa at codespeak.net Fri May 28 19:07:51 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 28 May 2010 19:07:51 +0200 (CEST) Subject: [pypy-svn] r74868 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100528170751.16BF5282BD6@codespeak.net> Author: afa Date: Fri May 28 19:07:49 2010 New Revision: 74868 Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Log: Fill PyObjectType.tp_setattro Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/slotdefs.py (original) +++ pypy/trunk/pypy/module/cpyext/slotdefs.py Fri May 28 19:07:49 2010 @@ -148,6 +148,14 @@ def slot_nb_int(space, w_self): return space.int(w_self) + at cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1, external=False) +def slot_tp_setattro(space, w_self, w_name, w_value): + if w_value is not None: + space.setattr(w_self, w_name, w_value) + else: + space.delattr(w_self, w_name) + return 0 + PyWrapperFlag_KEYWORDS = 1 # adopted from typeobject.c Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Fri May 28 19:07:49 2010 @@ -211,6 +211,21 @@ assert api.PyErr_Occurred() is None class AppTestSlots(AppTestCpythonExtensionBase): + def test_some_slots(self): + module = self.import_extension('foo', [ + ("test_type", "METH_O", + ''' + if (!args->ob_type->tp_setattro) + { + PyErr_SetString(PyExc_ValueError, "missing tp_setattro"); + return NULL; + } + Py_RETURN_TRUE; + ''' + ) + ]) + assert module.test_type(None) + def test_nb_int(self): module = self.import_extension('foo', [ ("nb_int", "METH_O", From afa at codespeak.net Fri May 28 19:17:33 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 28 May 2010 19:17:33 +0200 (CEST) Subject: [pypy-svn] r74869 - pypy/trunk/pypy/module/cpyext Message-ID: <20100528171733.86507282BD6@codespeak.net> Author: afa Date: Fri May 28 19:17:32 2010 New Revision: 74869 Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py Log: Add a note to self. Probably a bug, certainly a test to write. Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/slotdefs.py (original) +++ pypy/trunk/pypy/module/cpyext/slotdefs.py Fri May 28 19:17:32 2010 @@ -150,6 +150,8 @@ @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1, external=False) def slot_tp_setattro(space, w_self, w_name, w_value): + # XXX this is probably wrong when a subtype's tp_setattro + # delegates to PyType_Type.tp_setattro! if w_value is not None: space.setattr(w_self, w_name, w_value) else: From getxsick at codespeak.net Fri May 28 20:14:10 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Fri, 28 May 2010 20:14:10 +0200 (CEST) Subject: [pypy-svn] r74870 - in pypy/branch/fast-ctypes/pypy/module/jitffi: . test Message-ID: <20100528181410.77213282BD6@codespeak.net> Author: getxsick Date: Fri May 28 20:14:06 2010 New Revision: 74870 Added: pypy/branch/fast-ctypes/pypy/module/jitffi/ (props changed) pypy/branch/fast-ctypes/pypy/module/jitffi/__init__.py (contents, props changed) pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py pypy/branch/fast-ctypes/pypy/module/jitffi/test/ (props changed) pypy/branch/fast-ctypes/pypy/module/jitffi/test/__init__.py (contents, props changed) pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py (contents, props changed) Log: just some init checkin. nothing worth to look yet. Added: pypy/branch/fast-ctypes/pypy/module/jitffi/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/__init__.py Fri May 28 20:14:06 2010 @@ -0,0 +1,8 @@ +from pypy.interpreter.mixedmodule import MixedModule + +class Module(MixedModule): + interpleveldefs = { + 'load' : 'interp_jitffi.W_load', + } + + appleveldefs = {} Added: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py ============================================================================== --- (empty file) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Fri May 28 20:14:06 2010 @@ -0,0 +1,4 @@ +from pypy.interpreter.baseobjspace import Wrappable + +class W_load(Wrappable): + pass Added: pypy/branch/fast-ctypes/pypy/module/jitffi/test/__init__.py ============================================================================== Added: pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py ============================================================================== --- (empty file) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py Fri May 28 20:14:06 2010 @@ -0,0 +1,35 @@ +from pypy.conftest import gettestobjspace +from pypy.translator.tool.cbuild import ExternalCompilationInfo +from pypy.translator.platform import platform + +import py + +class AppTestJitffi(object): + @staticmethod + def preprare_c_example(): + from pypy.tool.udir import udir + c_file = udir.ensure("test_jitffi", dir=True).join("xlib.c") + c_file.write(py.code.Source(''' + int add_integers(int a, int b) + { + return a+b; + } + ''')) + + symbols = ["sum"] + eci = ExternalCompilationInfo(export_symbols=symbols) + + return str(platform.compile([c_file], eci, 'x', standalone=False)) + + def setup_class(cls): + from pypy.rlib.libffi import get_libc_name + space = gettestobjspace(usemodules=('jitffi',)) + cls.space = space + cls.w_lib_name = space.wrap(cls.preprare_c_example()) + cls.w_libc_name = space.wrap(get_libc_name()) + + def test_raw_callable(self): + import jitffi + lib = jitffi.load(self.lib_name) + res = lib.call('add_integers', 1, 2) + assert res == 3 From arigo at codespeak.net Fri May 28 20:46:52 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 20:46:52 +0200 (CEST) Subject: [pypy-svn] r74871 - in pypy/branch/blackhole-improvement/pypy: jit/backend/llsupport jit/backend/llsupport/test rpython/lltypesystem Message-ID: <20100528184652.32B07282BD6@codespeak.net> Author: arigo Date: Fri May 28 20:46:50 2010 New Revision: 74871 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/descr.py pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/gc.py pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/test/test_descr.py pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/test/test_gc.py pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/ll2ctypes.py Log: Start implementing bh_call_x. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/descr.py Fri May 28 20:46:50 2010 @@ -179,7 +179,6 @@ # CallDescrs class BaseCallDescr(AbstractDescr): - empty_box = BoxInt(0) _clsname = '' loop_token = None arg_classes = '' # <-- annotation hack @@ -207,19 +206,11 @@ def get_result_size(self, translate_support_code): raise NotImplementedError - def get_call_stub(self): - return self.call_stub - def create_call_stub(self, rtyper, RESULT): - def process(no, c): - if c == 'i': - return 'args[%d].getint()' % (no,) - elif c == 'f': - return 'args[%d].getfloat()' % (no,) - elif c == 'r': - return 'args[%d].getref_base()' % (no,) - else: - raise Exception("Unknown type %s for type %s" % (c, TP)) + def process(c): + arg = 'args_%s[%d]' % (c, seen[c]) + seen[c] += 1 + return arg def TYPE(arg): if arg == 'i': @@ -230,21 +221,21 @@ return llmemory.GCREF elif arg == 'v': return lltype.Void - - args = ", ".join([process(i + 1, c) for i, c in - enumerate(self.arg_classes)]) + + seen = {'i': 0, 'r': 0, 'f': 0} + args = ", ".join([process(c) for c in self.arg_classes]) if self.returns_a_pointer(): - result = 'history.BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, res))' + result = 'lltype.cast_opaque_ptr(llmemory.GCREF, res)' elif self.returns_a_float(): - result = 'history.BoxFloat(res)' + result = 'res' elif self.returns_a_void(): result = 'None' else: - result = 'history.BoxInt(rffi.cast(lltype.Signed, res))' + result = 'rffi.cast(lltype.Signed, res)' source = py.code.Source(""" - def call_stub(args): - fnptr = rffi.cast(lltype.Ptr(FUNC), args[0].getint()) + def call_stub(func, args_i, args_r, args_f): + fnptr = rffi.cast(lltype.Ptr(FUNC), func) res = support.maybe_on_top_of_llinterp(rtyper, fnptr)(%(args)s) return %(result)s """ % locals()) @@ -255,35 +246,61 @@ exec source.compile() in d self.call_stub = d['call_stub'] + def verify_types(self, args_i, args_r, args_f, return_type): + assert self._returns_a_pointer == (return_type == 'ref') + assert self._returns_a_float == (return_type == 'float') + assert self._returns_a_void == (return_type == 'void') + assert self.arg_classes.count('i') == len(args_i or ()) + assert self.arg_classes.count('r') == len(args_r or ()) + assert self.arg_classes.count('f') == len(args_f or ()) + def repr_of_descr(self): return '<%s>' % self._clsname -class NonGcPtrCallDescr(BaseCallDescr): +class BaseIntCallDescr(BaseCallDescr): + # Base class of the various subclasses of descrs corresponding to + # calls having a return kind of 'int' (including non-gc pointers). + # The inheritance hierarchy is a bit different than with other Descr + # classes because of the 'call_stub' attribute, which is of type + # + # lambda args_i, args_r, args_f --> int/ref/float/void + # + # The purpose of BaseIntCallDescr is to be the parent of all classes + # in which 'call_stub' has a return kind of 'int'. + pass + +class NonGcPtrCallDescr(BaseIntCallDescr): _clsname = 'NonGcPtrCallDescr' - def get_result_size(self, translate_support_code): return symbolic.get_size_of_ptr(translate_support_code) -class GcPtrCallDescr(NonGcPtrCallDescr): - empty_box = BoxPtr(lltype.nullptr(llmemory.GCREF.TO)) +class GcPtrCallDescr(BaseCallDescr): _clsname = 'GcPtrCallDescr' _returns_a_pointer = True + def get_result_size(self, translate_support_code): + return symbolic.get_size_of_ptr(translate_support_code) + +class FloatCallDescr(BaseCallDescr): + _clsname = 'FloatCallDescr' + _returns_a_float = True + def get_result_size(self, translate_support_code): + return symbolic.get_size(lltype.Float, translate_support_code) -class VoidCallDescr(NonGcPtrCallDescr): - empty_box = None +class VoidCallDescr(BaseCallDescr): _clsname = 'VoidCallDescr' _returns_a_void = True - def get_result_size(self, translate_support_code): return 0 def getCallDescrClass(RESULT): if RESULT is lltype.Void: return VoidCallDescr - return getDescrClass(RESULT, BaseCallDescr, GcPtrCallDescr, + if RESULT is lltype.Float: + return FloatCallDescr + return getDescrClass(RESULT, BaseIntCallDescr, GcPtrCallDescr, NonGcPtrCallDescr, 'Call', 'get_result_size', - '_returns_a_float') + Ellipsis) # <= floatattrname should not be used here def get_call_descr(gccache, ARGS, RESULT, extrainfo=None): arg_classes = [] @@ -330,7 +347,6 @@ # if TYPE is lltype.Float: setattr(Descr, floatattrname, True) - Descr.empty_box = BoxFloat(0.0) # _cache[nameprefix, TYPE] = Descr return Descr Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/gc.py Fri May 28 20:46:50 2010 @@ -171,6 +171,7 @@ # the object moves) addr = llmemory.cast_ptr_to_adr(gcref) hash = llmemory.cast_adr_to_int(addr) + hash = llmemory.get_inthash_from_int(hash) hash -= hash >> self.HASHTABLE_BITS hash &= self.HASHTABLE_SIZE - 1 addr_ref = self.hashtable[hash] Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py Fri May 28 20:46:50 2010 @@ -12,7 +12,9 @@ from pypy.jit.backend.llsupport.descr import get_size_descr, BaseSizeDescr from pypy.jit.backend.llsupport.descr import get_field_descr, BaseFieldDescr from pypy.jit.backend.llsupport.descr import get_array_descr, BaseArrayDescr -from pypy.jit.backend.llsupport.descr import get_call_descr, BaseCallDescr +from pypy.jit.backend.llsupport.descr import get_call_descr +from pypy.jit.backend.llsupport.descr import BaseIntCallDescr, GcPtrCallDescr +from pypy.jit.backend.llsupport.descr import FloatCallDescr, VoidCallDescr from pypy.rpython.annlowlevel import cast_instance_to_base_ptr empty_int_box = BoxInt(0) @@ -471,32 +473,29 @@ basesize = basesize // itemsize rffi.cast(rffi.CArrayPtr(lltype.UniChar), a)[index + basesize] = unichr(v) - def do_call(self, args, calldescr): - assert isinstance(calldescr, BaseCallDescr) - assert len(args) == 1 + len(calldescr.arg_classes) + def bh_call_i(self, func, calldescr, args_i, args_r, args_f): + assert isinstance(calldescr, BaseIntCallDescr) if not we_are_translated(): - assert (list(calldescr.arg_classes) == - [arg.type for arg in args[1:]]) - callstub = calldescr.get_call_stub() - try: - return callstub(args) - except Exception, e: - if not we_are_translated(): - if not type(e) is LLException: - raise - self.saved_exc_value = lltype.cast_opaque_ptr(llmemory.GCREF, - e.args[1]) - self.saved_exception = rffi.cast(lltype.Signed, e.args[0]) - else: - ptr = cast_instance_to_base_ptr(e) - self.saved_exc_value = lltype.cast_opaque_ptr(llmemory.GCREF, - ptr) - self.saved_exception = rffi.cast(lltype.Signed, ptr.typeptr) - return calldescr.empty_box - - def do_cast_ptr_to_int(self, ptrbox): - return BoxInt(self.cast_gcref_to_int(ptrbox.getref_base())) + calldescr.verify_types(args_i, args_r, args_f, 'int') + return calldescr.call_stub(func, args_i, args_r, args_f) + def bh_call_r(self, func, calldescr, args_i, args_r, args_f): + assert isinstance(calldescr, GcPtrCallDescr) + if not we_are_translated(): + calldescr.verify_types(args_i, args_r, args_f, 'ref') + return calldescr.call_stub(func, args_i, args_r, args_f) + + def bh_call_f(self, func, calldescr, args_i, args_r, args_f): + assert isinstance(calldescr, FloatCallDescr) + if not we_are_translated(): + calldescr.verify_types(args_i, args_r, args_f, 'float') + return calldescr.call_stub(func, args_i, args_r, args_f) -import pypy.jit.metainterp.executor -pypy.jit.metainterp.executor.make_execute_list(AbstractLLCPU) + def bh_call_v(self, func, calldescr, args_i, args_r, args_f): + assert isinstance(calldescr, VoidCallDescr) + if not we_are_translated(): + calldescr.verify_types(args_i, args_r, args_f, 'void') + return calldescr.call_stub(func, args_i, args_r, args_f) + + def do_cast_ptr_to_int(self, ptrbox): + return BoxInt(self.cast_gcref_to_int(ptrbox.getref_base())) Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/test/test_descr.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/test/test_descr.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/test/test_descr.py Fri May 28 20:46:50 2010 @@ -148,30 +148,29 @@ assert not descr1.returns_a_pointer() assert not descr1.returns_a_float() assert descr1.arg_classes == "ii" - assert isinstance(descr1.empty_box, BoxInt) # T = lltype.GcStruct('T') descr2 = get_call_descr(c0, [lltype.Ptr(T)], lltype.Ptr(T)) assert descr2.get_result_size(False) == rffi.sizeof(lltype.Ptr(T)) assert descr2.returns_a_pointer() assert not descr2.returns_a_float() + assert not descr2.returns_a_void() assert descr2.arg_classes == "r" - assert isinstance(descr2.empty_box, BoxPtr) # U = lltype.GcStruct('U', ('x', lltype.Signed)) assert descr2 == get_call_descr(c0, [lltype.Ptr(U)], lltype.Ptr(U)) # V = lltype.Struct('V', ('x', lltype.Signed)) - assert isinstance(get_call_descr(c0, [], lltype.Ptr(V)).empty_box, BoxInt) + assert not get_call_descr(c0, [], lltype.Ptr(V)).returns_a_pointer() # - assert get_call_descr(c0, [], lltype.Void).empty_box is None + assert get_call_descr(c0, [], lltype.Void).returns_a_void() # descr4 = get_call_descr(c0, [lltype.Float, lltype.Float], lltype.Float) assert descr4.get_result_size(False) == rffi.sizeof(lltype.Float) assert not descr4.returns_a_pointer() assert descr4.returns_a_float() + assert not descr4.returns_a_void() assert descr4.arg_classes == "ff" - assert isinstance(descr4.empty_box, BoxFloat) def test_get_call_descr_translated(): c1 = GcCache(True) @@ -245,12 +244,11 @@ def f(a, b): return 'c' - call_stub = descr1.get_call_stub() + call_stub = descr1.call_stub fnptr = llhelper(lltype.Ptr(lltype.FuncType(ARGS, RES)), f) - res = call_stub([BoxInt(rffi.cast(lltype.Signed, fnptr)), - BoxInt(1), BoxInt(2)]) - assert res.getint() == ord('c') + res = call_stub(rffi.cast(lltype.Signed, fnptr), [1, 2], None, None) + assert res == ord('c') ARRAY = lltype.GcArray(lltype.Signed) ARGS = [lltype.Float, lltype.Ptr(ARRAY)] @@ -264,6 +262,6 @@ a = lltype.malloc(ARRAY, 3) opaquea = lltype.cast_opaque_ptr(llmemory.GCREF, a) a[0] = 1 - res = descr2.get_call_stub()([BoxInt(rffi.cast(lltype.Signed, fnptr)), - BoxFloat(3.5), BoxPtr(opaquea)]) - assert res.getfloat() == 4.5 + res = descr2.call_stub(rffi.cast(lltype.Signed, fnptr), + [], [opaquea], [3.5]) + assert res == 4.5 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/test/test_gc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/test/test_gc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/test/test_gc.py Fri May 28 20:46:50 2010 @@ -291,7 +291,7 @@ v_random_box = BoxPtr() v_result = BoxInt() operations = [ - ResOperation(rop.OOIS, [v_random_box, ConstPtr(s_gcref)], + ResOperation(rop.PTR_EQ, [v_random_box, ConstPtr(s_gcref)], v_result), ] gc_ll_descr = self.gc_ll_descr @@ -303,7 +303,7 @@ assert operations[0].descr == gc_ll_descr.single_gcref_descr v_box = operations[0].result assert isinstance(v_box, BoxPtr) - assert operations[1].opnum == rop.OOIS + assert operations[1].opnum == rop.PTR_EQ assert operations[1].args == [v_random_box, v_box] assert operations[1].result == v_result @@ -324,7 +324,7 @@ v_random_box = BoxPtr() v_result = BoxInt() operations = [ - ResOperation(rop.OOIS, [v_random_box, ConstPtr(s_gcref)], + ResOperation(rop.PTR_EQ, [v_random_box, ConstPtr(s_gcref)], v_result), ] gc_ll_descr = self.gc_ll_descr @@ -336,7 +336,7 @@ finally: rgc.can_move = old_can_move assert len(operations) == 1 - assert operations[0].opnum == rop.OOIS + assert operations[0].opnum == rop.PTR_EQ assert operations[0].args == [v_random_box, ConstPtr(s_gcref)] assert operations[0].result == v_result # check that s_gcref gets added to the list anyway, to make sure Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/ll2ctypes.py Fri May 28 20:46:50 2010 @@ -511,6 +511,10 @@ """ if isinstance(llobj, lltype._uninitialized): return uninitialized2ctypes(llobj.TYPE) + if isinstance(llobj, llmemory.AddressAsInt): + llobj = llobj.adr + if isinstance(llobj, llmemory.fakeaddress): + llobj = llobj.ptr or 0 T = lltype.typeOf(llobj) @@ -975,6 +979,8 @@ """Cast a value to a result type, trying to use the same rules as C.""" if not isinstance(RESTYPE, lltype.LowLevelType): raise TypeError("rffi.cast() first arg should be a TYPE") + if isinstance(value, llmemory.AddressAsInt): + value = value.adr if isinstance(value, llmemory.fakeaddress): value = value.ptr or 0 TYPE1 = lltype.typeOf(value) From arigo at codespeak.net Fri May 28 20:55:18 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 20:55:18 +0200 (CEST) Subject: [pypy-svn] r74872 - pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport Message-ID: <20100528185518.A1AA5282BD6@codespeak.net> Author: arigo Date: Fri May 28 20:55:17 2010 New Revision: 74872 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py Log: Progress. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py Fri May 28 20:55:17 2010 @@ -229,9 +229,7 @@ assert isinstance(arraydescr, BaseArrayDescr) ofs = arraydescr.get_base_size(self.translate_support_code) size = arraydescr.get_item_size(self.translate_support_code) - ptr = arraydescr.is_array_of_pointers() - float = arraydescr.is_array_of_floats() - return ofs, size, ptr, float + return ofs, size unpack_arraydescr._always_inline_ = True def calldescrof(self, FUNC, ARGS, RESULT, extrainfo=None): @@ -251,77 +249,72 @@ # ____________________________________________________________ - def do_arraylen_gc(self, arraybox, arraydescr): + def bh_arraylen_gc(self, arraydescr, array): assert isinstance(arraydescr, BaseArrayDescr) ofs = arraydescr.get_ofs_length(self.translate_support_code) - gcref = arraybox.getref_base() - length = rffi.cast(rffi.CArrayPtr(lltype.Signed), gcref)[ofs/WORD] - return BoxInt(length) - - def do_getarrayitem_gc(self, arraybox, indexbox, arraydescr): - itemindex = indexbox.getint() - gcref = arraybox.getref_base() - ofs, size, ptr, float = self.unpack_arraydescr(arraydescr) + return rffi.cast(rffi.CArrayPtr(lltype.Signed), array)[ofs/WORD] + + def bh_getarrayitem_gc_i(self, arraydescr, gcref, itemindex): + ofs, size = self.unpack_arraydescr(arraydescr) # --- start of GC unsafe code (no GC operation!) --- items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) - # - if ptr: - items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items) - pval = self._cast_int_to_gcref(items[itemindex]) - # --- end of GC unsafe code --- - return BoxPtr(pval) - # - if float: - items = rffi.cast(rffi.CArrayPtr(lltype.Float), items) - fval = items[itemindex] - # --- end of GC unsafe code --- - return BoxFloat(fval) - # for TYPE, itemsize in unroll_basic_sizes: if size == itemsize: items = rffi.cast(rffi.CArrayPtr(TYPE), items) val = items[itemindex] # --- end of GC unsafe code --- - return BoxInt(rffi.cast(lltype.Signed, val)) + return rffi.cast(lltype.Signed, val) else: raise NotImplementedError("size = %d" % size) - def do_setarrayitem_gc(self, arraybox, indexbox, vbox, arraydescr): - itemindex = indexbox.getint() - gcref = arraybox.getref_base() - ofs, size, ptr, float = self.unpack_arraydescr(arraydescr) - # - if ptr: - vboxptr = vbox.getref_base() - self.gc_ll_descr.do_write_barrier(gcref, vboxptr) - # --- start of GC unsafe code (no GC operation!) --- - items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) - items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items) - items[itemindex] = self.cast_gcref_to_int(vboxptr) - # --- end of GC unsafe code --- - return - # - if float: - fval = vbox.getfloat() - # --- start of GC unsafe code (no GC operation!) --- - items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) - items = rffi.cast(rffi.CArrayPtr(lltype.Float), items) - items[itemindex] = fval - # --- end of GC unsafe code --- - return - # - val = vbox.getint() + def bh_getarrayitem_gc_r(self, arraydescr, gcref, itemindex): + ofs, size = self.unpack_arraydescr(arraydescr) + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items) + pval = self._cast_int_to_gcref(items[itemindex]) + # --- end of GC unsafe code --- + return pval + + def bh_getarrayitem_gc_f(self, arraydescr, gcref, itemindex): + ofs, size = self.unpack_arraydescr(arraydescr) + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + items = rffi.cast(rffi.CArrayPtr(lltype.Float), items) + fval = items[itemindex] + # --- end of GC unsafe code --- + return fval + + def bh_setarrayitem_gc_i(self, arraydescr, gcref, itemindex, newvalue): + ofs, size = self.unpack_arraydescr(arraydescr) for TYPE, itemsize in unroll_basic_sizes: if size == itemsize: # --- start of GC unsafe code (no GC operation!) --- items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) items = rffi.cast(rffi.CArrayPtr(TYPE), items) - items[itemindex] = rffi.cast(TYPE, val) + items[itemindex] = rffi.cast(TYPE, newvalue) # --- end of GC unsafe code --- return else: raise NotImplementedError("size = %d" % size) + def bh_setarrayitem_gc_r(self, arraydescr, gcref, itemindex, newvalue): + ofs, size = self.unpack_arraydescr(arraydescr) + self.gc_ll_descr.do_write_barrier(gcref, newvalue) + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items) + items[itemindex] = self.cast_gcref_to_int(newvalue) + # --- end of GC unsafe code --- + + def bh_setarrayitem_gc_f(self, arraydescr, gcref, itemindex, newvalue): + ofs, size = self.unpack_arraydescr(arraydescr) + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + items = rffi.cast(rffi.CArrayPtr(lltype.Float), items) + items[itemindex] = newvalue + # --- end of GC unsafe code --- + def _new_do_len(TP): def do_strlen(self, stringbox): basesize, itemsize, ofs_length = symbolic.get_array_token(TP, @@ -342,14 +335,9 @@ v = rffi.cast(rffi.CArrayPtr(lltype.Char), gcref)[basesize + i] return BoxInt(ord(v)) - def do_unicodegetitem(self, stringbox, indexbox): - basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, - self.translate_support_code) - gcref = stringbox.getref_base() - i = indexbox.getint() - basesize = basesize // itemsize - v = rffi.cast(rffi.CArrayPtr(lltype.UniChar), gcref)[basesize + i] - return BoxInt(ord(v)) + def bh_unicodegetitem(self, string, index): + u = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string) + return ord(u.chars[index]) @specialize.argtype(1) def _base_do_getfield(self, gcref, fielddescr): @@ -446,15 +434,11 @@ res = self.gc_ll_descr.gc_malloc_array(arraydescr, num_elem) return BoxPtr(res) - def do_newstr(self, countbox): - num_elem = countbox.getint() - res = self.gc_ll_descr.gc_malloc_str(num_elem) - return BoxPtr(res) + def bh_newstr(self, length): + return self.gc_ll_descr.gc_malloc_str(length) - def do_newunicode(self, countbox): - num_elem = countbox.getint() - res = self.gc_ll_descr.gc_malloc_unicode(num_elem) - return BoxPtr(res) + def bh_newunicode(self, length): + return self.gc_ll_descr.gc_malloc_unicode(length) def do_strsetitem(self, stringbox, indexbox, vbox): basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, @@ -464,14 +448,9 @@ a = stringbox.getref_base() rffi.cast(rffi.CArrayPtr(lltype.Char), a)[index + basesize] = chr(v) - def do_unicodesetitem(self, stringbox, indexbox, vbox): - basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, - self.translate_support_code) - index = indexbox.getint() - v = vbox.getint() - a = stringbox.getref_base() - basesize = basesize // itemsize - rffi.cast(rffi.CArrayPtr(lltype.UniChar), a)[index + basesize] = unichr(v) + def bh_unicodesetitem(self, string, index, newvalue): + u = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string) + u.chars[index] = unichr(newvalue) def bh_call_i(self, func, calldescr, args_i, args_r, args_f): assert isinstance(calldescr, BaseIntCallDescr) From arigo at codespeak.net Fri May 28 21:12:04 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 21:12:04 +0200 (CEST) Subject: [pypy-svn] r74873 - in pypy/branch/blackhole-improvement/pypy: jit/backend/llsupport rpython/lltypesystem Message-ID: <20100528191204.CBD2F282BD6@codespeak.net> Author: arigo Date: Fri May 28 21:11:32 2010 New Revision: 74873 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/ll2ctypes.py Log: Progress. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py Fri May 28 21:11:32 2010 @@ -217,9 +217,7 @@ assert isinstance(fielddescr, BaseFieldDescr) ofs = fielddescr.offset size = fielddescr.get_field_size(self.translate_support_code) - ptr = fielddescr.is_pointer_field() - float = fielddescr.is_float_field() - return ofs, size, ptr, float + return ofs, size unpack_fielddescr._always_inline_ = True def arraydescrof(self, A): @@ -287,10 +285,10 @@ def bh_setarrayitem_gc_i(self, arraydescr, gcref, itemindex, newvalue): ofs, size = self.unpack_arraydescr(arraydescr) + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) for TYPE, itemsize in unroll_basic_sizes: if size == itemsize: - # --- start of GC unsafe code (no GC operation!) --- - items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) items = rffi.cast(rffi.CArrayPtr(TYPE), items) items[itemindex] = rffi.cast(TYPE, newvalue) # --- end of GC unsafe code --- @@ -315,124 +313,115 @@ items[itemindex] = newvalue # --- end of GC unsafe code --- - def _new_do_len(TP): - def do_strlen(self, stringbox): - basesize, itemsize, ofs_length = symbolic.get_array_token(TP, - self.translate_support_code) - gcref = stringbox.getref_base() - v = rffi.cast(rffi.CArrayPtr(lltype.Signed), gcref)[ofs_length/WORD] - return BoxInt(v) - return do_strlen - - do_strlen = _new_do_len(rstr.STR) - do_unicodelen = _new_do_len(rstr.UNICODE) - - def do_strgetitem(self, stringbox, indexbox): - basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, - self.translate_support_code) - gcref = stringbox.getref_base() - i = indexbox.getint() - v = rffi.cast(rffi.CArrayPtr(lltype.Char), gcref)[basesize + i] - return BoxInt(ord(v)) + def bh_strlen(self, string): + s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string) + return len(s.chars) + + def bh_unicodelen(self, string): + u = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string) + return len(u.chars) + + def bh_strgetitem(self, string, index): + s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string) + return ord(s.chars[index]) def bh_unicodegetitem(self, string, index): u = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string) return ord(u.chars[index]) @specialize.argtype(1) - def _base_do_getfield(self, gcref, fielddescr): - ofs, size, ptr, float = self.unpack_fielddescr(fielddescr) + def _base_do_getfield_i(self, struct, fielddescr): + ofs, size = self.unpack_fielddescr(fielddescr) # --- start of GC unsafe code (no GC operation!) --- - field = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) - # - if ptr: - pval = rffi.cast(rffi.CArrayPtr(lltype.Signed), field)[0] - pval = self._cast_int_to_gcref(pval) - # --- end of GC unsafe code --- - return BoxPtr(pval) - # - if float: - fval = rffi.cast(rffi.CArrayPtr(lltype.Float), field)[0] - # --- end of GC unsafe code --- - return BoxFloat(fval) - # + fieldptr = rffi.ptradd(rffi.cast(rffi.CCHARP, struct), ofs) for TYPE, itemsize in unroll_basic_sizes: if size == itemsize: - val = rffi.cast(rffi.CArrayPtr(TYPE), field)[0] + val = rffi.cast(rffi.CArrayPtr(TYPE), fieldptr)[0] # --- end of GC unsafe code --- - val = rffi.cast(lltype.Signed, val) - return BoxInt(val) + return rffi.cast(lltype.Signed, val) else: raise NotImplementedError("size = %d" % size) - def do_getfield_gc(self, structbox, fielddescr): - gcref = structbox.getref_base() - return self._base_do_getfield(gcref, fielddescr) + @specialize.argtype(1) + def _base_do_getfield_r(self, struct, fielddescr): + ofs, size = self.unpack_fielddescr(fielddescr) + # --- start of GC unsafe code (no GC operation!) --- + fieldptr = rffi.ptradd(rffi.cast(rffi.CCHARP, struct), ofs) + pval = rffi.cast(rffi.CArrayPtr(lltype.Signed), fieldptr)[0] + pval = self._cast_int_to_gcref(pval) + # --- end of GC unsafe code --- + return pval - def do_getfield_raw(self, structbox, fielddescr): - return self._base_do_getfield(structbox.getint(), fielddescr) + @specialize.argtype(1) + def _base_do_getfield_f(self, struct, fielddescr): + ofs, size = self.unpack_fielddescr(fielddescr) + # --- start of GC unsafe code (no GC operation!) --- + fieldptr = rffi.ptradd(rffi.cast(rffi.CCHARP, struct), ofs) + fval = rffi.cast(rffi.CArrayPtr(lltype.Float), fieldptr)[0] + # --- end of GC unsafe code --- + return fval + + bh_getfield_gc_i = _base_do_getfield_i + bh_getfield_gc_r = _base_do_getfield_r + bh_getfield_gc_f = _base_do_getfield_f + bh_getfield_raw_i = _base_do_getfield_i + bh_getfield_raw_r = _base_do_getfield_r + bh_getfield_raw_f = _base_do_getfield_f @specialize.argtype(1) - def _base_do_setfield(self, gcref, vbox, fielddescr): - ofs, size, ptr, float = self.unpack_fielddescr(fielddescr) - # - if ptr: - assert lltype.typeOf(gcref) is not lltype.Signed, ( - "can't handle write barriers for setfield_raw") - ptr = vbox.getref_base() - self.gc_ll_descr.do_write_barrier(gcref, ptr) - # --- start of GC unsafe code (no GC operation!) --- - field = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) - field = rffi.cast(rffi.CArrayPtr(lltype.Signed), field) - field[0] = self.cast_gcref_to_int(ptr) - # --- end of GC unsafe code --- - return - # - if float: - fval = vbox.getfloat() - # --- start of GC unsafe code (no GC operation!) --- - field = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) - field = rffi.cast(rffi.CArrayPtr(lltype.Float), field) - field[0] = fval - # --- end of GC unsafe code --- - return - # - val = vbox.getint() + def _base_do_setfield_i(self, struct, fielddescr, newvalue): + ofs, size = self.unpack_fielddescr(fielddescr) + # --- start of GC unsafe code (no GC operation!) --- + fieldptr = rffi.ptradd(rffi.cast(rffi.CCHARP, struct), ofs) for TYPE, itemsize in unroll_basic_sizes: if size == itemsize: - # --- start of GC unsafe code (no GC operation!) --- - field = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) - field = rffi.cast(rffi.CArrayPtr(TYPE), field) - field[0] = rffi.cast(TYPE, val) + fieldptr = rffi.cast(rffi.CArrayPtr(TYPE), fieldptr) + fieldptr[0] = rffi.cast(TYPE, newvalue) # --- end of GC unsafe code --- return else: raise NotImplementedError("size = %d" % size) - def do_setfield_gc(self, structbox, vbox, fielddescr): - gcref = structbox.getref_base() - self._base_do_setfield(gcref, vbox, fielddescr) + @specialize.argtype(1) + def _base_do_setfield_r(self, struct, fielddescr, newvalue): + ofs, size = self.unpack_fielddescr(fielddescr) + assert lltype.typeOf(struct) is not lltype.Signed, ( + "can't handle write barriers for setfield_raw") + self.gc_ll_descr.do_write_barrier(struct, newvalue) + # --- start of GC unsafe code (no GC operation!) --- + fieldptr = rffi.ptradd(rffi.cast(rffi.CCHARP, struct), ofs) + fieldptr = rffi.cast(rffi.CArrayPtr(lltype.Signed), fieldptr) + fieldptr[0] = self.cast_gcref_to_int(newvalue) + # --- end of GC unsafe code --- + + @specialize.argtype(1) + def _base_do_setfield_f(self, struct, fielddescr, newvalue): + ofs, size = self.unpack_fielddescr(fielddescr) + # --- start of GC unsafe code (no GC operation!) --- + fieldptr = rffi.ptradd(rffi.cast(rffi.CCHARP, struct), ofs) + fieldptr = rffi.cast(rffi.CArrayPtr(lltype.Float), fieldptr) + fieldptr[0] = newvalue + # --- end of GC unsafe code --- + + bh_setfield_gc_i = _base_do_setfield_i + bh_setfield_gc_r = _base_do_setfield_r + bh_setfield_gc_f = _base_do_setfield_f + bh_setfield_raw_i = _base_do_setfield_i + bh_setfield_raw_r = _base_do_setfield_r + bh_setfield_raw_f = _base_do_setfield_f - def do_setfield_raw(self, structbox, vbox, fielddescr): - self._base_do_setfield(structbox.getint(), vbox, fielddescr) + def bh_new(self, sizedescr): + return self.gc_ll_descr.gc_malloc(sizedescr) - def do_new(self, sizedescr): + def bh_new_with_vtable(self, sizedescr, vtable): res = self.gc_ll_descr.gc_malloc(sizedescr) - return BoxPtr(res) - - def do_new_with_vtable(self, classbox): - classint = classbox.getint() - descrsize = self.class_sizes[classint] - res = self.gc_ll_descr.gc_malloc(descrsize) if self.vtable_offset is not None: as_array = rffi.cast(rffi.CArrayPtr(lltype.Signed), res) - as_array[self.vtable_offset/WORD] = classint - return BoxPtr(res) + as_array[self.vtable_offset/WORD] = vtable + return res - def do_new_array(self, countbox, arraydescr): - num_elem = countbox.getint() - res = self.gc_ll_descr.gc_malloc_array(arraydescr, num_elem) - return BoxPtr(res) + def bh_new_array(self, arraydescr, length): + return self.gc_ll_descr.gc_malloc_array(arraydescr, length) def bh_newstr(self, length): return self.gc_ll_descr.gc_malloc_str(length) @@ -440,13 +429,9 @@ def bh_newunicode(self, length): return self.gc_ll_descr.gc_malloc_unicode(length) - def do_strsetitem(self, stringbox, indexbox, vbox): - basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, - self.translate_support_code) - index = indexbox.getint() - v = vbox.getint() - a = stringbox.getref_base() - rffi.cast(rffi.CArrayPtr(lltype.Char), a)[index + basesize] = chr(v) + def bh_strsetitem(self, string, index, newvalue): + s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string) + s.chars[index] = chr(newvalue) def bh_unicodesetitem(self, string, index, newvalue): u = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string) Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/ll2ctypes.py Fri May 28 21:11:32 2010 @@ -512,7 +512,7 @@ if isinstance(llobj, lltype._uninitialized): return uninitialized2ctypes(llobj.TYPE) if isinstance(llobj, llmemory.AddressAsInt): - llobj = llobj.adr + return ctypes.cast(lltype2ctypes(llobj.adr), ctypes.c_void_p).value if isinstance(llobj, llmemory.fakeaddress): llobj = llobj.ptr or 0 From arigo at codespeak.net Fri May 28 21:29:45 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 21:29:45 +0200 (CEST) Subject: [pypy-svn] r74874 - in pypy/branch/blackhole-improvement/pypy/jit: backend/llsupport backend/test metainterp/test Message-ID: <20100528192945.5272836C538@codespeak.net> Author: arigo Date: Fri May 28 21:29:42 2010 New Revision: 74874 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_resume.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_warmspot.py Log: Test fixes. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py Fri May 28 21:29:42 2010 @@ -461,5 +461,5 @@ calldescr.verify_types(args_i, args_r, args_f, 'void') return calldescr.call_stub(func, args_i, args_r, args_f) - def do_cast_ptr_to_int(self, ptrbox): - return BoxInt(self.cast_gcref_to_int(ptrbox.getref_base())) + def bh_cast_ptr_to_int(self, ptr): + return self.cast_gcref_to_int(ptr) Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py Fri May 28 21:29:42 2010 @@ -1662,7 +1662,8 @@ # x = cpu.bh_newstr(5) y = cpu.bh_cast_ptr_to_int(x) - assert y == llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(x)) + z = cpu.bh_cast_ptr_to_int(x) + assert type(y) == type(z) == int and y == z def test_sorting_of_fields(self): S = self.S Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py Fri May 28 21:29:42 2010 @@ -130,6 +130,7 @@ def end_blackhole(): pass class DoneWithThisFrameInt(Exception): pass + last_exc_value_box = None framestack = [MyMIFrame()] MyMetaInterp.staticdata.blackholeinterpbuilder = getblackholeinterp( {'int_add/ii>i': 0, 'int_return/i': 1}).builder Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_resume.py Fri May 28 21:29:42 2010 @@ -103,7 +103,8 @@ def get_current_position_info(self): class MyInfo: - def enumerate_vars(_, callback_i, callback_r, callback_f): + @staticmethod + def enumerate_vars(callback_i, callback_r, callback_f, _): count_i = count_r = count_f = 0 for index, ARG in enumerate(self.ARGS): if ARG == lltype.Signed: @@ -210,7 +211,7 @@ _already_allocated_resume_virtuals = None cpu = None reader = ResumeDataDirectReader(None, FakeStorage()) - assert reader.virtuals == ["allocated", None] + assert reader.virtuals == ["allocated", reader.virtual_default] # ____________________________________________________________ @@ -835,7 +836,8 @@ def __eq__(self, other): return True class MyInfo: - def enumerate_vars(_, callback_i, callback_r, callback_f): + @staticmethod + def enumerate_vars(callback_i, callback_r, callback_f, _): for index, tagged in enumerate(self.cur_numb.nums): box = self.decode_box(tagged, Whatever()) if box.type == INT: Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_warmspot.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_warmspot.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_warmspot.py Fri May 28 21:29:42 2010 @@ -313,6 +313,9 @@ lltype.cast_opaque_ptr(llmemory.GCREF, exc)) return self.no + class FakeDescr: + pass + class FakeCPU(object): supports_floats = False ts = llhelper @@ -326,7 +329,7 @@ pass def nodescr(self, *args, **kwds): - return "this is a descr" + return FakeDescr() fielddescrof = nodescr calldescrof = nodescr sizeof = nodescr From arigo at codespeak.net Fri May 28 21:31:26 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 21:31:26 +0200 (CEST) Subject: [pypy-svn] r74875 - pypy/branch/blackhole-improvement/pypy/jit/backend/test Message-ID: <20100528193126.6228336C538@codespeak.net> Author: arigo Date: Fri May 28 21:31:25 2010 New Revision: 74875 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py Log: Test fix. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py Fri May 28 21:31:25 2010 @@ -1663,6 +1663,8 @@ x = cpu.bh_newstr(5) y = cpu.bh_cast_ptr_to_int(x) z = cpu.bh_cast_ptr_to_int(x) + y = llmemory.get_inthash_from_int(y) + z = llmemory.get_inthash_from_int(z) assert type(y) == type(z) == int and y == z def test_sorting_of_fields(self): From afa at codespeak.net Fri May 28 22:09:31 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 28 May 2010 22:09:31 +0200 (CEST) Subject: [pypy-svn] r74876 - pypy/trunk/pypy/objspace Message-ID: <20100528200931.258AA282BD6@codespeak.net> Author: afa Date: Fri May 28 22:09:29 2010 New Revision: 74876 Modified: pypy/trunk/pypy/objspace/descroperation.py Log: add missing object_delattr, copied from object_setattr Modified: pypy/trunk/pypy/objspace/descroperation.py ============================================================================== --- pypy/trunk/pypy/objspace/descroperation.py (original) +++ pypy/trunk/pypy/objspace/descroperation.py Fri May 28 22:09:29 2010 @@ -21,6 +21,13 @@ return w_setattr object_setattr._annspecialcase_ = 'specialize:memo' +def object_delattr(space): + "Utility that returns the app-level descriptor object.__delattr__." + w_src, w_delattr = space.lookup_in_type_where(space.w_object, + '__delattr__') + return w_delattr +object_delattr._annspecialcase_ = 'specialize:memo' + def raiseattrerror(space, w_obj, name, w_descr=None): w_type = space.type(w_obj) typename = w_type.getname(space, '?') From arigo at codespeak.net Fri May 28 22:33:52 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 22:33:52 +0200 (CEST) Subject: [pypy-svn] r74877 - in pypy/branch/blackhole-improvement/pypy/jit: backend backend/llgraph backend/test backend/x86 backend/x86/test metainterp metainterp/test Message-ID: <20100528203352.95394282BD6@codespeak.net> Author: arigo Date: Fri May 28 22:33:50 2010 New Revision: 74877 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py pypy/branch/blackhole-improvement/pypy/jit/backend/model.py pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py pypy/branch/blackhole-improvement/pypy/jit/backend/test/test_random.py pypy/branch/blackhole-improvement/pypy/jit/backend/x86/assembler.py pypy/branch/blackhole-improvement/pypy/jit/backend/x86/regalloc.py pypy/branch/blackhole-improvement/pypy/jit/backend/x86/ri386.py pypy/branch/blackhole-improvement/pypy/jit/backend/x86/runner.py pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_basic.py pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_regalloc.py pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_regalloc2.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizeopt.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualref.py Log: In-progress. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py Fri May 28 22:33:50 2010 @@ -1080,8 +1080,9 @@ frame = _from_opaque(frame) return len(frame.fail_args) -def frame_clear_latest_values(frame): +def frame_clear_latest_values(frame, count): frame = _from_opaque(frame) + assert count == len(frame.fail_args) del frame.fail_args _last_exception = None Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py Fri May 28 22:33:50 2010 @@ -247,8 +247,8 @@ token = llimpl.get_frame_forced_token(self.latest_frame) return llmemory.cast_adr_to_int(token) - def clear_latest_values(self): - llimpl.frame_clear_latest_values(self.latest_frame) + def clear_latest_values(self, count): + llimpl.frame_clear_latest_values(self.latest_frame, count) # ---------- Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/model.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/model.py Fri May 28 22:33:50 2010 @@ -87,7 +87,8 @@ def get_latest_value_count(self): """Return how many values are ready to be returned by - get_latest_value_xxx().""" + get_latest_value_xxx(). Only after a guard failure; not + necessarily correct after a FINISH.""" raise NotImplementedError def get_latest_force_token(self): @@ -95,9 +96,10 @@ same FORCE_TOKEN result as the one in the just-failed loop.""" raise NotImplementedError - def clear_latest_values(self): - """Clear the latest values (at least the ref ones, so that - they no longer keep objects alive).""" + def clear_latest_values(self, count): + """Clear the latest values (at least the ref ones), so that + they no longer keep objects alive. 'count' is the number of + values -- normally get_latest_value_count().""" raise NotImplementedError def grab_exc_value(self): Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py Fri May 28 22:33:50 2010 @@ -265,7 +265,7 @@ assert count == 3 assert self.cpu.get_latest_value_int(1) == 10 assert self.cpu.get_latest_value_int(1) == 10 # multiple reads ok - self.cpu.clear_latest_values() + self.cpu.clear_latest_values(3) def test_finish(self): i0 = BoxInt() Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/test/test_random.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/test/test_random.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/test/test_random.py Fri May 28 22:33:50 2010 @@ -413,7 +413,7 @@ OPERATIONS.append(UnaryOperation(_op)) OPERATIONS.append(UnaryOperation(rop.INT_IS_TRUE, boolres=True)) -OPERATIONS.append(BooleanUnaryOperation(rop.BOOL_NOT, boolres=True)) +OPERATIONS.append(UnaryOperation(rop.INT_IS_ZERO, boolres=True)) OPERATIONS.append(ConstUnaryOperation(rop.SAME_AS, boolres='sometimes')) for _op in [rop.INT_ADD_OVF, @@ -434,8 +434,6 @@ ]: OPERATIONS.append(UnaryFloatOperation(_op)) -OPERATIONS.append(UnaryFloatOperation(rop.FLOAT_IS_TRUE, boolres=True)) - OPERATIONS.append(CastFloatToIntOperation(rop.CAST_FLOAT_TO_INT)) OPERATIONS.append(CastIntToFloatOperation(rop.CAST_INT_TO_FLOAT)) Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/x86/assembler.py Fri May 28 22:33:50 2010 @@ -639,8 +639,8 @@ genop_int_ne = _cmpop("NE", "NE") genop_int_gt = _cmpop("G", "L") genop_int_ge = _cmpop("GE", "LE") - genop_oois = genop_int_eq - genop_ooisnot = genop_int_ne + genop_ptr_eq = genop_int_eq + genop_ptr_ne = genop_int_ne genop_float_lt = _cmpop_float('B') genop_float_le = _cmpop_float('BE') @@ -660,8 +660,8 @@ genop_guard_int_ne = _cmpop_guard("NE", "NE", "E", "E") genop_guard_int_gt = _cmpop_guard("G", "L", "LE", "GE") genop_guard_int_ge = _cmpop_guard("GE", "LE", "L", "G") - genop_guard_oois = genop_guard_int_eq - genop_guard_ooisnot = genop_guard_int_ne + genop_guard_ptr_eq = genop_guard_int_eq + genop_guard_ptr_ne = genop_guard_int_ne genop_guard_uint_gt = _cmpop_guard("A", "B", "BE", "AE") genop_guard_uint_lt = _cmpop_guard("B", "A", "AE", "BE") @@ -733,7 +733,7 @@ self.mc.SETNE(rl) self.mc.MOVZX(resloc, rl) - def genop_guard_bool_not(self, op, guard_op, addr, arglocs, resloc): + def genop_guard_int_is_zero(self, op, guard_op, addr, arglocs, resloc): guard_opnum = guard_op.opnum self.mc.CMP(arglocs[0], imm8(0)) if guard_opnum == rop.GUARD_TRUE: @@ -741,8 +741,11 @@ else: return self.implement_guard(addr, self.mc.JZ) - def genop_bool_not(self, op, arglocs, resloc): - self.mc.XOR(arglocs[0], imm8(1)) + def genop_int_is_zero(self, op, arglocs, resloc): + self.mc.CMP(arglocs[0], imm8(0)) + rl = resloc.lowest8bits() + self.mc.SETE(rl) + self.mc.MOVZX(resloc, rl) def genop_same_as(self, op, arglocs, resloc): self.mov(arglocs[0], resloc) @@ -1125,39 +1128,6 @@ arglocs.append(loc) return arglocs[:] - def make_boxes_from_latest_values(self, bytecode): - bytecode = rffi.cast(rffi.UCHARP, bytecode) - boxes = [] - while 1: - # decode the next instruction from the bytecode - code = rffi.cast(lltype.Signed, bytecode[0]) - bytecode = rffi.ptradd(bytecode, 1) - kind = code & 3 - while code > 0x7F: - code = rffi.cast(lltype.Signed, bytecode[0]) - bytecode = rffi.ptradd(bytecode, 1) - index = len(boxes) - if kind == self.DESCR_INT: - box = BoxInt(self.fail_boxes_int.getitem(index)) - elif kind == self.DESCR_REF: - box = BoxPtr(self.fail_boxes_ptr.getitem(index)) - # clear after reading (xxx duplicates - # get_latest_value_ref()) - self.fail_boxes_ptr.setitem(index, lltype.nullptr( - llmemory.GCREF.TO)) - elif kind == self.DESCR_FLOAT: - box = BoxFloat(self.fail_boxes_float.getitem(index)) - else: - assert kind == self.DESCR_SPECIAL - if code == self.CODE_STOP: - break - elif code == self.CODE_HOLE: - box = None - else: - assert 0, "bad code" - boxes.append(box) - return boxes - @rgc.no_collect def grab_frame_values(self, bytecode, frame_addr, allregisters): # no malloc allowed here!! Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/x86/regalloc.py Fri May 28 22:33:50 2010 @@ -3,7 +3,7 @@ """ from pypy.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr, - ResOperation, ConstAddr, BoxPtr, + ResOperation, BoxPtr, LoopToken, INT, REF, FLOAT) from pypy.jit.backend.x86.ri386 import * from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi, rstr @@ -11,6 +11,7 @@ from pypy.rlib import rgc from pypy.jit.backend.llsupport import symbolic from pypy.jit.backend.x86.jump import remap_frame_layout +from pypy.jit.codewriter import heaptracker from pypy.jit.metainterp.resoperation import rop from pypy.jit.backend.llsupport.descr import BaseFieldDescr, BaseArrayDescr from pypy.jit.backend.llsupport.descr import BaseCallDescr, BaseSizeDescr @@ -45,8 +46,6 @@ print "convert_to_imm: ConstPtr needs special care" raise AssertionError return imm(rffi.cast(lltype.Signed, c.value)) - elif isinstance(c, ConstAddr): - return imm(ll2ctypes.cast_adr_to_int(c.value)) else: print "convert_to_imm: got a %s" % c raise AssertionError @@ -540,8 +539,8 @@ consider_uint_lt = _consider_compop consider_uint_le = _consider_compop consider_uint_ge = _consider_compop - consider_oois = _consider_compop - consider_ooisnot = _consider_compop + consider_ptr_eq = _consider_compop + consider_ptr_ne = _consider_compop def _consider_float_op(self, op): loc1 = self.xrm.loc(op.args[1]) @@ -583,20 +582,6 @@ self.Perform(op, [loc0], loc0) self.xrm.possibly_free_var(op.args[0]) - def consider_float_is_true(self, op, guard_op): - # doesn't need arg to be in a register - tmpbox0 = TempBox() - loc0 = self.xrm.force_allocate_reg(tmpbox0) - loc1 = self.xrm.loc(op.args[0]) - arglocs = [loc0, loc1] - self.xrm.possibly_free_var(op.args[0]) - self.xrm.possibly_free_var(tmpbox0) - if guard_op is not None: - self.perform_with_guard(op, guard_op, arglocs, None) - else: - loc2 = self.rm.force_allocate_reg(op.result, need_lower_byte=True) - self.Perform(op, arglocs, loc2) - def consider_cast_float_to_int(self, op): loc0 = self.xrm.make_sure_var_in_reg(op.args[0], imm_fine=False) loc1 = self.rm.force_allocate_reg(op.result) @@ -715,7 +700,7 @@ def consider_new_with_vtable(self, op): classint = op.args[0].getint() - descrsize = self.assembler.cpu.class_sizes[classint] + descrsize = heaptracker.vtable2descr(self.assembler.cpu, classint) if self.assembler.cpu.gc_ll_descr.can_inline_malloc(descrsize): self._fastpath_malloc(op, descrsize) self.assembler.set_vtable(eax, imm(classint)) @@ -879,14 +864,7 @@ resloc = self.rm.force_allocate_reg(op.result, need_lower_byte=True) self.Perform(op, [argloc], resloc) - def consider_bool_not(self, op, guard_op): - if guard_op is not None: - # doesn't need arg to be in a register - argloc = self.loc(op.args[0]) - self.rm.possibly_free_var(op.args[0]) - self.perform_with_guard(op, guard_op, [argloc], None) - else: - self.consider_int_neg(op) + consider_int_is_zero = consider_int_is_true def consider_same_as(self, op): argloc = self.loc(op.args[0]) Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/ri386.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/x86/ri386.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/x86/ri386.py Fri May 28 22:33:50 2010 @@ -1,6 +1,7 @@ from pypy.rlib.rarithmetic import intmask from pypy.rlib.objectmodel import ComputedIntSymbolic, we_are_translated from pypy.rlib.debug import make_sure_not_resized +from pypy.rpython.lltypesystem import llmemory class OPERAND(object): _attrs_ = [] @@ -289,8 +290,7 @@ def imm(value): if isinstance(value, ComputedIntSymbolic): value = value.compute_fn() - if not we_are_translated(): - assert type(value) is int + value = llmemory.get_inthash_from_int(value) if single_byte(value): return imm8(value) else: Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/x86/runner.py Fri May 28 22:33:50 2010 @@ -70,19 +70,20 @@ return self.assembler.fail_boxes_float.getitem(index) def get_latest_value_ref(self, index): - ptrvalue = self.assembler.fail_boxes_ptr.getitem(index) - # clear after reading - self.assembler.fail_boxes_ptr.setitem(index, lltype.nullptr( - llmemory.GCREF.TO)) - return ptrvalue + return self.assembler.fail_boxes_ptr.getitem(index) + + def get_latest_value_count(self): + XXX + + def clear_latest_values(self, count): + setitem = self.assembler.fail_boxes_ptr.setitem + null = lltype.nullptr(llmemory.GCREF.TO) + for index in range(count): + setitem(index, null) def get_latest_force_token(self): return self.assembler.fail_ebp + FORCE_INDEX_OFS - def make_boxes_from_latest_values(self, faildescr): - return self.assembler.make_boxes_from_latest_values( - faildescr._x86_failure_recovery_bytecode) - def execute_token(self, executable_token): addr = executable_token._x86_bootstrap_code func = rffi.cast(lltype.Ptr(self.BOOTSTRAP_TP), addr) @@ -139,9 +140,6 @@ CPU = CPU386 -import pypy.jit.metainterp.executor -pypy.jit.metainterp.executor.make_execute_list(CPU) - # silence warnings history.LoopToken._x86_param_depth = 0 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_basic.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_basic.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_basic.py Fri May 28 22:33:50 2010 @@ -2,7 +2,7 @@ from pypy.jit.backend.x86.runner import CPU386 from pypy.jit.metainterp.warmspot import ll_meta_interp from pypy.jit.metainterp.test import test_basic -from pypy.jit.metainterp.policy import StopAtXPolicy +from pypy.jit.codewriter.policy import StopAtXPolicy from pypy.rlib.jit import JitDriver, OPTIMIZER_SIMPLE class Jit386Mixin(test_basic.LLJitMixin): Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_regalloc.py Fri May 28 22:33:50 2010 @@ -522,18 +522,20 @@ assert self.getint(0) == 0 def test_bug_float_is_true_stack(self): + # NB. float_is_true no longer exists. Unsure if keeping this test + # makes sense any more. ops = ''' [f0, f1, f2, f3, f4, f5, f6, f7, f8, f9] - i0 = float_is_true(f0) - i1 = float_is_true(f1) - i2 = float_is_true(f2) - i3 = float_is_true(f3) - i4 = float_is_true(f4) - i5 = float_is_true(f5) - i6 = float_is_true(f6) - i7 = float_is_true(f7) - i8 = float_is_true(f8) - i9 = float_is_true(f9) + i0 = float_ne(f0, ConstFloat(0.0)) + i1 = float_ne(f1, ConstFloat(0.0)) + i2 = float_ne(f2, ConstFloat(0.0)) + i3 = float_ne(f3, ConstFloat(0.0)) + i4 = float_ne(f4, ConstFloat(0.0)) + i5 = float_ne(f5, ConstFloat(0.0)) + i6 = float_ne(f6, ConstFloat(0.0)) + i7 = float_ne(f7, ConstFloat(0.0)) + i8 = float_ne(f8, ConstFloat(0.0)) + i9 = float_ne(f9, ConstFloat(0.0)) finish(i0, i1, i2, i3, i4, i5, i6, i7, i8, i9) ''' loop = self.interpret(ops, [0.0, .1, .2, .3, .4, .5, .6, .7, .8, .9]) Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_regalloc2.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_regalloc2.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_regalloc2.py Fri May 28 22:33:50 2010 @@ -35,7 +35,7 @@ ResOperation(rop.INT_MUL, [v1, v1], v2), ResOperation(rop.INT_MUL, [v2, v1], v3), ResOperation(rop.INT_IS_TRUE, [v2], tmp5), - ResOperation(rop.BOOL_NOT, [tmp5], v4), + ResOperation(rop.INT_IS_ZERO, [tmp5], v4), ResOperation(rop.FINISH, [v4, v3, tmp5], None, descr=BasicFailDescr()), ] cpu = CPU(None, None) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Fri May 28 22:33:50 2010 @@ -550,9 +550,6 @@ @arguments("f", returns="f") def bhimpl_float_abs(a): return abs(a) - @arguments("f", returns="i") - def bhimpl_float_is_true(a): - return bool(a) @arguments("f", "f", returns="f") def bhimpl_float_add(a, b): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py Fri May 28 22:33:50 2010 @@ -156,6 +156,7 @@ assert metainterp_sd.result_type == 'ref' cpu = metainterp_sd.cpu result = cpu.get_latest_value_ref(0) + cpu.clear_latest_values(1) raise metainterp_sd.DoneWithThisFrameRef(cpu, result) class DoneWithThisFrameDescrFloat(_DoneWithThisFrameDescr): @@ -168,6 +169,7 @@ def handle_fail(self, metainterp_sd): cpu = metainterp_sd.cpu value = cpu.get_latest_value_ref(0) + cpu.clear_latest_values(1) raise metainterp_sd.ExitFrameWithExceptionRef(cpu, value) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/optimizeopt.py Fri May 28 22:33:50 2010 @@ -757,6 +757,9 @@ def optimize_INT_IS_TRUE(self, op): self._optimize_nullness(op, op.args[0], True) + def optimize_INT_IS_ZERO(self, op): + self._optimize_nullness(op, op.args[0], False) + def _optimize_oois_ooisnot(self, op, expect_isnot): value0 = self.getvalue(op.args[0]) value1 = self.getvalue(op.args[1]) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py Fri May 28 22:33:50 2010 @@ -548,7 +548,7 @@ self.write_a_float(register_index, value) def done(self): - self.cpu.clear_latest_values() + self.cpu.clear_latest_values(self.cpu.get_latest_value_count()) # ---------- when resuming for pyjitpl.py, make boxes ---------- Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualref.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualref.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_virtualref.py Fri May 28 22:33:50 2010 @@ -87,7 +87,7 @@ cpu.get_latest_value_count = lambda : len(guard_op.fail_args) cpu.get_latest_value_int = lambda i:guard_op.fail_args[i].getint() cpu.get_latest_value_ref = lambda i:guard_op.fail_args[i].getref_base() - cpu.clear_latest_values = lambda : None + cpu.clear_latest_values = lambda count: None resumereader = ResumeDataDirectReader(cpu, guard_op.descr) vrefinfo = self.metainterp.staticdata.virtualref_info lst = [] From arigo at codespeak.net Fri May 28 22:45:36 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 22:45:36 +0200 (CEST) Subject: [pypy-svn] r74878 - in pypy/branch/blackhole-improvement/pypy/jit: backend/llsupport backend/llsupport/test backend/x86 backend/x86/test metainterp Message-ID: <20100528204536.2F637282BD6@codespeak.net> Author: arigo Date: Fri May 28 22:45:34 2010 New Revision: 74878 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/descr.py pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/test/test_descr.py pypy/branch/blackhole-improvement/pypy/jit/backend/x86/assembler.py pypy/branch/blackhole-improvement/pypy/jit/backend/x86/runner.py pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_assembler.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py Log: Still in-progress... Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/descr.py Fri May 28 22:45:34 2010 @@ -190,18 +190,8 @@ def get_extra_info(self): return self.extrainfo - _returns_a_pointer = False # unless overridden by GcPtrCallDescr - _returns_a_float = False # unless overridden by FloatCallDescr - _returns_a_void = False # unless overridden by VoidCallDescr - - def returns_a_pointer(self): - return self._returns_a_pointer - - def returns_a_float(self): - return self._returns_a_float - - def returns_a_void(self): - return self._returns_a_void + def get_return_type(self): + return self._return_type def get_result_size(self, translate_support_code): raise NotImplementedError @@ -225,14 +215,16 @@ seen = {'i': 0, 'r': 0, 'f': 0} args = ", ".join([process(c) for c in self.arg_classes]) - if self.returns_a_pointer(): + if self.get_return_type() == history.INT: + result = 'rffi.cast(lltype.Signed, res)' + elif self.get_return_type() == history.REF: result = 'lltype.cast_opaque_ptr(llmemory.GCREF, res)' - elif self.returns_a_float(): + elif self.get_return_type() == history.FLOAT: result = 'res' - elif self.returns_a_void(): + elif self.get_return_type() == history.VOID: result = 'None' else: - result = 'rffi.cast(lltype.Signed, res)' + assert 0 source = py.code.Source(""" def call_stub(func, args_i, args_r, args_f): fnptr = rffi.cast(lltype.Ptr(FUNC), func) @@ -247,9 +239,7 @@ self.call_stub = d['call_stub'] def verify_types(self, args_i, args_r, args_f, return_type): - assert self._returns_a_pointer == (return_type == 'ref') - assert self._returns_a_float == (return_type == 'float') - assert self._returns_a_void == (return_type == 'void') + assert self._return_type == return_type assert self.arg_classes.count('i') == len(args_i or ()) assert self.arg_classes.count('r') == len(args_r or ()) assert self.arg_classes.count('f') == len(args_f or ()) @@ -268,7 +258,7 @@ # # The purpose of BaseIntCallDescr is to be the parent of all classes # in which 'call_stub' has a return kind of 'int'. - pass + _return_type = history.INT class NonGcPtrCallDescr(BaseIntCallDescr): _clsname = 'NonGcPtrCallDescr' @@ -277,19 +267,19 @@ class GcPtrCallDescr(BaseCallDescr): _clsname = 'GcPtrCallDescr' - _returns_a_pointer = True + _return_type = history.REF def get_result_size(self, translate_support_code): return symbolic.get_size_of_ptr(translate_support_code) class FloatCallDescr(BaseCallDescr): _clsname = 'FloatCallDescr' - _returns_a_float = True + _return_type = history.FLOAT def get_result_size(self, translate_support_code): return symbolic.get_size(lltype.Float, translate_support_code) class VoidCallDescr(BaseCallDescr): _clsname = 'VoidCallDescr' - _returns_a_void = True + _return_type = history.VOID def get_result_size(self, translate_support_code): return 0 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py Fri May 28 22:45:34 2010 @@ -6,6 +6,7 @@ from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.jit.metainterp.history import BoxInt, BoxPtr, set_future_values,\ BoxFloat +from pypy.jit.metainterp import history from pypy.jit.backend.model import AbstractCPU from pypy.jit.backend.llsupport import symbolic from pypy.jit.backend.llsupport.symbolic import WORD, unroll_basic_sizes @@ -440,25 +441,25 @@ def bh_call_i(self, func, calldescr, args_i, args_r, args_f): assert isinstance(calldescr, BaseIntCallDescr) if not we_are_translated(): - calldescr.verify_types(args_i, args_r, args_f, 'int') + calldescr.verify_types(args_i, args_r, args_f, history.INT) return calldescr.call_stub(func, args_i, args_r, args_f) def bh_call_r(self, func, calldescr, args_i, args_r, args_f): assert isinstance(calldescr, GcPtrCallDescr) if not we_are_translated(): - calldescr.verify_types(args_i, args_r, args_f, 'ref') + calldescr.verify_types(args_i, args_r, args_f, history.REF) return calldescr.call_stub(func, args_i, args_r, args_f) def bh_call_f(self, func, calldescr, args_i, args_r, args_f): assert isinstance(calldescr, FloatCallDescr) if not we_are_translated(): - calldescr.verify_types(args_i, args_r, args_f, 'float') + calldescr.verify_types(args_i, args_r, args_f, history.FLOAT) return calldescr.call_stub(func, args_i, args_r, args_f) def bh_call_v(self, func, calldescr, args_i, args_r, args_f): assert isinstance(calldescr, VoidCallDescr) if not we_are_translated(): - calldescr.verify_types(args_i, args_r, args_f, 'void') + calldescr.verify_types(args_i, args_r, args_f, history.VOID) return calldescr.call_stub(func, args_i, args_r, args_f) def bh_cast_ptr_to_int(self, ptr): Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/test/test_descr.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/test/test_descr.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/test/test_descr.py Fri May 28 22:45:34 2010 @@ -4,6 +4,7 @@ from pypy.rlib.objectmodel import Symbolic from pypy.rpython.annlowlevel import llhelper from pypy.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr +from pypy.jit.metainterp import history def test_get_size_descr(): c0 = GcCache(False) @@ -145,31 +146,28 @@ c0 = GcCache(False) descr1 = get_call_descr(c0, [lltype.Char, lltype.Signed], lltype.Char) assert descr1.get_result_size(False) == rffi.sizeof(lltype.Char) - assert not descr1.returns_a_pointer() - assert not descr1.returns_a_float() + assert descr1.get_return_type() == history.INT assert descr1.arg_classes == "ii" # T = lltype.GcStruct('T') descr2 = get_call_descr(c0, [lltype.Ptr(T)], lltype.Ptr(T)) assert descr2.get_result_size(False) == rffi.sizeof(lltype.Ptr(T)) - assert descr2.returns_a_pointer() - assert not descr2.returns_a_float() - assert not descr2.returns_a_void() + assert descr2.get_return_type() == history.REF assert descr2.arg_classes == "r" # U = lltype.GcStruct('U', ('x', lltype.Signed)) assert descr2 == get_call_descr(c0, [lltype.Ptr(U)], lltype.Ptr(U)) # V = lltype.Struct('V', ('x', lltype.Signed)) - assert not get_call_descr(c0, [], lltype.Ptr(V)).returns_a_pointer() + assert (get_call_descr(c0, [], lltype.Ptr(V)).get_return_type() == + history.INT) # - assert get_call_descr(c0, [], lltype.Void).returns_a_void() + assert (get_call_descr(c0, [], lltype.Void).get_return_type() == + history.VOID) # descr4 = get_call_descr(c0, [lltype.Float, lltype.Float], lltype.Float) assert descr4.get_result_size(False) == rffi.sizeof(lltype.Float) - assert not descr4.returns_a_pointer() - assert descr4.returns_a_float() - assert not descr4.returns_a_void() + assert descr4.get_return_type() == history.FLOAT assert descr4.arg_classes == "ff" def test_get_call_descr_translated(): @@ -178,14 +176,12 @@ U = lltype.GcStruct('U', ('x', lltype.Signed)) descr3 = get_call_descr(c1, [lltype.Ptr(T)], lltype.Ptr(U)) assert isinstance(descr3.get_result_size(True), Symbolic) - assert descr3.returns_a_pointer() - assert not descr3.returns_a_float() + assert descr3.get_return_type() == history.REF assert descr3.arg_classes == "r" # descr4 = get_call_descr(c1, [lltype.Float, lltype.Float], lltype.Float) assert isinstance(descr4.get_result_size(True), Symbolic) - assert not descr4.returns_a_pointer() - assert descr4.returns_a_float() + assert descr4.get_return_type() == history.FLOAT assert descr4.arg_classes == "ff" def test_call_descr_extra_info(): Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/x86/assembler.py Fri May 28 22:45:34 2010 @@ -1188,6 +1188,7 @@ # if not we_are_translated(): assert bytecode[4] == 0xCC + self.fail_boxes_count = num fail_index = rffi.cast(rffi.LONGP, bytecode)[0] return fail_index Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/x86/runner.py Fri May 28 22:45:34 2010 @@ -73,7 +73,7 @@ return self.assembler.fail_boxes_ptr.getitem(index) def get_latest_value_count(self): - XXX + return self.assembler.fail_boxes_count def clear_latest_values(self, count): setitem = self.assembler.fail_boxes_ptr.setitem Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_assembler.py Fri May 28 22:45:34 2010 @@ -75,35 +75,6 @@ assert ([loc.assembler() for loc in newlocs] == [loc.assembler() for loc in locs if loc is not None]) - # finally, test make_boxes_from_latest_values(), which should - # reproduce the holes - expected_classes = [BoxInt, BoxPtr, BoxFloat, - BoxInt, BoxPtr, BoxFloat, - type(None), type(None), - BoxInt, BoxPtr, BoxFloat] - ptrvalues = {} - S = lltype.GcStruct('S') - for i, cls in enumerate(expected_classes): - if cls == BoxInt: - assembler.fail_boxes_int.setitem(i, 1000 + i) - elif cls == BoxPtr: - s = lltype.malloc(S) - s_ref = lltype.cast_opaque_ptr(llmemory.GCREF, s) - ptrvalues[i] = s_ref - assembler.fail_boxes_ptr.setitem(i, s_ref) - elif cls == BoxFloat: - assembler.fail_boxes_float.setitem(i, 42.5 + i) - boxes = assembler.make_boxes_from_latest_values(bytecode_addr) - assert len(boxes) == len(locs) == len(expected_classes) - for i, (box, expected_class) in enumerate(zip(boxes, expected_classes)): - assert type(box) is expected_class - if expected_class == BoxInt: - assert box.value == 1000 + i - elif expected_class == BoxPtr: - assert box.value == ptrvalues[i] - elif expected_class == BoxFloat: - assert box.value == 42.5 + i - # ____________________________________________________________ def test_failure_recovery_func_no_floats(): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py Fri May 28 22:45:34 2010 @@ -8,7 +8,7 @@ from pypy.rlib.rarithmetic import ovfcheck, r_uint, intmask from pypy.rlib.unroll import unrolling_iterable from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, check_descr -from pypy.jit.metainterp.history import INT, REF, FLOAT, AbstractDescr +from pypy.jit.metainterp.history import INT, REF, FLOAT, VOID, AbstractDescr from pypy.jit.metainterp import resoperation from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.blackhole import BlackholeInterpreter, NULL @@ -75,7 +75,7 @@ else: metainterp.execute_did_not_raise() return BoxFloat(result) - if rettype == 'v': # void + if rettype == VOID: try: cpu.bh_call_v(func, descr, args_i, args_r, args_f) except Exception, e: Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py Fri May 28 22:45:34 2010 @@ -16,6 +16,7 @@ REF = 'r' FLOAT = 'f' HOLE = '_' +VOID = 'v' FAILARGS_LIMIT = 1000 From agaynor at codespeak.net Fri May 28 22:52:55 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Fri, 28 May 2010 22:52:55 +0200 (CEST) Subject: [pypy-svn] r74879 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100528205255.56E51282BD6@codespeak.net> Author: agaynor Date: Fri May 28 22:52:53 2010 New Revision: 74879 Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py pypy/trunk/pypy/module/cpyext/test/foo.c pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Log: Fixed exception raising in __init__ for cpyext (previously they were swallowed). Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/slotdefs.py (original) +++ pypy/trunk/pypy/module/cpyext/slotdefs.py Fri May 28 22:52:53 2010 @@ -28,7 +28,9 @@ def wrap_init(space, w_self, w_args, func, w_kwargs): func_init = rffi.cast(initproc, func) - generic_cpy_call(space, func_init, w_self, w_args, w_kwargs) + res = generic_cpy_call(space, func_init, w_self, w_args, w_kwargs) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception() return None def wrap_unaryfunc(space, w_self, w_args, func): Modified: pypy/trunk/pypy/module/cpyext/test/foo.c ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/foo.c (original) +++ pypy/trunk/pypy/module/cpyext/test/foo.c Fri May 28 22:52:53 2010 @@ -193,8 +193,9 @@ } FuuObject; -void Fuu_init(FuuObject *self, PyObject *args, PyObject *kwargs) { +static int Fuu_init(FuuObject *self, PyObject *args, PyObject *kwargs) { self->val = 42; + return 0; } static PyObject * @@ -417,6 +418,72 @@ }; +static int initerrtype_init(PyObject *self, PyObject *args, PyObject *kwargs) { + PyErr_SetString(PyExc_ValueError, "init raised an error!"); + return -1; +} + + +PyTypeObject InitErrType = { + PyObject_HEAD_INIT(NULL) + 0, + "foo.InitErr", + sizeof(PyObject), + 0, + 0, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_CHECKTYPES, /*tp_flags*/ + 0, /*tp_doc*/ + + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + + /* Attribute descriptor and subclassing stuff */ + + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + + initerrtype_init, /*tp_init*/ + 0, /*tp_alloc will be set to PyType_GenericAlloc in module init*/ + 0, /*tp_new*/ + 0, /*tp_free Low-level free-memory routine */ + 0, /*tp_is_gc For PyObject_IS_GC */ + 0, /*tp_bases*/ + 0, /*tp_mro method resolution order */ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0 /*tp_weaklist*/ +}; + + /* Initialize this module. */ void initfoo(void) @@ -437,6 +504,8 @@ return; if (PyType_Ready(&MetaType) < 0) return; + if (PyType_Ready(&InitErrType) < 0) + return; m = Py_InitModule("foo", foo_functions); if (m == NULL) return; @@ -451,4 +520,6 @@ return; if (PyDict_SetItemString(d, "MetaType", (PyObject *) &MetaType) < 0) return; + if (PyDict_SetItemString(d, "InitErrType", (PyObject *) &InitErrType) < 0) + return; } Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Fri May 28 22:52:53 2010 @@ -173,6 +173,11 @@ re._cache.clear() re._cache_repl.clear() + def test_init_error(self): + module = self.import_module("foo") + raises(ValueError, module.InitErrType) + + class TestTypes(BaseApiTest): def test_type_attributes(self, space, api): w_class = space.appexec([], """(): @@ -209,7 +214,7 @@ w_obj = api._PyType_Lookup(w_type, space.wrap("__invalid")) assert w_obj is None assert api.PyErr_Occurred() is None - + class AppTestSlots(AppTestCpythonExtensionBase): def test_some_slots(self): module = self.import_extension('foo', [ From arigo at codespeak.net Fri May 28 23:57:07 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 May 2010 23:57:07 +0200 (CEST) Subject: [pypy-svn] r74880 - in pypy/branch/blackhole-improvement/pypy: annotation jit/backend/llsupport jit/backend/test jit/backend/x86 jit/metainterp rpython rpython/lltypesystem translator/c/src Message-ID: <20100528215707.DA4D5282BD6@codespeak.net> Author: arigo Date: Fri May 28 23:57:05 2010 New Revision: 74880 Modified: pypy/branch/blackhole-improvement/pypy/annotation/builtin.py pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/gc.py pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py pypy/branch/blackhole-improvement/pypy/jit/backend/x86/ri386.py pypy/branch/blackhole-improvement/pypy/jit/backend/x86/runner.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/logger.py pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/lloperation.py pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/opimpl.py pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/rffi.py pypy/branch/blackhole-improvement/pypy/rpython/raddress.py pypy/branch/blackhole-improvement/pypy/rpython/rbuiltin.py pypy/branch/blackhole-improvement/pypy/translator/c/src/int.h Log: General progress. (Quite boring checkin messages, I know, but it's hard to describe it more precisely than: I spent some time tracking obscure bugs and fixing them) Modified: pypy/branch/blackhole-improvement/pypy/annotation/builtin.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/annotation/builtin.py (original) +++ pypy/branch/blackhole-improvement/pypy/annotation/builtin.py Fri May 28 23:57:05 2010 @@ -346,9 +346,6 @@ def llmemory_cast_int_to_adr(s): return SomeAddress() -def llmemory_get_inthash_from_int(s): - return SomeInteger() # xxx - ##def rarith_ovfcheck(s_obj): ## if isinstance(s_obj, SomeInteger) and s_obj.unsigned: @@ -390,7 +387,6 @@ BUILTIN_ANALYZERS[pypy.rpython.lltypesystem.llmemory.cast_adr_to_ptr] = llmemory_cast_adr_to_ptr BUILTIN_ANALYZERS[pypy.rpython.lltypesystem.llmemory.cast_adr_to_int] = llmemory_cast_adr_to_int BUILTIN_ANALYZERS[pypy.rpython.lltypesystem.llmemory.cast_int_to_adr] = llmemory_cast_int_to_adr -BUILTIN_ANALYZERS[pypy.rpython.lltypesystem.llmemory.get_inthash_from_int] = llmemory_get_inthash_from_int BUILTIN_ANALYZERS[getattr(OSError.__init__, 'im_func', OSError.__init__)] = ( OSError_init) Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/gc.py Fri May 28 23:57:05 2010 @@ -171,7 +171,7 @@ # the object moves) addr = llmemory.cast_ptr_to_adr(gcref) hash = llmemory.cast_adr_to_int(addr) - hash = llmemory.get_inthash_from_int(hash) + hash = rffi.get_real_int(hash) hash -= hash >> self.HASHTABLE_BITS hash &= self.HASHTABLE_SIZE - 1 addr_ref = self.hashtable[hash] Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py Fri May 28 23:57:05 2010 @@ -1663,8 +1663,8 @@ x = cpu.bh_newstr(5) y = cpu.bh_cast_ptr_to_int(x) z = cpu.bh_cast_ptr_to_int(x) - y = llmemory.get_inthash_from_int(y) - z = llmemory.get_inthash_from_int(z) + y = rffi.get_real_int(y) + z = rffi.get_real_int(z) assert type(y) == type(z) == int and y == z def test_sorting_of_fields(self): Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/ri386.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/x86/ri386.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/x86/ri386.py Fri May 28 23:57:05 2010 @@ -1,7 +1,7 @@ from pypy.rlib.rarithmetic import intmask from pypy.rlib.objectmodel import ComputedIntSymbolic, we_are_translated from pypy.rlib.debug import make_sure_not_resized -from pypy.rpython.lltypesystem import llmemory +from pypy.rpython.lltypesystem.rffi import get_real_int class OPERAND(object): _attrs_ = [] @@ -211,7 +211,7 @@ class REL32(OPERAND): width = 4 def __init__(self, absolute_target): - self.absolute_target = absolute_target + self.absolute_target = get_real_int(absolute_target) def assembler(self): return '%d' % (self.absolute_target,) @@ -290,7 +290,7 @@ def imm(value): if isinstance(value, ComputedIntSymbolic): value = value.compute_fn() - value = llmemory.get_inthash_from_int(value) + value = get_real_int(value) if single_byte(value): return imm8(value) else: Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/x86/runner.py Fri May 28 23:57:05 2010 @@ -2,6 +2,7 @@ import ctypes import py from pypy.rpython.lltypesystem import lltype, llmemory, rffi +from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.llinterp import LLInterpreter from pypy.rlib.objectmodel import we_are_translated from pypy.jit.metainterp import history, compile Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/logger.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/logger.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/logger.py Fri May 28 23:57:05 2010 @@ -2,7 +2,7 @@ from pypy.rlib.debug import have_debug_prints from pypy.rlib.debug import debug_start, debug_stop, debug_print from pypy.rlib.objectmodel import we_are_translated -from pypy.rpython.lltypesystem import llmemory +from pypy.rpython.lltypesystem import llmemory, rffi from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.history import Const, ConstInt, Box, \ BoxInt, ConstFloat, BoxFloat, AbstractFailDescr @@ -108,7 +108,7 @@ def int_could_be_an_address(x): if we_are_translated(): - x = llmemory.get_inthash_from_int(x) + x = rffi.get_real_int(x) return not (-32768 <= x <= 32767) else: return isinstance(x, llmemory.AddressAsInt) Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py Fri May 28 23:57:05 2010 @@ -635,14 +635,6 @@ ptr = lltype.cast_int_to_ptr(_NONGCREF, int) return cast_ptr_to_adr(ptr) -def get_inthash_from_int(int): - """Just returns 'int' after translation. Before translation, it convert - an AddressAsInt into a real integer, to use e.g. as a hash. Do not store - the real integer (or the dict using this as a hash) across translation.""" - if isinstance(int, AddressAsInt): - int = lltype.cast_ptr_to_int(int.adr.ptr) - return int - # ____________________________________________________________ # Weakrefs. # Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/lloperation.py Fri May 28 23:57:05 2010 @@ -411,7 +411,6 @@ 'cast_adr_to_ptr': LLOp(canfold=True), 'cast_adr_to_int': LLOp(sideeffects=False), 'cast_int_to_adr': LLOp(canfold=True), - 'get_inthash_from_int': LLOp(canfold=True), 'get_group_member': LLOp(canfold=True), 'get_next_group_member':LLOp(canfold=True), Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/opimpl.py Fri May 28 23:57:05 2010 @@ -340,9 +340,6 @@ def op_cast_int_to_adr(int): return llmemory.cast_int_to_adr(int) -def op_get_inthash_from_int(int): - return llmemory.get_inthash_from_int(int) - ##def op_cast_int_to_adr(x): ## assert type(x) is int ## return llmemory.cast_int_to_adr(x) Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/rffi.py Fri May 28 23:57:05 2010 @@ -905,3 +905,11 @@ """ return cast(lltype.Signed, getattr(pdst, fieldname)) getintfield._annspecialcase_ = 'specialize:ll_and_arg(1)' + +def get_real_int(i): + """No-op after translation. Before translation, it convert an + AddressAsInt into a real integer, to use e.g. as a hash. Do not store + the real integer (or the dict using this as a hash) across translation.""" + # although it's not really a cast, this force_cast forces + # the AddressAsInt to be evaluated with ctypes + return cast(lltype.Signed, i) Modified: pypy/branch/blackhole-improvement/pypy/rpython/raddress.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/raddress.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/raddress.py Fri May 28 23:57:05 2010 @@ -2,7 +2,7 @@ from pypy.tool.pairtype import pairtype from pypy.annotation import model as annmodel from pypy.rpython.lltypesystem.llmemory import NULL, Address, \ - cast_adr_to_int, get_inthash_from_int, fakeaddress + cast_adr_to_int, fakeaddress from pypy.rpython.rmodel import Repr, IntegerRepr from pypy.rpython.rptr import PtrRepr from pypy.rpython.lltypesystem import lltype @@ -57,9 +57,9 @@ get_ll_fasthash_function = get_ll_hash_function def ll_addrhash(addr1): - result = cast_adr_to_int(addr1) # we don't want to have an AddressAsInt instance in this case - return get_inthash_from_int(result) + from pypy.rpython.lltypesystem import rffi + return rffi.get_real_int(cast_adr_to_int(addr1)) address_repr = AddressRepr() Modified: pypy/branch/blackhole-improvement/pypy/rpython/rbuiltin.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/rbuiltin.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/rbuiltin.py Fri May 28 23:57:05 2010 @@ -615,15 +615,9 @@ return hop.genop('cast_int_to_adr', [v_input], resulttype = llmemory.Address) -def rtype_get_inthash_from_int(hop): - v_input, = hop.inputargs(lltype.Signed) - hop.exception_cannot_occur() - return hop.genop('get_inthash_from_int', [v_input], - resulttype = lltype.Signed) - BUILTIN_TYPER[llmemory.cast_ptr_to_adr] = rtype_cast_ptr_to_adr BUILTIN_TYPER[llmemory.cast_adr_to_ptr] = rtype_cast_adr_to_ptr BUILTIN_TYPER[llmemory.cast_adr_to_int] = rtype_cast_adr_to_int BUILTIN_TYPER[llmemory.cast_int_to_adr] = rtype_cast_int_to_adr -BUILTIN_TYPER[llmemory.get_inthash_from_int] = rtype_get_inthash_from_int + Modified: pypy/branch/blackhole-improvement/pypy/translator/c/src/int.h ============================================================================== --- pypy/branch/blackhole-improvement/pypy/translator/c/src/int.h (original) +++ pypy/branch/blackhole-improvement/pypy/translator/c/src/int.h Fri May 28 23:57:05 2010 @@ -297,5 +297,3 @@ #define OP_ULLONG_AND OP_LLONG_AND #define OP_ULLONG_OR OP_LLONG_OR #define OP_ULLONG_XOR OP_LLONG_XOR - -#define OP_GET_INTHASH_FROM_INT(x, r) r = x; From arigo at codespeak.net Sat May 29 00:03:25 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 29 May 2010 00:03:25 +0200 (CEST) Subject: [pypy-svn] r74881 - pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport Message-ID: <20100528220325.B7EE1282BD6@codespeak.net> Author: arigo Date: Sat May 29 00:03:23 2010 New Revision: 74881 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py Log: Add this. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py Sat May 29 00:03:23 2010 @@ -421,6 +421,12 @@ as_array[self.vtable_offset/WORD] = vtable return res + def bh_classof(self, struct): + struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct) + result = struct.typeptr + result_adr = llmemory.cast_ptr_to_adr(struct.typeptr) + return llmemory.cast_adr_to_int(result_adr) + def bh_new_array(self, arraydescr, length): return self.gc_ll_descr.gc_malloc_array(arraydescr, length) From arigo at codespeak.net Sat May 29 12:09:21 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 29 May 2010 12:09:21 +0200 (CEST) Subject: [pypy-svn] r74886 - in pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem: . test Message-ID: <20100529100921.1A237282BAD@codespeak.net> Author: arigo Date: Sat May 29 12:09:20 2010 New Revision: 74886 Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/ll2ctypes.py pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/lltype.py pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Log: Fixes to ll2ctypes handling of AddressAsInt. Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/ll2ctypes.py Sat May 29 12:09:20 2010 @@ -494,7 +494,7 @@ # additionally, this adds mess to __del__ "semantics" _all_callbacks = {} _all_callbacks_results = [] -_callback2obj = {} +_int2obj = {} _callback_exc_info = None def get_rtyper(): @@ -512,7 +512,10 @@ if isinstance(llobj, lltype._uninitialized): return uninitialized2ctypes(llobj.TYPE) if isinstance(llobj, llmemory.AddressAsInt): - return ctypes.cast(lltype2ctypes(llobj.adr), ctypes.c_void_p).value + cobj = ctypes.cast(lltype2ctypes(llobj.adr), ctypes.c_void_p) + res = intmask(cobj.value) + _int2obj[res] = llobj.adr.ptr._obj + return res if isinstance(llobj, llmemory.fakeaddress): llobj = llobj.ptr or 0 @@ -607,8 +610,9 @@ else: ctypes_func_type = get_ctypes_type(T) res = ctypes_func_type(callback) - _callback2obj[ctypes.cast(res, ctypes.c_void_p).value] = container _all_callbacks[key] = res + key2 = intmask(ctypes.cast(res, ctypes.c_void_p).value) + _int2obj[key2] = container return res index = 0 @@ -711,9 +715,9 @@ container = _array_of_known_length(T.TO) container._storage = cobj.contents elif isinstance(T.TO, lltype.FuncType): - cobjkey = ctypes.cast(cobj, ctypes.c_void_p).value - if cobjkey in _callback2obj: - container = _callback2obj[cobjkey] + cobjkey = intmask(ctypes.cast(cobj, ctypes.c_void_p).value) + if cobjkey in _int2obj: + container = _int2obj[cobjkey] else: _callable = get_ctypes_trampoline(T.TO, cobj) return lltype.functionptr(T.TO, getattr(cobj, '__name__', '?'), Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py Sat May 29 12:09:20 2010 @@ -632,7 +632,11 @@ def cast_int_to_adr(int): if isinstance(int, AddressAsInt): return int.adr - ptr = lltype.cast_int_to_ptr(_NONGCREF, int) + try: + ptr = lltype.cast_int_to_ptr(_NONGCREF, int) + except ValueError: + from pypy.rpython.lltypesystem import ll2ctypes + ptr = ll2ctypes._int2obj[int]._as_ptr() return cast_ptr_to_adr(ptr) # ____________________________________________________________ Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/lltype.py Sat May 29 12:09:20 2010 @@ -1840,7 +1840,8 @@ def cast_int_to_ptr(PTRTYPE, oddint): if oddint == 0: return nullptr(PTRTYPE.TO) - assert oddint & 1, "only odd integers can be cast back to ptr" + if not (oddint & 1): + raise ValueError("only odd integers can be cast back to ptr") return _ptr(PTRTYPE, oddint, solid=True) def attachRuntimeTypeInfo(GCSTRUCT, funcptr=None, destrptr=None): Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Sat May 29 12:09:20 2010 @@ -1204,6 +1204,18 @@ res = interpret(f, []) assert res == 6 + def test_force_to_int_and_back(self): + S = lltype.Struct('S') + p = lltype.malloc(S, flavor='raw') + a = llmemory.cast_ptr_to_adr(p) + i = llmemory.cast_adr_to_int(a) + i = rffi.get_real_int(i) + # + a2 = llmemory.cast_int_to_adr(i) + assert a2 == a + # + lltype.free(p) + class TestPlatform(object): def test_lib_on_libpaths(self): from pypy.translator.platform import platform From arigo at codespeak.net Sat May 29 13:53:49 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 29 May 2010 13:53:49 +0200 (CEST) Subject: [pypy-svn] r74887 - pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph Message-ID: <20100529115349.7169A282BD6@codespeak.net> Author: arigo Date: Sat May 29 13:53:47 2010 New Revision: 74887 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py Log: Improve the tests making sure that the CALL operations are given arguments in the expected order. Fails because right now they are always in the order all-INTs, all-REFs, all-FLOATs. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py Sat May 29 13:53:47 2010 @@ -763,18 +763,22 @@ global _last_exception assert _last_exception is None, "exception left behind" assert _call_args_i == _call_args_r == _call_args_f == [] + args_in_order = [] for x in args: T = lltype.typeOf(x) if T is lltype.Signed: + args_in_order.append('i') _call_args_i.append(x) elif T == llmemory.GCREF: + args_in_order.append('r') _call_args_r.append(x) elif T is lltype.Float: + args_in_order.append('f') _call_args_f.append(x) else: raise TypeError(x) try: - return _do_call_common(func) + return _do_call_common(func, args_in_order) except LLException, lle: _last_exception = lle d = {'v': None, @@ -1334,11 +1338,12 @@ def do_call_pushfloat(x): _call_args_f.append(x) -def _do_call_common(f): +def _do_call_common(f, args_in_order=None): ptr = llmemory.cast_int_to_adr(f).ptr FUNC = lltype.typeOf(ptr).TO ARGS = FUNC.ARGS - args = cast_call_args(ARGS, _call_args_i, _call_args_r, _call_args_f) + args = cast_call_args(ARGS, _call_args_i, _call_args_r, _call_args_f, + args_in_order) del _call_args_i[:] del _call_args_r[:] del _call_args_f[:] @@ -1366,25 +1371,39 @@ x = _do_call_common(f) return cast_to_ptr(x) -def cast_call_args(ARGS, args_i, args_r, args_f): +def cast_call_args(ARGS, args_i, args_r, args_f, args_in_order=None): argsiter_i = iter(args_i) argsiter_r = iter(args_r) argsiter_f = iter(args_f) + if args_in_order is not None: + orderiter = iter(args_in_order) args = [] for TYPE in ARGS: if TYPE is lltype.Void: x = None else: if isinstance(TYPE, ootype.OOType): + if args_in_order is not None: + n = orderiter.next() + assert n == 'r' x = argsiter_r.next() x = ootype.cast_from_object(TYPE, x) elif isinstance(TYPE, lltype.Ptr) and TYPE.TO._gckind == 'gc': + if args_in_order is not None: + n = orderiter.next() + assert n == 'r' x = argsiter_r.next() x = cast_from_ptr(TYPE, x) elif TYPE is lltype.Float: + if args_in_order is not None: + n = orderiter.next() + assert n == 'f' x = argsiter_f.next() x = cast_from_float(TYPE, x) else: + if args_in_order is not None: + n = orderiter.next() + assert n == 'i' x = argsiter_i.next() x = cast_from_int(TYPE, x) args.append(x) From arigo at codespeak.net Sat May 29 13:53:59 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 29 May 2010 13:53:59 +0200 (CEST) Subject: [pypy-svn] r74888 - in pypy/branch/blackhole-improvement/pypy/jit: backend/llgraph backend/llsupport metainterp Message-ID: <20100529115359.7322D282BD8@codespeak.net> Author: arigo Date: Sat May 29 13:53:57 2010 New Revision: 74888 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/descr.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: Fix the previous failure. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py Sat May 29 13:53:57 2010 @@ -23,11 +23,16 @@ class Descr(history.AbstractDescr): - def __init__(self, ofs, typeinfo, extrainfo=None, name=None): + def __init__(self, ofs, typeinfo, extrainfo=None, name=None, + arg_types=None): self.ofs = ofs self.typeinfo = typeinfo self.extrainfo = extrainfo self.name = name + self.arg_types = arg_types + + def get_arg_types(self): + return self.arg_types def get_return_type(self): return self.typeinfo @@ -98,12 +103,13 @@ assert self.translate_support_code return False - def getdescr(self, ofs, typeinfo='?', extrainfo=None, name=None): - key = (ofs, typeinfo, extrainfo, name) + def getdescr(self, ofs, typeinfo='?', extrainfo=None, name=None, + arg_types=None): + key = (ofs, typeinfo, extrainfo, name, arg_types) try: return self._descrs[key] except KeyError: - descr = Descr(ofs, typeinfo, extrainfo, name) + descr = Descr(ofs, typeinfo, extrainfo, name, arg_types) self._descrs[key] = descr return descr @@ -271,8 +277,14 @@ return self.getdescr(ofs, token[0], name=fieldname) def calldescrof(self, FUNC, ARGS, RESULT, extrainfo=None): + arg_types = [] + for ARG in ARGS: + token = history.getkind(ARG) + if token != 'void': + arg_types.append(token[0]) token = history.getkind(RESULT) - return self.getdescr(0, token[0], extrainfo=extrainfo) + return self.getdescr(0, token[0], extrainfo=extrainfo, + arg_types=''.join(arg_types)) def grab_exc_value(self): return llimpl.grab_exc_value() Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/descr.py Sat May 29 13:53:57 2010 @@ -190,6 +190,9 @@ def get_extra_info(self): return self.extrainfo + def get_arg_types(self): + return self.arg_classes + def get_return_type(self): return self._return_type Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py Sat May 29 13:53:57 2010 @@ -127,6 +127,12 @@ def _clone_if_mutable(self): return self + def get_arg_types(self): + """ Implement in call descr. + Must return a string of INT, REF and FLOAT ('i', 'r', 'f'). + """ + raise NotImplementedError + def get_return_type(self): """ Implement in call descr. Must return INT, REF, FLOAT, or 'v' for void. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Sat May 29 13:53:57 2010 @@ -985,7 +985,38 @@ def do_residual_call(self, funcbox, descr, argboxes, assembler_call_token=None): - allboxes = [funcbox] + argboxes + # First build allboxes: it may need some reordering from the + # list provided in argboxes, depending on the order in which + # the arguments are expected by the function + allboxes = [None] * (len(argboxes)+1) + allboxes[0] = funcbox + src_i = src_r = src_f = 0 + i = 1 + for kind in descr.get_arg_types(): + if kind == history.INT: + while True: + box = argboxes[src_i] + src_i += 1 + if box.type == history.INT: + break + elif kind == history.REF: + while True: + box = argboxes[src_r] + src_r += 1 + if box.type == history.REF: + break + elif kind == history.FLOAT: + while True: + box = argboxes[src_f] + src_f += 1 + if box.type == history.FLOAT: + break + else: + raise AssertionError + allboxes[i] = box + i += 1 + assert i == len(allboxes) + # effectinfo = descr.get_extra_info() if (effectinfo is None or effectinfo.extraeffect == @@ -995,11 +1026,12 @@ self.metainterp.vable_and_vrefs_before_residual_call() resbox = self.metainterp.execute_and_record_varargs( rop.CALL_MAY_FORCE, allboxes, descr=descr) + self.metainterp.vrefs_after_residual_call() if assembler_call_token is not None: self.metainterp.direct_assembler_call(assembler_call_token) if resbox is not None: self.make_result_of_lastop(resbox) - self.metainterp.vable_and_vrefs_after_residual_call() + self.metainterp.vable_after_residual_call() self.generate_guard(rop.GUARD_NOT_FORCED, None) self.metainterp.handle_possible_exception() return resbox @@ -1768,7 +1800,7 @@ force_token_box], None, descr=vinfo.vable_token_descr) - def vable_and_vrefs_after_residual_call(self): + def vrefs_after_residual_call(self): vrefinfo = self.staticdata.virtualref_info for i in range(0, len(self.virtualref_boxes), 2): virtualbox = self.virtualref_boxes[i] @@ -1780,7 +1812,8 @@ # generating a VIRTUAL_REF_FINISH on it and replacing # it by ConstPtr(NULL). self.stop_tracking_virtualref(i) - # + + def vable_after_residual_call(self): vinfo = self.staticdata.virtualizable_info if vinfo is not None: virtualizable_box = self.virtualizable_boxes[-1] From arigo at codespeak.net Sat May 29 14:03:46 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 29 May 2010 14:03:46 +0200 (CEST) Subject: [pypy-svn] r74889 - pypy/branch/blackhole-improvement/pypy/jit/metainterp/test Message-ID: <20100529120346.AB7C5282BD6@codespeak.net> Author: arigo Date: Sat May 29 14:03:45 2010 New Revision: 74889 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Log: Yay. test_basic passes in backend/x86/. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Sat May 29 14:03:45 2010 @@ -1519,7 +1519,7 @@ if not self.basic: py.test.skip("test written in a style that " "means it's frontend only") - from pypy.rpython.lltypesystem import lltype, llmemory + from pypy.rpython.lltypesystem import lltype, llmemory, rffi TP = lltype.GcStruct('S1') def f(p): @@ -1528,9 +1528,9 @@ x = lltype.malloc(TP) xref = lltype.cast_opaque_ptr(llmemory.GCREF, x) res = self.interp_operations(f, [xref]) - y = llmemory.cast_int_to_adr(res) - y = llmemory.cast_adr_to_ptr(y, lltype.Ptr(TP)) - assert x == y + y = llmemory.cast_ptr_to_adr(x) + y = llmemory.cast_adr_to_int(y) + assert rffi.get_real_int(res) == rffi.get_real_int(y) # TP = lltype.Struct('S2') prebuilt = [lltype.malloc(TP, immortal=True), @@ -1540,9 +1540,9 @@ n = lltype.cast_ptr_to_int(p) return n res = self.interp_operations(f, [1]) - y = llmemory.cast_int_to_adr(res) - y = llmemory.cast_adr_to_ptr(y, lltype.Ptr(TP)) - assert prebuilt[1] == y + y = llmemory.cast_ptr_to_adr(prebuilt[1]) + y = llmemory.cast_adr_to_int(y) + assert rffi.get_real_int(res) == rffi.get_real_int(y) def test_collapsing_ptr_eq(self): S = lltype.GcStruct('S') From getxsick at codespeak.net Sat May 29 18:55:22 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Sat, 29 May 2010 18:55:22 +0200 (CEST) Subject: [pypy-svn] r74890 - in pypy/branch/fast-ctypes/pypy/module/jitffi: . test Message-ID: <20100529165522.EA2FD282BAD@codespeak.net> Author: getxsick Date: Sat May 29 18:55:21 2010 New Revision: 74890 Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/__init__.py pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py Log: first lines of the new CDLL class Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/__init__.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/__init__.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/__init__.py Sat May 29 18:55:21 2010 @@ -2,7 +2,7 @@ class Module(MixedModule): interpleveldefs = { - 'load' : 'interp_jitffi.W_load', + 'CDLL' : 'interp_jitffi.W_CDLL', } appleveldefs = {} Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Sat May 29 18:55:21 2010 @@ -1,4 +1,41 @@ -from pypy.interpreter.baseobjspace import Wrappable +from pypy.interpreter.baseobjspace import ObjSpace, W_Root, Wrappable +from pypy.interpreter.error import operationerrfmt +from pypy.interpreter.gateway import interp2app +from pypy.interpreter.typedef import TypeDef +from pypy.rlib import libffi +from pypy.rpython.lltypesystem import rffi, lltype -class W_load(Wrappable): - pass +class W_CDLL(Wrappable): + def __init__(self, space, name): + try: + self.cdll = libffi.CDLL(name) + except libffi.DLOpenError, e: + raise operationerrfmt(space.w_OSError, '%s: %s', name, + e.msg or 'unspecified error') + self.name = name + self.space = space + + def call(self, space, func, a, b): # XXX temporary fixed number of func args + try: + addr = rffi.cast(lltype.Unsigned, self.cdll.getaddressindll(func)) + except KeyError: + raise operationerrfmt(space.w_ValueError, + "Cannot find symbol %s", func) + # XXX unfinished yet + call.unwrap_spec = ['self', ObjSpace, str, int, int] + +def descr_new_cdll(space, w_type, name): + try: + return space.wrap(W_CDLL(space, name)) + except OSError, e: + raise wrap_oserror(space, e) +descr_new_cdll.unwrap_spec = [ObjSpace, W_Root, str] + +W_CDLL.typedef = TypeDef( + 'CDLL', + __new__ = interp2app(descr_new_cdll), + call = interp2app(W_CDLL.call), + __doc__ = """ C Dynamically loaded library +use CDLL(libname) to create a handle to a C library (the argument is processed +the same way as dlopen processes it).""" +) Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py Sat May 29 18:55:21 2010 @@ -16,7 +16,7 @@ } ''')) - symbols = ["sum"] + symbols = ["add_integers"] eci = ExternalCompilationInfo(export_symbols=symbols) return str(platform.compile([c_file], eci, 'x', standalone=False)) @@ -30,6 +30,6 @@ def test_raw_callable(self): import jitffi - lib = jitffi.load(self.lib_name) + lib = jitffi.CDLL(self.lib_name) res = lib.call('add_integers', 1, 2) assert res == 3 From getxsick at codespeak.net Sat May 29 19:05:52 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Sat, 29 May 2010 19:05:52 +0200 (CEST) Subject: [pypy-svn] r74891 - pypy/trunk/pypy/jit/backend/test Message-ID: <20100529170552.D9135282BAD@codespeak.net> Author: getxsick Date: Sat May 29 19:05:51 2010 New Revision: 74891 Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py Log: kill redundant imports Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/runner_test.py (original) +++ pypy/trunk/pypy/jit/backend/test/runner_test.py Sat May 29 19:05:51 2010 @@ -1,4 +1,3 @@ - import py, sys, random, os, struct, operator from pypy.jit.metainterp.history import (AbstractFailDescr, BasicFailDescr, @@ -9,13 +8,11 @@ ConstObj, BoxFloat, ConstFloat) from pypy.jit.metainterp.resoperation import ResOperation, rop from pypy.jit.metainterp.typesystem import deref +from pypy.jit.metainterp.test.oparser import parse from pypy.rpython.lltypesystem import lltype, llmemory, rstr, rffi, rclass from pypy.rpython.ootypesystem import ootype -from pypy.rlib.rarithmetic import r_uint, intmask -from pypy.jit.metainterp.test.oparser import parse from pypy.rpython.annlowlevel import llhelper from pypy.rpython.llinterp import LLException -from pypy.jit.metainterp.test.oparser import parse class Runner(object): From fijal at codespeak.net Sat May 29 19:29:03 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 29 May 2010 19:29:03 +0200 (CEST) Subject: [pypy-svn] r74892 - pypy/trunk/pypy/objspace/std/test Message-ID: <20100529172903.DAC5F282BAD@codespeak.net> Author: fijal Date: Sat May 29 19:29:02 2010 New Revision: 74892 Modified: pypy/trunk/pypy/objspace/std/test/test_floatobject.py Log: Fix tests for windows (hopefully) Modified: pypy/trunk/pypy/objspace/std/test/test_floatobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_floatobject.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_floatobject.py Sat May 29 19:29:02 2010 @@ -92,9 +92,10 @@ def test_float_string(self): assert 42 == float("42") assert 42.25 == float("42.25") - assert str(float("inf")).startswith("inf") - assert str(float("-INf")).startswith("-inf") - assert str(float("-nAn")).startswith("nan") + inf = 1e200*1e200 + assert float("inf") == inf + assert float("-INf") == -inf + assert 'nan' in str(float("-nAn")).lower() def test_float_unicode(self): # u00A0 and u2000 are some kind of spaces From fijal at codespeak.net Sat May 29 20:44:18 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 29 May 2010 20:44:18 +0200 (CEST) Subject: [pypy-svn] r74893 - pypy/release/1.2.1.x Message-ID: <20100529184418.82F3A282BAD@codespeak.net> Author: fijal Date: Sat May 29 20:44:17 2010 New Revision: 74893 Added: pypy/release/1.2.1.x/ (props changed) - copied from r74892, pypy/trunk/ Log: Create a branch (from trunk) to try to release 1.2.1 ASAP (sans cpyext for now) From fijal at codespeak.net Sat May 29 20:48:18 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 29 May 2010 20:48:18 +0200 (CEST) Subject: [pypy-svn] r74894 - pypy/release/1.2.1.x/pypy/config Message-ID: <20100529184818.39C85282BAD@codespeak.net> Author: fijal Date: Sat May 29 20:48:16 2010 New Revision: 74894 Modified: pypy/release/1.2.1.x/pypy/config/pypyoption.py Log: Disable cpyext for the release Modified: pypy/release/1.2.1.x/pypy/config/pypyoption.py ============================================================================== --- pypy/release/1.2.1.x/pypy/config/pypyoption.py (original) +++ pypy/release/1.2.1.x/pypy/config/pypyoption.py Sat May 29 20:48:16 2010 @@ -29,7 +29,7 @@ "rctime" , "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios", "zlib", "struct", "md5", "sha", "bz2", "_minimal_curses", "cStringIO", - "thread", "itertools", "pyexpat", "_ssl", "cpyext"] + "thread", "itertools", "pyexpat", "_ssl" ]#, "cpyext"] disabled for release )) working_oo_modules = default_modules.copy() From fijal at codespeak.net Sat May 29 20:49:10 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 29 May 2010 20:49:10 +0200 (CEST) Subject: [pypy-svn] r74895 - pypy/release/1.2.1.x/pypy/rpython/lltypesystem Message-ID: <20100529184910.798BF282BAD@codespeak.net> Author: fijal Date: Sat May 29 20:49:09 2010 New Revision: 74895 Modified: pypy/release/1.2.1.x/pypy/rpython/lltypesystem/rffi.py Log: Revert a change that broke benchmarks. Modified: pypy/release/1.2.1.x/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/release/1.2.1.x/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/release/1.2.1.x/pypy/rpython/lltypesystem/rffi.py Sat May 29 20:49:09 2010 @@ -645,7 +645,7 @@ """ str_chars_offset = (offsetof(STRTYPE, 'chars') + \ itemoffsetof(STRTYPE.chars, 0)) - gc_buf = lltype.nullptr(STRTYPE) # rgc.malloc_nonmovable(STRTYPE, count) + gc_buf = rgc.malloc_nonmovable(STRTYPE, count) if gc_buf: realbuf = cast_ptr_to_adr(gc_buf) + str_chars_offset raw_buf = cast(TYPEP, realbuf) From fijal at codespeak.net Sat May 29 20:53:43 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 29 May 2010 20:53:43 +0200 (CEST) Subject: [pypy-svn] r74896 - pypy/release/1.2.1.x/pypy/module/sys Message-ID: <20100529185343.598BA282BAD@codespeak.net> Author: fijal Date: Sat May 29 20:53:40 2010 New Revision: 74896 Modified: pypy/release/1.2.1.x/pypy/module/sys/version.py Log: bump version revision number Modified: pypy/release/1.2.1.x/pypy/module/sys/version.py ============================================================================== --- pypy/release/1.2.1.x/pypy/module/sys/version.py (original) +++ pypy/release/1.2.1.x/pypy/module/sys/version.py Sat May 29 20:53:40 2010 @@ -7,7 +7,7 @@ CPYTHON_VERSION = (2, 5, 2, "beta", 42) CPYTHON_API_VERSION = 1012 -PYPY_VERSION = (1, 2, 0, "beta", '?') +PYPY_VERSION = (1, 2, 1, "beta", '?') # the last item is replaced by the svn revision ^^^ TRIM_URL_UP_TO = 'svn/pypy/' From fijal at codespeak.net Sat May 29 20:58:33 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 29 May 2010 20:58:33 +0200 (CEST) Subject: [pypy-svn] r74897 - pypy/trunk/pypy/doc Message-ID: <20100529185833.BC0DA282BAD@codespeak.net> Author: fijal Date: Sat May 29 20:58:31 2010 New Revision: 74897 Modified: pypy/trunk/pypy/doc/how-to-release.txt Log: Update a bit this. Needs more Modified: pypy/trunk/pypy/doc/how-to-release.txt ============================================================================== --- pypy/trunk/pypy/doc/how-to-release.txt (original) +++ pypy/trunk/pypy/doc/how-to-release.txt Sat May 29 20:58:31 2010 @@ -25,9 +25,10 @@ necessary * update pypy/doc/contributor.txt (and possibly LICENSE) * update README +* update pypy.org (under extradoc/pypy.org), rebuild and commit +XXX stuff below is out-of-date or at least needs some review * write release announcement pypy/doc/release-x.y(.z).txt the release announcement should contain a direct link to the download page - (which is getting started). * update pypy/doc/getting-started.txt links at the top and release number references, make sure it is generally up-to-date * use, after the necessary updates, pypy/tool/makerelease.py to From fijal at codespeak.net Sat May 29 21:00:37 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 29 May 2010 21:00:37 +0200 (CEST) Subject: [pypy-svn] r74898 - pypy/trunk/pypy/doc Message-ID: <20100529190037.33F7D282BAD@codespeak.net> Author: fijal Date: Sat May 29 21:00:36 2010 New Revision: 74898 Modified: pypy/trunk/pypy/doc/how-to-release.txt Log: Supported binaries Modified: pypy/trunk/pypy/doc/how-to-release.txt ============================================================================== --- pypy/trunk/pypy/doc/how-to-release.txt (original) +++ pypy/trunk/pypy/doc/how-to-release.txt Sat May 29 21:00:36 2010 @@ -26,6 +26,12 @@ * update pypy/doc/contributor.txt (and possibly LICENSE) * update README * update pypy.org (under extradoc/pypy.org), rebuild and commit +* build following binaries (supported versions for now): + JIT: windows, linux, os/x + no JIT: windows, linux, os/x + sandbox: linux, os/x + stackless: windows, linux, os/x + XXX stuff below is out-of-date or at least needs some review * write release announcement pypy/doc/release-x.y(.z).txt the release announcement should contain a direct link to the download page From fijal at codespeak.net Sat May 29 21:02:08 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 29 May 2010 21:02:08 +0200 (CEST) Subject: [pypy-svn] r74899 - pypy/release/1.2.1.x/pypy/config Message-ID: <20100529190208.196D0282BAD@codespeak.net> Author: fijal Date: Sat May 29 21:01:44 2010 New Revision: 74899 Modified: pypy/release/1.2.1.x/pypy/config/translationoption.py Log: Turn off jit profiling for the release (by default) Modified: pypy/release/1.2.1.x/pypy/config/translationoption.py ============================================================================== --- pypy/release/1.2.1.x/pypy/config/translationoption.py (original) +++ pypy/release/1.2.1.x/pypy/config/translationoption.py Sat May 29 21:01:44 2010 @@ -112,7 +112,7 @@ default="auto", cmdline="--jit-backend"), ChoiceOption("jit_debug", "the amount of debugging dumps made by the JIT", ["off", "profile", "steps", "detailed"], - default="profile", # XXX for now + default="off", cmdline="--jit-debug"), ChoiceOption("jit_profiler", "integrate profiler support into the JIT", ["off", "oprofile"], From fijal at codespeak.net Sat May 29 22:00:48 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 29 May 2010 22:00:48 +0200 (CEST) Subject: [pypy-svn] r74900 - pypy/build/bot2/pypybuildbot Message-ID: <20100529200048.67CD3282BAD@codespeak.net> Author: fijal Date: Sat May 29 22:00:46 2010 New Revision: 74900 Modified: pypy/build/bot2/pypybuildbot/builds.py Log: Try to harness buildbot into creating different sets of nightly builds Modified: pypy/build/bot2/pypybuildbot/builds.py ============================================================================== --- pypy/build/bot2/pypybuildbot/builds.py (original) +++ pypy/build/bot2/pypybuildbot/builds.py Sat May 29 22:00:46 2010 @@ -110,15 +110,6 @@ if pypyjit: # upload nightly build, if we're running jit tests - nightly = os.path.expanduser('~/nightly/pypy-c-jit-%(got_revision)s-' + platform + '.bz2') - self.addStep(ShellCmd( - description="compress pypy-c", - command=["bzip2", "-kf", "pypy/translator/goal/pypy-c"])) - pypy_c_rel = 'build/pypy/translator/goal/pypy-c.bz2' - self.addStep(transfer.FileUpload(slavesrc=pypy_c_rel, - masterdest=WithProperties(nightly), - workdir='.', - blocksize=100*1024)) self.addStep(ShellCmd( description="pypyjit tests", command=["python", "pypy/test_all.py", @@ -126,6 +117,22 @@ "--resultlog=pypyjit.log", "pypy/module/pypyjit/test"], logfiles={'pytestLog': 'pypyjit.log'})) + self.addStep(ShellCmd( + description="compress pypy-c", + command=["bzip2", "-kf", "pypy/translator/goal/pypy-c"])) + if pypyjit: + kind = 'jit' + else: + if '--stackless' in translationArgs: + kind = 'stackless' + else: + kind = 'nojit' + nightly = os.path.expanduser('~/nightly/pypy-c-' + kind + '-%(got_revision)s-' + platform + '.bz2') + pypy_c_rel = 'build/pypy/translator/goal/pypy-c.bz2' + self.addStep(transfer.FileUpload(slavesrc=pypy_c_rel, + masterdest=WithProperties(nightly), + workdir='.', + blocksize=100*1024)) class JITBenchmark(factory.BuildFactory): From fijal at codespeak.net Sat May 29 22:08:11 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 29 May 2010 22:08:11 +0200 (CEST) Subject: [pypy-svn] r74901 - pypy/trunk/pypy/lib/test2 Message-ID: <20100529200811.E43C6282BAD@codespeak.net> Author: fijal Date: Sat May 29 22:08:10 2010 New Revision: 74901 Modified: pypy/trunk/pypy/lib/test2/test_stackless.py Log: Hopefully fix this test. We need _socket module (I don't even want to know why faking does not work here) Modified: pypy/trunk/pypy/lib/test2/test_stackless.py ============================================================================== --- pypy/trunk/pypy/lib/test2/test_stackless.py (original) +++ pypy/trunk/pypy/lib/test2/test_stackless.py Sat May 29 22:08:10 2010 @@ -3,7 +3,7 @@ class AppTest_Stackless: def setup_class(cls): - space = gettestobjspace(usemodules=('_stackless',)) + space = gettestobjspace(usemodules=('_stackless','_socket')) cls.space = space # cannot test the unpickle part on top of py.py cls.w_can_unpickle = space.wrap(bool(option.runappdirect)) From fijal at codespeak.net Sat May 29 22:10:18 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 29 May 2010 22:10:18 +0200 (CEST) Subject: [pypy-svn] r74902 - in pypy/release/1.2.1.x: . pypy/lib/test2 Message-ID: <20100529201018.3BCE0282BAD@codespeak.net> Author: fijal Date: Sat May 29 22:10:16 2010 New Revision: 74902 Modified: pypy/release/1.2.1.x/ (props changed) pypy/release/1.2.1.x/pypy/lib/test2/test_stackless.py Log: Merge 74900 from trunk Modified: pypy/release/1.2.1.x/pypy/lib/test2/test_stackless.py ============================================================================== --- pypy/release/1.2.1.x/pypy/lib/test2/test_stackless.py (original) +++ pypy/release/1.2.1.x/pypy/lib/test2/test_stackless.py Sat May 29 22:10:16 2010 @@ -3,7 +3,7 @@ class AppTest_Stackless: def setup_class(cls): - space = gettestobjspace(usemodules=('_stackless',)) + space = gettestobjspace(usemodules=('_stackless','_socket')) cls.space = space # cannot test the unpickle part on top of py.py cls.w_can_unpickle = space.wrap(bool(option.runappdirect)) From fijal at codespeak.net Sun May 30 01:40:19 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 30 May 2010 01:40:19 +0200 (CEST) Subject: [pypy-svn] r74903 - pypy/trunk/pypy/objspace/std Message-ID: <20100529234019.83E65282BAD@codespeak.net> Author: fijal Date: Sun May 30 01:40:16 2010 New Revision: 74903 Modified: pypy/trunk/pypy/objspace/std/unicodetype.py Log: Leave a comment that we might want to have a perf shortcut here Modified: pypy/trunk/pypy/objspace/std/unicodetype.py ============================================================================== --- pypy/trunk/pypy/objspace/std/unicodetype.py (original) +++ pypy/trunk/pypy/objspace/std/unicodetype.py Sun May 30 01:40:16 2010 @@ -183,6 +183,8 @@ return encoding, errors def encode_object(space, w_object, encoding, errors): + # XXX write down shortcuts for performance for common encodings, + # just like CPython if encoding is None: # Get the encoder functions as a wrapped object. # This lookup is cached. From arigo at codespeak.net Sun May 30 12:27:02 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 12:27:02 +0200 (CEST) Subject: [pypy-svn] r74904 - pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport Message-ID: <20100530102702.66B5B282B9C@codespeak.net> Author: arigo Date: Sun May 30 12:27:00 2010 New Revision: 74904 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py Log: Fix exception handling. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py Sun May 30 12:27:00 2010 @@ -50,7 +50,7 @@ self._setup_exception_handling_translated() else: self._setup_exception_handling_untranslated() - self.clear_exception() + self.saved_exc_value = lltype.nullptr(rclass.OBJECT) self.setup() if translate_support_code: self._setup_on_leave_jitted_translated() @@ -107,12 +107,10 @@ def save_exception(): # copy from _exception_emulator to the real attributes on self - tp_i = _exception_emulator[0] - v_i = _exception_emulator[1] + v_i = _exception_emulator[1] _exception_emulator[0] = 0 _exception_emulator[1] = 0 - self.saved_exception = tp_i - self.saved_exc_value = self._cast_int_to_gcref(v_i) + self.saved_exc_value = rffi.cast(rclass.OBJECTPTR, v_i) self.pos_exception = pos_exception self.pos_exc_value = pos_exc_value @@ -131,15 +129,13 @@ def save_exception(): addr = llop.get_exception_addr(llmemory.Address) - exception = rffi.cast(lltype.Signed, addr.address[0]) addr.address[0] = llmemory.NULL addr = llop.get_exc_value_addr(llmemory.Address) - exc_value = rffi.cast(llmemory.GCREF, addr.address[0]) + exc_value = rffi.cast(rclass.OBJECTPTR, addr.address[0]) addr.address[0] = llmemory.NULL # from now on, the state is again consistent -- no more RPython # exception is set. The following code produces a write barrier # in the assignment to self.saved_exc_value, as needed. - self.saved_exception = exception self.saved_exc_value = exc_value self.pos_exception = pos_exception @@ -177,16 +173,10 @@ f = llhelper(self._ON_JIT_LEAVE_FUNC, self.on_leave_jitted_noexc) return rffi.cast(lltype.Signed, f) - def get_exception(self): - return self.saved_exception - - def get_exc_value(self): - return self.saved_exc_value - - def clear_exception(self): - self.saved_exception = 0 - self.saved_exc_value = lltype.nullptr(llmemory.GCREF.TO) - + def grab_exc_value(self): + exc = self.saved_exc_value + self.saved_exc_value = lltype.nullptr(rclass.OBJECT) + return exc # ------------------- helpers and descriptions -------------------- From arigo at codespeak.net Sun May 30 12:31:09 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 12:31:09 +0200 (CEST) Subject: [pypy-svn] r74905 - in pypy/branch/blackhole-improvement/pypy/jit/backend: llsupport x86/test Message-ID: <20100530103109.7DD4F282B9C@codespeak.net> Author: arigo Date: Sun May 30 12:31:07 2010 New Revision: 74905 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_gc_integration.py Log: Fix test. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py Sun May 30 12:31:07 2010 @@ -60,9 +60,6 @@ def setup(self): pass - def set_class_sizes(self, class_sizes): - self.class_sizes = class_sizes - def _setup_prebuilt_error(self, prefix, Class): if self.rtyper is not None: # normal case bk = self.rtyper.annotator.bookkeeper Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_gc_integration.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_gc_integration.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_gc_integration.py Sun May 30 12:31:07 2010 @@ -6,6 +6,7 @@ from pypy.jit.metainterp.history import ResOperation, BoxInt, ConstInt,\ BoxPtr, ConstPtr, TreeLoop from pypy.jit.metainterp.resoperation import rop, ResOperation +from pypy.jit.codewriter import heaptracker from pypy.jit.backend.llsupport.descr import GcCache from pypy.jit.backend.llsupport.gc import GcLLDescription from pypy.jit.backend.x86.runner import CPU @@ -228,7 +229,7 @@ NODE2 = lltype.Struct('node2', ('tid', lltype.Signed), ('vtable', lltype.Ptr(rclass.OBJECT_VTABLE))) descrsize = cpu.sizeof(NODE2) - cpu.set_class_sizes({vtable_int: descrsize}) + heaptracker.register_known_gctype(cpu, vtable, NODE2) self.descrsize = descrsize self.vtable_int = vtable_int From arigo at codespeak.net Sun May 30 12:43:58 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 12:43:58 +0200 (CEST) Subject: [pypy-svn] r74906 - pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test Message-ID: <20100530104358.A9DF8282B9C@codespeak.net> Author: arigo Date: Sun May 30 12:43:57 2010 New Revision: 74906 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_assembler.py pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_regalloc.py Log: Fix tests. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_assembler.py Sun May 30 12:43:57 2010 @@ -215,6 +215,13 @@ # bits of the float were correctly saved and restored. assert assembler.fail_boxes_float.getitem(i) == expected_floats[i] + # verify that until clear_latest_values() is called, reading the + # same values multiple times work + for i in range(len(content)): + assert assembler.fail_boxes_int.getitem(i) == expected_ints[i] + assert assembler.fail_boxes_ptr.getitem(i) == expected_ptrs[i] + assert assembler.fail_boxes_float.getitem(i) == expected_floats[i] + class FakeProfileAgent(object): def __init__(self): self.functions = [] Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_regalloc.py Sun May 30 12:43:57 2010 @@ -211,10 +211,9 @@ ''' S = lltype.GcStruct('S') ptr = lltype.malloc(S) + self.cpu.clear_latest_values(2) self.interpret(ops, [0, ptr]) assert self.getptr(0, lltype.Ptr(S)) == ptr - assert not self.cpu.assembler.fail_boxes_ptr.getitem(0) - assert not self.cpu.assembler.fail_boxes_ptr.getitem(1) def test_exception_bridge_no_exception(self): ops = ''' @@ -526,16 +525,16 @@ # makes sense any more. ops = ''' [f0, f1, f2, f3, f4, f5, f6, f7, f8, f9] - i0 = float_ne(f0, ConstFloat(0.0)) - i1 = float_ne(f1, ConstFloat(0.0)) - i2 = float_ne(f2, ConstFloat(0.0)) - i3 = float_ne(f3, ConstFloat(0.0)) - i4 = float_ne(f4, ConstFloat(0.0)) - i5 = float_ne(f5, ConstFloat(0.0)) - i6 = float_ne(f6, ConstFloat(0.0)) - i7 = float_ne(f7, ConstFloat(0.0)) - i8 = float_ne(f8, ConstFloat(0.0)) - i9 = float_ne(f9, ConstFloat(0.0)) + i0 = float_ne(f0, 0.0) + i1 = float_ne(f1, 0.0) + i2 = float_ne(f2, 0.0) + i3 = float_ne(f3, 0.0) + i4 = float_ne(f4, 0.0) + i5 = float_ne(f5, 0.0) + i6 = float_ne(f6, 0.0) + i7 = float_ne(f7, 0.0) + i8 = float_ne(f8, 0.0) + i9 = float_ne(f9, 0.0) finish(i0, i1, i2, i3, i4, i5, i6, i7, i8, i9) ''' loop = self.interpret(ops, [0.0, .1, .2, .3, .4, .5, .6, .7, .8, .9]) From arigo at codespeak.net Sun May 30 12:48:13 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 12:48:13 +0200 (CEST) Subject: [pypy-svn] r74907 - pypy/branch/blackhole-improvement/pypy/jit/backend/test Message-ID: <20100530104813.EBF11282B9C@codespeak.net> Author: arigo Date: Sun May 30 12:48:12 2010 New Revision: 74907 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py Log: Fix test. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/test/runner_test.py Sun May 30 12:48:12 2010 @@ -1163,10 +1163,12 @@ x = lltype.cast_opaque_ptr(llmemory.GCREF, x) res = self.execute_operation(rop.CAST_PTR_TO_INT, [BoxPtr(x)], 'int').value - assert res == llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(x)) + expected = llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(x)) + assert rffi.get_real_int(res) == rffi.get_real_int(expected) res = self.execute_operation(rop.CAST_PTR_TO_INT, [ConstPtr(x)], 'int').value - assert res == llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(x)) + expected = llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(x)) + assert rffi.get_real_int(res) == rffi.get_real_int(expected) def test_ooops_non_gc(self): x = lltype.malloc(lltype.Struct('x'), flavor='raw') From arigo at codespeak.net Sun May 30 12:56:05 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 12:56:05 +0200 (CEST) Subject: [pypy-svn] r74908 - pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test Message-ID: <20100530105605.C404B36C537@codespeak.net> Author: arigo Date: Sun May 30 12:56:04 2010 New Revision: 74908 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_runner.py Log: Fix test. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_runner.py Sun May 30 12:56:04 2010 @@ -259,9 +259,11 @@ self.cpu.execute_token(looptoken) result = self.cpu.get_latest_value_int(0) if guard == rop.GUARD_FALSE: - assert result == execute(self.cpu, op, None, b).value + assert result == execute(self.cpu, None, + op, None, b).value else: - assert result != execute(self.cpu, op, None, b).value + assert result != execute(self.cpu, None, + op, None, b).value def test_stuff_followed_by_guard(self): @@ -304,7 +306,7 @@ self.cpu.set_future_value_int(i, box.value) self.cpu.execute_token(looptoken) result = self.cpu.get_latest_value_int(0) - expected = execute(self.cpu, op, None, a, b).value + expected = execute(self.cpu, None, op, None, a, b).value if guard == rop.GUARD_FALSE: assert result == expected else: From arigo at codespeak.net Sun May 30 12:56:20 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 12:56:20 +0200 (CEST) Subject: [pypy-svn] r74909 - in pypy/branch/blackhole-improvement/pypy/jit: backend backend/llgraph backend/llsupport metainterp Message-ID: <20100530105620.4A8AC282B9E@codespeak.net> Author: arigo Date: Sun May 30 12:56:18 2010 New Revision: 74909 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py pypy/branch/blackhole-improvement/pypy/jit/backend/model.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: Change grab_exc_value() to return a GCREF. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py Sun May 30 12:56:18 2010 @@ -1096,9 +1096,9 @@ if _last_exception is not None: result = _last_exception.args[1] _last_exception = None - return result + return lltype.cast_opaque_ptr(llmemory.GCREF, result) else: - return lltype.nullptr(rclass.OBJECTPTR.TO) + return lltype.nullptr(llmemory.GCREF.TO) ##_pseudo_exceptions = {} @@ -1519,7 +1519,7 @@ setannotation(frame_get_value_count, annmodel.SomeInteger()) setannotation(frame_clear_latest_values, annmodel.s_None) -setannotation(grab_exc_value, annmodel.SomePtr(rclass.OBJECTPTR)) +setannotation(grab_exc_value, annmodel.SomePtr(llmemory.GCREF)) setannotation(force, annmodel.SomeInteger()) setannotation(get_forced_token_frame, s_Frame) setannotation(get_frame_forced_token, annmodel.SomeAddress()) Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py Sun May 30 12:56:18 2010 @@ -50,7 +50,7 @@ self._setup_exception_handling_translated() else: self._setup_exception_handling_untranslated() - self.saved_exc_value = lltype.nullptr(rclass.OBJECT) + self.saved_exc_value = lltype.nullptr(llmemory.GCREF.TO) self.setup() if translate_support_code: self._setup_on_leave_jitted_translated() @@ -107,7 +107,7 @@ v_i = _exception_emulator[1] _exception_emulator[0] = 0 _exception_emulator[1] = 0 - self.saved_exc_value = rffi.cast(rclass.OBJECTPTR, v_i) + self.saved_exc_value = rffi.cast(llmemory.GCREF, v_i) self.pos_exception = pos_exception self.pos_exc_value = pos_exc_value @@ -128,7 +128,7 @@ addr = llop.get_exception_addr(llmemory.Address) addr.address[0] = llmemory.NULL addr = llop.get_exc_value_addr(llmemory.Address) - exc_value = rffi.cast(rclass.OBJECTPTR, addr.address[0]) + exc_value = rffi.cast(llmemory.GCREF, addr.address[0]) addr.address[0] = llmemory.NULL # from now on, the state is again consistent -- no more RPython # exception is set. The following code produces a write barrier @@ -172,7 +172,7 @@ def grab_exc_value(self): exc = self.saved_exc_value - self.saved_exc_value = lltype.nullptr(rclass.OBJECT) + self.saved_exc_value = lltype.nullptr(llmemory.GCREF.TO) return exc # ------------------- helpers and descriptions -------------------- Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/model.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/model.py Sun May 30 12:56:18 2010 @@ -105,7 +105,7 @@ def grab_exc_value(self): """Return and clear the exception set by the latest execute_token(), when it exits due to a failure of a GUARD_EXCEPTION or - GUARD_NO_EXCEPTION.""" # XXX remove me + GUARD_NO_EXCEPTION. (Returns a GCREF)""" # XXX remove me raise NotImplementedError @staticmethod Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Sun May 30 12:56:18 2010 @@ -1221,7 +1221,8 @@ elif (opnum == rop.GUARD_NO_EXCEPTION or opnum == rop.GUARD_EXCEPTION or opnum == rop.GUARD_NOT_FORCED): - return self.cpu.grab_exc_value() + return lltype.cast_opaque_ptr(rclass.OBJECTPTR, + self.cpu.grab_exc_value()) # elif opnum == rop.GUARD_NO_OVERFLOW: # Produced by int_xxx_ovf(). The pc is just after the opcode. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Sun May 30 12:56:18 2010 @@ -1,5 +1,5 @@ import py, os -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem import lltype, llmemory, rclass from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.debug import debug_start, debug_stop, debug_print @@ -1642,7 +1642,8 @@ elif opnum == rop.GUARD_NO_EXCEPTION or opnum == rop.GUARD_EXCEPTION: exception = self.cpu.grab_exc_value() if exception: - self.execute_ll_raised(exception) + self.execute_ll_raised(lltype.cast_opaque_ptr(rclass.OBJECTPTR, + exception)) else: self.execute_did_not_raise() try: From arigo at codespeak.net Sun May 30 13:18:19 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 13:18:19 +0200 (CEST) Subject: [pypy-svn] r74910 - pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test Message-ID: <20100530111819.9A605282B9E@codespeak.net> Author: arigo Date: Sun May 30 13:18:18 2010 New Revision: 74910 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_send.py Log: Fix test. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_send.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_send.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_send.py Sun May 30 13:18:18 2010 @@ -2,7 +2,6 @@ import py from pypy.jit.metainterp.test.test_send import SendTests from pypy.jit.backend.x86.test.test_basic import Jit386Mixin -from pypy.jit.metainterp.policy import StopAtXPolicy from pypy.rlib import jit class TestSend(Jit386Mixin, SendTests): From arigo at codespeak.net Sun May 30 13:18:38 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 13:18:38 +0200 (CEST) Subject: [pypy-svn] r74911 - pypy/branch/blackhole-improvement/pypy/jit/backend/test Message-ID: <20100530111838.C3CEE282B9E@codespeak.net> Author: arigo Date: Sun May 30 13:18:37 2010 New Revision: 74911 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/test/test_ll_random.py pypy/branch/blackhole-improvement/pypy/jit/backend/test/test_random.py Log: Fix test_(ll)_random. Seems to pass fine on x86 backend. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/test/test_ll_random.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/test/test_ll_random.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/test/test_ll_random.py Sun May 30 13:18:37 2010 @@ -3,8 +3,9 @@ from pypy.jit.backend.test import test_random from pypy.jit.metainterp.resoperation import ResOperation, rop from pypy.jit.metainterp.history import ConstInt, ConstPtr -from pypy.jit.metainterp.history import ConstAddr, BoxPtr, BoxInt +from pypy.jit.metainterp.history import BoxPtr, BoxInt from pypy.jit.metainterp.history import BasicFailDescr +from pypy.jit.codewriter import heaptracker from pypy.rpython.annlowlevel import llhelper from pypy.rlib.rarithmetic import intmask from pypy.rpython.llinterp import LLException @@ -19,13 +20,11 @@ self.runicodes = [] self.structure_types = [] self.structure_types_and_vtables = [] - self.class_sizes_cache = [] def fork(self, cpu, loop, vars): fork = test_random.OperationBuilder.fork(self, cpu, loop, vars) fork.structure_types = self.structure_types fork.structure_types_and_vtables = self.structure_types_and_vtables - fork.class_sizes_cache = self.class_sizes_cache return fork def get_structptr_var(self, r, must_have_vtable=False, type=lltype.Struct): @@ -107,11 +106,7 @@ vtable.name[len(name)] = '\x00' self.structure_types_and_vtables.append((S, vtable)) # - vtable_adr = llmemory.cast_ptr_to_adr(vtable) - vtable_int = self.cpu.cast_adr_to_int(vtable_adr) - descr = self.cpu.sizeof(S) - self.class_sizes_cache.append((vtable_int, descr)) - self.cpu.set_class_sizes(dict(self.class_sizes_cache)) + heaptracker.register_known_gctype(self.cpu, vtable, S) # return S, vtable @@ -186,6 +181,9 @@ # ____________________________________________________________ +def ConstAddr(addr, cpu): + return ConstInt(llmemory.cast_adr_to_int(addr)) + class GuardClassOperation(test_random.GuardOperation): def gen_guard(self, builder, r): ptrvars = [(v, S) for (v, S) in builder.ptrvars @@ -507,8 +505,6 @@ descr = builder.cpu.calldescrof(TP, TP.ARGS, TP.RESULT) self.put(builder, args, descr) exc_box = ConstAddr(llmemory.cast_ptr_to_adr(exc), builder.cpu) - assert builder.cpu.get_exception() - builder.cpu.clear_exception() op = ResOperation(rop.GUARD_EXCEPTION, [exc_box], BoxPtr(), descr=BasicFailDescr()) op.fail_args = fail_subset @@ -525,8 +521,6 @@ args = [c_addr] + subset descr = builder.cpu.calldescrof(TP, TP.ARGS, TP.RESULT) self.put(builder, args, descr) - assert builder.cpu.get_exception() - builder.cpu.clear_exception() op = ResOperation(rop.GUARD_NO_EXCEPTION, [], BoxPtr(), descr=BasicFailDescr()) op._exc_box = ConstAddr(llmemory.cast_ptr_to_adr(exc), builder.cpu) @@ -546,8 +540,6 @@ args = [c_addr] + subset descr = builder.cpu.calldescrof(TP, TP.ARGS, TP.RESULT) self.put(builder, args, descr) - assert builder.cpu.get_exception() - builder.cpu.clear_exception() while True: _, vtableptr = builder.get_random_structure_type_and_vtable(r) if vtableptr != exc: Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/test/test_random.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/test/test_random.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/test/test_random.py Sun May 30 13:18:37 2010 @@ -4,7 +4,7 @@ from pypy.jit.backend.test import conftest as demo_conftest from pypy.jit.metainterp.history import BasicFailDescr, TreeLoop from pypy.jit.metainterp.history import BoxInt, ConstInt, LoopToken -from pypy.jit.metainterp.history import BoxPtr, ConstPtr, ConstAddr +from pypy.jit.metainterp.history import BoxPtr, ConstPtr from pypy.jit.metainterp.history import BoxFloat, ConstFloat, Const from pypy.jit.metainterp.resoperation import ResOperation, rop from pypy.jit.metainterp.executor import execute_nonspec @@ -17,9 +17,16 @@ def __init__(self, subops): self.operations = subops +class FakeMetaInterp(object): + def execute_raised(self, exc, constant=False): + self._got_exc = exc + def execute_did_not_raise(self): + self._got_exc = None + class OperationBuilder(object): def __init__(self, cpu, loop, vars): self.cpu = cpu + self.fakemetainterp = FakeMetaInterp() self.loop = loop self.intvars = [box for box in vars if isinstance(box, BoxInt)] self.boolvars = [] # subset of self.intvars @@ -40,7 +47,8 @@ return fork def do(self, opnum, argboxes, descr=None): - v_result = execute_nonspec(self.cpu, opnum, argboxes, descr) + v_result = execute_nonspec(self.cpu, self.fakemetainterp, + opnum, argboxes, descr) if isinstance(v_result, Const): v_result = v_result.clonebox() self.loop.operations.append(ResOperation(opnum, argboxes, v_result, @@ -56,11 +64,14 @@ if S == S2 and not (isinstance(v, ConstPtr) and isinstance(v2, ConstPtr)): if r.random() < 0.5: - return self.do(rop.OOIS, [v, v2]) + return self.do(rop.PTR_EQ, [v, v2]) else: - return self.do(rop.OOISNOT, [v, v2]) + return self.do(rop.PTR_NE, [v, v2]) v = r.choice(self.intvars) - return self.do(rop.INT_IS_TRUE, [v]) + if r.random() < 0.7: + return self.do(rop.INT_IS_TRUE, [v]) + else: + return self.do(rop.INT_IS_ZERO, [v]) def subset_of_intvars(self, r): subset = [] @@ -79,16 +90,16 @@ for v in op.args: if v in names: args.append(names[v]) - elif isinstance(v, ConstAddr): - try: - name = ''.join([v.value.ptr.name[i] - for i in range(len(v.value.ptr.name)-1)]) - except AttributeError: - args.append('ConstAddr(...)') - else: - args.append( - 'ConstAddr(llmemory.cast_ptr_to_adr(%s_vtable), cpu)' - % name) +## elif isinstance(v, ConstAddr): +## try: +## name = ''.join([v.value.ptr.name[i] +## for i in range(len(v.value.ptr.name)-1)]) +## except AttributeError: +## args.append('ConstAddr(...)') +## else: +## args.append( +## 'ConstAddr(llmemory.cast_ptr_to_adr(%s_vtable), cpu)' +## % name) elif isinstance(v, ConstFloat): args.append('ConstFloat(%r)' % v.value) elif isinstance(v, ConstInt): @@ -264,8 +275,8 @@ fail_subset = builder.subset_of_intvars(r) original_intvars = builder.intvars[:] super(AbstractOvfOperation, self).produce_into(builder, r) - if builder.cpu._overflow_flag: # overflow detected - del builder.cpu._overflow_flag + if builder.fakemetainterp._got_exc: # overflow detected + assert isinstance(builder.fakemetainterp._got_exc, OverflowError) op = ResOperation(rop.GUARD_OVERFLOW, [], None) # the overflowed result should not be used any more, but can # be used on the failure path: recompute fail_subset including @@ -274,6 +285,7 @@ builder.intvars[:] = original_intvars else: op = ResOperation(rop.GUARD_NO_OVERFLOW, [], None) + del builder.fakemetainterp._got_exc op.descr = BasicFailDescr() op.fail_args = fail_subset builder.loop.operations.append(op) @@ -585,8 +597,8 @@ def run_loop(self): cpu = self.builder.cpu self.clear_state() - assert not cpu.get_exception() - assert not cpu.get_exc_value() + exc = cpu.grab_exc_value() + assert not exc for i, box in enumerate(self.startvars): if isinstance(box, BoxInt): @@ -607,15 +619,13 @@ self.expected[v], i) ) + exc = cpu.grab_exc_value() if (self.guard_op is not None and self.guard_op.is_guard_exception()): if self.guard_op.opnum == rop.GUARD_NO_EXCEPTION: - assert cpu.get_exception() - assert cpu.get_exc_value() - cpu.clear_exception() + assert exc else: - assert not cpu.get_exception() - assert not cpu.get_exc_value() + assert not exc def build_bridge(self): def exc_handling(guard_op): From arigo at codespeak.net Sun May 30 14:13:43 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 14:13:43 +0200 (CEST) Subject: [pypy-svn] r74912 - in pypy/branch/blackhole-improvement/pypy/jit: backend/llgraph backend/llsupport codewriter metainterp Message-ID: <20100530121343.D0A78282B9E@codespeak.net> Author: arigo Date: Sun May 30 14:13:42 2010 New Revision: 74912 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/descr.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: Annotation fixes. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py Sun May 30 14:13:42 2010 @@ -59,6 +59,9 @@ def is_array_of_floats(self): return self.typeinfo == FLOAT + def as_vtable_size_descr(self): + return self + def __lt__(self, other): raise TypeError("cannot use comparison on Descrs") def __le__(self, other): Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/descr.py Sun May 30 14:13:42 2010 @@ -1,10 +1,11 @@ import py -from pypy.rpython.lltypesystem import lltype, rffi, llmemory +from pypy.rpython.lltypesystem import lltype, rffi, llmemory, rclass from pypy.jit.backend.llsupport import symbolic, support from pypy.jit.metainterp.history import AbstractDescr, getkind, BoxInt, BoxPtr from pypy.jit.metainterp.history import BasicFailDescr, LoopToken, BoxFloat from pypy.jit.metainterp import history from pypy.jit.metainterp.resoperation import ResOperation, rop +from pypy.jit.codewriter import heaptracker # The point of the class organization in this file is to make instances # as compact as possible. This is done by not storing the field size or @@ -40,6 +41,10 @@ def repr_of_descr(self): return '' % self.size +class SizeDescrWithVTable(SizeDescr): + def as_vtable_size_descr(self): + return self + BaseSizeDescr = SizeDescr def get_size_descr(gccache, STRUCT): @@ -48,7 +53,10 @@ return cache[STRUCT] except KeyError: size = symbolic.get_size(STRUCT, gccache.translate_support_code) - sizedescr = SizeDescr(size) + if heaptracker.has_gcstruct_a_vtable(STRUCT): + sizedescr = SizeDescrWithVTable(size) + else: + sizedescr = SizeDescr(size) gccache.init_size_descr(STRUCT, sizedescr) cache[STRUCT] = sizedescr return sizedescr @@ -257,11 +265,12 @@ # The inheritance hierarchy is a bit different than with other Descr # classes because of the 'call_stub' attribute, which is of type # - # lambda args_i, args_r, args_f --> int/ref/float/void + # lambda func, args_i, args_r, args_f --> int/ref/float/void # # The purpose of BaseIntCallDescr is to be the parent of all classes # in which 'call_stub' has a return kind of 'int'. _return_type = history.INT + call_stub = staticmethod(lambda func, args_i, args_r, args_f: 0) class NonGcPtrCallDescr(BaseIntCallDescr): _clsname = 'NonGcPtrCallDescr' @@ -271,18 +280,22 @@ class GcPtrCallDescr(BaseCallDescr): _clsname = 'GcPtrCallDescr' _return_type = history.REF + call_stub = staticmethod(lambda func, args_i, args_r, args_f: + lltype.nullptr(llmemory.GCREF.TO)) def get_result_size(self, translate_support_code): return symbolic.get_size_of_ptr(translate_support_code) class FloatCallDescr(BaseCallDescr): _clsname = 'FloatCallDescr' _return_type = history.FLOAT + call_stub = staticmethod(lambda func, args_i, args_r, args_f: 0.0) def get_result_size(self, translate_support_code): return symbolic.get_size(lltype.Float, translate_support_code) class VoidCallDescr(BaseCallDescr): _clsname = 'VoidCallDescr' _return_type = history.VOID + call_stub = staticmethod(lambda func, args_i, args_r, args_f: None) def get_result_size(self, translate_support_code): return 0 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py Sun May 30 14:13:42 2010 @@ -2,16 +2,20 @@ from pypy.rlib.objectmodel import we_are_translated +def has_gcstruct_a_vtable(GCSTRUCT): + assert isinstance(GCSTRUCT, lltype.GcStruct) + while not GCSTRUCT._hints.get('typeptr'): + _, GCSTRUCT = GCSTRUCT._first_struct() + if GCSTRUCT is None: + return False + return True + def get_vtable_for_gcstruct(cpu, GCSTRUCT): # xxx hack: from a GcStruct representing an instance's # lowleveltype, return the corresponding vtable pointer. # Returns None if the GcStruct does not belong to an instance. - assert isinstance(GCSTRUCT, lltype.GcStruct) - HEAD = GCSTRUCT - while not HEAD._hints.get('typeptr'): - _, HEAD = HEAD._first_struct() - if HEAD is None: - return None + if not has_gcstruct_a_vtable(GCSTRUCT): + return None setup_cache_gcstruct2vtable(cpu) return cpu._cache_gcstruct2vtable[GCSTRUCT] @@ -42,15 +46,18 @@ def register_known_gctype(cpu, vtable, STRUCT): # register the correspondance 'vtable' <-> 'STRUCT' in the cpu sizedescr = cpu.sizeof(STRUCT) - if hasattr(sizedescr, '_corresponding_vtable'): + assert sizedescr.as_vtable_size_descr() is sizedescr + try: assert sizedescr._corresponding_vtable == vtable - else: - assert lltype.typeOf(vtable) == VTABLETYPE - if not hasattr(cpu, '_all_size_descrs_with_vtable'): - cpu._all_size_descrs_with_vtable = [] - cpu._vtable_to_descr_dict = None - cpu._all_size_descrs_with_vtable.append(sizedescr) - sizedescr._corresponding_vtable = vtable + return + except AttributeError: + pass + assert lltype.typeOf(vtable) == VTABLETYPE + if not hasattr(cpu, '_all_size_descrs_with_vtable'): + cpu._all_size_descrs_with_vtable = [] + cpu._vtable_to_descr_dict = None + cpu._all_size_descrs_with_vtable.append(sizedescr) + sizedescr._corresponding_vtable = vtable def finish_registering(cpu): # annotation hack for small examples which have no vtable at all @@ -84,7 +91,7 @@ def descr2vtable(cpu, descr): from pypy.jit.metainterp import history assert isinstance(descr, history.AbstractDescr) - vtable = descr._corresponding_vtable + vtable = descr.as_vtable_size_descr()._corresponding_vtable vtable = llmemory.cast_ptr_to_adr(vtable) vtable = llmemory.cast_adr_to_int(vtable) return vtable Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Sun May 30 14:13:42 2010 @@ -258,6 +258,9 @@ self.registers_i = [default_i] * 256 self.registers_r = [default_r] * 256 self.registers_f = [default_f] * 256 + self.tmpreg_i = default_i + self.tmpreg_r = default_r + self.tmpreg_f = default_f self.jitcode = None def __repr__(self): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py Sun May 30 14:13:42 2010 @@ -164,6 +164,12 @@ """ raise NotImplementedError + def as_vtable_size_descr(self): + """ Implement for size descr representing objects with vtables. + Returns self. (it's an annotation hack) + """ + raise NotImplementedError + class AbstractFailDescr(AbstractDescr): index = -1 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Sun May 30 14:13:42 2010 @@ -1038,14 +1038,15 @@ else: effect = effectinfo.extraeffect if effect == effectinfo.EF_CANNOT_RAISE: - opnum, exc = rop.CALL, False + return self.execute_varargs(rop.CALL, allboxes, descr, False) elif effect == effectinfo.EF_PURE: - opnum, exc = rop.CALL_PURE, False + return self.execute_varargs(rop.CALL_PURE, allboxes, + descr, False) elif effect == effectinfo.EF_LOOPINVARIANT: - opnum, exc = rop.CALL_LOOPINVARIANT, True + return self.execute_varargs(rop.CALL_LOOPINVARIANT, allboxes, + descr, True) else: - opnum, exc = rop.CALL, True - return self.execute_varargs(opnum, allboxes, descr, exc) + return self.execute_varargs(rop.CALL, allboxes, descr, True) def do_residual_or_indirect_call(self, funcbox, calldescr, argboxes): """The 'residual_call' operation is emitted in two cases: From arigo at codespeak.net Sun May 30 14:16:10 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 14:16:10 +0200 (CEST) Subject: [pypy-svn] r74913 - in pypy/branch/blackhole-improvement/pypy/jit: backend/x86/test codewriter Message-ID: <20100530121610.D7D64282B9E@codespeak.net> Author: arigo Date: Sun May 30 14:16:09 2010 New Revision: 74913 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_ztranslation.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py Log: Fixes. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_ztranslation.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_ztranslation.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_ztranslation.py Sun May 30 14:16:09 2010 @@ -5,7 +5,7 @@ from pypy.jit.metainterp.jitprof import Profiler from pypy.jit.backend.x86.runner import CPU386 from pypy.jit.backend.test.support import CCompiledMixin -from pypy.jit.metainterp.policy import StopAtXPolicy +from pypy.jit.codewriter.policy import StopAtXPolicy from pypy.translator.translator import TranslationContext class TestTranslationX86(CCompiledMixin): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py Sun May 30 14:16:09 2010 @@ -62,7 +62,7 @@ def finish_registering(cpu): # annotation hack for small examples which have no vtable at all if not hasattr(cpu, '_all_size_descrs_with_vtable'): - STRUCT = lltype.GcStruct('empty') + STRUCT = lltype.GcStruct('empty', ('parent', rclass.OBJECT)) vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) register_known_gctype(cpu, vtable, STRUCT) From agaynor at codespeak.net Sun May 30 15:58:52 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sun, 30 May 2010 15:58:52 +0200 (CEST) Subject: [pypy-svn] r74914 - in pypy/trunk/pypy/module/cpyext: . include test Message-ID: <20100530135852.ADCD3282BAD@codespeak.net> Author: agaynor Date: Sun May 30 15:58:51 2010 New Revision: 74914 Added: pypy/trunk/pypy/module/cpyext/test/comparisons.c Modified: pypy/trunk/pypy/module/cpyext/include/object.h pypy/trunk/pypy/module/cpyext/slotdefs.py pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Log: Implement richcmp_eq for cpyext. Modified: pypy/trunk/pypy/module/cpyext/include/object.h ============================================================================== --- pypy/trunk/pypy/module/cpyext/include/object.h (original) +++ pypy/trunk/pypy/module/cpyext/include/object.h Sun May 30 15:58:51 2010 @@ -70,6 +70,9 @@ #define Py_NotImplemented (&_Py_NotImplementedStruct) /* Rich comparison opcodes */ +/* + XXX: Also defined in slotdefs.py +*/ #define Py_LT 0 #define Py_LE 1 #define Py_EQ 2 Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/slotdefs.py (original) +++ pypy/trunk/pypy/module/cpyext/slotdefs.py Sun May 30 15:58:51 2010 @@ -5,7 +5,7 @@ from pypy.module.cpyext.typeobjectdefs import ( unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc, getattrfunc, setattrofunc, lenfunc, ssizeargfunc, ssizessizeargfunc, - ssizeobjargproc, iternextfunc, initproc) + ssizeobjargproc, iternextfunc, initproc, richcmpfunc) from pypy.module.cpyext.pyobject import from_ref from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.state import State @@ -15,6 +15,15 @@ from pypy.tool.sourcetools import func_with_new_name +# XXX: Also defined in object.h +Py_LT = 0 +Py_LE = 1 +Py_EQ = 2 +Py_NE = 3 +Py_GT = 4 +Py_GE = 5 + + def check_num_args(space, ob, n): from pypy.module.cpyext.tupleobject import PyTuple_CheckExact, \ PyTuple_GET_SIZE @@ -124,6 +133,14 @@ raise OperationError(space.w_StopIteration, space.w_None) return w_res +def richcmp_eq(space, w_self, w_args, func): + func_target = rffi.cast(richcmpfunc, func) + check_num_args(space, w_args, 1) + args_w = space.fixedview(w_args) + other_w = args_w[0] + return generic_cpy_call(space, func_target, + w_self, other_w, rffi.cast(rffi.INT_real, Py_EQ)) + @cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject, external=False) def slot_tp_new(space, type, w_args, w_kwds): from pypy.module.cpyext.tupleobject import PyTuple_Check Added: pypy/trunk/pypy/module/cpyext/test/comparisons.c ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/cpyext/test/comparisons.c Sun May 30 15:58:51 2010 @@ -0,0 +1,75 @@ +#include "Python.h" + +typedef struct CmpObject { + PyObject_HEAD +} CmpObject; + + +static PyObject* cmp_richcmp(PyObject *self, PyObject *other, int opid) { + if (opid != Py_EQ || !PyInt_CheckExact(other)) { + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + long val = PyLong_AsLong(other); + return PyBool_FromLong(val == 3); +} + + +PyTypeObject CmpType = { + PyVarObject_HEAD_INIT(NULL, 0) + "comparisons.CmpType", /* tp_name */ + sizeof(CmpObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, /* tp_flags */ + 0, /* tp_doc */ + (traverseproc)0, /* tp_traverse */ + 0, /* tp_clear */ + (richcmpfunc)cmp_richcmp, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ + 0 /* tp_free */ +}; + + +void initcomparisons(void) +{ + PyObject *m, *d; + + if (PyType_Ready(&CmpType) < 0) + return; + m = Py_InitModule("comparisons", NULL); + if (m == NULL) + return; + d = PyModule_GetDict(m); + if (d == NULL) + return; + if (PyDict_SetItemString(d, "CmpType", (PyObject *)&CmpType) < 0) + return; +} Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Sun May 30 15:58:51 2010 @@ -176,6 +176,11 @@ def test_init_error(self): module = self.import_module("foo") raises(ValueError, module.InitErrType) + + def test_cmps(self): + module = self.import_module("comparisons") + cmpr = module.CmpType() + assert cmpr == 3 class TestTypes(BaseApiTest): From arigo at codespeak.net Sun May 30 15:59:39 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 15:59:39 +0200 (CEST) Subject: [pypy-svn] r74915 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100530135939.300F9282BAD@codespeak.net> Author: arigo Date: Sun May 30 15:59:37 2010 New Revision: 74915 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py Log: Shut off some translation warnings. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/compile.py Sun May 30 15:59:37 2010 @@ -280,21 +280,28 @@ else: index = self._counter & self.CNT_MASK typetag = self._counter & ~ self.CNT_MASK + counters = self._counters if typetag == self.CNT_INT: intvalue = metainterp_sd.cpu.get_latest_value_int(index) - if self._counters is None: - self._counters = ResumeGuardCountersInt() - counter = self._counters.see_int(intvalue) + if counters is None: + self._counters = counters = ResumeGuardCountersInt() + else: + assert isinstance(counters, ResumeGuardCountersInt) + counter = counters.see_int(intvalue) elif typetag == self.CNT_REF: refvalue = metainterp_sd.cpu.get_latest_value_ref(index) - if self._counters is None: - self._counters = ResumeGuardCountersRef() - counter = self._counters.see_ref(refvalue) + if counters is None: + self._counters = counters = ResumeGuardCountersRef() + else: + assert isinstance(counters, ResumeGuardCountersRef) + counter = counters.see_ref(refvalue) elif typetag == self.CNT_FLOAT: floatvalue = metainterp_sd.cpu.get_latest_value_float(index) - if self._counters is None: - self._counters = ResumeGuardCountersFloat() - counter = self._counters.see_float(floatvalue) + if counters is None: + self._counters = counters = ResumeGuardCountersFloat() + else: + assert isinstance(counters, ResumeGuardCountersFloat) + counter = counters.see_float(floatvalue) else: assert 0, typetag return counter >= trace_eagerness Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py Sun May 30 15:59:37 2010 @@ -120,6 +120,7 @@ return BoxInt(cpu.bh_getfield_gc_i(struct, fielddescr)) def do_getfield_raw(cpu, _, structbox, fielddescr): + check_descr(fielddescr) struct = structbox.getint() if fielddescr.is_pointer_field(): return BoxPtr(cpu.bh_getfield_raw_r(struct, fielddescr)) @@ -245,11 +246,10 @@ # find which list to store the operation in, based on num_args num_args = resoperation.oparity[value] withdescr = resoperation.opwithdescr[value] - if withdescr and num_args >= 0: - num_args += 1 - if num_args not in execute_by_num_args: - execute_by_num_args[num_args] = [None] * (rop._LAST+1) - execute = execute_by_num_args[num_args] + dictkey = num_args, withdescr + if dictkey not in execute_by_num_args: + execute_by_num_args[dictkey] = [None] * (rop._LAST+1) + execute = execute_by_num_args[dictkey] # if execute[value] is not None: raise AssertionError("duplicate entry for op number %d"% value) @@ -328,19 +328,19 @@ do.func_name = 'do_' + name return do -def get_execute_funclist(num_args): +def get_execute_funclist(num_args, withdescr): # workaround, similar to the next one - return EXECUTE_BY_NUM_ARGS[num_args] + return EXECUTE_BY_NUM_ARGS[num_args, withdescr] get_execute_funclist._annspecialcase_ = 'specialize:memo' -def get_execute_function(opnum, num_args): +def get_execute_function(opnum, num_args, withdescr): # workaround for an annotation limitation: putting this code in # a specialize:memo function makes sure the following line is # constant-folded away. Only works if opnum and num_args are # constants, of course. - func = EXECUTE_BY_NUM_ARGS[num_args][opnum] - assert func is not None, "EXECUTE_BY_NUM_ARGS[%s][%s]" % ( - num_args, resoperation.opname[opnum]) + func = EXECUTE_BY_NUM_ARGS[num_args, withdescr][opnum] + assert func is not None, "EXECUTE_BY_NUM_ARGS[%s, %s][%s]" % ( + num_args, withdescr, resoperation.opname[opnum]) return func get_execute_function._annspecialcase_ = 'specialize:memo' @@ -352,12 +352,14 @@ def execute(cpu, metainterp, opnum, descr, *argboxes): # only for opnums with a fixed arity - if has_descr(opnum): + num_args = len(argboxes) + withdescr = has_descr(opnum) + if withdescr: check_descr(descr) argboxes = argboxes + (descr,) else: assert descr is None - func = get_execute_function(opnum, len(argboxes)) + func = get_execute_function(opnum, num_args, withdescr) return func(cpu, metainterp, *argboxes) # note that the 'argboxes' tuple # optionally ends with the descr execute._annspecialcase_ = 'specialize:arg(2)' @@ -365,7 +367,7 @@ def execute_varargs(cpu, metainterp, opnum, argboxes, descr): # only for opnums with a variable arity (calls, typically) check_descr(descr) - func = get_execute_function(opnum, -1) + func = get_execute_function(opnum, -1, True) return func(cpu, metainterp, argboxes, descr) execute_varargs._annspecialcase_ = 'specialize:arg(2)' @@ -376,31 +378,31 @@ if resoperation.opwithdescr[opnum]: check_descr(descr) if arity == -1: - func = get_execute_funclist(-1)[opnum] + func = get_execute_funclist(-1, True)[opnum] return func(cpu, metainterp, argboxes, descr) if arity == 0: - func = get_execute_funclist(1)[opnum] + func = get_execute_funclist(0, True)[opnum] return func(cpu, metainterp, descr) if arity == 1: - func = get_execute_funclist(2)[opnum] + func = get_execute_funclist(1, True)[opnum] return func(cpu, metainterp, argboxes[0], descr) if arity == 2: - func = get_execute_funclist(3)[opnum] + func = get_execute_funclist(2, True)[opnum] return func(cpu, metainterp, argboxes[0], argboxes[1], descr) if arity == 3: - func = get_execute_funclist(4)[opnum] + func = get_execute_funclist(3, True)[opnum] return func(cpu, metainterp, argboxes[0], argboxes[1], argboxes[2], descr) else: assert descr is None if arity == 1: - func = get_execute_funclist(1)[opnum] + func = get_execute_funclist(1, False)[opnum] return func(cpu, metainterp, argboxes[0]) if arity == 2: - func = get_execute_funclist(2)[opnum] + func = get_execute_funclist(2, False)[opnum] return func(cpu, metainterp, argboxes[0], argboxes[1]) if arity == 3: - func = get_execute_funclist(3)[opnum] + func = get_execute_funclist(3, False)[opnum] return func(cpu, metainterp, argboxes[0], argboxes[1], argboxes[2]) raise NotImplementedError From arigo at codespeak.net Sun May 30 16:03:56 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 16:03:56 +0200 (CEST) Subject: [pypy-svn] r74916 - in pypy/branch/blackhole-improvement/pypy/jit/backend/x86: . test Message-ID: <20100530140356.9D009282BAD@codespeak.net> Author: arigo Date: Sun May 30 16:03:55 2010 New Revision: 74916 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/regalloc.py pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_zrpy_gc.py Log: Fix for test_zrpy_gc. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/x86/regalloc.py Sun May 30 16:03:55 2010 @@ -937,6 +937,8 @@ assert isinstance(val, MODRM) gcrootmap.add_ebp_offset(shape, get_ebp_ofs(val.position)) for v, reg in self.rm.reg_bindings.items(): + if reg is eax: + continue # ok to ignore this one if (isinstance(v, BoxPtr) and self.rm.stays_alive(v)): if reg is ebx: gcrootmap.add_ebx(shape) @@ -945,7 +947,8 @@ elif reg is edi: gcrootmap.add_edi(shape) else: - assert reg is eax # ok to ignore this one + print "[get_mark_gc_roots] bogus register", reg + assert False return gcrootmap.compress_callshape(shape) def consider_force_token(self, op): Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_zrpy_gc.py Sun May 30 16:03:55 2010 @@ -94,7 +94,7 @@ def run(cbuilder, args=''): # pypylog = udir.join('test_zrpy_gc.log') - data = cbuilder.cmdexec(args, env={'PYPYLOG': str(pypylog)}) + data = cbuilder.cmdexec(args, env={'PYPYLOG': ':%s' % pypylog}) return data.strip() def compile_and_run(f, gc, **kwds): @@ -174,7 +174,7 @@ def run(self, name, n=2000): pypylog = udir.join('TestCompileHybrid.log') res = self.cbuilder.cmdexec("%s %d" %(name, n), - env={'PYPYLOG': str(pypylog)}) + env={'PYPYLOG': ':%s' % pypylog}) assert int(res) == 20 def run_orig(self, name, n, x): From arigo at codespeak.net Sun May 30 16:24:25 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 16:24:25 +0200 (CEST) Subject: [pypy-svn] r74917 - in pypy/branch/blackhole-improvement/pypy/jit: backend/x86/test codewriter metainterp Message-ID: <20100530142425.24E49282BAD@codespeak.net> Author: arigo Date: Sun May 30 16:24:23 2010 New Revision: 74917 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_ztranslation.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: Last fixes for that round. It seems that all tests pass (or passed at some point). Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_ztranslation.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_ztranslation.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_ztranslation.py Sun May 30 16:24:23 2010 @@ -38,7 +38,7 @@ return abs(x) jitdriver = JitDriver(greens = [], - reds = ['frame', 'total', 'j'], + reds = ['total', 'frame', 'j'], virtualizables = ['frame']) def f(i, j): for param in unroll_parameters: @@ -72,7 +72,7 @@ class Frame(object): _virtualizable2_ = ['thing'] - driver = JitDriver(greens = ['codeno'], reds = ['frame', 'i'], + driver = JitDriver(greens = ['codeno'], reds = ['i', 'frame'], virtualizables = ['frame'], get_printable_location = lambda codeno : str(codeno), can_inline = lambda codeno : False) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py Sun May 30 16:24:23 2010 @@ -62,9 +62,8 @@ def finish_registering(cpu): # annotation hack for small examples which have no vtable at all if not hasattr(cpu, '_all_size_descrs_with_vtable'): - STRUCT = lltype.GcStruct('empty', ('parent', rclass.OBJECT)) vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) - register_known_gctype(cpu, vtable, STRUCT) + register_known_gctype(cpu, vtable, rclass.OBJECT) def vtable2descr(cpu, vtable): assert lltype.typeOf(vtable) is lltype.Signed Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Sun May 30 16:24:23 2010 @@ -2009,7 +2009,8 @@ if self.staticdata.virtualizable_info is not None: vindex = self.staticdata.virtualizable_info.index_of_virtualizable vbox = args[vindex - num_green_args] - args += self.gen_load_from_other_virtualizable(vbox) + args = args + self.gen_load_from_other_virtualizable(vbox) + # ^^^ and not "+=", which makes 'args' a resizable list op.opnum = rop.CALL_ASSEMBLER op.args = args op.descr = token From arigo at codespeak.net Sun May 30 16:30:27 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 16:30:27 +0200 (CEST) Subject: [pypy-svn] r74918 - in pypy/branch/blackhole-improvement/pypy/jit: backend/x86/test codewriter Message-ID: <20100530143027.B62F2282BAD@codespeak.net> Author: arigo Date: Sun May 30 16:30:26 2010 New Revision: 74918 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_gc_integration.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py Log: Fix again some tests. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_gc_integration.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_gc_integration.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_gc_integration.py Sun May 30 16:30:26 2010 @@ -226,7 +226,9 @@ self.nodedescr = nodedescr vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) vtable_int = cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(vtable)) - NODE2 = lltype.Struct('node2', ('tid', lltype.Signed), + NODE2 = lltype.GcStruct('node2', + ('parent', rclass.OBJECT), + ('tid', lltype.Signed), ('vtable', lltype.Ptr(rclass.OBJECT_VTABLE))) descrsize = cpu.sizeof(NODE2) heaptracker.register_known_gctype(cpu, vtable, NODE2) @@ -283,4 +285,4 @@ assert gc_ll_descr.nursery[0] == self.descrsize.tid assert gc_ll_descr.nursery[1] == self.vtable_int nurs_adr = rffi.cast(lltype.Signed, gc_ll_descr.nursery) - assert gc_ll_descr.addrs[0] == nurs_adr + 8 + assert gc_ll_descr.addrs[0] == nurs_adr + 12 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py Sun May 30 16:30:26 2010 @@ -3,7 +3,8 @@ def has_gcstruct_a_vtable(GCSTRUCT): - assert isinstance(GCSTRUCT, lltype.GcStruct) + if not isinstance(GCSTRUCT, lltype.GcStruct): + return False while not GCSTRUCT._hints.get('typeptr'): _, GCSTRUCT = GCSTRUCT._first_struct() if GCSTRUCT is None: @@ -14,6 +15,7 @@ # xxx hack: from a GcStruct representing an instance's # lowleveltype, return the corresponding vtable pointer. # Returns None if the GcStruct does not belong to an instance. + assert isinstance(GCSTRUCT, lltype.GcStruct) if not has_gcstruct_a_vtable(GCSTRUCT): return None setup_cache_gcstruct2vtable(cpu) From arigo at codespeak.net Sun May 30 17:19:28 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 17:19:28 +0200 (CEST) Subject: [pypy-svn] r74919 - pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/test Message-ID: <20100530151928.826B4282BAD@codespeak.net> Author: arigo Date: Sun May 30 17:19:27 2010 New Revision: 74919 Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Log: Fix test. Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Sun May 30 17:19:27 2010 @@ -1204,17 +1204,14 @@ res = interpret(f, []) assert res == 6 - def test_force_to_int_and_back(self): + def test_force_to_int(self): S = lltype.Struct('S') p = lltype.malloc(S, flavor='raw') a = llmemory.cast_ptr_to_adr(p) - i = llmemory.cast_adr_to_int(a) - i = rffi.get_real_int(i) - # - a2 = llmemory.cast_int_to_adr(i) - assert a2 == a - # - lltype.free(p) + i = rffi.get_real_int(llmemory.cast_adr_to_int(a)) + assert type(i) is int + assert i == rffi.get_real_int(llmemory.cast_adr_to_int(a)) + lltype.free(p, flavor='raw') class TestPlatform(object): def test_lib_on_libpaths(self): From arigo at codespeak.net Sun May 30 17:25:51 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 17:25:51 +0200 (CEST) Subject: [pypy-svn] r74920 - in pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem: . test Message-ID: <20100530152551.AB526282BAD@codespeak.net> Author: arigo Date: Sun May 30 17:25:50 2010 New Revision: 74920 Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/lltype.py pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/test/test_lltype.py Log: Test and fix. Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/lltype.py Sun May 30 17:25:50 2010 @@ -64,17 +64,30 @@ class LowLevelType(object): - # the following line prevents '__cached_hash' to be in the __dict__ of - # the instance, which is needed for __eq__() and __hash__() to work. - __slots__ = ['__dict__', '__cached_hash'] def __eq__(self, other): - return self.__class__ is other.__class__ and ( - self is other or safe_equal(self.__dict__, other.__dict__)) + if self.__class__ is not other.__class__: + return False + if self is other: + return True + items1 = self._get_dict_as_list() + items2 = other._get_dict_as_list() + if len(items1) != len(items2): + return False + for (k1, v1), (k2, v2) in zip(items1, items2): + if k1 != k2 or not safe_equal(v1, v2): + return False + return True def __ne__(self, other): return not (self == other) + def _get_dict_as_list(self): + result = [(key, value) for (key, value) in self.__dict__.items() + if not key.startswith('_HIDDEN_')] + result.sort() + return result + _is_compatible = __eq__ def _enforce(self, value): @@ -84,27 +97,26 @@ def __hash__(self): # cannot use saferecursive() -- see test_lltype.test_hash(). - # NB. the __cached_hash should neither be used nor updated + # NB. the _cached_hash should neither be used nor updated # if we enter with hash_level > 0, because the computed # __hash__ can be different in this situation. hash_level = 0 try: hash_level = TLS.nested_hash_level if hash_level == 0: - return self.__cached_hash + return self._HIDDEN_cached_hash except AttributeError: pass if hash_level >= 3: return 0 - items = self.__dict__.items() - items.sort() + items = self._get_dict_as_list() TLS.nested_hash_level = hash_level + 1 try: result = hash((self.__class__,) + tuple(items)) finally: TLS.nested_hash_level = hash_level if hash_level == 0: - self.__cached_hash = result + self._HIDDEN_cached_hash = result return result # due to this dynamic hash value, we should forbid @@ -283,11 +295,13 @@ return _struct(self, n, initialization='example') class RttiStruct(Struct): - _runtime_type_info = None + _HIDDEN_runtime_type_info = None def _attach_runtime_type_info_funcptr(self, funcptr, destrptr): - if self._runtime_type_info is None: - self._runtime_type_info = opaqueptr(RuntimeTypeInfo, name=self._name, about=self)._obj + if self._HIDDEN_runtime_type_info is None: + self._HIDDEN_runtime_type_info = opaqueptr(RuntimeTypeInfo, + name=self._name, + about=self)._obj if funcptr is not None: T = typeOf(funcptr) if (not isinstance(T, Ptr) or @@ -297,7 +311,7 @@ castable(T.TO.ARGS[0], Ptr(self)) < 0): raise TypeError("expected a runtime type info function " "implementation, got: %s" % funcptr) - self._runtime_type_info.query_funcptr = funcptr + self._HIDDEN_runtime_type_info.query_funcptr = funcptr if destrptr is not None : T = typeOf(destrptr) if (not isinstance(T, Ptr) or @@ -307,7 +321,7 @@ castable(T.TO.ARGS[0], Ptr(self)) < 0): raise TypeError("expected a destructor function " "implementation, got: %s" % destrptr) - self._runtime_type_info.destructor_funcptr = destrptr + self._HIDDEN_runtime_type_info.destructor_funcptr = destrptr class GcStruct(RttiStruct): _gckind = 'gc' @@ -1848,15 +1862,15 @@ if not isinstance(GCSTRUCT, RttiStruct): raise TypeError, "expected a RttiStruct: %s" % GCSTRUCT GCSTRUCT._attach_runtime_type_info_funcptr(funcptr, destrptr) - return _ptr(Ptr(RuntimeTypeInfo), GCSTRUCT._runtime_type_info) + return _ptr(Ptr(RuntimeTypeInfo), GCSTRUCT._HIDDEN_runtime_type_info) def getRuntimeTypeInfo(GCSTRUCT): if not isinstance(GCSTRUCT, RttiStruct): raise TypeError, "expected a RttiStruct: %s" % GCSTRUCT - if GCSTRUCT._runtime_type_info is None: + if GCSTRUCT._HIDDEN_runtime_type_info is None: raise ValueError, ("no attached runtime type info for GcStruct %s" % GCSTRUCT._name) - return _ptr(Ptr(RuntimeTypeInfo), GCSTRUCT._runtime_type_info) + return _ptr(Ptr(RuntimeTypeInfo), GCSTRUCT._HIDDEN_runtime_type_info) def runtime_type_info(p): T = typeOf(p) Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/test/test_lltype.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/test/test_lltype.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/test/test_lltype.py Sun May 30 17:25:50 2010 @@ -373,7 +373,13 @@ Sbis = GcStruct('s', ('x', Signed)) attachRuntimeTypeInfo(Sbis) assert getRuntimeTypeInfo(Sbis) != pinf0 - assert Sbis != S # the attached runtime type info distinguishes them + #assert Sbis != S # the attached runtime type info distinguishes them + # --- disabled for now, not really useful. Instead we have the more + # important property: + Ster = GcStruct('s', ('x', Signed)) + h = hash(Ptr(Ster)) + attachRuntimeTypeInfo(Ster) + assert h == hash(Ptr(Ster)) def test_getRuntimeTypeInfo_destrpointer(): S = GcStruct('s', ('x', Signed)) From agaynor at codespeak.net Sun May 30 17:32:59 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sun, 30 May 2010 17:32:59 +0200 (CEST) Subject: [pypy-svn] r74921 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100530153259.4BE4C282BAD@codespeak.net> Author: agaynor Date: Sun May 30 17:32:57 2010 New Revision: 74921 Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py pypy/trunk/pypy/module/cpyext/test/comparisons.c pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Log: Implemented != support for cpyext (and refactored cmp op support a little) Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/slotdefs.py (original) +++ pypy/trunk/pypy/module/cpyext/slotdefs.py Sun May 30 17:32:57 2010 @@ -133,13 +133,18 @@ raise OperationError(space.w_StopIteration, space.w_None) return w_res -def richcmp_eq(space, w_self, w_args, func): - func_target = rffi.cast(richcmpfunc, func) - check_num_args(space, w_args, 1) - args_w = space.fixedview(w_args) - other_w = args_w[0] - return generic_cpy_call(space, func_target, - w_self, other_w, rffi.cast(rffi.INT_real, Py_EQ)) +def get_richcmp_func(OP_CONST): + def inner(space, w_self, w_args, func): + func_target = rffi.cast(richcmpfunc, func) + check_num_args(space, w_args, 1) + args_w = space.fixedview(w_args) + other_w = args_w[0] + return generic_cpy_call(space, func_target, + w_self, other_w, rffi.cast(rffi.INT_real, OP_CONST)) + return inner + +richcmp_eq = get_richcmp_func(Py_EQ) +richcmp_ne = get_richcmp_func(Py_NE) @cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject, external=False) def slot_tp_new(space, type, w_args, w_kwds): Modified: pypy/trunk/pypy/module/cpyext/test/comparisons.c ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/comparisons.c (original) +++ pypy/trunk/pypy/module/cpyext/test/comparisons.c Sun May 30 17:32:57 2010 @@ -6,12 +6,17 @@ static PyObject* cmp_richcmp(PyObject *self, PyObject *other, int opid) { - if (opid != Py_EQ || !PyInt_CheckExact(other)) { + if ((opid != Py_EQ && opid != Py_NE) || !PyInt_CheckExact(other)) { Py_INCREF(Py_NotImplemented); return Py_NotImplemented; } long val = PyLong_AsLong(other); - return PyBool_FromLong(val == 3); + if (opid == Py_EQ) { + return PyBool_FromLong(val == 3); + } + else if (opid == Py_NE) { + return PyBool_FromLong(val != 3); + } } Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Sun May 30 17:32:57 2010 @@ -181,6 +181,7 @@ module = self.import_module("comparisons") cmpr = module.CmpType() assert cmpr == 3 + assert cmpr != 42 class TestTypes(BaseApiTest): From arigo at codespeak.net Sun May 30 17:44:40 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 17:44:40 +0200 (CEST) Subject: [pypy-svn] r74922 - pypy/branch/blackhole-improvement/pypy/jit/metainterp/test Message-ID: <20100530154440.F1319282BAD@codespeak.net> Author: arigo Date: Sun May 30 17:44:39 2010 New Revision: 74922 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_warmspot.py Log: Fix test. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_warmspot.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_warmspot.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_warmspot.py Sun May 30 17:44:39 2010 @@ -314,7 +314,8 @@ return self.no class FakeDescr: - pass + def as_vtable_size_descr(self): + return self class FakeCPU(object): supports_floats = False From arigo at codespeak.net Sun May 30 18:03:50 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 18:03:50 +0200 (CEST) Subject: [pypy-svn] r74923 - pypy/branch/blackhole-improvement/pypy/module/pypyjit Message-ID: <20100530160350.1F3DD282BAD@codespeak.net> Author: arigo Date: Sun May 30 18:03:49 2010 New Revision: 74923 Modified: pypy/branch/blackhole-improvement/pypy/module/pypyjit/policy.py Log: Fix import. Modified: pypy/branch/blackhole-improvement/pypy/module/pypyjit/policy.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/module/pypyjit/policy.py (original) +++ pypy/branch/blackhole-improvement/pypy/module/pypyjit/policy.py Sun May 30 18:03:49 2010 @@ -1,4 +1,4 @@ -from pypy.jit.metainterp.policy import JitPolicy +from pypy.jit.codewriter.policy import JitPolicy class PyPyJitPolicy(JitPolicy): From arigo at codespeak.net Sun May 30 18:13:13 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 18:13:13 +0200 (CEST) Subject: [pypy-svn] r74924 - pypy/branch/blackhole-improvement/pypy/jit/codewriter Message-ID: <20100530161313.D3281282BAD@codespeak.net> Author: arigo Date: Sun May 30 18:13:12 2010 New Revision: 74924 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Log: Fixes. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Sun May 30 18:13:12 2010 @@ -3,6 +3,7 @@ from pypy.jit.codewriter.flatten import ListOfKind, IndirectCallTargets from pypy.jit.codewriter.format import format_assembler from pypy.jit.codewriter.jitcode import SwitchDictDescr, JitCode +from pypy.rlib.objectmodel import ComputedIntSymbolic from pypy.objspace.flow.model import Constant from pypy.rpython.lltypesystem import lltype, llmemory, rclass @@ -71,6 +72,8 @@ value = llmemory.cast_adr_to_int(value) else: value = lltype.cast_primitive(lltype.Signed, value) + if isinstance(value, ComputedIntSymbolic): + value = value.compute_fn() if allow_short and -128 <= value <= 127: # xxx symbolic # emit the constant as a small integer self.code.append(chr(value & 0xFF)) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Sun May 30 18:13:12 2010 @@ -106,7 +106,7 @@ def _killed_exception_raising_operation(self, block): assert block.exits[0].exitcase is None - del block.exits[1:] + block.exits = block.exits[:1] block.exitswitch = None # ---------- From arigo at codespeak.net Sun May 30 18:35:47 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 18:35:47 +0200 (CEST) Subject: [pypy-svn] r74925 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100530163547.11B9A282BAD@codespeak.net> Author: arigo Date: Sun May 30 18:35:46 2010 New Revision: 74925 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Log: Fix. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Sun May 30 18:35:46 2010 @@ -1042,15 +1042,15 @@ @arguments("cpu", "r", "d", "d", "i", "i") def bhimpl_setarrayitem_vable_i(cpu, vable, fdescr, adescr, index, newval): array = cpu.bh_getfield_gc_r(vable, fdescr) - cpu.bh_setarrayitem_gc_i(a, array, index, newval) + cpu.bh_setarrayitem_gc_i(adescr, array, index, newval) @arguments("cpu", "r", "d", "d", "i", "r") def bhimpl_setarrayitem_vable_r(cpu, vable, fdescr, adescr, index, newval): array = cpu.bh_getfield_gc_r(vable, fdescr) - cpu.bh_setarrayitem_gc_r(a, array, index, newval) + cpu.bh_setarrayitem_gc_r(adescr, array, index, newval) @arguments("cpu", "r", "d", "d", "i", "f") def bhimpl_setarrayitem_vable_f(cpu, vable, fdescr, adescr, index, newval): array = cpu.bh_getfield_gc_r(vable, fdescr) - cpu.bh_setarrayitem_gc_f(a, array, index, newval) + cpu.bh_setarrayitem_gc_f(adescr, array, index, newval) @arguments("cpu", "r", "d", "d", returns="i") def bhimpl_arraylen_vable(cpu, vable, fdescr, adescr): From arigo at codespeak.net Sun May 30 18:47:33 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 18:47:33 +0200 (CEST) Subject: [pypy-svn] r74926 - pypy/branch/blackhole-improvement/pypy/jit/codewriter/test Message-ID: <20100530164733.90F83282BAD@codespeak.net> Author: arigo Date: Sun May 30 18:47:32 2010 New Revision: 74926 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Log: Fix tests (thanks Alex Gaynor). Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py Sun May 30 18:47:32 2010 @@ -21,6 +21,8 @@ class FakeSizeDescr(AbstractDescr): def __init__(self, STRUCT): self.STRUCT = STRUCT + def as_vtable_size_descr(self): + return self class FakeCPU: def __init__(self, rtyper): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Sun May 30 18:47:32 2010 @@ -32,6 +32,8 @@ class FakeDescr(AbstractDescr): def __repr__(self): return '' + def as_vtable_size_descr(self): + return self class FakeDict(object): def __getitem__(self, key): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Sun May 30 18:47:32 2010 @@ -21,7 +21,8 @@ return FakeDescr(('sizedescr', STRUCT)) class FakeDescr(tuple): - pass + def as_vtable_size_descr(self): + return self class FakeLink: args = [] From arigo at codespeak.net Sun May 30 19:21:33 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 19:21:33 +0200 (CEST) Subject: [pypy-svn] r74927 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100530172133.47836282BD8@codespeak.net> Author: arigo Date: Sun May 30 19:21:31 2010 New Revision: 74927 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py Log: Fix a bit the comment. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py Sun May 30 19:21:31 2010 @@ -22,10 +22,11 @@ self.setdescr(descr) def setdescr(self, descr): - # for 'call', 'new', 'getfield_gc'...: the descr is a number provided - # by the backend holding details about the type of the operation -- - # actually an instance of a class, typically Descr, that inherits - # from AbstractDescr + # for 'call', 'new', 'getfield_gc'...: the descr is a prebuilt + # instance provided by the backend holding details about the type + # of the operation. It must inherit from AbstractDescr. The + # backend provides it with cpu.fielddescrof(), cpu.arraydescrof(), + # cpu.calldescrof(), and cpu.typedescrof(). from pypy.jit.metainterp.history import check_descr check_descr(descr) self.descr = descr From arigo at codespeak.net Sun May 30 19:26:21 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 19:26:21 +0200 (CEST) Subject: [pypy-svn] r74928 - pypy/branch/blackhole-improvement/pypy/jit/codewriter Message-ID: <20100530172621.608C3282BD8@codespeak.net> Author: arigo Date: Sun May 30 19:26:19 2010 New Revision: 74928 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Log: Don't force the symbolics; it appears to still be too early. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Sun May 30 19:26:19 2010 @@ -70,11 +70,9 @@ TYPE = llmemory.Address if TYPE == llmemory.Address: value = llmemory.cast_adr_to_int(value) - else: + elif not isinstance(value, ComputedIntSymbolic): value = lltype.cast_primitive(lltype.Signed, value) - if isinstance(value, ComputedIntSymbolic): - value = value.compute_fn() - if allow_short and -128 <= value <= 127: # xxx symbolic + if allow_short and -128 <= value <= 127: # emit the constant as a small integer self.code.append(chr(value & 0xFF)) return True From arigo at codespeak.net Sun May 30 19:27:05 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 19:27:05 +0200 (CEST) Subject: [pypy-svn] r74929 - pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport Message-ID: <20100530172705.2091F282BD8@codespeak.net> Author: arigo Date: Sun May 30 19:27:03 2010 New Revision: 74929 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py Log: Don't compute unnecessary stuff for 'size', which is not used half of the time. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llsupport/llmodel.py Sun May 30 19:27:03 2010 @@ -203,20 +203,30 @@ def unpack_fielddescr(self, fielddescr): assert isinstance(fielddescr, BaseFieldDescr) + return fielddescr.offset + unpack_fielddescr._always_inline_ = True + + def unpack_fielddescr_size(self, fielddescr): + assert isinstance(fielddescr, BaseFieldDescr) ofs = fielddescr.offset size = fielddescr.get_field_size(self.translate_support_code) return ofs, size - unpack_fielddescr._always_inline_ = True + unpack_fielddescr_size._always_inline_ = True def arraydescrof(self, A): return get_array_descr(self.gc_ll_descr, A) def unpack_arraydescr(self, arraydescr): assert isinstance(arraydescr, BaseArrayDescr) + return arraydescr.get_base_size(self.translate_support_code) + unpack_arraydescr._always_inline_ = True + + def unpack_arraydescr_size(self, arraydescr): + assert isinstance(arraydescr, BaseArrayDescr) ofs = arraydescr.get_base_size(self.translate_support_code) size = arraydescr.get_item_size(self.translate_support_code) return ofs, size - unpack_arraydescr._always_inline_ = True + unpack_arraydescr_size._always_inline_ = True def calldescrof(self, FUNC, ARGS, RESULT, extrainfo=None): return get_call_descr(self.gc_ll_descr, ARGS, RESULT, extrainfo) @@ -241,7 +251,7 @@ return rffi.cast(rffi.CArrayPtr(lltype.Signed), array)[ofs/WORD] def bh_getarrayitem_gc_i(self, arraydescr, gcref, itemindex): - ofs, size = self.unpack_arraydescr(arraydescr) + ofs, size = self.unpack_arraydescr_size(arraydescr) # --- start of GC unsafe code (no GC operation!) --- items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) for TYPE, itemsize in unroll_basic_sizes: @@ -254,7 +264,7 @@ raise NotImplementedError("size = %d" % size) def bh_getarrayitem_gc_r(self, arraydescr, gcref, itemindex): - ofs, size = self.unpack_arraydescr(arraydescr) + ofs = self.unpack_arraydescr(arraydescr) # --- start of GC unsafe code (no GC operation!) --- items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items) @@ -263,7 +273,7 @@ return pval def bh_getarrayitem_gc_f(self, arraydescr, gcref, itemindex): - ofs, size = self.unpack_arraydescr(arraydescr) + ofs = self.unpack_arraydescr(arraydescr) # --- start of GC unsafe code (no GC operation!) --- items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) items = rffi.cast(rffi.CArrayPtr(lltype.Float), items) @@ -272,7 +282,7 @@ return fval def bh_setarrayitem_gc_i(self, arraydescr, gcref, itemindex, newvalue): - ofs, size = self.unpack_arraydescr(arraydescr) + ofs, size = self.unpack_arraydescr_size(arraydescr) # --- start of GC unsafe code (no GC operation!) --- items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) for TYPE, itemsize in unroll_basic_sizes: @@ -285,7 +295,7 @@ raise NotImplementedError("size = %d" % size) def bh_setarrayitem_gc_r(self, arraydescr, gcref, itemindex, newvalue): - ofs, size = self.unpack_arraydescr(arraydescr) + ofs = self.unpack_arraydescr(arraydescr) self.gc_ll_descr.do_write_barrier(gcref, newvalue) # --- start of GC unsafe code (no GC operation!) --- items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) @@ -294,7 +304,7 @@ # --- end of GC unsafe code --- def bh_setarrayitem_gc_f(self, arraydescr, gcref, itemindex, newvalue): - ofs, size = self.unpack_arraydescr(arraydescr) + ofs = self.unpack_arraydescr(arraydescr) # --- start of GC unsafe code (no GC operation!) --- items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) items = rffi.cast(rffi.CArrayPtr(lltype.Float), items) @@ -319,7 +329,7 @@ @specialize.argtype(1) def _base_do_getfield_i(self, struct, fielddescr): - ofs, size = self.unpack_fielddescr(fielddescr) + ofs, size = self.unpack_fielddescr_size(fielddescr) # --- start of GC unsafe code (no GC operation!) --- fieldptr = rffi.ptradd(rffi.cast(rffi.CCHARP, struct), ofs) for TYPE, itemsize in unroll_basic_sizes: @@ -332,7 +342,7 @@ @specialize.argtype(1) def _base_do_getfield_r(self, struct, fielddescr): - ofs, size = self.unpack_fielddescr(fielddescr) + ofs = self.unpack_fielddescr(fielddescr) # --- start of GC unsafe code (no GC operation!) --- fieldptr = rffi.ptradd(rffi.cast(rffi.CCHARP, struct), ofs) pval = rffi.cast(rffi.CArrayPtr(lltype.Signed), fieldptr)[0] @@ -342,7 +352,7 @@ @specialize.argtype(1) def _base_do_getfield_f(self, struct, fielddescr): - ofs, size = self.unpack_fielddescr(fielddescr) + ofs = self.unpack_fielddescr(fielddescr) # --- start of GC unsafe code (no GC operation!) --- fieldptr = rffi.ptradd(rffi.cast(rffi.CCHARP, struct), ofs) fval = rffi.cast(rffi.CArrayPtr(lltype.Float), fieldptr)[0] @@ -358,7 +368,7 @@ @specialize.argtype(1) def _base_do_setfield_i(self, struct, fielddescr, newvalue): - ofs, size = self.unpack_fielddescr(fielddescr) + ofs, size = self.unpack_fielddescr_size(fielddescr) # --- start of GC unsafe code (no GC operation!) --- fieldptr = rffi.ptradd(rffi.cast(rffi.CCHARP, struct), ofs) for TYPE, itemsize in unroll_basic_sizes: @@ -372,7 +382,7 @@ @specialize.argtype(1) def _base_do_setfield_r(self, struct, fielddescr, newvalue): - ofs, size = self.unpack_fielddescr(fielddescr) + ofs = self.unpack_fielddescr(fielddescr) assert lltype.typeOf(struct) is not lltype.Signed, ( "can't handle write barriers for setfield_raw") self.gc_ll_descr.do_write_barrier(struct, newvalue) @@ -384,7 +394,7 @@ @specialize.argtype(1) def _base_do_setfield_f(self, struct, fielddescr, newvalue): - ofs, size = self.unpack_fielddescr(fielddescr) + ofs = self.unpack_fielddescr(fielddescr) # --- start of GC unsafe code (no GC operation!) --- fieldptr = rffi.ptradd(rffi.cast(rffi.CCHARP, struct), ofs) fieldptr = rffi.cast(rffi.CArrayPtr(lltype.Float), fieldptr) From arigo at codespeak.net Sun May 30 20:30:08 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 20:30:08 +0200 (CEST) Subject: [pypy-svn] r74930 - pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test Message-ID: <20100530183008.EF365282BAD@codespeak.net> Author: arigo Date: Sun May 30 20:30:07 2010 New Revision: 74930 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_ztranslation.py Log: A failing test. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_ztranslation.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_ztranslation.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/x86/test/test_ztranslation.py Sun May 30 20:30:07 2010 @@ -127,8 +127,14 @@ def test_external_exception_handling_translates(self): jitdriver = JitDriver(greens = [], reds = ['n', 'total']) + class ImDone(Exception): + def __init__(self, resvalue): + self.resvalue = resvalue + @dont_look_inside - def f(x): + def f(x, total): + if x <= 3: + raise ImDone(total * 10) if x > 20: return 2 raise ValueError @@ -149,16 +155,16 @@ return Sub() else: return Base() - def main(i): + def myportal(i): jitdriver.set_param("threshold", 3) jitdriver.set_param("trace_eagerness", 2) total = 0 n = i - while n > 3: + while True: jitdriver.can_enter_jit(n=n, total=total) jitdriver.jit_merge_point(n=n, total=total) try: - total += f(n) + total += f(n, total) except ValueError: total += 1 try: @@ -166,7 +172,11 @@ except ValueError: total -= 1 n -= h(n).meth() # this is to force a GUARD_CLASS - return total * 10 + def main(i): + try: + myportal(i) + except ImDone, e: + return e.resvalue # XXX custom fishing, depends on the exact env var and format logfile = udir.join('test_ztranslation.log') From arigo at codespeak.net Sun May 30 20:36:39 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 20:36:39 +0200 (CEST) Subject: [pypy-svn] r74931 - pypy/branch/blackhole-improvement/pypy/rpython/test Message-ID: <20100530183639.CD1D7282BAD@codespeak.net> Author: arigo Date: Sun May 30 20:36:38 2010 New Revision: 74931 Modified: pypy/branch/blackhole-improvement/pypy/rpython/test/test_exception.py Log: This is the root of the failure of the previous checkin. It should either work, or clearly complain that it's invalid (and then we need another way to do that). Modified: pypy/branch/blackhole-improvement/pypy/rpython/test/test_exception.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/test/test_exception.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/test/test_exception.py Sun May 30 20:36:38 2010 @@ -125,7 +125,19 @@ class TestLLtype(BaseTestException, LLRtypeMixin): - pass + def test_raise_ll_exception(self): + from pypy.rpython.annlowlevel import cast_instance_to_base_ptr + def g(): + e = OverflowError() + lle = cast_instance_to_base_ptr(e) + raise Exception, lle + def f(): + try: + g() + except OverflowError: + return 42 + res = self.interpret(f, []) + assert res == 42 class TestOOtype(BaseTestException, OORtypeMixin): pass From agaynor at codespeak.net Sun May 30 20:51:06 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sun, 30 May 2010 20:51:06 +0200 (CEST) Subject: [pypy-svn] r74932 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100530185106.D72E8282BAD@codespeak.net> Author: agaynor Date: Sun May 30 20:51:05 2010 New Revision: 74932 Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py pypy/trunk/pypy/module/cpyext/test/comparisons.c pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Log: Implemented hashfunc wrapper for cpyext Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/slotdefs.py (original) +++ pypy/trunk/pypy/module/cpyext/slotdefs.py Sun May 30 20:51:05 2010 @@ -5,7 +5,7 @@ from pypy.module.cpyext.typeobjectdefs import ( unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc, getattrfunc, setattrofunc, lenfunc, ssizeargfunc, ssizessizeargfunc, - ssizeobjargproc, iternextfunc, initproc, richcmpfunc) + ssizeobjargproc, iternextfunc, initproc, richcmpfunc, hashfunc) from pypy.module.cpyext.pyobject import from_ref from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.state import State @@ -133,6 +133,11 @@ raise OperationError(space.w_StopIteration, space.w_None) return w_res +def wrap_hashfunc(space, w_self, w_args, func): + func_target = rffi.cast(hashfunc, func) + check_num_args(space, w_args, 0) + return space.wrap(generic_cpy_call(space, func_target, w_self)) + def get_richcmp_func(OP_CONST): def inner(space, w_self, w_args, func): func_target = rffi.cast(richcmpfunc, func) Modified: pypy/trunk/pypy/module/cpyext/test/comparisons.c ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/comparisons.c (original) +++ pypy/trunk/pypy/module/cpyext/test/comparisons.c Sun May 30 20:51:05 2010 @@ -19,6 +19,10 @@ } } +static long cmp_hashfunc(PyObject *self) { + return 3; +} + PyTypeObject CmpType = { PyVarObject_HEAD_INIT(NULL, 0) @@ -34,7 +38,7 @@ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ - 0, /* tp_hash */ + cmp_hashfunc, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ 0, /* tp_getattro */ Modified: pypy/trunk/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_typeobject.py Sun May 30 20:51:05 2010 @@ -182,6 +182,15 @@ cmpr = module.CmpType() assert cmpr == 3 assert cmpr != 42 + + def test_hash(self): + module = self.import_module("comparisons") + cmpr = module.CmpType() + assert hash(cmpr) == 3 + d = {} + d[cmpr] = 72 + assert d[cmpr] == 72 + assert d[3] == 72 class TestTypes(BaseApiTest): From arigo at codespeak.net Sun May 30 20:51:36 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 20:51:36 +0200 (CEST) Subject: [pypy-svn] r74933 - in pypy/branch/blackhole-improvement/pypy: annotation annotation/test rpython/test Message-ID: <20100530185136.ADEE7282BAD@codespeak.net> Author: arigo Date: Sun May 30 20:51:34 2010 New Revision: 74933 Modified: pypy/branch/blackhole-improvement/pypy/annotation/description.py pypy/branch/blackhole-improvement/pypy/annotation/test/test_annrpython.py pypy/branch/blackhole-improvement/pypy/rpython/test/test_exception.py Log: Add negative tests, and add an explicit check to detect the bogus case. Modified: pypy/branch/blackhole-improvement/pypy/annotation/description.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/annotation/description.py (original) +++ pypy/branch/blackhole-improvement/pypy/annotation/description.py Sun May 30 20:51:34 2010 @@ -538,9 +538,18 @@ try: args.fixedunpack(0) except ValueError: - raise Exception("default __init__ takes no argument" " (class %s)" % (self.name,)) + elif self.pyobj is Exception: + # check explicitly against "raise Exception, x" where x + # is a low-level exception pointer + try: + [s_arg] = args.fixedunpack(1) + except ValueError: + pass + else: + from pypy.annotation.model import SomePtr + assert not isinstance(s_arg, SomePtr) else: # call the constructor args = args.prepend(s_instance) Modified: pypy/branch/blackhole-improvement/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/annotation/test/test_annrpython.py (original) +++ pypy/branch/blackhole-improvement/pypy/annotation/test/test_annrpython.py Sun May 30 20:51:34 2010 @@ -3283,6 +3283,17 @@ s = a.build_types(f, [int]) assert s.knowntype is int + def test_cannot_raise_ll_exception(self): + from pypy.rpython.annlowlevel import cast_instance_to_base_ptr + # + def f(): + e = OverflowError() + lle = cast_instance_to_base_ptr(e) + raise Exception, lle + # ^^^ instead, must cast back from a base ptr to an instance + a = self.RPythonAnnotator() + py.test.raises(AssertionError, a.build_types, f, []) + def g(n): return [0,1,2,n] Modified: pypy/branch/blackhole-improvement/pypy/rpython/test/test_exception.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/test/test_exception.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/test/test_exception.py Sun May 30 20:51:34 2010 @@ -3,6 +3,7 @@ from pypy.rpython.lltypesystem.lltype import * from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin from pypy.rpython.llinterp import LLException +from pypy.rpython.error import MissingRTypeOperation class MyException(Exception): pass @@ -125,19 +126,18 @@ class TestLLtype(BaseTestException, LLRtypeMixin): - def test_raise_ll_exception(self): + def test_cannot_raise_ll_exception(self): from pypy.rpython.annlowlevel import cast_instance_to_base_ptr def g(): e = OverflowError() lle = cast_instance_to_base_ptr(e) - raise Exception, lle + raise lle # instead, must cast back from a base ptr to an instance def f(): try: g() except OverflowError: return 42 - res = self.interpret(f, []) - assert res == 42 + py.test.raises(MissingRTypeOperation, self.interpret, f, []) class TestOOtype(BaseTestException, OORtypeMixin): pass From arigo at codespeak.net Sun May 30 20:56:47 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 20:56:47 +0200 (CEST) Subject: [pypy-svn] r74934 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100530185647.32529282BAD@codespeak.net> Author: arigo Date: Sun May 30 20:56:45 2010 New Revision: 74934 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/jitexc.py Log: Fix. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Sun May 30 20:56:45 2010 @@ -8,7 +8,7 @@ from pypy.rpython.llinterp import LLException from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr from pypy.jit.codewriter import heaptracker -from pypy.jit.metainterp.jitexc import JitException, get_llexception +from pypy.jit.metainterp.jitexc import JitException, get_llexception, reraise def arguments(*argtypes, **kwds): @@ -361,11 +361,7 @@ self.position = target return # no 'catch_exception' insn follows: just reraise - if we_are_translated(): - raise Exception, e - else: - etype = rclass.ll_type(e) - raise LLException(etype, e) + reraise(e) def copy_constants(self, registers, constants): """Copy jitcode.constants[0] to registers[255], @@ -758,21 +754,13 @@ def bhimpl_raise(self, excvalue): e = lltype.cast_opaque_ptr(rclass.OBJECTPTR, excvalue) assert e - if we_are_translated(): - raise Exception, e - else: - etype = rclass.ll_type(e) - raise LLException(etype, e) + reraise(e) @arguments("self") def bhimpl_reraise(self): e = self.exception_last_value assert e - if we_are_translated(): - raise Exception, e - else: - etype = rclass.ll_type(e) - raise LLException(etype, e) + reraise(e) # ---------- # the main hints and recursive calls Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/jitexc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/jitexc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/jitexc.py Sun May 30 20:56:45 2010 @@ -1,4 +1,6 @@ from pypy.rpython.annlowlevel import cast_instance_to_base_ptr +from pypy.rpython.annlowlevel import cast_base_ptr_to_instance +from pypy.rpython.lltypesystem import rclass from pypy.rpython.llinterp import LLException from pypy.rlib.objectmodel import we_are_translated @@ -26,3 +28,11 @@ if isinstance(e, OverflowError): return _get_standard_error(cpu.rtyper, OverflowError) raise # leave other exceptions to be propagated + +def reraise(lle): + if we_are_translated(): + e = cast_base_ptr_to_instance(Exception, lle) + raise e + else: + etype = rclass.ll_type(lle) + raise LLException(etype, lle) From getxsick at codespeak.net Sun May 30 22:07:17 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Sun, 30 May 2010 22:07:17 +0200 (CEST) Subject: [pypy-svn] r74935 - pypy/branch/fast-ctypes/pypy/module/jitffi Message-ID: <20100530200717.8FA3A282B9C@codespeak.net> Author: getxsick Date: Sun May 30 22:07:15 2010 New Revision: 74935 Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Log: implement call() it's working with some limitations (e.g. fixed number of arguments etc.) Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Sun May 30 22:07:15 2010 @@ -4,6 +4,10 @@ from pypy.interpreter.typedef import TypeDef from pypy.rlib import libffi from pypy.rpython.lltypesystem import rffi, lltype +from pypy.jit.backend.x86.runner import CPU +from pypy.jit.metainterp.history import LoopToken, BasicFailDescr, BoxInt +from pypy.jit.metainterp.resoperation import ResOperation, rop +from pypy.jit.metainterp.typesystem import deref class W_CDLL(Wrappable): def __init__(self, space, name): @@ -14,14 +18,45 @@ e.msg or 'unspecified error') self.name = name self.space = space + self.cpu = CPU(None, None) - def call(self, space, func, a, b): # XXX temporary fixed number of func args + def call(self, space, func, a, b): # XXX temporary fixed number of func args (ints) + # result_type argument? try: - addr = rffi.cast(lltype.Unsigned, self.cdll.getaddressindll(func)) + addr = rffi.cast(lltype.Signed, self.cdll.getaddressindll(func)) except KeyError: raise operationerrfmt(space.w_ValueError, "Cannot find symbol %s", func) - # XXX unfinished yet + + bfuncaddr = BoxInt(addr) + barg0 = BoxInt(a) + barg1 = BoxInt(b) + bres = BoxInt() + + FPTR = lltype.Ptr(lltype.FuncType([lltype.Signed, lltype.Signed], + lltype.Signed)) + FUNC = deref(FPTR) + calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + + oplist = [ResOperation(rop.CALL, [bfuncaddr, barg0, barg1], bres, + descr=calldescr), + ResOperation(rop.FINISH, [bres], None, + descr=BasicFailDescr(0))] + inputargs = [bfuncaddr, barg0, barg1] + looptoken = LoopToken() + self.cpu.compile_loop(inputargs, oplist, looptoken) + + i = 0 + for box in inputargs: + self.cpu.set_future_value_int(i, box.getint()) + i += 1 + + res = self.cpu.execute_token(looptoken) + if res is oplist[-1].descr: + self.guard_failed = False + else: + self.guard_failed = True + return space.wrap(BoxInt(self.cpu.get_latest_value_int(0)).getint()) call.unwrap_spec = ['self', ObjSpace, str, int, int] def descr_new_cdll(space, w_type, name): From arigo at codespeak.net Sun May 30 22:20:38 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 22:20:38 +0200 (CEST) Subject: [pypy-svn] r74936 - pypy/branch/blackhole-improvement/pypy/jit/codewriter Message-ID: <20100530202038.4124D282B9C@codespeak.net> Author: arigo Date: Sun May 30 22:20:36 2010 New Revision: 74936 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Log: Missing ---. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Sun May 30 22:20:36 2010 @@ -152,6 +152,7 @@ if link.target.operations == () and link.args == [link.last_exception, link.last_exc_value]: self.emitline("reraise") + self.emitline("---") return # done if link.last_exception in link.args: self.emitline("last_exception", From arigo at codespeak.net Sun May 30 22:21:03 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 22:21:03 +0200 (CEST) Subject: [pypy-svn] r74937 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter metainterp Message-ID: <20100530202103.77371282B9C@codespeak.net> Author: arigo Date: Sun May 30 22:21:01 2010 New Revision: 74937 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: Improve debugging dumps. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py Sun May 30 22:21:01 2010 @@ -52,14 +52,14 @@ # step 3b: compute the liveness around certain operations compute_liveness(ssarepr) # - # print the resulting assembler - self.print_ssa_repr(ssarepr, portal, verbose) - # # step 4: "assemble" it into a JitCode, which contains a sequence # of bytes and lists of constants. It's during this step that # constants are cast to their normalized type (Signed, GCREF or # Float). self.assembler.assemble(ssarepr, jitcode) + # + # print the resulting assembler + self.print_ssa_repr(ssarepr, portal, verbose) def make_jitcodes(self, verbose=False): log.info("making JitCodes...") @@ -90,7 +90,7 @@ def print_ssa_repr(self, ssarepr, portal, verbose): if verbose: print '%s:' % (ssarepr.name,) - print indent(format_assembler(ssarepr), 4) + print format_assembler(ssarepr) else: dir = udir.ensure("jitcodes", dir=1) if portal: @@ -101,8 +101,3 @@ name = 'unnamed_%x' % id(ssarepr) dir.join(name).write(format_assembler(ssarepr)) log.dot() - - -def indent(s, indent): - indent = ' ' * indent - return indent + s.replace('\n', '\n'+indent).rstrip(' ') Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitcode.py Sun May 30 22:21:01 2010 @@ -51,9 +51,10 @@ # 'pc' gives a position in this bytecode. This returns an object # of class LiveVarsInfo that describes all variables that are live # across the instruction boundary at 'pc'. - if not we_are_translated() and pc not in self.liveness: + try: + return self.liveness[pc] # XXX compactify!! + except KeyError: self._missing_liveness(pc) - return self.liveness[pc] # XXX compactify!! def _live_vars(self, pc): # for testing only @@ -67,7 +68,11 @@ return ' '.join(lst_i + lst_r + lst_f) def _missing_liveness(self, pc): - raise MissingLiveness("missing liveness[%d]\n%s" % (pc, self.dump())) + msg = "missing liveness[%d] in %s" % (pc, self.name) + if we_are_translated(): + print msg + raise AssertionError + raise MissingLiveness("%s\n%s" % (msg, self.dump())) def follow_jump(self, position): """Assuming that 'position' points just after a bytecode @@ -83,7 +88,7 @@ def dump(self): if self._ssarepr is None: - return '' + return '' % (self.name,) else: from pypy.jit.codewriter.format import format_assembler return format_assembler(self._ssarepr) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Sun May 30 22:21:01 2010 @@ -44,7 +44,6 @@ assert isinstance(jitcode, JitCode) self.jitcode = jitcode self.bytecode = jitcode.code - self.name = jitcode.name # purely for having name attribute # this is not None for frames that are recursive portal calls self.greenkey = greenkey # copy the constants in place From arigo at codespeak.net Sun May 30 23:02:50 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 23:02:50 +0200 (CEST) Subject: [pypy-svn] r74938 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100530210250.824E5282B9C@codespeak.net> Author: arigo Date: Sun May 30 23:02:48 2010 New Revision: 74938 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Log: Add new non-negative flags to help some more bytecode[pc]. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Sun May 30 23:02:48 2010 @@ -68,6 +68,7 @@ all_funcs = unrolling_iterable(enumerate(all_funcs)) # def dispatch_loop(self, code, position): + assert position >= 0 while True: if (not we_are_translated() and self.jitcode._startpoints is not None): @@ -207,6 +208,7 @@ self.registers_f[ord(code[position])] = result position += 1 elif resulttype == 'L': + assert result >= 0 position = result else: assert resulttype is None From arigo at codespeak.net Sun May 30 23:03:32 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 23:03:32 +0200 (CEST) Subject: [pypy-svn] r74939 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100530210332.8B90C282B9C@codespeak.net> Author: arigo Date: Sun May 30 23:03:31 2010 New Revision: 74939 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Log: Update the test along with the reason for it as comment. Fix. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jtransform.py Sun May 30 23:03:31 2010 @@ -275,14 +275,14 @@ else: raise AssertionError(kind) lst.append(v) - def handle_residual_call(self, op, extraargs=[]): + def handle_residual_call(self, op, extraargs=[], may_call_jitcodes=False): """A direct_call turns into the operation 'residual_call_xxx' if it is calling a function that we don't want to JIT. The initial args of 'residual_call_xxx' are the function to call, and its calldescr.""" calldescr = self.callcontrol.getcalldescr(op) op1 = self.rewrite_call(op, 'residual_call', [op.args[0], calldescr] + extraargs) - if self.callcontrol.calldescr_canraise(calldescr): + if may_call_jitcodes or self.callcontrol.calldescr_canraise(calldescr): op1 = [op1, SpaceOperation('-live-', [], None)] return op1 @@ -338,7 +338,7 @@ lst.append(jitcode) op0 = SpaceOperation('-live-', [], None) op1 = SpaceOperation('int_guard_value', [op.args[0]], None) - op2 = self.handle_residual_call(op, [IndirectCallTargets(lst)]) + op2 = self.handle_residual_call(op, [IndirectCallTargets(lst)], True) result = [op0, op1] if isinstance(op2, list): result += op2 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jtransform.py Sun May 30 23:03:31 2010 @@ -65,7 +65,7 @@ assert graph in ('somegraph1', 'somegraph2') return 'somejitcode' + graph[-1] def calldescr_canraise(self, calldescr): - return True + return False def test_optimize_goto_if_not(): @@ -325,6 +325,9 @@ for v in op.args[1:]: kind = getkind(v.concretetype) assert kind == 'void' or kind[0] in expectedkind + # Note: we still expect a -live- here, even though canraise() returns + # False, because this 'residual_call' will likely call further jitcodes + # which can do e.g. guard_class or other stuff requiring anyway a -live-. assert op1.opname == '-live-' assert op1.args == [] From arigo at codespeak.net Sun May 30 23:06:00 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 May 2010 23:06:00 +0200 (CEST) Subject: [pypy-svn] r74940 - pypy/branch/blackhole-improvement/pypy/jit/codewriter Message-ID: <20100530210600.8AFBE282B9C@codespeak.net> Author: arigo Date: Sun May 30 23:05:59 2010 New Revision: 74940 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py Log: Detect name collisions and create files called xxx.2, xxx.3, ... Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py Sun May 30 23:05:59 2010 @@ -98,6 +98,11 @@ elif ssarepr.name and ssarepr.name != '?': name = ssarepr.name else: - name = 'unnamed_%x' % id(ssarepr) - dir.join(name).write(format_assembler(ssarepr)) + name = 'unnamed' % id(ssarepr) + i = 1 + extra = '' + while dir.join(name+extra).check(exists=1): + i += 1 + extra = '.%d' % i + dir.join(name+extra).write(format_assembler(ssarepr)) log.dot() From arigo at codespeak.net Mon May 31 00:11:03 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 31 May 2010 00:11:03 +0200 (CEST) Subject: [pypy-svn] r74941 - pypy/branch/blackhole-improvement/pypy/jit/metainterp/test Message-ID: <20100530221103.2FE1B282B9C@codespeak.net> Author: arigo Date: Mon May 31 00:11:01 2010 New Revision: 74941 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_exception.py Log: Test for a bug. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_exception.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_exception.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_exception.py Mon May 31 00:11:01 2010 @@ -555,6 +555,26 @@ optimizer=OPTIMIZER_SIMPLE) assert res == 8 + def test_bug_1(self): + def h(i): + if i > 10: + raise ValueError + if i > 5: + raise KeyError + return 5 + def g(i): + try: + return h(i) + except ValueError: + return 21 + def f(i): + try: + return g(i) + except KeyError: + return 42 + res = self.interp_operations(f, [99]) + assert res == 21 + class MyError(Exception): def __init__(self, n): self.n = n From arigo at codespeak.net Mon May 31 00:34:44 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 31 May 2010 00:34:44 +0200 (CEST) Subject: [pypy-svn] r74942 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100530223444.C784F282B9C@codespeak.net> Author: arigo Date: Mon May 31 00:34:42 2010 New Revision: 74942 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: Fix the bug of r74941. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Mon May 31 00:34:42 2010 @@ -1287,6 +1287,8 @@ self.free_frames_list.append(frame) def finishframe(self, resultbox): + # handle a non-exceptional return from the current frame + self.last_exc_value_box = None self.popframe() if self.framestack: self.framestack[-1].make_result_of_lastop(resultbox) From arigo at codespeak.net Mon May 31 00:47:42 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 31 May 2010 00:47:42 +0200 (CEST) Subject: [pypy-svn] r74943 - pypy/branch/blackhole-improvement/pypy/jit/metainterp/test Message-ID: <20100530224742.268D6282B9C@codespeak.net> Author: arigo Date: Mon May 31 00:47:39 2010 New Revision: 74943 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Log: The next failing test. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Mon May 31 00:47:39 2010 @@ -366,6 +366,38 @@ assert res == 42 self.check_operations_history(int_add=0, int_mul=0, call=0) + def test_constfold_call_pure(self): + myjitdriver = JitDriver(greens = ['m'], reds = ['n']) + def externfn(x): + return x - 3 + externfn._pure_function_ = True + def f(n, m): + while n > 0: + myjitdriver.can_enter_jit(n=n, m=m) + myjitdriver.jit_merge_point(n=n, m=m) + n -= externfn(m) + return n + res = self.meta_interp(f, [21, 5]) + assert res == -1 + + def test_constfold_call_pure_2(self): + myjitdriver = JitDriver(greens = ['m'], reds = ['n']) + def externfn(x): + return x - 3 + externfn._pure_function_ = True + class V: + def __init__(self, value): + self.value = value + def f(n, m): + while n > 0: + myjitdriver.can_enter_jit(n=n, m=m) + myjitdriver.jit_merge_point(n=n, m=m) + v = V(m) + n -= externfn(v.value) + return n + res = self.meta_interp(f, [21, 5]) + assert res == -1 + def test_constant_across_mp(self): myjitdriver = JitDriver(greens = [], reds = ['n']) class X(object): From getxsick at codespeak.net Mon May 31 00:57:20 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Mon, 31 May 2010 00:57:20 +0200 (CEST) Subject: [pypy-svn] r74944 - in pypy/trunk/pypy/rlib: . test Message-ID: <20100530225720.CF07D282B9C@codespeak.net> Author: getxsick Date: Mon May 31 00:57:19 2010 New Revision: 74944 Added: pypy/trunk/pypy/rlib/rdynload.py pypy/trunk/pypy/rlib/test/test_rdynload.py Modified: pypy/trunk/pypy/rlib/libffi.py pypy/trunk/pypy/rlib/test/test_libffi.py Log: move dl* functions to pypy.rlib.rdynload Modified: pypy/trunk/pypy/rlib/libffi.py ============================================================================== --- pypy/trunk/pypy/rlib/libffi.py (original) +++ pypy/trunk/pypy/rlib/libffi.py Mon May 31 00:57:19 2010 @@ -1,5 +1,4 @@ - -""" Various rpython-level functions for dlopen and libffi wrapping +""" Libffi wrapping """ from pypy.rpython.tool import rffi_platform @@ -7,15 +6,17 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rarithmetic import intmask, r_uint from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.rmmap import alloc +from pypy.rlib.rdynload import dlopen, dlclose, dlsym, dlsym_byordinal +from pypy.rlib.rdynload import DLOpenError from pypy.tool.autopath import pypydir from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.rlib.rmmap import alloc +from pypy.translator.platform import platform import py import os import sys import ctypes.util -from pypy.translator.platform import platform # maaaybe isinstance here would be better. Think _MSVC = platform.name == "msvc" @@ -55,18 +56,13 @@ if _MINGW: includes = ['windows.h', 'ffi.h'] else: - includes = ['dlfcn.h', 'ffi.h'] + includes = ['ffi.h'] if _MAC_OS: pre_include_bits = ['#define MACOSX'] else: pre_include_bits = [] - if _FREEBSD_7 or _MINGW: - libraries = [] - else: - libraries = ['dl'] - def find_libffi_a(): dirlist = platform.library_dirs_for_libffi_a() for dir in dirlist: @@ -77,10 +73,11 @@ if hasattr(platform, 'library_dirs_for_libffi_a'): # platforms on which we want static linking + libraries = [] link_files = [find_libffi_a()] else: # platforms on which we want dynamic linking - libraries = ['ffi'] + libraries + libraries = ['ffi'] link_files = [] eci = ExternalCompilationInfo( @@ -115,10 +112,6 @@ class CConfig: _compilation_info_ = eci - RTLD_LOCAL = rffi_platform.DefinedConstantInteger('RTLD_LOCAL') - RTLD_GLOBAL = rffi_platform.DefinedConstantInteger('RTLD_GLOBAL') - RTLD_NOW = rffi_platform.DefinedConstantInteger('RTLD_NOW') - FFI_OK = rffi_platform.ConstantInteger('FFI_OK') FFI_BAD_TYPEDEF = rffi_platform.ConstantInteger('FFI_BAD_TYPEDEF') FFI_DEFAULT_ABI = rffi_platform.ConstantInteger('FFI_DEFAULT_ABI') @@ -216,61 +209,8 @@ def winexternal(name, args, result): return rffi.llexternal(name, args, result, compilation_info=eci, calling_conv='win') -class DLOpenError(Exception): - def __init__(self, msg): - self.msg = msg - def __str__(self): - return repr(self.msg) - if not _WIN32: - c_dlopen = external('dlopen', [rffi.CCHARP, rffi.INT], rffi.VOIDP) - c_dlclose = external('dlclose', [rffi.VOIDP], rffi.INT) - c_dlerror = external('dlerror', [], rffi.CCHARP) - c_dlsym = external('dlsym', [rffi.VOIDP, rffi.CCHARP], rffi.VOIDP) - - RTLD_LOCAL = cConfig.RTLD_LOCAL - RTLD_GLOBAL = cConfig.RTLD_GLOBAL - RTLD_NOW = cConfig.RTLD_NOW - - def dlerror(): - # XXX this would never work on top of ll2ctypes, because - # ctypes are calling dlerror itself, unsure if I can do much in this - # area (nor I would like to) - res = c_dlerror() - if not res: - return "" - return rffi.charp2str(res) - - def dlopen(name, mode=-1): - """ Wrapper around C-level dlopen - """ - if mode == -1: - if RTLD_LOCAL is not None: - mode = RTLD_LOCAL | RTLD_NOW - else: - mode = RTLD_NOW - res = c_dlopen(name, rffi.cast(rffi.INT, mode)) - if not res: - err = dlerror() - raise DLOpenError(err) - return res - - dlclose = c_dlclose - - def dlsym(libhandle, name): - """ Wrapper around C-level dlsym - """ - res = c_dlsym(libhandle, name) - if not res: - raise KeyError(name) - # XXX rffi.cast here... - return res - - def dlsym_byordinal(handle, index): - # Never called - raise KeyError(index) - def check_fficall_result(result, flags): pass # No check @@ -281,36 +221,6 @@ return libc_name if _WIN32: - def dlopen(name): - res = rwin32.LoadLibrary(name) - if not res: - err = rwin32.GetLastError() - raise DLOpenError(rwin32.FormatError(err)) - return res - - def dlclose(handle): - res = rwin32.FreeLibrary(handle) - if res: - return -1 - else: - return 0 - - def dlsym(handle, name): - res = rwin32.GetProcAddress(handle, name) - if not res: - raise KeyError(name) - # XXX rffi.cast here... - return res - - def dlsym_byordinal(handle, index): - # equivalent to MAKEINTRESOURCEA - intresource = rffi.cast(rffi.CCHARP, r_uint(index) & 0xFFFF) - res = rwin32.GetProcAddress(handle, intresource) - if not res: - raise KeyError(name) - # XXX rffi.cast here... - return res - def check_fficall_result(result, flags): if result == 0: return Added: pypy/trunk/pypy/rlib/rdynload.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/rlib/rdynload.py Mon May 31 00:57:19 2010 @@ -0,0 +1,150 @@ +""" Various rpython-level functions for dlopen +""" + +from pypy.rpython.tool import rffi_platform +from pypy.rpython.lltypesystem import rffi +from pypy.rlib.rarithmetic import r_uint +from pypy.translator.tool.cbuild import ExternalCompilationInfo +from pypy.translator.platform import platform + +import sys + +# maaaybe isinstance here would be better. Think +_MSVC = platform.name == "msvc" +_MINGW = platform.name == "mingw32" +_WIN32 = _MSVC or _MINGW +_MAC_OS = platform.name == "darwin" +_FREEBSD_7 = platform.name == "freebsd7" + +if _WIN32: + from pypy.rlib import rwin32 + +if _MINGW: + includes = ['windows.h'] +else: + includes = ['dlfcn.h'] + +if _MAC_OS: + pre_include_bits = ['#define MACOSX'] +else: + pre_include_bits = [] + +if _FREEBSD_7 or _MINGW: # no need of library or not supported? + libraries = [] +elif _MSVC: + libraries = ['kernel32'] # not sure if needed +else: + libraries = ['dl'] + +eci = ExternalCompilationInfo( + pre_include_bits = pre_include_bits, + includes = includes, + libraries = libraries, +) + +class CConfig: + _compilation_info_ = eci + + RTLD_LOCAL = rffi_platform.DefinedConstantInteger('RTLD_LOCAL') + RTLD_GLOBAL = rffi_platform.DefinedConstantInteger('RTLD_GLOBAL') + RTLD_NOW = rffi_platform.DefinedConstantInteger('RTLD_NOW') + +class cConfig: + pass + +for k, v in rffi_platform.configure(CConfig).items(): + setattr(cConfig, k, v) + +def external(name, args, result, **kwds): + return rffi.llexternal(name, args, result, compilation_info=eci, **kwds) + +def winexternal(name, args, result): + return rffi.llexternal(name, args, result, compilation_info=eci, calling_conv='win') + +class DLOpenError(Exception): + def __init__(self, msg): + self.msg = msg + def __str__(self): + return repr(self.msg) + + +if not _WIN32: + c_dlopen = external('dlopen', [rffi.CCHARP, rffi.INT], rffi.VOIDP) + c_dlclose = external('dlclose', [rffi.VOIDP], rffi.INT) + c_dlerror = external('dlerror', [], rffi.CCHARP) + c_dlsym = external('dlsym', [rffi.VOIDP, rffi.CCHARP], rffi.VOIDP) + + RTLD_LOCAL = cConfig.RTLD_LOCAL + RTLD_GLOBAL = cConfig.RTLD_GLOBAL + RTLD_NOW = cConfig.RTLD_NOW + + def dlerror(): + # XXX this would never work on top of ll2ctypes, because + # ctypes are calling dlerror itself, unsure if I can do much in this + # area (nor I would like to) + res = c_dlerror() + if not res: + return "" + return rffi.charp2str(res) + + def dlopen(name, mode=-1): + """ Wrapper around C-level dlopen + """ + if mode == -1: + if RTLD_LOCAL is not None: + mode = RTLD_LOCAL | RTLD_NOW + else: + mode = RTLD_NOW + res = c_dlopen(name, rffi.cast(rffi.INT, mode)) + if not res: + err = dlerror() + raise DLOpenError(err) + return res + + dlclose = c_dlclose + + def dlsym(libhandle, name): + """ Wrapper around C-level dlsym + """ + res = c_dlsym(libhandle, name) + if not res: + raise KeyError(name) + # XXX rffi.cast here... + return res + + def dlsym_byordinal(handle, index): + # Never called + raise KeyError(index) + +if _WIN32: + def dlopen(name): + res = rwin32.LoadLibrary(name) + if not res: + err = rwin32.GetLastError() + raise DLOpenError(rwin32.FormatError(err)) + return res + + def dlclose(handle): + res = rwin32.FreeLibrary(handle) + if res: + return -1 + else: + return 0 + + def dlsym(handle, name): + res = rwin32.GetProcAddress(handle, name) + if not res: + raise KeyError(name) + # XXX rffi.cast here... + return res + + def dlsym_byordinal(handle, index): + # equivalent to MAKEINTRESOURCEA + intresource = rffi.cast(rffi.CCHARP, r_uint(index) & 0xFFFF) + res = rwin32.GetProcAddress(handle, intresource) + if not res: + raise KeyError(name) + # XXX rffi.cast here... + return res + + LoadLibrary = rwin32.LoadLibrary Modified: pypy/trunk/pypy/rlib/test/test_libffi.py ============================================================================== --- pypy/trunk/pypy/rlib/test/test_libffi.py (original) +++ pypy/trunk/pypy/rlib/test/test_libffi.py Mon May 31 00:57:19 2010 @@ -1,15 +1,14 @@ -""" Tests of libffi wrappers and dl* friends +""" Tests of libffi wrapper """ -from pypy.rpython.test.test_llinterp import interpret from pypy.translator.c.test.test_genc import compile from pypy.rlib.libffi import * from pypy.rlib.objectmodel import keepalive_until_here from pypy.rpython.lltypesystem.ll2ctypes import ALLOCATED from pypy.rpython.lltypesystem import rffi, lltype -import os, sys import py +import sys import time def setup_module(mod): @@ -19,14 +18,10 @@ ffistruct = globals()[name] rffi.cast(rffi.VOIDP, ffistruct) -class TestDLOperations: +class TestLibffi: def setup_method(self, meth): ALLOCATED.clear() - def test_dlopen(self): - py.test.raises(DLOpenError, "dlopen(rffi.str2charp('xxxxxxxxxxxx'))") - assert dlopen(rffi.str2charp(get_libc_name())) - def get_libc(self): return CDLL(get_libc_name()) Added: pypy/trunk/pypy/rlib/test/test_rdynload.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/rlib/test/test_rdynload.py Mon May 31 00:57:19 2010 @@ -0,0 +1,16 @@ +from pypy.rlib.rdynload import * +from pypy.rlib.libffi import get_libc_name +from pypy.rpython.lltypesystem import rffi, lltype +import py + +class TestDLOperations: + def test_dlopen(self): + py.test.raises(DLOpenError, "dlopen(rffi.str2charp('xxxxxxxxxxxx'))") + assert dlopen(rffi.str2charp(get_libc_name())) + + def test_dlsym(self): + lib = dlopen(rffi.str2charp(get_libc_name())) + handle = rffi.cast(lltype.Ptr(lltype.FuncType([lltype.Signed], + lltype.Signed)), dlsym(lib, 'abs')) + assert 1 == handle(1) + assert 1 == handle(-1) From arigo at codespeak.net Mon May 31 00:58:46 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 31 May 2010 00:58:46 +0200 (CEST) Subject: [pypy-svn] r74945 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100530225846.7858C282B9C@codespeak.net> Author: arigo Date: Mon May 31 00:58:44 2010 New Revision: 74945 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py Log: Fix the test of r74943. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py Mon May 31 00:58:44 2010 @@ -15,8 +15,7 @@ # ____________________________________________________________ -def do_call(cpu, metainterp, argboxes, descr): - assert metainterp is not None +def _prepare_call(argboxes, descr): # count the number of arguments of the different types count_i = count_r = count_f = 0 for i in range(1, len(argboxes)): @@ -48,6 +47,11 @@ func = argboxes[0].getint() # do the call using the correct function from the cpu rettype = descr.get_return_type() + return func, args_i, args_r, args_f, rettype + +def do_call(cpu, metainterp, argboxes, descr): + assert metainterp is not None + func, args_i, args_r, args_f, rettype = _prepare_call(argboxes, descr) if rettype == INT: try: result = cpu.bh_call_i(func, descr, args_i, args_r, args_f) @@ -85,10 +89,24 @@ return None raise AssertionError("bad rettype") -do_call_pure = do_call do_call_loopinvariant = do_call do_call_may_force = do_call +def do_call_pure(cpu, _, argboxes, descr): + # this version does not deal with exceptions at all + # xxx in case of MemoryError, it crashes the JIT + func, args_i, args_r, args_f, rettype = _prepare_call(argboxes, descr) + if rettype == INT: + return BoxInt(cpu.bh_call_i(func, descr, args_i, args_r, args_f)) + if rettype == REF: + return BoxPtr(cpu.bh_call_r(func, descr, args_i, args_r, args_f)) + if rettype == FLOAT: + return BoxFloat(cpu.bh_call_f(func, descr, args_i, args_r, args_f)) + if rettype == VOID: + cpu.bh_call_v(func, descr, args_i, args_r, args_f) + return None + raise AssertionError("bad rettype") + def do_getarrayitem_gc(cpu, _, arraybox, indexbox, arraydescr): array = arraybox.getref_base() index = indexbox.getint() @@ -253,8 +271,6 @@ # if execute[value] is not None: raise AssertionError("duplicate entry for op number %d"% value) - if key.endswith('_PURE'): - key = key[:-5] # # Fish for a way for the pyjitpl interpreter to delegate # really running the operation to the blackhole interpreter @@ -264,6 +280,15 @@ if name in globals(): execute[value] = globals()[name] continue + # + # Maybe the same without the _PURE suffix? + if key.endswith('_PURE'): + key = key[:-5] + name = 'do_' + key.lower() + if name in globals(): + execute[value] = globals()[name] + continue + # # If missing, fallback to the bhimpl_xxx() method of the # blackhole interpreter. This only works if there is a # method of the exact same name and it accepts simple From benjamin at codespeak.net Mon May 31 01:22:38 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 31 May 2010 01:22:38 +0200 (CEST) Subject: [pypy-svn] r74946 - in pypy/trunk/pypy/rlib: . test Message-ID: <20100530232238.4200E282B9C@codespeak.net> Author: benjamin Date: Mon May 31 01:22:36 2010 New Revision: 74946 Modified: pypy/trunk/pypy/rlib/rdynload.py (props changed) pypy/trunk/pypy/rlib/test/test_rdynload.py (props changed) Log: set svn:eol-style on new rlib files From getxsick at codespeak.net Mon May 31 01:31:11 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Mon, 31 May 2010 01:31:11 +0200 (CEST) Subject: [pypy-svn] r74947 - pypy/trunk/pypy/module/cpyext Message-ID: <20100530233111.1D28B282B9C@codespeak.net> Author: getxsick Date: Mon May 31 01:31:10 2010 New Revision: 74947 Modified: pypy/trunk/pypy/module/cpyext/api.py Log: use rdynload instead libffi Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Mon May 31 01:31:10 2010 @@ -802,16 +802,18 @@ state = space.fromcache(State) state.package_context = name try: - from pypy.rlib import libffi + from pypy.rlib import rdynload try: - dll = libffi.CDLL(path, False) - except libffi.DLOpenError, e: + ll_libname = rffi.str2charp(path) + dll = rdynload.dlopen(ll_libname) + lltype.free(ll_libname, flavor='raw') + except rdynload.DLOpenError, e: raise operationerrfmt( space.w_ImportError, "unable to load extension module '%s': %s", path, e.msg) try: - initptr = libffi.dlsym(dll.lib, 'init%s' % (name.split('.')[-1],)) + initptr = rdynload.dlsym(dll, 'init%s' % (name.split('.')[-1],)) except KeyError: raise operationerrfmt( space.w_ImportError, From afa at codespeak.net Mon May 31 01:36:35 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 31 May 2010 01:36:35 +0200 (CEST) Subject: [pypy-svn] r74948 - pypy/trunk/pypy/rlib Message-ID: <20100530233635.9A884282B9C@codespeak.net> Author: afa Date: Mon May 31 01:36:33 2010 New Revision: 74948 Modified: pypy/trunk/pypy/rlib/rdynload.py Log: Fix rdynload on Windows Modified: pypy/trunk/pypy/rlib/rdynload.py ============================================================================== --- pypy/trunk/pypy/rlib/rdynload.py (original) +++ pypy/trunk/pypy/rlib/rdynload.py Mon May 31 01:36:33 2010 @@ -19,7 +19,7 @@ if _WIN32: from pypy.rlib import rwin32 -if _MINGW: +if _WIN32: includes = ['windows.h'] else: includes = ['dlfcn.h'] From afa at codespeak.net Mon May 31 01:40:04 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 31 May 2010 01:40:04 +0200 (CEST) Subject: [pypy-svn] r74949 - in pypy/trunk/pypy/module/cpyext: . include Message-ID: <20100530234004.4421C282B9C@codespeak.net> Author: afa Date: Mon May 31 01:40:02 2010 New Revision: 74949 Modified: pypy/trunk/pypy/module/cpyext/include/object.h pypy/trunk/pypy/module/cpyext/object.py Log: Reduce C compilation warnings: - a macro will accept all types that extend PyObject - PyObject_GC_Track actually accept void*. Modified: pypy/trunk/pypy/module/cpyext/include/object.h ============================================================================== --- pypy/trunk/pypy/module/cpyext/include/object.h (original) +++ pypy/trunk/pypy/module/cpyext/include/object.h Mon May 31 01:40:02 2010 @@ -475,6 +475,9 @@ } \ } while (0) +#define PyObject_TypeCheck(ob, tp) \ + ((ob)->ob_type == (tp) || PyType_IsSubtype((ob)->ob_type, (tp))) + /* Copied from CPython ----------------------------- */ int PyObject_AsReadBuffer(PyObject *, const void **, Py_ssize_t *); int PyObject_AsWriteBuffer(PyObject *, void **, Py_ssize_t *); Modified: pypy/trunk/pypy/module/cpyext/object.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/object.py (original) +++ pypy/trunk/pypy/module/cpyext/object.py Mon May 31 01:40:02 2010 @@ -61,7 +61,7 @@ def PyObject_GC_Del(space, obj): PyObject_Del(space, obj) - at cpython_api([PyObject], lltype.Void) + at cpython_api([rffi.VOIDP_real], lltype.Void) def PyObject_GC_Track(space, op): """Adds the object op to the set of container objects tracked by the collector. The collector can run at unexpected times so objects must be @@ -70,7 +70,7 @@ end of the constructor.""" pass - at cpython_api([rffi.VOIDP], lltype.Void) + at cpython_api([rffi.VOIDP_real], lltype.Void) def PyObject_GC_UnTrack(space, op): """Remove the object op from the set of container objects tracked by the collector. Note that PyObject_GC_Track() can be called again on @@ -284,16 +284,6 @@ w_res = PyObject_RichCompare(space, ref1, ref2, opid) return int(space.is_true(w_res)) - at cpython_api([PyObject, PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyObject_TypeCheck(space, w_obj, w_type): - """Return true if the object o is of type type or a subtype of type. Both - parameters must be non-NULL. - """ - w_obj_type = space.type(w_obj) - assert isinstance(w_type, W_TypeObject) - return int(space.is_w(w_obj_type, w_type) or - space.is_true(space.issubtype(w_obj_type, w_type))) - @cpython_api([PyObject], PyObject) def PyObject_SelfIter(space, ref): """Undocumented function, this is wat CPython does.""" From afa at codespeak.net Mon May 31 01:41:04 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 31 May 2010 01:41:04 +0200 (CEST) Subject: [pypy-svn] r74950 - in pypy/trunk/pypy/module/cpyext: include test Message-ID: <20100530234104.95BD3282B9C@codespeak.net> Author: afa Date: Mon May 31 01:41:02 2010 New Revision: 74950 Modified: pypy/trunk/pypy/module/cpyext/include/stringobject.h pypy/trunk/pypy/module/cpyext/test/test_cell.py pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Log: kill warnings in cpyext tests Modified: pypy/trunk/pypy/module/cpyext/include/stringobject.h ============================================================================== --- pypy/trunk/pypy/module/cpyext/include/stringobject.h (original) +++ pypy/trunk/pypy/module/cpyext/include/stringobject.h Mon May 31 01:41:02 2010 @@ -18,7 +18,7 @@ Py_ssize_t size; } PyStringObject; -PyObject *PyString_FromFormatV(char *format, va_list vargs); +PyObject *PyString_FromFormatV(const char *format, va_list vargs); PyObject *PyString_FromFormat(const char *format, ...); #ifdef __cplusplus Modified: pypy/trunk/pypy/module/cpyext/test/test_cell.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_cell.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_cell.py Mon May 31 01:41:02 2010 @@ -6,7 +6,7 @@ module = self.import_extension('foo', [ ("cell_type", "METH_O", """ - PyDict_SetItemString(args, "cell", &PyCell_Type); + PyDict_SetItemString(args, "cell", (PyObject*)&PyCell_Type); Py_RETURN_NONE; """)]) d = {} Modified: pypy/trunk/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Mon May 31 01:41:02 2010 @@ -582,8 +582,8 @@ Py_InitModule("foo", methods); """ body = """ - PyObject* PyPy_Crash1(void); - long PyPy_Crash2(void); + PyAPI_FUNC(PyObject*) PyPy_Crash1(void); + PyAPI_FUNC(long) PyPy_Crash2(void); static PyObject* foo_crash1(PyObject* self, PyObject *args) { return PyPy_Crash1(); From arigo at codespeak.net Mon May 31 01:48:00 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 31 May 2010 01:48:00 +0200 (CEST) Subject: [pypy-svn] r74951 - pypy/branch/blackhole-improvement/pypy/jit/codewriter/test Message-ID: <20100530234800.63BBE282B9C@codespeak.net> Author: arigo Date: Mon May 31 01:47:58 2010 New Revision: 74951 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Log: An extra test. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Mon May 31 01:47:58 2010 @@ -140,6 +140,27 @@ int_return %i1 """) + def test_if(self): + def f(n): + if n > 0: + n -= 3 + return n + 1 + self.encoding_test(f, [10], """ + int_gt %i0, $0 -> %i1 + goto_if_not %i1, L1 + -live- L1 + int_copy %i0 -> %i2 + int_sub %i2, $3 -> %i3 + int_copy %i3 -> %i4 + L2: + int_add %i4, $1 -> %i5 + int_return %i5 + --- + L1: + int_copy %i0 -> %i4 + goto L2 + """) + def test_loop(self): def f(a, b): while a > 0: From arigo at codespeak.net Mon May 31 01:48:36 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 31 May 2010 01:48:36 +0200 (CEST) Subject: [pypy-svn] r74952 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100530234836.078DA282B9C@codespeak.net> Author: arigo Date: Mon May 31 01:48:35 2010 New Revision: 74952 Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/opt.py (contents, props changed) pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_opt.py (contents, props changed) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Log: Just for the fun of it (will be removed, too much complexity). Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py Mon May 31 01:48:35 2010 @@ -4,6 +4,7 @@ from pypy.jit.codewriter.assembler import Assembler, JitCode from pypy.jit.codewriter.jtransform import transform_graph from pypy.jit.codewriter.format import format_assembler +from pypy.jit.codewriter.opt import optimize_ssarepr from pypy.jit.codewriter.liveness import compute_liveness from pypy.jit.codewriter.call import CallControl from pypy.jit.codewriter.policy import log @@ -48,6 +49,7 @@ # inserting jumps or conditional jumps. This is a list of tuples # of the shape ("opname", arg1, ..., argN) or (Label(...),). ssarepr = flatten_graph(graph, regallocs) + optimize_ssarepr(ssarepr) # # step 3b: compute the liveness around certain operations compute_liveness(ssarepr) Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/opt.py ============================================================================== --- (empty file) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/opt.py Mon May 31 01:48:35 2010 @@ -0,0 +1,24 @@ +from pypy.jit.codewriter.flatten import Label + +def optimize_ssarepr(ssarepr): + # For now, just optimize "L2: goto L1" by moving the label L2 + # at the same place as L1, and possibly killing the "goto L1" + # if it becomes unreachable. + i = 0 + insns = ssarepr.insns + while i < len(insns): + if (isinstance(insns[i][0], Label) and + insns[i+1][0] == 'goto'): + movinglabelinsn = insns[i] + targettlabel = insns[i+1][1] + if movinglabelinsn[0].name != targettlabel.name: + del insns[i] + if i > 0: + if insns[i-1][0] == '---': + del insns[i] + else: + i -= 1 + targetindex = insns.index((Label(targettlabel.name),)) + insns.insert(targetindex, movinglabelinsn) + continue + i += 1 Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_opt.py ============================================================================== --- (empty file) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_opt.py Mon May 31 01:48:35 2010 @@ -0,0 +1,130 @@ +from pypy.jit.codewriter.opt import optimize_ssarepr +from pypy.jit.codewriter.format import assert_format, unformat_assembler + + +def test_opt_noop(): + test = """ + int_add %i0, %i1 -> %i2 + int_return %i2 + """ + ssarepr = unformat_assembler(test) + optimize_ssarepr(ssarepr) + assert_format(ssarepr, """ + int_add %i0, %i1 -> %i2 + int_return %i2 + """) + +def test_goto_1(): + test = """ + goto L2 + --- + L3: + int_return %i0 + --- + L2: + goto L3 + """ + ssarepr = unformat_assembler(test) + optimize_ssarepr(ssarepr) + assert_format(ssarepr, """ + goto L1 + --- + L1: + int_return %i0 + """) + +def test_goto_1bis(): + test = """ + goto L2 + --- + L2: + goto L3 + --- + L3: + int_return %i0 + """ + ssarepr = unformat_assembler(test) + optimize_ssarepr(ssarepr) + assert_format(ssarepr, """ + goto L1 + --- + --- + L1: + int_return %i0 + """) + +def test_goto_2(): + test = """ + goto L2 + --- + L3: + foobar + L2: + goto L3 + """ + ssarepr = unformat_assembler(test) + optimize_ssarepr(ssarepr) + assert_format(ssarepr, """ + goto L1 + --- + L1: + L2: + foobar + goto L2 + """) + +def test_goto_3(): + test = """ + foobar + L1: + goto L1 + """ + ssarepr = unformat_assembler(test) + optimize_ssarepr(ssarepr) + assert_format(ssarepr, """ + foobar + L1: + goto L1 + """) + +def test_goto_4(): + test = """ + foobar + L1: + goto L2 + --- + L2: + goto L1 + """ + ssarepr = unformat_assembler(test) + optimize_ssarepr(ssarepr) + assert_format(ssarepr, """ + foobar + goto L1 + --- + L1: + L2: + goto L2 + """) + +def test_goto_5(): + test = """ + goto_if_some_condition L2 + foobar + L2: + goto L3 + --- + L3: + int_return %i0 + """ + ssarepr = unformat_assembler(test) + optimize_ssarepr(ssarepr) + assert_format(ssarepr, """ + goto_if_some_condition L1 + foobar + goto L2 + --- + L1: + L2: + int_return %i0 + """) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Mon May 31 01:48:35 2010 @@ -17,7 +17,8 @@ self.rtyper = support.annotate(func, values, type_system=type_system) return self.rtyper.annotator.translator.graphs - def check_assembler(self, graph, expected, transform=False): + def check_assembler(self, graph, expected, transform=False, + optimize_ssarepr=False): # 'transform' can be False only for simple graphs. More complex # graphs must first be transformed by jtransform.py before they can be # subjected to register allocation and flattening. @@ -26,6 +27,9 @@ transform_graph(graph) regalloc = perform_register_allocation(graph, 'int') ssarepr = flatten_graph(graph, {'int': regalloc}) + if optimize_ssarepr: + from pypy.jit.codewriter.opt import optimize_ssarepr + optimize_ssarepr(ssarepr) asm = format_assembler(ssarepr) assert asm == str(py.code.Source(expected).strip()) + '\n' @@ -240,3 +244,21 @@ L1: int_return %i2 """, transform=True) + + def test_regalloc_if(self): + def f(n): + if n > 0: + n -= 3 + return n + 1 + graph = self.make_graphs(f, [10])[0] + # this also checks that "L2: goto L1" is optimized + self.check_assembler(graph, """ + int_gt %i0, $0 -> %i1 + goto_if_not %i1, L1 + -live- L1 + int_sub %i0, $3 -> %i0 + L1: + int_add %i0, $1 -> %i1 + int_return %i1 + --- + """, optimize_ssarepr=True) From arigo at codespeak.net Mon May 31 01:49:02 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 31 May 2010 01:49:02 +0200 (CEST) Subject: [pypy-svn] r74953 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100530234902.325F2282B9C@codespeak.net> Author: arigo Date: Mon May 31 01:49:00 2010 New Revision: 74953 Removed: pypy/branch/blackhole-improvement/pypy/jit/codewriter/opt.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_opt.py Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Log: Revert r74952. Too much complexity. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py Mon May 31 01:49:00 2010 @@ -4,7 +4,6 @@ from pypy.jit.codewriter.assembler import Assembler, JitCode from pypy.jit.codewriter.jtransform import transform_graph from pypy.jit.codewriter.format import format_assembler -from pypy.jit.codewriter.opt import optimize_ssarepr from pypy.jit.codewriter.liveness import compute_liveness from pypy.jit.codewriter.call import CallControl from pypy.jit.codewriter.policy import log @@ -49,7 +48,6 @@ # inserting jumps or conditional jumps. This is a list of tuples # of the shape ("opname", arg1, ..., argN) or (Label(...),). ssarepr = flatten_graph(graph, regallocs) - optimize_ssarepr(ssarepr) # # step 3b: compute the liveness around certain operations compute_liveness(ssarepr) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Mon May 31 01:49:00 2010 @@ -17,8 +17,7 @@ self.rtyper = support.annotate(func, values, type_system=type_system) return self.rtyper.annotator.translator.graphs - def check_assembler(self, graph, expected, transform=False, - optimize_ssarepr=False): + def check_assembler(self, graph, expected, transform=False): # 'transform' can be False only for simple graphs. More complex # graphs must first be transformed by jtransform.py before they can be # subjected to register allocation and flattening. @@ -27,9 +26,6 @@ transform_graph(graph) regalloc = perform_register_allocation(graph, 'int') ssarepr = flatten_graph(graph, {'int': regalloc}) - if optimize_ssarepr: - from pypy.jit.codewriter.opt import optimize_ssarepr - optimize_ssarepr(ssarepr) asm = format_assembler(ssarepr) assert asm == str(py.code.Source(expected).strip()) + '\n' @@ -244,21 +240,3 @@ L1: int_return %i2 """, transform=True) - - def test_regalloc_if(self): - def f(n): - if n > 0: - n -= 3 - return n + 1 - graph = self.make_graphs(f, [10])[0] - # this also checks that "L2: goto L1" is optimized - self.check_assembler(graph, """ - int_gt %i0, $0 -> %i1 - goto_if_not %i1, L1 - -live- L1 - int_sub %i0, $3 -> %i0 - L1: - int_add %i0, $1 -> %i1 - int_return %i1 - --- - """, optimize_ssarepr=True) From getxsick at codespeak.net Mon May 31 02:10:06 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Mon, 31 May 2010 02:10:06 +0200 (CEST) Subject: [pypy-svn] r74954 - in pypy/branch/fast-ctypes: . pypy/rlib pypy/rlib/test Message-ID: <20100531001006.9EE0B282B9C@codespeak.net> Author: getxsick Date: Mon May 31 02:10:05 2010 New Revision: 74954 Added: pypy/branch/fast-ctypes/pypy/rlib/rdynload.py - copied unchanged from r74944, pypy/trunk/pypy/rlib/rdynload.py pypy/branch/fast-ctypes/pypy/rlib/test/test_rdynload.py - copied unchanged from r74944, pypy/trunk/pypy/rlib/test/test_rdynload.py Modified: pypy/branch/fast-ctypes/ (props changed) pypy/branch/fast-ctypes/pypy/rlib/libffi.py pypy/branch/fast-ctypes/pypy/rlib/test/test_libffi.py Log: merge r74944 from trunk Modified: pypy/branch/fast-ctypes/pypy/rlib/libffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/libffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/libffi.py Mon May 31 02:10:05 2010 @@ -1,5 +1,4 @@ - -""" Various rpython-level functions for dlopen and libffi wrapping +""" Libffi wrapping """ from pypy.rpython.tool import rffi_platform @@ -7,15 +6,17 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rarithmetic import intmask, r_uint from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.rmmap import alloc +from pypy.rlib.rdynload import dlopen, dlclose, dlsym, dlsym_byordinal +from pypy.rlib.rdynload import DLOpenError from pypy.tool.autopath import pypydir from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.rlib.rmmap import alloc +from pypy.translator.platform import platform import py import os import sys import ctypes.util -from pypy.translator.platform import platform # maaaybe isinstance here would be better. Think _MSVC = platform.name == "msvc" @@ -55,18 +56,13 @@ if _MINGW: includes = ['windows.h', 'ffi.h'] else: - includes = ['dlfcn.h', 'ffi.h'] + includes = ['ffi.h'] if _MAC_OS: pre_include_bits = ['#define MACOSX'] else: pre_include_bits = [] - if _FREEBSD_7 or _MINGW: - libraries = [] - else: - libraries = ['dl'] - def find_libffi_a(): dirlist = platform.library_dirs_for_libffi_a() for dir in dirlist: @@ -77,10 +73,11 @@ if hasattr(platform, 'library_dirs_for_libffi_a'): # platforms on which we want static linking + libraries = [] link_files = [find_libffi_a()] else: # platforms on which we want dynamic linking - libraries = ['ffi'] + libraries + libraries = ['ffi'] link_files = [] eci = ExternalCompilationInfo( @@ -115,10 +112,6 @@ class CConfig: _compilation_info_ = eci - RTLD_LOCAL = rffi_platform.DefinedConstantInteger('RTLD_LOCAL') - RTLD_GLOBAL = rffi_platform.DefinedConstantInteger('RTLD_GLOBAL') - RTLD_NOW = rffi_platform.DefinedConstantInteger('RTLD_NOW') - FFI_OK = rffi_platform.ConstantInteger('FFI_OK') FFI_BAD_TYPEDEF = rffi_platform.ConstantInteger('FFI_BAD_TYPEDEF') FFI_DEFAULT_ABI = rffi_platform.ConstantInteger('FFI_DEFAULT_ABI') @@ -216,61 +209,8 @@ def winexternal(name, args, result): return rffi.llexternal(name, args, result, compilation_info=eci, calling_conv='win') -class DLOpenError(Exception): - def __init__(self, msg): - self.msg = msg - def __str__(self): - return repr(self.msg) - if not _WIN32: - c_dlopen = external('dlopen', [rffi.CCHARP, rffi.INT], rffi.VOIDP) - c_dlclose = external('dlclose', [rffi.VOIDP], rffi.INT) - c_dlerror = external('dlerror', [], rffi.CCHARP) - c_dlsym = external('dlsym', [rffi.VOIDP, rffi.CCHARP], rffi.VOIDP) - - RTLD_LOCAL = cConfig.RTLD_LOCAL - RTLD_GLOBAL = cConfig.RTLD_GLOBAL - RTLD_NOW = cConfig.RTLD_NOW - - def dlerror(): - # XXX this would never work on top of ll2ctypes, because - # ctypes are calling dlerror itself, unsure if I can do much in this - # area (nor I would like to) - res = c_dlerror() - if not res: - return "" - return rffi.charp2str(res) - - def dlopen(name, mode=-1): - """ Wrapper around C-level dlopen - """ - if mode == -1: - if RTLD_LOCAL is not None: - mode = RTLD_LOCAL | RTLD_NOW - else: - mode = RTLD_NOW - res = c_dlopen(name, rffi.cast(rffi.INT, mode)) - if not res: - err = dlerror() - raise DLOpenError(err) - return res - - dlclose = c_dlclose - - def dlsym(libhandle, name): - """ Wrapper around C-level dlsym - """ - res = c_dlsym(libhandle, name) - if not res: - raise KeyError(name) - # XXX rffi.cast here... - return res - - def dlsym_byordinal(handle, index): - # Never called - raise KeyError(index) - def check_fficall_result(result, flags): pass # No check @@ -281,36 +221,6 @@ return libc_name if _WIN32: - def dlopen(name): - res = rwin32.LoadLibrary(name) - if not res: - err = rwin32.GetLastError() - raise DLOpenError(rwin32.FormatError(err)) - return res - - def dlclose(handle): - res = rwin32.FreeLibrary(handle) - if res: - return -1 - else: - return 0 - - def dlsym(handle, name): - res = rwin32.GetProcAddress(handle, name) - if not res: - raise KeyError(name) - # XXX rffi.cast here... - return res - - def dlsym_byordinal(handle, index): - # equivalent to MAKEINTRESOURCEA - intresource = rffi.cast(rffi.CCHARP, r_uint(index) & 0xFFFF) - res = rwin32.GetProcAddress(handle, intresource) - if not res: - raise KeyError(name) - # XXX rffi.cast here... - return res - def check_fficall_result(result, flags): if result == 0: return Modified: pypy/branch/fast-ctypes/pypy/rlib/test/test_libffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/test/test_libffi.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/test/test_libffi.py Mon May 31 02:10:05 2010 @@ -1,15 +1,14 @@ -""" Tests of libffi wrappers and dl* friends +""" Tests of libffi wrapper """ -from pypy.rpython.test.test_llinterp import interpret from pypy.translator.c.test.test_genc import compile from pypy.rlib.libffi import * from pypy.rlib.objectmodel import keepalive_until_here from pypy.rpython.lltypesystem.ll2ctypes import ALLOCATED from pypy.rpython.lltypesystem import rffi, lltype -import os, sys import py +import sys import time def setup_module(mod): @@ -19,14 +18,10 @@ ffistruct = globals()[name] rffi.cast(rffi.VOIDP, ffistruct) -class TestDLOperations: +class TestLibffi: def setup_method(self, meth): ALLOCATED.clear() - def test_dlopen(self): - py.test.raises(DLOpenError, "dlopen(rffi.str2charp('xxxxxxxxxxxx'))") - assert dlopen(rffi.str2charp(get_libc_name())) - def get_libc(self): return CDLL(get_libc_name()) From getxsick at codespeak.net Mon May 31 02:11:12 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Mon, 31 May 2010 02:11:12 +0200 (CEST) Subject: [pypy-svn] r74955 - in pypy/branch/fast-ctypes: . pypy/rlib pypy/rlib/test Message-ID: <20100531001112.CA51E36C223@codespeak.net> Author: getxsick Date: Mon May 31 02:11:10 2010 New Revision: 74955 Modified: pypy/branch/fast-ctypes/ (props changed) pypy/branch/fast-ctypes/pypy/rlib/rdynload.py (props changed) pypy/branch/fast-ctypes/pypy/rlib/test/test_rdynload.py (props changed) Log: merge r74946 from trunk From getxsick at codespeak.net Mon May 31 02:12:01 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Mon, 31 May 2010 02:12:01 +0200 (CEST) Subject: [pypy-svn] r74956 - in pypy/branch/fast-ctypes: . pypy/rlib Message-ID: <20100531001201.90A9C282B9C@codespeak.net> Author: getxsick Date: Mon May 31 02:12:00 2010 New Revision: 74956 Modified: pypy/branch/fast-ctypes/ (props changed) pypy/branch/fast-ctypes/pypy/rlib/rdynload.py Log: merge r74948 from trunk Modified: pypy/branch/fast-ctypes/pypy/rlib/rdynload.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rdynload.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rdynload.py Mon May 31 02:12:00 2010 @@ -19,7 +19,7 @@ if _WIN32: from pypy.rlib import rwin32 -if _MINGW: +if _WIN32: includes = ['windows.h'] else: includes = ['dlfcn.h'] From arigo at codespeak.net Mon May 31 02:15:20 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 31 May 2010 02:15:20 +0200 (CEST) Subject: [pypy-svn] r74957 - in pypy/branch/blackhole-improvement/pypy/jit/metainterp: . test Message-ID: <20100531001520.BC069282B9C@codespeak.net> Author: arigo Date: Mon May 31 02:15:19 2010 New Revision: 74957 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_executor.py Log: Some more fixes for when last_exc_value_box is expected to be cleared or not. Now clear it a bit more aggressively before running any operation that might set it --- this includes CALL_PURE, even though it cannot actually set it; previously, it would fail to clear the exception in that case, and crash because one exception is still set. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py Mon May 31 02:15:19 2010 @@ -58,8 +58,6 @@ except Exception, e: metainterp.execute_raised(e) result = 0 - else: - metainterp.execute_did_not_raise() return BoxInt(result) if rettype == REF: try: @@ -67,8 +65,6 @@ except Exception, e: metainterp.execute_raised(e) result = NULL - else: - metainterp.execute_did_not_raise() return BoxPtr(result) if rettype == FLOAT: try: @@ -76,16 +72,12 @@ except Exception, e: metainterp.execute_raised(e) result = 0.0 - else: - metainterp.execute_did_not_raise() return BoxFloat(result) if rettype == VOID: try: cpu.bh_call_v(func, descr, args_i, args_r, args_f) except Exception, e: metainterp.execute_raised(e) - else: - metainterp.execute_did_not_raise() return None raise AssertionError("bad rettype") @@ -189,8 +181,6 @@ except OverflowError: metainterp.execute_raised(OverflowError(), constant=True) z = 0 - else: - metainterp.execute_did_not_raise() return BoxInt(z) def do_int_sub_ovf(cpu, metainterp, box1, box2): @@ -202,8 +192,6 @@ except OverflowError: metainterp.execute_raised(OverflowError(), constant=True) z = 0 - else: - metainterp.execute_did_not_raise() return BoxInt(z) def do_int_mul_ovf(cpu, metainterp, box1, box2): @@ -215,8 +203,6 @@ except OverflowError: metainterp.execute_raised(OverflowError(), constant=True) z = 0 - else: - metainterp.execute_did_not_raise() return BoxInt(z) def do_same_as(cpu, _, box): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Mon May 31 02:15:19 2010 @@ -184,6 +184,7 @@ exec py.code.Source(''' @arguments("box", "box") def opimpl_%s(self, b1, b2): + self.metainterp.clear_exception() resbox = self.execute(rop.%s, b1, b2) self.make_result_of_lastop(resbox) # same as execute_varargs() self.metainterp.handle_possible_overflow_error() @@ -971,6 +972,7 @@ @specialize.arg(1) def execute_varargs(self, opnum, argboxes, descr, exc): + self.metainterp.clear_exception() resbox = self.metainterp.execute_and_record_varargs(opnum, argboxes, descr=descr) self.make_result_of_lastop(resbox) @@ -1022,6 +1024,7 @@ effectinfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE or assembler_call_token is not None): # residual calls require attention to keep virtualizables in-sync + self.metainterp.clear_exception() self.metainterp.vable_and_vrefs_before_residual_call() resbox = self.metainterp.execute_and_record_varargs( rop.CALL_MAY_FORCE, allboxes, descr=descr) @@ -1451,7 +1454,7 @@ # stored in the exc_value Box can be assumed to be a Const. This # is only True after a GUARD_EXCEPTION or GUARD_CLASS. - def execute_did_not_raise(self): + def clear_exception(self): self.last_exc_value_box = None def aborted_tracing(self, reason): @@ -1647,7 +1650,7 @@ self.execute_ll_raised(lltype.cast_opaque_ptr(rclass.OBJECTPTR, exception)) else: - self.execute_did_not_raise() + self.clear_exception() try: self.handle_possible_exception() except ChangeFrame: @@ -1659,7 +1662,7 @@ except ChangeFrame: pass elif opnum == rop.GUARD_OVERFLOW: # no longer overflowing - self.execute_did_not_raise() + self.clear_exception() else: from pypy.jit.metainterp.resoperation import opname raise NotImplementedError(opname[opnum]) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_executor.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_executor.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_executor.py Mon May 31 02:15:19 2010 @@ -38,8 +38,7 @@ self.fakeargs = args class FakeMetaInterp: - def execute_did_not_raise(self): - self.didraise = False + pass class FakeCPU(AbstractCPU): supports_floats = True @@ -76,15 +75,13 @@ def test_execute_varargs(): cpu = FakeCPU() - metainterp = FakeMetaInterp() descr = FakeCallDescr() argboxes = [BoxInt(99999), BoxInt(321), ConstFloat(2.25), ConstInt(123), BoxPtr(), BoxFloat(5.5)] - box = execute_varargs(cpu, metainterp, rop.CALL, argboxes, descr) + box = execute_varargs(cpu, FakeMetaInterp(), rop.CALL, argboxes, descr) assert box.value == 42.5 assert cpu.fakecalled == (99999, descr, [321, 123], [ConstPtr.value], [2.25, 5.5]) - assert not metainterp.didraise def test_execute_nonspec(): cpu = FakeCPU() From getxsick at codespeak.net Mon May 31 02:19:54 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Mon, 31 May 2010 02:19:54 +0200 (CEST) Subject: [pypy-svn] r74958 - pypy/branch/fast-ctypes/pypy/module/jitffi Message-ID: <20100531001954.4F926282B9C@codespeak.net> Author: getxsick Date: Mon May 31 02:19:52 2010 New Revision: 74958 Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Log: use rdynload instead of libffi Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Mon May 31 02:19:52 2010 @@ -2,7 +2,7 @@ from pypy.interpreter.error import operationerrfmt from pypy.interpreter.gateway import interp2app from pypy.interpreter.typedef import TypeDef -from pypy.rlib import libffi +from pypy.rlib import rdynload from pypy.rpython.lltypesystem import rffi, lltype from pypy.jit.backend.x86.runner import CPU from pypy.jit.metainterp.history import LoopToken, BasicFailDescr, BoxInt @@ -12,7 +12,7 @@ class W_CDLL(Wrappable): def __init__(self, space, name): try: - self.cdll = libffi.CDLL(name) + self.lib = rdynload.dlopen(name) except libffi.DLOpenError, e: raise operationerrfmt(space.w_OSError, '%s: %s', name, e.msg or 'unspecified error') @@ -23,7 +23,7 @@ def call(self, space, func, a, b): # XXX temporary fixed number of func args (ints) # result_type argument? try: - addr = rffi.cast(lltype.Signed, self.cdll.getaddressindll(func)) + addr = rffi.cast(lltype.Signed, rdynload.dlsym(self.lib, func)) except KeyError: raise operationerrfmt(space.w_ValueError, "Cannot find symbol %s", func) From afa at codespeak.net Mon May 31 10:06:31 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 31 May 2010 10:06:31 +0200 (CEST) Subject: [pypy-svn] r74959 - pypy/trunk/pypy/module/cpyext/src Message-ID: <20100531080631.6A3B3282B9C@codespeak.net> Author: afa Date: Mon May 31 10:06:29 2010 New Revision: 74959 Modified: pypy/trunk/pypy/module/cpyext/src/stringobject.c Log: Fix compilation: signature in .c must match the declaration Modified: pypy/trunk/pypy/module/cpyext/src/stringobject.c ============================================================================== --- pypy/trunk/pypy/module/cpyext/src/stringobject.c (original) +++ pypy/trunk/pypy/module/cpyext/src/stringobject.c Mon May 31 10:06:29 2010 @@ -2,7 +2,7 @@ #include "Python.h" PyObject * -PyString_FromFormatV(char *format, va_list vargs) +PyString_FromFormatV(const char *format, va_list vargs) { va_list count; Py_ssize_t n = 0; From afa at codespeak.net Mon May 31 10:59:42 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 31 May 2010 10:59:42 +0200 (CEST) Subject: [pypy-svn] r74963 - pypy/trunk/pypy/module/cpyext/test Message-ID: <20100531085942.D55DF282B9C@codespeak.net> Author: afa Date: Mon May 31 10:59:41 2010 New Revision: 74963 Modified: pypy/trunk/pypy/module/cpyext/test/test_object.py Log: Fix test: PyObject_TypeCheck is now a macro. Modified: pypy/trunk/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_object.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_object.py Mon May 31 10:59:41 2010 @@ -127,17 +127,6 @@ assert api.PyErr_Occurred() is space.w_SystemError api.PyErr_Clear() - def test_TypeCheck(self, space, api): - assert api.PyObject_TypeCheck(space.wrap(1), space.w_int) - assert api.PyObject_TypeCheck(space.wrap(1), api.PyInt_Type) - assert api.PyObject_TypeCheck(space.wrap('foo'), space.w_str) - assert api.PyObject_TypeCheck(space.wrap('foo'), api.PyString_Type) - assert api.PyObject_TypeCheck(space.wrap('foo'), space.w_object) - assert api.PyObject_TypeCheck(space.wrap(1L), api.PyLong_Type) - assert api.PyObject_TypeCheck(space.wrap(True), api.PyBool_Type) - assert api.PyObject_TypeCheck(space.wrap(1.2), api.PyFloat_Type) - assert api.PyObject_TypeCheck(space.w_int, api.PyType_Type) - def test_IsInstance(self, space, api): assert api.PyObject_IsInstance(space.wrap(1), space.w_int) == 1 assert api.PyObject_IsInstance(space.wrap(1), space.w_float) == 0 @@ -203,11 +192,27 @@ space.call_method(w_file, "close") assert (udir / "_test_file").read() == "text" -class AppTestObjectPrint(AppTestCpythonExtensionBase): +class AppTestObject(AppTestCpythonExtensionBase): def setup_class(cls): AppTestCpythonExtensionBase.setup_class.im_func(cls) cls.w_tmpname = cls.space.wrap(str(py.test.ensuretemp("out", dir=0))) + def test_TypeCheck(self): + module = self.import_extension('foo', [ + ("typecheck", "METH_VARARGS", + """ + PyObject *obj = PyTuple_GET_ITEM(args, 0); + PyObject *type = PyTuple_GET_ITEM(args, 1); + return PyBool_FromLong(PyObject_TypeCheck(obj, type)); + """)]) + assert module.typecheck(1, int) + assert module.typecheck('foo', str) + assert module.typecheck('foo', object) + assert module.typecheck(1L, long) + assert module.typecheck(True, bool) + assert module.typecheck(1.2, float) + assert module.typecheck(int, type) + def test_print(self): module = self.import_extension('foo', [ ("dump", "METH_VARARGS", From antocuni at codespeak.net Mon May 31 11:15:39 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 31 May 2010 11:15:39 +0200 (CEST) Subject: [pypy-svn] r74964 - pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test Message-ID: <20100531091539.2F1B3282B9C@codespeak.net> Author: antocuni Date: Mon May 31 11:15:37 2010 New Revision: 74964 Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_runpy.py Log: fix test_runpy, which relied on runpy to be accessible via pypy.lib.runpy Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_runpy.py ============================================================================== --- pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_runpy.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_runpy.py Mon May 31 11:15:37 2010 @@ -1,12 +1,16 @@ # Test the runpy module from __future__ import absolute_import +import py import unittest import os import os.path import sys import tempfile +from .. import runpy from ..runpy import _run_module_code, run_module +sys.modules['my_runpy'] = runpy # so that code in exec can freely import it + # without messing with __name__ and/or sys.path verbose = 0 @@ -14,22 +18,22 @@ class TestRunModuleCode: expected_result = ["Top level assignment", "Lower level reference"] - test_source = ( - "# Check basic code execution\n" - "result = ['Top level assignment']\n" - "def f():\n" - " result.append('Lower level reference')\n" - "f()\n" - "# Check the sys module\n" - "import sys\n" - "run_argv0 = sys.argv[0]\n" - "if __name__ in sys.modules:\n" - " run_name = sys.modules[__name__].__name__\n" - "# Check nested operation\n" - "import pypy.lib.runpy\n" - "nested = pypy.lib.runpy._run_module_code('x=1\\n', mod_name='',\n" - " alter_sys=True)\n" - ) + test_source = py.code.Source(""" + # Check basic code execution + result = ['Top level assignment'] + def f(): + result.append('Lower level reference') + f() + # Check the sys module + import sys + run_argv0 = sys.argv[0] + if __name__ in sys.modules: + run_name = sys.modules[__name__].__name__ + # Check nested operation + import my_runpy # this is imported as ..runpy as manually saved into sys.modules, see above + nested = my_runpy._run_module_code('x=1\', mod_name='', + alter_sys=True) + """).compile() def test_run_module_code(self): @@ -87,7 +91,7 @@ self.expect_import_error("..eaten") def test_library_module(self): - run_module("pypy.lib.runpy") + run_module("runpy") def _make_pkg(self, source, depth): pkg_name = "__runpy_pkg__" From afa at codespeak.net Mon May 31 11:36:02 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 31 May 2010 11:36:02 +0200 (CEST) Subject: [pypy-svn] r74965 - pypy/trunk/pypy/module/cpyext/test Message-ID: <20100531093602.E5FE0282B9C@codespeak.net> Author: afa Date: Mon May 31 11:36:01 2010 New Revision: 74965 Modified: pypy/trunk/pypy/module/cpyext/test/comparisons.c Log: remove c99-ism. Modified: pypy/trunk/pypy/module/cpyext/test/comparisons.c ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/comparisons.c (original) +++ pypy/trunk/pypy/module/cpyext/test/comparisons.c Mon May 31 11:36:01 2010 @@ -6,11 +6,12 @@ static PyObject* cmp_richcmp(PyObject *self, PyObject *other, int opid) { + long val; if ((opid != Py_EQ && opid != Py_NE) || !PyInt_CheckExact(other)) { Py_INCREF(Py_NotImplemented); return Py_NotImplemented; } - long val = PyLong_AsLong(other); + val = PyLong_AsLong(other); if (opid == Py_EQ) { return PyBool_FromLong(val == 3); } From antocuni at codespeak.net Mon May 31 12:53:36 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 31 May 2010 12:53:36 +0200 (CEST) Subject: [pypy-svn] r74966 - in pypy/branch/sys-prefix/pypy: interpreter interpreter/test module/sys tool tool/test translator Message-ID: <20100531105336.C7BCB282B9C@codespeak.net> Author: antocuni Date: Mon May 31 12:53:35 2010 New Revision: 74966 Added: pypy/branch/sys-prefix/pypy/tool/lib_pypy.py (contents, props changed) pypy/branch/sys-prefix/pypy/tool/test/test_lib_pypy.py (contents, props changed) Modified: pypy/branch/sys-prefix/pypy/interpreter/baseobjspace.py pypy/branch/sys-prefix/pypy/interpreter/test/test_module.py pypy/branch/sys-prefix/pypy/interpreter/test/test_objspace.py pypy/branch/sys-prefix/pypy/module/sys/state.py pypy/branch/sys-prefix/pypy/module/sys/version.py pypy/branch/sys-prefix/pypy/translator/geninterplevel.py Log: fix a couple of places that hardcoded the path to pypy/lib, and add a tool function to get the path to lib_pypy Modified: pypy/branch/sys-prefix/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/sys-prefix/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/sys-prefix/pypy/interpreter/baseobjspace.py Mon May 31 12:53:35 2010 @@ -385,6 +385,7 @@ def get_builtinmodule_to_install(self): """NOT_RPYTHON""" + from pypy.tool.lib_pypy import get_lib_pypy_dir try: return self._builtinmodule_list except AttributeError: @@ -402,11 +403,10 @@ if ('time2' in modules or 'rctime' in modules) and 'time' in modules: modules.remove('time') + lib_pypy = get_lib_pypy_dir() if not self.config.objspace.nofaking: for modname in self.ALL_BUILTIN_MODULES: - if not (os.path.exists( - os.path.join(os.path.dirname(pypy.__file__), - 'lib', modname+'.py'))): + if not lib_pypy.join(modname+'.py').check(file=True): modules.append('faked+'+modname) self._builtinmodule_list = modules Modified: pypy/branch/sys-prefix/pypy/interpreter/test/test_module.py ============================================================================== --- pypy/branch/sys-prefix/pypy/interpreter/test/test_module.py (original) +++ pypy/branch/sys-prefix/pypy/interpreter/test/test_module.py Mon May 31 12:53:35 2010 @@ -57,8 +57,8 @@ import _pypy_interact # known to be in pypy/lib r = repr(_pypy_interact) assert (r.startswith("')) nofile = type(_pypy_interact)('nofile', 'foo') assert repr(nofile) == "" Modified: pypy/branch/sys-prefix/pypy/interpreter/test/test_objspace.py ============================================================================== --- pypy/branch/sys-prefix/pypy/interpreter/test/test_objspace.py (original) +++ pypy/branch/sys-prefix/pypy/interpreter/test/test_objspace.py Mon May 31 12:53:35 2010 @@ -259,3 +259,23 @@ def test_sys_import(self): from pypy.interpreter.main import run_string run_string('import sys', space=self.space) + + def test_get_builtinmodule_to_install(self): + space = self.space + try: + # force rebuilding with this fake builtin + space.ALL_BUILTIN_MODULES.append('this_doesnt_exist') + del space._builtinmodule_list + mods = space.get_builtinmodule_to_install() + + assert '__pypy__' in mods # real builtin + assert 'array' not in mods # in lib_pypy + assert 'faked+array' not in mods # in lib_pypy + assert 'this_doesnt_exist' not in mods # not in lib_pypy + assert 'faked+this_doesnt_exist' in mods # not in lib_pypy, but in + # ALL_BUILTIN_MODULES + finally: + # rebuild the original list + space.ALL_BUILTIN_MODULES.pop() + del space._builtinmodule_list + mods = space.get_builtinmodule_to_install() Modified: pypy/branch/sys-prefix/pypy/module/sys/state.py ============================================================================== --- pypy/branch/sys-prefix/pypy/module/sys/state.py (original) +++ pypy/branch/sys-prefix/pypy/module/sys/state.py Mon May 31 12:53:35 2010 @@ -35,7 +35,7 @@ def getinitialpath(srcdir): # build the initial path from the srcdir, which is the path of # the "dist" directory of a PyPy checkout. - from pypy.module.sys.version import CPYTHON_VERSION + from pypy.module.sys.version import CPYTHON_VERSION, PYPY_VERSION dirname = '%d.%d.%d' % (CPYTHON_VERSION[0], CPYTHON_VERSION[1], @@ -46,12 +46,14 @@ checkdir(python_std_lib) python_std_lib_modified = os.path.join(lib_python, 'modified-' + dirname) checkdir(python_std_lib_modified) - pypydir = os.path.join(srcdir, 'pypy') - pypy_lib = os.path.join(pypydir, 'lib') - checkdir(pypy_lib) + + libdir = os.path.join(srcdir, 'lib') + pypyxy_dir = os.path.join(libdir, 'pypy%d.%d' % PYPY_VERSION[:2]) + lib_pypy = os.path.join(pypyxy_dir, 'lib_pypy') + checkdir(lib_pypy) importlist = [] - importlist.append(pypy_lib) + importlist.append(lib_pypy) importlist.append(python_std_lib_modified) importlist.append(python_std_lib) return importlist Modified: pypy/branch/sys-prefix/pypy/module/sys/version.py ============================================================================== --- pypy/branch/sys-prefix/pypy/module/sys/version.py (original) +++ pypy/branch/sys-prefix/pypy/module/sys/version.py Mon May 31 12:53:35 2010 @@ -7,6 +7,7 @@ CPYTHON_VERSION = (2, 5, 2, "beta", 42) CPYTHON_API_VERSION = 1012 +# when changing the pypy version, remind to also change the name of trunk/lib/pypyX.Y PYPY_VERSION = (1, 2, 0, "beta", '?') # the last item is replaced by the svn revision ^^^ Added: pypy/branch/sys-prefix/pypy/tool/lib_pypy.py ============================================================================== --- (empty file) +++ pypy/branch/sys-prefix/pypy/tool/lib_pypy.py Mon May 31 12:53:35 2010 @@ -0,0 +1,8 @@ +import py +import pypy +from pypy.module.sys.version import PYPY_VERSION + +def get_lib_pypy_dir(): + prefix = py.path.local(pypy.__path__[0]).dirpath() + pypy_ver = 'pypy%d.%d' % PYPY_VERSION[:2] + return prefix.join('lib', pypy_ver, 'lib_pypy') Added: pypy/branch/sys-prefix/pypy/tool/test/test_lib_pypy.py ============================================================================== --- (empty file) +++ pypy/branch/sys-prefix/pypy/tool/test/test_lib_pypy.py Mon May 31 12:53:35 2010 @@ -0,0 +1,6 @@ +import py +from pypy.tool import lib_pypy + +def test_lib_pypy_exists(): + dirname = lib_pypy.get_lib_pypy_dir() + assert dirname.check(dir=1) Modified: pypy/branch/sys-prefix/pypy/translator/geninterplevel.py ============================================================================== --- pypy/branch/sys-prefix/pypy/translator/geninterplevel.py (original) +++ pypy/branch/sys-prefix/pypy/translator/geninterplevel.py Mon May 31 12:53:35 2010 @@ -1473,6 +1473,7 @@ dic = initfunc(space) # and now use the members of the dict """ + from pypy.tool.lib_pypy import get_lib_pypy_dir # create something like a module if type(sourcetext) is str: code = py.code.Source(sourcetext).compile() @@ -1484,12 +1485,7 @@ dic['__file__'] = filename # XXX allow the app-level code to contain e.g. "import _formatting" - for pkgdir in pypy.__path__: - libdir = os.path.join(pkgdir, "lib") - if os.path.isdir(libdir): - break - else: - raise Exception, "cannot find pypy/lib directory" + libdir = str(get_lib_pypy_dir()) sys.path.insert(0, libdir) try: if faked_set: From afa at codespeak.net Mon May 31 13:08:52 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 31 May 2010 13:08:52 +0200 (CEST) Subject: [pypy-svn] r74967 - pypy/trunk/pypy/rlib Message-ID: <20100531110852.7324F282BDB@codespeak.net> Author: afa Date: Mon May 31 13:08:51 2010 New Revision: 74967 Modified: pypy/trunk/pypy/rlib/rdynload.py Log: rdynload does not need any extra library on Windows Modified: pypy/trunk/pypy/rlib/rdynload.py ============================================================================== --- pypy/trunk/pypy/rlib/rdynload.py (original) +++ pypy/trunk/pypy/rlib/rdynload.py Mon May 31 13:08:51 2010 @@ -29,10 +29,8 @@ else: pre_include_bits = [] -if _FREEBSD_7 or _MINGW: # no need of library or not supported? +if _FREEBSD_7 or _WIN32: libraries = [] -elif _MSVC: - libraries = ['kernel32'] # not sure if needed else: libraries = ['dl'] @@ -58,9 +56,6 @@ def external(name, args, result, **kwds): return rffi.llexternal(name, args, result, compilation_info=eci, **kwds) -def winexternal(name, args, result): - return rffi.llexternal(name, args, result, compilation_info=eci, calling_conv='win') - class DLOpenError(Exception): def __init__(self, msg): self.msg = msg From getxsick at codespeak.net Mon May 31 13:11:50 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Mon, 31 May 2010 13:11:50 +0200 (CEST) Subject: [pypy-svn] r74968 - in pypy/branch/fast-ctypes: . pypy/rlib Message-ID: <20100531111150.93A89282BDB@codespeak.net> Author: getxsick Date: Mon May 31 13:11:49 2010 New Revision: 74968 Modified: pypy/branch/fast-ctypes/ (props changed) pypy/branch/fast-ctypes/pypy/rlib/rdynload.py Log: merge r74967 from trunk Modified: pypy/branch/fast-ctypes/pypy/rlib/rdynload.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/rlib/rdynload.py (original) +++ pypy/branch/fast-ctypes/pypy/rlib/rdynload.py Mon May 31 13:11:49 2010 @@ -29,10 +29,8 @@ else: pre_include_bits = [] -if _FREEBSD_7 or _MINGW: # no need of library or not supported? +if _FREEBSD_7 or _WIN32: libraries = [] -elif _MSVC: - libraries = ['kernel32'] # not sure if needed else: libraries = ['dl'] @@ -58,9 +56,6 @@ def external(name, args, result, **kwds): return rffi.llexternal(name, args, result, compilation_info=eci, **kwds) -def winexternal(name, args, result): - return rffi.llexternal(name, args, result, compilation_info=eci, calling_conv='win') - class DLOpenError(Exception): def __init__(self, msg): self.msg = msg From antocuni at codespeak.net Mon May 31 14:22:32 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 31 May 2010 14:22:32 +0200 (CEST) Subject: [pypy-svn] r74969 - in pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy: app_test test2 Message-ID: <20100531122232.8F25A282B9C@codespeak.net> Author: antocuni Date: Mon May 31 14:22:31 2010 New Revision: 74969 Added: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_collections.py - copied, changed from r74962, pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/test2/test_collections.py Removed: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/test2/test_collections.py Log: move test_collections to app_test, as it contains only direct tests (the naming is confusing here: test2 contains app-level tests, while app_test contains direct tests. app_test will be renamed later) Copied: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_collections.py (from r74962, pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/test2/test_collections.py) ============================================================================== --- pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/test2/test_collections.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_collections.py Mon May 31 14:22:31 2010 @@ -1,4 +1,5 @@ -from pypy.lib import collections +from __future__ import absolute_import +from .. import collections import py def test_deque_remove_empty(): From arigo at codespeak.net Mon May 31 15:26:45 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 31 May 2010 15:26:45 +0200 (CEST) Subject: [pypy-svn] r74970 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100531132645.C72FE282BD8@codespeak.net> Author: arigo Date: Mon May 31 15:26:44 2010 New Revision: 74970 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Log: Test and fix. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Mon May 31 15:26:44 2010 @@ -137,8 +137,10 @@ self.emitline("---") def make_link(self, link): - if link.target.exits == (): - self.make_return(link.args) + if (link.target.exits == () + and link.last_exception not in link.args + and link.last_exc_value not in link.args): + self.make_return(link.args) # optimization only return self.insert_renamings(link) self.make_bytecode_block(link.target) @@ -154,12 +156,6 @@ self.emitline("reraise") self.emitline("---") return # done - if link.last_exception in link.args: - self.emitline("last_exception", - "->", self.getcolor(link.last_exception)) - if link.last_exc_value in link.args: - self.emitline("last_exc_value", - "->", self.getcolor(link.last_exc_value)) self.make_link(link) def insert_exits(self, block): @@ -298,7 +294,8 @@ renamings = {} lst = [(self.getcolor(v), self.getcolor(link.target.inputargs[i])) for i, v in enumerate(link.args) - if v.concretetype is not lltype.Void] + if v.concretetype is not lltype.Void and + v not in (link.last_exception, link.last_exc_value)] lst.sort(key=lambda(v, w): w.index) for v, w in lst: if v == w: @@ -321,6 +318,17 @@ self.emitline('%s_pop' % kind, "->", w) else: self.emitline('%s_copy' % kind, v, "->", w) + self.generate_last_exc(link, link.target.inputargs) + + def generate_last_exc(self, link, inputargs): + # Write 'last_exc_xxx' operations that load the last exception + # directly into the locations specified by 'inputargs'. This + # must be done at the end of the link renamings. + for v, w in zip(link.args, inputargs): + if v is link.last_exception: + self.emitline("last_exception", "->", self.getcolor(w)) + if v is link.last_exc_value: + self.emitline("last_exc_value", "->", self.getcolor(w)) def emitline(self, *line): self.ssarepr.insns.append(line) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Mon May 31 15:26:44 2010 @@ -372,8 +372,7 @@ L1: goto_if_exception_mismatch $<* struct object_vtable>, L2 last_exc_value -> %r0 - ref_copy %r0 -> %r1 - getfield_gc_i %r1, -> %i1 + getfield_gc_i %r0, -> %i1 int_return %i1 --- L2: @@ -431,13 +430,11 @@ void_return --- L1: - last_exception -> %i1 last_exc_value -> %r0 - int_copy %i1 -> %i2 - ref_copy %r0 -> %r1 + last_exception -> %i1 setfield_gc_i $<* struct test.Foo>, , $5 -live- - raise %r1 + raise %r0 """, transform=True) def test_goto_if_not_int_is_true(self): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Mon May 31 15:26:44 2010 @@ -2,11 +2,12 @@ from pypy.jit.codewriter import support from pypy.jit.codewriter.regalloc import perform_register_allocation from pypy.jit.codewriter.flatten import flatten_graph, ListOfKind -from pypy.jit.codewriter.format import format_assembler +from pypy.jit.codewriter.format import assert_format +from pypy.jit.metainterp.history import AbstractDescr from pypy.objspace.flow.model import Variable, Constant, SpaceOperation from pypy.objspace.flow.model import FunctionGraph, Block, Link from pypy.objspace.flow.model import c_last_exception -from pypy.rpython.lltypesystem import lltype, rclass +from pypy.rpython.lltypesystem import lltype, llmemory, rclass from pypy.rlib.rarithmetic import ovfcheck from pypy.rlib.objectmodel import keepalive_until_here @@ -17,17 +18,19 @@ self.rtyper = support.annotate(func, values, type_system=type_system) return self.rtyper.annotator.translator.graphs - def check_assembler(self, graph, expected, transform=False): + def check_assembler(self, graph, expected, transform=False, + callcontrol=None): # 'transform' can be False only for simple graphs. More complex # graphs must first be transformed by jtransform.py before they can be # subjected to register allocation and flattening. if transform: from pypy.jit.codewriter.jtransform import transform_graph - transform_graph(graph) + transform_graph(graph, callcontrol=callcontrol) regalloc = perform_register_allocation(graph, 'int') - ssarepr = flatten_graph(graph, {'int': regalloc}) - asm = format_assembler(ssarepr) - assert asm == str(py.code.Source(expected).strip()) + '\n' + regalloc2 = perform_register_allocation(graph, 'ref') + ssarepr = flatten_graph(graph, {'int': regalloc, + 'ref': regalloc2}) + assert_format(ssarepr, expected) def test_regalloc_simple(self): def f(a, b): @@ -240,3 +243,61 @@ L1: int_return %i2 """, transform=True) + + def test_regalloc_bug_2(self): + class FakeDescr(AbstractDescr): + def __repr__(self): + return '' + class FakeCallControl: + def guess_call_kind(self, op): + return 'residual' + def getcalldescr(self, op): + return FakeDescr() + def calldescr_canraise(self, calldescr): + return True + class FooError(Exception): + def __init__(self, num): + self.num = num + def g(n): + if n > 100: + raise FooError(n) + return lltype.nullptr(llmemory.GCREF.TO) + def foo(e): + print "hello" + return e + def bar(e): + print "world" + return e + def f(n, kref): + kref2 = bar(kref) + try: + return g(n) + except FooError, e: + if foo(e): + return kref + else: + return kref2 + graph = self.make_graphs(f, [5, lltype.nullptr(llmemory.GCREF.TO)])[0] + # this used to produce bogus code, containing these two + # lines in the following broken order: + # last_exc_value -> %r0 + # ref_copy %r0 -> %r2 -- but expect to read the old value of %r0! + self.check_assembler(graph, """ + residual_call_r_r $<* fn bar>, , R[%r0] -> %r1 + -live- + residual_call_ir_r $<* fn g>, , I[%i0], R[] -> %r2 + -live- + catch_exception L1 + ref_return %r2 + --- + L1: + goto_if_exception_mismatch $<* struct object_vtable>, L2 + ref_copy %r0 -> %r2 + last_exc_value -> %r0 + residual_call_r_r $<* fn foo>, , R[%r0] -> %r0 + -live- + ref_return %r2 + --- + L2: + reraise + """, transform=True, callcontrol=FakeCallControl()) From arigo at codespeak.net Mon May 31 15:49:49 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 31 May 2010 15:49:49 +0200 (CEST) Subject: [pypy-svn] r74971 - pypy/trunk/pypy/module/bz2 Message-ID: <20100531134949.2D65636C227@codespeak.net> Author: arigo Date: Mon May 31 15:49:47 2010 New Revision: 74971 Modified: pypy/trunk/pypy/module/bz2/interp_bz2.py Log: Mess: forgot a free(). Modified: pypy/trunk/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/trunk/pypy/module/bz2/interp_bz2.py (original) +++ pypy/trunk/pypy/module/bz2/interp_bz2.py Mon May 31 15:49:47 2010 @@ -569,6 +569,7 @@ temp.append(data) out_bufsize = _new_buffer_size(out_bufsize) + lltype.free(out_buf, flavor='raw') out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize, flavor='raw', zero=True) self.bzs.c_next_out = out_buf From antocuni at codespeak.net Mon May 31 15:53:50 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 31 May 2010 15:53:50 +0200 (CEST) Subject: [pypy-svn] r74972 - in pypy/branch/sys-prefix: lib/pypy1.2/lib_pypy lib/pypy1.2/lib_pypy/app_test lib/pypy1.2/lib_pypy/pypy_test lib/pypy1.2/lib_pypy/test2 pypy/config pypy/module/test_lib_pypy Message-ID: <20100531135350.6F866282B90@codespeak.net> Author: antocuni Date: Mon May 31 15:53:48 2010 New Revision: 74972 Added: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/README.tests pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/pypy_test/ - copied from r74962, pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/pypy_test/test_collections.py - copied unchanged from r74969, pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_collections.py pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/pypy_test/test_runpy.py - copied unchanged from r74964, pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/test_runpy.py pypy/branch/sys-prefix/pypy/module/test_lib_pypy/ - copied from r74962, pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/test2/ Removed: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/app_test/ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/test2/ pypy/branch/sys-prefix/pypy/module/test_lib_pypy/autopath.py pypy/branch/sys-prefix/pypy/module/test_lib_pypy/test_collections.py Modified: pypy/branch/sys-prefix/pypy/config/pypyoption.py pypy/branch/sys-prefix/pypy/module/test_lib_pypy/test_array.py pypy/branch/sys-prefix/pypy/module/test_lib_pypy/test_stackless.py Log: Tidy up a bit the mess of lib_pypy tests: - despite the name, the old "app_test" directory contained direct tests; hence, it has been renamed to pypy_test (we cannot name it "test" because else it would hide the test package of lib-python) - test2 contained AppLevel tests, that need to stay somewhere under pypy/. Move them to module/test_lib_pypy Added: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/README.tests ============================================================================== --- (empty file) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/README.tests Mon May 31 15:53:48 2010 @@ -0,0 +1,9 @@ +There are two sets of tests for lib_pypy: + + - pypy_test contains direct tests for the module in this directory, which + are supposed to be run by either cpython or pypy (or even jython or + ironpython, in theory) + + - moreover, there are app-level tests that check how the modules work on + py.py. Being app-level, they need to stay somewhere below the main pypy/ + package, so they are in module/test_lib_pypy Modified: pypy/branch/sys-prefix/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/sys-prefix/pypy/config/pypyoption.py (original) +++ pypy/branch/sys-prefix/pypy/config/pypyoption.py Mon May 31 15:53:48 2010 @@ -8,7 +8,8 @@ modulepath = py.path.local(__file__).dirpath().dirpath().join("module") all_modules = [p.basename for p in modulepath.listdir() if p.check(dir=True, dotfile=False) - and p.join('__init__.py').check()] + and p.join('__init__.py').check() + and not p.basename.startswith('test')] essential_modules = dict.fromkeys( ["exceptions", "_file", "sys", "__builtin__", "posix"] Modified: pypy/branch/sys-prefix/pypy/module/test_lib_pypy/test_array.py ============================================================================== --- pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/test2/test_array.py (original) +++ pypy/branch/sys-prefix/pypy/module/test_lib_pypy/test_array.py Mon May 31 15:53:48 2010 @@ -1,6 +1,5 @@ # minimal tests. See also lib-python/modified-2.4.1/test/test_array. -import autopath import py from py.test import raises import struct @@ -74,10 +73,11 @@ class TestArrayOnTopOfCPython(BaseArrayTests): def setup_class(cls): + from pypy.tool.lib_pypy import get_lib_pypy_dir if not hasattr(struct, 'pack_into'): py.test.skip("requires CPython >= 2.5") import new - path = py.path.local(autopath.this_dir).dirpath().join('array.py') + path = get_lib_pypy_dir().join('array.py') myarraymodule = new.module('array') execfile(str(path), myarraymodule.__dict__) cls.array = myarraymodule Modified: pypy/branch/sys-prefix/pypy/module/test_lib_pypy/test_stackless.py ============================================================================== --- pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/test2/test_stackless.py (original) +++ pypy/branch/sys-prefix/pypy/module/test_lib_pypy/test_stackless.py Mon May 31 15:53:48 2010 @@ -3,7 +3,7 @@ class AppTest_Stackless: def setup_class(cls): - space = gettestobjspace(usemodules=('_stackless',)) + space = gettestobjspace(usemodules=('_stackless', '_socket')) cls.space = space # cannot test the unpickle part on top of py.py cls.w_can_unpickle = space.wrap(bool(option.runappdirect)) From antocuni at codespeak.net Mon May 31 16:01:44 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 31 May 2010 16:01:44 +0200 (CEST) Subject: [pypy-svn] r74973 - in pypy/branch/sys-prefix: lib/pypy1.2/lib_pypy/pypy_test pypy/module/test_lib_pypy Message-ID: <20100531140144.93311282B9C@codespeak.net> Author: antocuni Date: Mon May 31 16:01:43 2010 New Revision: 74973 Added: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/pypy_test/test_grp_extra.py - copied, changed from r74972, pypy/branch/sys-prefix/pypy/module/test_lib_pypy/test_grp_extra.py Removed: pypy/branch/sys-prefix/pypy/module/test_lib_pypy/test_grp_extra.py Log: this test should really go here Copied: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/pypy_test/test_grp_extra.py (from r74972, pypy/branch/sys-prefix/pypy/module/test_lib_pypy/test_grp_extra.py) ============================================================================== --- pypy/branch/sys-prefix/pypy/module/test_lib_pypy/test_grp_extra.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/pypy_test/test_grp_extra.py Mon May 31 16:01:43 2010 @@ -1,6 +1,7 @@ +from __future__ import absolute_import import py try: - from pypy.lib import grp + from .. import grp except ImportError: py.test.skip("No grp module on this platform") From arigo at codespeak.net Mon May 31 16:04:31 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 31 May 2010 16:04:31 +0200 (CEST) Subject: [pypy-svn] r74974 - pypy/trunk/pypy/module/bz2/test Message-ID: <20100531140431.E533736C22E@codespeak.net> Author: arigo Date: Mon May 31 16:04:30 2010 New Revision: 74974 Added: pypy/trunk/pypy/module/bz2/test/largetest.bz2 (contents, props changed) pypy/trunk/pypy/module/bz2/test/test_large.py Modified: pypy/trunk/pypy/module/bz2/test/test_bz2_compdecomp.py Log: issue541 in-progress Add the test file, and found out that the multiple buffers code in interp_bz2 is rather broken anyway, as shown by random crashes when making the buffer size small. Added: pypy/trunk/pypy/module/bz2/test/largetest.bz2 ============================================================================== Binary file. No diff available. Modified: pypy/trunk/pypy/module/bz2/test/test_bz2_compdecomp.py ============================================================================== --- pypy/trunk/pypy/module/bz2/test/test_bz2_compdecomp.py (original) +++ pypy/trunk/pypy/module/bz2/test/test_bz2_compdecomp.py Mon May 31 16:04:30 2010 @@ -1,5 +1,6 @@ from pypy.conftest import gettestobjspace from pypy.module.bz2.test.support import CheckAllocation +from pypy.module.bz2 import interp_bz2 import os, py HUGE_OK = False @@ -27,6 +28,13 @@ mod.DATA = DATA mod.BUGGY_DATA = py.path.local(__file__).dirpath().join('data.bz2').read() mod.decompress = decompress + # + # For tests, patch the value of SMALLCHUNK + mod.OLD_SMALLCHUNK = interp_bz2.SMALLCHUNK + interp_bz2.SMALLCHUNK = 32 + +def teardown_module(mod): + interp_bz2.SMALLCHUNK = mod.OLD_SMALLCHUNK class AppTestBZ2Compressor(CheckAllocation): def setup_class(cls): Added: pypy/trunk/pypy/module/bz2/test/test_large.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/bz2/test/test_large.py Mon May 31 16:04:30 2010 @@ -0,0 +1,14 @@ +import py +from pypy.conftest import gettestobjspace + + +class AppTestBZ2File: + def setup_class(cls): + cls.space = gettestobjspace(usemodules=('bz2',)) + largetest_bz2 = py.path.local(__file__).dirpath().join("largetest.bz2") + cls.w_compressed_data = cls.space.wrap(largetest_bz2.read()) + + def test_decompress(self): + from bz2 import decompress + result = decompress(self.compressed_data) + assert len(result) == 901179 From arigo at codespeak.net Mon May 31 16:18:12 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 31 May 2010 16:18:12 +0200 (CEST) Subject: [pypy-svn] r74975 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100531141812.021F2282BD8@codespeak.net> Author: arigo Date: Mon May 31 16:18:11 2010 New Revision: 74975 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: Add commented out debugging dumps. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Mon May 31 16:18:11 2010 @@ -154,6 +154,8 @@ if resultbox.type == history.INT: self.registers_i[target_index] = resultbox elif resultbox.type == history.REF: + #debug_print(' ->', + # llmemory.cast_ptr_to_adr(resultbox.getref_base())) self.registers_r[target_index] = resultbox elif resultbox.type == history.FLOAT: self.registers_f[target_index] = resultbox @@ -909,6 +911,7 @@ while True: pc = self.pc op = ord(self.bytecode[pc]) + #debug_print(self.jitcode.name, pc) #print staticdata.opcode_names[op] staticdata.opcode_implementations[op](self, pc) except ChangeFrame: From antocuni at codespeak.net Mon May 31 17:02:23 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 31 May 2010 17:02:23 +0200 (CEST) Subject: [pypy-svn] r74976 - in pypy/branch/sys-prefix: lib/pypy1.2/lib_pypy/ctypes_config_cache pypy/interpreter/test pypy/rlib pypy/tool pypy/tool/test pypy/translator/goal pypy/translator/sandbox Message-ID: <20100531150223.33D88282B90@codespeak.net> Author: antocuni Date: Mon May 31 17:02:20 2010 New Revision: 74976 Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/ctypes_config_cache/rebuild.py pypy/branch/sys-prefix/pypy/interpreter/test/test_module.py pypy/branch/sys-prefix/pypy/rlib/rmd5.py pypy/branch/sys-prefix/pypy/rlib/rsha.py pypy/branch/sys-prefix/pypy/rlib/rzipfile.py pypy/branch/sys-prefix/pypy/tool/compat.py pypy/branch/sys-prefix/pypy/tool/lib_pypy.py pypy/branch/sys-prefix/pypy/tool/test/test_lib_pypy.py pypy/branch/sys-prefix/pypy/translator/goal/targetpypystandalone.py pypy/branch/sys-prefix/pypy/translator/sandbox/pypy_interact.py pypy/branch/sys-prefix/pypy/translator/sandbox/sandlib.py Log: remove most of the remaining references to pypy/lib, and make them pointing to lib_pypy Modified: pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/ctypes_config_cache/rebuild.py ============================================================================== --- pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/ctypes_config_cache/rebuild.py (original) +++ pypy/branch/sys-prefix/lib/pypy1.2/lib_pypy/ctypes_config_cache/rebuild.py Mon May 31 17:02:20 2010 @@ -4,7 +4,10 @@ # hack: we cannot directly import autopath, as we are outside the pypy # package. However, we pretend to be inside pypy/tool and manually run it, to # get the correct path -autopath_py = '../../../../pypy/tool/autopath.py' +import os.path +this_dir = os.path.dirname(__file__) +autopath_py = os.path.join(this_dir, '../../../../pypy/tool/autopath.py') +autopath_py = os.path.abspath(autopath_py) execfile(autopath_py, dict(__name__='autopath', __file__=autopath_py)) import os, sys Modified: pypy/branch/sys-prefix/pypy/interpreter/test/test_module.py ============================================================================== --- pypy/branch/sys-prefix/pypy/interpreter/test/test_module.py (original) +++ pypy/branch/sys-prefix/pypy/interpreter/test/test_module.py Mon May 31 17:02:20 2010 @@ -54,7 +54,7 @@ r = repr(sys) assert r == "" - import _pypy_interact # known to be in pypy/lib + import _pypy_interact # known to be in lib_pypy r = repr(_pypy_interact) assert (r.startswith(" Author: getxsick Date: Mon May 31 17:25:38 2010 New Revision: 74977 Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py Log: more asserts Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py Mon May 31 17:25:38 2010 @@ -28,8 +28,13 @@ cls.w_lib_name = space.wrap(cls.preprare_c_example()) cls.w_libc_name = space.wrap(get_libc_name()) - def test_raw_callable(self): + def test_call(self): import jitffi lib = jitffi.CDLL(self.lib_name) + res = lib.call('add_integers', 1, 2) - assert res == 3 + assert 3 == res + res = lib.call('add_integers', -1, 2) + assert 1 == res + res = lib.call('add_integers', 0, 0) + assert 0 == res From getxsick at codespeak.net Mon May 31 18:35:59 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Mon, 31 May 2010 18:35:59 +0200 (CEST) Subject: [pypy-svn] r74978 - pypy/trunk/pypy/module/cpyext Message-ID: <20100531163559.BBE93282B90@codespeak.net> Author: getxsick Date: Mon May 31 18:35:57 2010 New Revision: 74978 Modified: pypy/trunk/pypy/module/cpyext/api.py pypy/trunk/pypy/module/cpyext/methodobject.py Log: clean up import statements Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Mon May 31 18:35:57 2010 @@ -15,8 +15,8 @@ from pypy.translator import platform from pypy.module.cpyext.state import State from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter.gateway import ObjSpace, unwrap_spec +from pypy.interpreter.baseobjspace import W_Root, ObjSpace +from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter.nestedscope import Cell from pypy.interpreter.module import Module from pypy.interpreter.function import StaticMethod Modified: pypy/trunk/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/methodobject.py (original) +++ pypy/trunk/pypy/module/cpyext/methodobject.py Mon May 31 18:35:57 2010 @@ -1,14 +1,13 @@ -from pypy.interpreter.baseobjspace import Wrappable, W_Root +from pypy.interpreter.baseobjspace import Wrappable, W_Root, ObjSpace from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.interpreter.gateway import ObjSpace, W_Root from pypy.interpreter.argument import Arguments from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.function import BuiltinFunction, Method, StaticMethod from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.pyobject import (PyObject, from_ref, make_ref, make_typedescr, - Py_DecRef) +from pypy.module.cpyext.pyobject import (PyObject, from_ref, make_ref, + make_typedescr, Py_DecRef) from pypy.module.cpyext.api import ( generic_cpy_call, cpython_api, PyObject, cpython_struct, METH_KEYWORDS, METH_O, CONST_STRING, METH_CLASS, METH_STATIC, METH_COEXIST, METH_NOARGS, From getxsick at codespeak.net Mon May 31 18:41:09 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Mon, 31 May 2010 18:41:09 +0200 (CEST) Subject: [pypy-svn] r74979 - in pypy/branch/fast-ctypes/pypy/module/jitffi: . test Message-ID: <20100531164109.13D80282B90@codespeak.net> Author: getxsick Date: Mon May 31 18:41:07 2010 New Revision: 74979 Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py Log: return correct type of result Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/interp_jitffi.py Mon May 31 18:41:07 2010 @@ -5,7 +5,8 @@ from pypy.rlib import rdynload from pypy.rpython.lltypesystem import rffi, lltype from pypy.jit.backend.x86.runner import CPU -from pypy.jit.metainterp.history import LoopToken, BasicFailDescr, BoxInt +from pypy.jit.metainterp.history import LoopToken, BasicFailDescr +from pypy.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr from pypy.jit.metainterp.resoperation import ResOperation, rop from pypy.jit.metainterp.typesystem import deref @@ -20,8 +21,18 @@ self.space = space self.cpu = CPU(None, None) - def call(self, space, func, a, b): # XXX temporary fixed number of func args (ints) - # result_type argument? + def call(self, space, func, a, b, res_type): # XXX temporary fixed number of func args (ints) + if res_type == 'int': + bres = BoxInt() + elif res_type == 'float': + bres = BoxFloat() + elif res_type == 'ref': + bres = BoxPtr() + elif res_type == 'void': + bres = None + else: + raise ValueError(res_type) + try: addr = rffi.cast(lltype.Signed, rdynload.dlsym(self.lib, func)) except KeyError: @@ -31,7 +42,6 @@ bfuncaddr = BoxInt(addr) barg0 = BoxInt(a) barg1 = BoxInt(b) - bres = BoxInt() FPTR = lltype.Ptr(lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)) @@ -56,8 +66,19 @@ self.guard_failed = False else: self.guard_failed = True - return space.wrap(BoxInt(self.cpu.get_latest_value_int(0)).getint()) - call.unwrap_spec = ['self', ObjSpace, str, int, int] + + if res_type == 'int': + r = BoxInt(self.cpu.get_latest_value_int(0)).getint() + elif res_type == 'float': + r = BoxFloat(self.cpu.get_latest_value_float(0)).getfloat() + elif res_type == 'ref': + r = BoxPtr(self.cpu.get_latest_value_ref(0)).getref() + elif res_type == 'void': + r = None + else: + raise ValueError(res_type) + return space.wrap(r) + call.unwrap_spec = ['self', ObjSpace, str, int, int, str] def descr_new_cdll(space, w_type, name): try: Modified: pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py ============================================================================== --- pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py (original) +++ pypy/branch/fast-ctypes/pypy/module/jitffi/test/test_jitffi.py Mon May 31 18:41:07 2010 @@ -14,7 +14,13 @@ { return a+b; } - ''')) + + float return_float(int a, int b) + { + return a+b; + } + ''' + )) symbols = ["add_integers"] eci = ExternalCompilationInfo(export_symbols=symbols) @@ -32,9 +38,14 @@ import jitffi lib = jitffi.CDLL(self.lib_name) - res = lib.call('add_integers', 1, 2) + res = lib.call('add_integers', 1, 2, 'int') assert 3 == res - res = lib.call('add_integers', -1, 2) + assert isinstance(res, int) + res = lib.call('add_integers', -1, 2, 'int') assert 1 == res - res = lib.call('add_integers', 0, 0) + res = lib.call('add_integers', 0, 0, 'int') assert 0 == res + + res = lib.call('return_float', 1, 2, 'float') + assert 3.0 == res + assert isinstance(res, float) From arigo at codespeak.net Mon May 31 20:37:47 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 31 May 2010 20:37:47 +0200 (CEST) Subject: [pypy-svn] r74980 - in pypy/trunk/pypy/module/bz2: . test Message-ID: <20100531183747.43B6F282B90@codespeak.net> Author: arigo Date: Mon May 31 20:37:43 2010 New Revision: 74980 Modified: pypy/trunk/pypy/module/bz2/interp_bz2.py pypy/trunk/pypy/module/bz2/test/test_large.py Log: issue541 resolved Kill a lot of "approximate code duplication" in interp_bz2, i.e. a lot of code that does all more or less the same thing with more or less bugs. Replace it with a single separate implementation. Fixes the crashes. Modified: pypy/trunk/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/trunk/pypy/module/bz2/interp_bz2.py (original) +++ pypy/trunk/pypy/module/bz2/interp_bz2.py Mon May 31 20:37:43 2010 @@ -172,14 +172,58 @@ space.wrap("wrong sequence of bz2 library commands used")) def _new_buffer_size(current_size): - if current_size > SMALLCHUNK: - # keep doubling until we reach BIGCHUNK - # then keep adding BIGCHUNK - if current_size <= BIGCHUNK: - return current_size + current_size + # keep doubling until we reach BIGCHUNK; then the buffer size is no + # longer increased + if current_size < BIGCHUNK: + return current_size + current_size + return current_size + +# ____________________________________________________________ + +class OutBuffer(object): + """Handler for the output buffer. A bit custom code trying to + encapsulate the logic of setting up the fields of 'bzs' and + allocating raw memory as needed. + """ + def __init__(self, bzs, initial_size=SMALLCHUNK): + # when the constructor is called, allocate a piece of memory + # of length 'piece_size' and make bzs ready to dump there. + self.temp = [] + self.bzs = bzs + self._allocate_chunk(initial_size) + + def _allocate_chunk(self, size): + self.raw_buf, self.gc_buf = rffi.alloc_buffer(size) + self.current_size = size + self.bzs.c_next_out = self.raw_buf + rffi.setintfield(self.bzs, 'c_avail_out', size) + + def _get_chunk(self, chunksize): + assert 0 <= chunksize <= self.current_size + raw_buf = self.raw_buf + gc_buf = self.gc_buf + s = rffi.str_from_buffer(raw_buf, gc_buf, self.current_size, chunksize) + rffi.keep_buffer_alive_until_here(raw_buf, gc_buf) + self.current_size = 0 + return s + + def prepare_next_chunk(self): + size = self.current_size + self.temp.append(self._get_chunk(size)) + self._allocate_chunk(_new_buffer_size(size)) + + def make_result_string(self): + count_unoccupied = rffi.getintfield(self.bzs, 'c_avail_out') + s = self._get_chunk(self.current_size - count_unoccupied) + if self.temp: + self.temp.append(s) + return ''.join(self.temp) else: - return current_size + BIGCHUNK - return current_size + SMALLCHUNK + return s + + def free(self): + if self.current_size > 0: + rffi.keep_buffer_alive_until_here(self.raw_buf, self.gc_buf) # ____________________________________________________________ # @@ -488,10 +532,7 @@ raise OperationError(self.space.w_ValueError, self.space.wrap("this object was already flushed")) - out_bufsize = SMALLCHUNK - out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize, flavor='raw', - zero=True) - + out = OutBuffer(self.bzs) in_bufsize = datasize in_buf = lltype.malloc(rffi.CCHARP.TO, in_bufsize, flavor='raw') for i in range(datasize): @@ -501,10 +542,7 @@ self.bzs.c_next_in = in_buf rffi.setintfield(self.bzs, 'c_avail_in', in_bufsize) - self.bzs.c_next_out = out_buf - rffi.setintfield(self.bzs, 'c_avail_out', out_bufsize) - - temp = [] + while True: bzerror = BZ2_bzCompress(self.bzs, BZ_RUN) if bzerror != BZ_RUN_OK: @@ -513,29 +551,13 @@ if rffi.getintfield(self.bzs, 'c_avail_in') == 0: break elif rffi.getintfield(self.bzs, 'c_avail_out') == 0: - total_out = _bzs_total_out(self.bzs) - data = "".join([out_buf[i] for i in range(total_out)]) - temp.append(data) - - out_bufsize = _new_buffer_size(out_bufsize) - lltype.free(out_buf, flavor='raw') - out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize, - flavor='raw', zero=True) - self.bzs.c_next_out = out_buf - rffi.setintfield(self.bzs, 'c_avail_out', out_bufsize) - - if temp: - total_out = _bzs_total_out(self.bzs) - data = "".join([out_buf[i] for i in range(total_out - len(temp[0]))]) - temp.append(data) - return self.space.wrap("".join(temp)) + out.prepare_next_chunk() - total_out = _bzs_total_out(self.bzs) - res = "".join([out_buf[i] for i in range(total_out)]) + res = out.make_result_string() return self.space.wrap(res) finally: lltype.free(in_buf, flavor='raw') - lltype.free(out_buf, flavor='raw') + out.free() compress.unwrap_spec = ['self', 'bufferstr'] @@ -545,18 +567,8 @@ self.space.wrap("this object was already flushed")) self.running = False - out_bufsize = SMALLCHUNK - out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize, flavor='raw', - zero=True) - + out = OutBuffer(self.bzs) try: - - self.bzs.c_next_out = out_buf - rffi.setintfield(self.bzs, 'c_avail_out', out_bufsize) - - total_out = _bzs_total_out(self.bzs) - - temp = [] while True: bzerror = BZ2_bzCompress(self.bzs, BZ_FINISH) if bzerror == BZ_STREAM_END: @@ -565,30 +577,12 @@ _catch_bz2_error(self.space, bzerror) if rffi.getintfield(self.bzs, 'c_avail_out') == 0: - data = "".join([out_buf[i] for i in range(_bzs_total_out(self.bzs))]) - temp.append(data) - - out_bufsize = _new_buffer_size(out_bufsize) - lltype.free(out_buf, flavor='raw') - out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize, - flavor='raw', zero=True) - self.bzs.c_next_out = out_buf - rffi.setintfield(self.bzs, 'c_avail_out', out_bufsize) - + out.prepare_next_chunk() - if rffi.getintfield(self.bzs, 'c_avail_out'): - size = _bzs_total_out(self.bzs) - total_out - res = "".join([out_buf[i] for i in range(size)]) - else: - total_out = _bzs_total_out(self.bzs) - res = "".join([out_buf[i] for i in range(total_out)]) - if not temp: - return self.space.wrap(res) - else: - temp.append(res) - return self.space.wrap("".join(temp)) + res = out.make_result_string() + return self.space.wrap(res) finally: - lltype.free(out_buf, flavor='raw') + out.free() flush.unwrap_spec = ['self'] W_BZ2Compressor.typedef = TypeDef("BZ2Compressor", @@ -653,20 +647,11 @@ for i in range(in_bufsize): in_buf[i] = data[i] - out_bufsize = SMALLCHUNK - out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize, flavor='raw', - zero=True) - + out = OutBuffer(self.bzs) try: - self.bzs.c_next_in = in_buf rffi.setintfield(self.bzs, 'c_avail_in', in_bufsize) - self.bzs.c_next_out = out_buf - rffi.setintfield(self.bzs, 'c_avail_out', out_bufsize) - total_out = _bzs_total_out(self.bzs) - - temp = [] while True: bzerror = BZ2_bzDecompress(self.bzs) if bzerror == BZ_STREAM_END: @@ -681,31 +666,13 @@ if rffi.getintfield(self.bzs, 'c_avail_in') == 0: break elif rffi.getintfield(self.bzs, 'c_avail_out') == 0: - new_total_out = _bzs_total_out(self.bzs) - extra_out = new_total_out - total_out - assert out_bufsize >= extra_out - total_out = new_total_out - data = "".join([out_buf[i] for i in range(extra_out)]) - temp.append(data) - lltype.free(out_buf, flavor='raw') - out_bufsize = _new_buffer_size(out_bufsize) - out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize, flavor='raw') - self.bzs.c_next_out = out_buf - rffi.setintfield(self.bzs, 'c_avail_out', out_bufsize) - - new_total_out = _bzs_total_out(self.bzs) - extra_out = new_total_out - total_out - - if temp: - data = "".join([out_buf[i] for i in range(extra_out)]) - temp.append(data) - return self.space.wrap("".join(temp)) + out.prepare_next_chunk() - res = "".join([out_buf[i] for i in range(extra_out)]) + res = out.make_result_string() return self.space.wrap(res) finally: lltype.free(in_buf, flavor='raw') - lltype.free(out_buf, flavor='raw') + out.free() decompress.unwrap_spec = ['self', 'bufferstr'] @@ -733,10 +700,8 @@ in_bufsize = len(data) # conforming to bz2 manual, this is large enough to fit compressed # data in one shot. We will check it later anyway. - out_bufsize = in_bufsize + (in_bufsize / 100 + 1) + 600 - - out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize, flavor='raw', - zero=True) + out = OutBuffer(bzs, in_bufsize + (in_bufsize / 100 + 1) + 600) + in_buf = lltype.malloc(rffi.CCHARP.TO, in_bufsize, flavor='raw') for i in range(in_bufsize): in_buf[i] = data[i] @@ -744,15 +709,11 @@ try: bzs.c_next_in = in_buf rffi.setintfield(bzs, 'c_avail_in', in_bufsize) - bzs.c_next_out = out_buf - rffi.setintfield(bzs, 'c_avail_out', out_bufsize) bzerror = BZ2_bzCompressInit(bzs, compresslevel, 0, 0) if bzerror != BZ_OK: _catch_bz2_error(space, bzerror) - - total_out = _bzs_total_out(bzs) - temp = [] + while True: bzerror = BZ2_bzCompress(bzs, BZ_FINISH) if bzerror == BZ_STREAM_END: @@ -762,32 +723,15 @@ _catch_bz2_error(space, bzerror) if rffi.getintfield(bzs, 'c_avail_out') == 0: - data = "".join([out_buf[i] for i in range(_bzs_total_out(bzs))]) - temp.append(data) + out.prepare_next_chunk() - lltype.free(out_buf, flavor='raw') - out_bufsize = _new_buffer_size(out_bufsize) - out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize, - flavor='raw', zero=True) - bzs.c_next_out = out_buf - rffi.setintfield(bzs, 'c_avail_out', out_bufsize) - - if temp: - res = "".join(temp) - - if rffi.getintfield(bzs, 'c_avail_out'): - size = _bzs_total_out(bzs) - total_out - res = "".join([out_buf[i] for i in range(size)]) - else: - total_out = _bzs_total_out(bzs) - res = "".join([out_buf[i] for i in range(total_out)]) - + res = out.make_result_string() BZ2_bzCompressEnd(bzs) return space.wrap(res) finally: lltype.free(bzs, flavor='raw') lltype.free(in_buf, flavor='raw') - lltype.free(out_buf, flavor='raw') + out.free() compress.unwrap_spec = [ObjSpace, 'bufferstr', int] def decompress(space, data): @@ -805,22 +749,15 @@ for i in range(in_bufsize): in_buf[i] = data[i] - out_bufsize = SMALLCHUNK - out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize, flavor='raw', - zero=True) + out = OutBuffer(bzs) try: - - bzs.c_next_in = in_buf rffi.setintfield(bzs, 'c_avail_in', in_bufsize) - bzs.c_next_out = out_buf - rffi.setintfield(bzs, 'c_avail_out', out_bufsize) bzerror = BZ2_bzDecompressInit(bzs, 0, 0) if bzerror != BZ_OK: _catch_bz2_error(space, bzerror) - temp = [] while True: bzerror = BZ2_bzDecompress(bzs) if bzerror == BZ_STREAM_END: @@ -834,29 +771,13 @@ raise OperationError(space.w_ValueError, space.wrap("couldn't find end of stream")) elif rffi.getintfield(bzs, 'c_avail_out') == 0: - total_out = _bzs_total_out(bzs) - data = "".join([out_buf[i] for i in range(total_out)]) - temp.append(data) - - lltype.free(out_buf, flavor='raw') - out_bufsize = _new_buffer_size(out_bufsize) - out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize, - flavor='raw', zero=True) - bzs.c_next_out = out_buf - rffi.setintfield(bzs, 'c_avail_out', out_bufsize) - - total_out = _bzs_total_out(bzs) - if temp: - data = "".join([out_buf[i] for i in range(total_out - len(temp[0]))]) - temp.append(data) - res = "".join(temp) - else: - res = "".join([out_buf[i] for i in range(total_out) if out_buf[i] != '\x00']) - + out.prepare_next_chunk() + + res = out.make_result_string() BZ2_bzDecompressEnd(bzs) return space.wrap(res) finally: lltype.free(bzs, flavor='raw') - lltype.free(out_buf, flavor='raw') lltype.free(in_buf, flavor='raw') + out.free() decompress.unwrap_spec = [ObjSpace, 'bufferstr'] Modified: pypy/trunk/pypy/module/bz2/test/test_large.py ============================================================================== --- pypy/trunk/pypy/module/bz2/test/test_large.py (original) +++ pypy/trunk/pypy/module/bz2/test/test_large.py Mon May 31 20:37:43 2010 @@ -1,9 +1,11 @@ import py -from pypy.conftest import gettestobjspace +from pypy.conftest import gettestobjspace, option class AppTestBZ2File: def setup_class(cls): + if not option.runappdirect: + py.test.skip("skipping this very slow test; try 'pypy-c -A'") cls.space = gettestobjspace(usemodules=('bz2',)) largetest_bz2 = py.path.local(__file__).dirpath().join("largetest.bz2") cls.w_compressed_data = cls.space.wrap(largetest_bz2.read()) From arigo at codespeak.net Mon May 31 22:45:46 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 31 May 2010 22:45:46 +0200 (CEST) Subject: [pypy-svn] r74981 - in pypy/branch/blackhole-improvement/pypy/jit/metainterp: . test Message-ID: <20100531204546.A9AE6282B9C@codespeak.net> Author: arigo Date: Mon May 31 22:45:44 2010 New Revision: 74981 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py Log: Test and fix: the nice caching of blackhole interpreters was only working on tests run by interp_operations()! Use it also on meta_interp() tests (and in the real translation). Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Mon May 31 22:45:44 2010 @@ -1281,7 +1281,10 @@ def _run_forever(blackholeinterp, current_exc): while True: - current_exc = blackholeinterp._resume_mainloop(current_exc) + try: + current_exc = blackholeinterp._resume_mainloop(current_exc) + finally: + blackholeinterp.builder.release_interp(blackholeinterp) blackholeinterp = blackholeinterp.nextblackholeinterp def resume_in_blackhole(metainterp_sd, resumedescr, all_virtuals=None): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py Mon May 31 22:45:44 2010 @@ -2,8 +2,9 @@ from pypy.rlib.jit import JitDriver from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin from pypy.jit.metainterp.blackhole import BlackholeInterpBuilder +from pypy.jit.metainterp.blackhole import BlackholeInterpreter from pypy.jit.metainterp.blackhole import convert_and_run_from_pyjitpl -from pypy.jit.metainterp import history +from pypy.jit.metainterp import history, pyjitpl from pypy.jit.codewriter.assembler import JitCode from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.llinterp import LLException @@ -140,3 +141,71 @@ d = py.test.raises(MyMetaInterp.staticdata.DoneWithThisFrameInt, convert_and_run_from_pyjitpl, MyMetaInterp()) assert d.value.args == (42,) + + +class TestBlackhole(LLJitMixin): + + def test_blackholeinterp_cache(self): + myjitdriver = JitDriver(greens = [], reds = ['x', 'y']) + def choices(x): + if x == 2: return 10 + if x == 3: return 199 + if x == 4: return 124 + if x == 5: return -521 + if x == 6: return 8917 + if x == 7: return -387 + return 34871 + def f(x): + y = 0 + while x > 0: + myjitdriver.can_enter_jit(x=x, y=y) + myjitdriver.jit_merge_point(x=x, y=y) + y += choices(x) + x -= 1 + return y + # + seen = [] + def my_copy_constants(self, *args): + seen.append(1) + return org_copy_constants(self, *args) + org_copy_constants = BlackholeInterpreter.copy_constants + BlackholeInterpreter.copy_constants = my_copy_constants + try: + res = self.meta_interp(f, [7], repeat=7) + finally: + BlackholeInterpreter.copy_constants = org_copy_constants + # + assert res == sum([choices(x) for x in range(1, 8)]) + builder = pyjitpl._warmrunnerdesc.metainterp_sd.blackholeinterpbuilder + assert builder.num_interpreters == 2 + assert len(seen) == 2 * 3 + + def test_blackholeinterp_cache_exc(self): + myjitdriver = JitDriver(greens = [], reds = ['x', 'y']) + class FooError(Exception): + def __init__(self, num): + self.num = num + def choices(x): + if x == 2: raise FooError(10) + if x == 3: raise FooError(199) + if x == 4: raise FooError(124) + if x == 5: raise FooError(-521) + if x == 6: raise FooError(8917) + if x == 7: raise FooError(-387) + raise FooError(34871) + def f(x): + y = 0 + while x > 0: + myjitdriver.can_enter_jit(x=x, y=y) + myjitdriver.jit_merge_point(x=x, y=y) + try: + choices(x) + except FooError, e: + y += e.num + x -= 1 + return y + res = self.meta_interp(f, [7], repeat=7) + assert res == sum([py.test.raises(FooError, choices, x).value.num + for x in range(1, 8)]) + builder = pyjitpl._warmrunnerdesc.metainterp_sd.blackholeinterpbuilder + assert builder.num_interpreters == 2 From fijal at codespeak.net Mon May 31 23:58:45 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 31 May 2010 23:58:45 +0200 (CEST) Subject: [pypy-svn] r74985 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100531215845.46897282B9C@codespeak.net> Author: fijal Date: Mon May 31 23:58:43 2010 New Revision: 74985 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Log: Fix translation (?) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Mon May 31 23:58:43 2010 @@ -344,7 +344,7 @@ # these are all prebuilt constants anyway. for i in range(self.jitcode.num_regs_r()): self.registers_r[i] = NULL - self.exception_last_value = None + self.exception_last_value = lltype.nullptr(rclass.OBJECT) def get_current_position_info(self): return self.jitcode.get_live_vars_info(self.position)